From dcfadceecb20c6d838ab98b9509ee023676377d4 Mon Sep 17 00:00:00 2001 From: lancelui-amzn Date: Fri, 15 May 2026 12:06:16 -0500 Subject: [PATCH] time range support for profiling * time range support for profiling * standardize to TimeEnum, time series diffs now relative to record start time * update profiler analytical rules to use processed data accessor * bugfix per profile caching * simplify cache, variable renaming, remove InitParams time_now * start collection immediately in collect_data_serial --- src/analytics/mod.rs | 14 +- .../key_value_key_expected_rule.rs | 2 +- .../key_value_key_run_comparison_rule.rs | 2 +- .../profile_metadata_comparison_rule.rs | 84 ++-- .../profile_metadata_expected_rule.rs | 27 +- .../profile_stack_frame_threshold_rule.rs | 33 +- .../time_series_data_point_threshold_rule.rs | 2 +- ...e_series_stat_intra_run_comparison_rule.rs | 2 +- .../time_series_stat_run_comparison_rule.rs | 2 +- .../time_series_stat_threshold_rule.rs | 2 +- src/data.rs | 9 +- src/data/aperf_stats.rs | 3 +- src/data/common/data_formats.rs | 15 +- src/data/common/processed_data_accessor.rs | 439 +++++++++++++++--- src/data/common/time_series_data_processor.rs | 30 +- src/data/cpu_utilization.rs | 9 +- src/data/diskstats.rs | 8 +- src/data/efa_stat.rs | 9 +- src/data/ena_stat.rs | 9 +- src/data/interrupts.rs | 9 +- src/data/memalloc.rs | 9 +- src/data/meminfo.rs | 9 +- src/data/netstat.rs | 9 +- src/data/numastat.rs | 9 +- src/data/perf_stat.rs | 9 +- src/data/processes.rs | 12 +- src/data/vmstat.rs | 9 +- src/lib.rs | 80 ++-- src/record.rs | 2 +- src/report.rs | 89 +++- .../src/definitions/data-config.ts | 13 +- src/visualizer.rs | 7 + .../test_key_value_key_expected_rule.rs | 20 +- .../test_key_value_key_run_comparison_rule.rs | 20 +- .../test_profile_metadata_comparison_rule.rs | 20 +- .../test_profile_metadata_expected_rule.rs | 20 +- ...test_profile_stack_frame_threshold_rule.rs | 16 +- ...t_time_series_data_point_threshold_rule.rs | 98 ++-- ...e_series_stat_intra_run_comparison_rule.rs | 68 +-- ...st_time_series_stat_run_comparison_rule.rs | 60 +-- .../test_time_series_stat_threshold_rule.rs | 60 +-- tests/test_aperf.rs | 1 - tests/test_aperf_stats.rs | 1 + tests/test_java_profile.rs | 1 + 44 files changed, 888 insertions(+), 464 deletions(-) diff --git a/src/analytics/mod.rs b/src/analytics/mod.rs index fc8741f0..9f07d352 100644 --- a/src/analytics/mod.rs +++ b/src/analytics/mod.rs @@ -36,7 +36,7 @@ fn get_base_run_name() -> String { pub struct AnalyticalEngine<'a> { // Map from a data name to its processed data across all runs - all_processed_data: HashMap, + all_processed_data: HashMap, // Map from a data name to its defined rules per_data_rules: HashMap>, // All rules that require multiple data types @@ -57,7 +57,7 @@ impl Default for AnalyticalEngine<'_> { } impl<'a> AnalyticalEngine<'a> { - pub fn add_processed_data(&mut self, data_name: String, processed_data: &'a ProcessedData) { + pub fn add_processed_data(&mut self, data_name: String, processed_data: &'a mut ProcessedData) { self.all_processed_data.insert(data_name, processed_data); } @@ -67,7 +67,7 @@ impl<'a> AnalyticalEngine<'a> { pub fn run(&mut self, processed_data_accessor: &mut ProcessedDataAccessor) { for (data_name, data_rules) in &self.per_data_rules { - if let Some(&processed_data) = self.all_processed_data.get(data_name) { + if let Some(processed_data) = self.all_processed_data.get_mut(data_name) { let data_findings = self .findings .entry(data_name.clone()) @@ -194,7 +194,7 @@ pub trait Analyze { fn analyze( &self, data_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ); } @@ -206,7 +206,7 @@ pub trait MultiDataAnalyze { fn analyze( &self, findings: &HashMap, - all_processed_data: &HashMap, + all_processed_data: &HashMap, processed_data_accessor: &mut ProcessedDataAccessor, ); } @@ -220,7 +220,7 @@ macro_rules! analytical_rules { } impl AnalyticalRule { - pub fn analyze(&self, data_findings: &mut DataFindings, processed_data: &ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor) { + pub fn analyze(&self, data_findings: &mut DataFindings, processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor) { match self { $( AnalyticalRule::$analytical_rule(ref analytical_rule) => analytical_rule.analyze(data_findings, processed_data, processed_data_accessor), @@ -253,7 +253,7 @@ macro_rules! multi_data_analytical_rules { } impl MultiDataAnalyticalRule { - pub fn analyze(&self, _findings: &mut HashMap, _all_processed_data: &HashMap, _processed_data_accessor: &mut ProcessedDataAccessor) { + pub fn analyze(&self, _findings: &mut HashMap, _all_processed_data: &HashMap, _processed_data_accessor: &mut ProcessedDataAccessor) { match self { $( MultiDataAnalyticalRule::$multi_data_analytical_rule(ref multi_data_analytical_rule) => multi_data_analytical_rule.analyze(findings, all_processed_data, processed_data_accessor), diff --git a/src/analytics/rule_templates/key_value_key_expected_rule.rs b/src/analytics/rule_templates/key_value_key_expected_rule.rs index c6a4a738..3ae3f332 100644 --- a/src/analytics/rule_templates/key_value_key_expected_rule.rs +++ b/src/analytics/rule_templates/key_value_key_expected_rule.rs @@ -48,7 +48,7 @@ impl Analyze for KeyValueKeyExpectedRule { fn analyze( &self, report_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { for run_name in processed_data.runs.keys() { diff --git a/src/analytics/rule_templates/key_value_key_run_comparison_rule.rs b/src/analytics/rule_templates/key_value_key_run_comparison_rule.rs index 6752c333..3165aaa8 100644 --- a/src/analytics/rule_templates/key_value_key_run_comparison_rule.rs +++ b/src/analytics/rule_templates/key_value_key_run_comparison_rule.rs @@ -48,7 +48,7 @@ impl Analyze for KeyValueKeyRunComparisonRule { fn analyze( &self, report_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { let base_run_name = analytics::get_base_run_name(); diff --git a/src/analytics/rule_templates/profile_metadata_comparison_rule.rs b/src/analytics/rule_templates/profile_metadata_comparison_rule.rs index f4910557..813ee596 100644 --- a/src/analytics/rule_templates/profile_metadata_comparison_rule.rs +++ b/src/analytics/rule_templates/profile_metadata_comparison_rule.rs @@ -1,6 +1,6 @@ use crate::analytics; use crate::analytics::{AnalyticalFinding, Analyze, DataFindings}; -use crate::data::common::data_formats::{AperfData, ProcessedData}; +use crate::data::common::data_formats::ProcessedData; use crate::data::common::processed_data_accessor::ProcessedDataAccessor; use log::debug; use std::collections::HashMap; @@ -56,39 +56,38 @@ impl Analyze for ProfileMetadataComparisonRule { fn analyze( &self, report_findings: &mut DataFindings, - processed_data: &ProcessedData, - _processed_data_accessor: &mut ProcessedDataAccessor, + processed_data: &mut ProcessedData, + processed_data_accessor: &mut ProcessedDataAccessor, ) { let base_run_name = analytics::get_base_run_name(); - let base_run_data = match processed_data.runs.get(&base_run_name) { - Some(data) => data, - None => { - if processed_data.runs.keys().len() > 0 { - debug!("{self} failed to analyze: base run does not exist"); - } - return; + if !processed_data.runs.contains_key(&base_run_name) { + if !processed_data.runs.is_empty() { + debug!("{self} failed to analyze: base run does not exist"); } + return; + } + + // Collect metadata values for a run: profiler_key -> value + let collect_values = |processed_data: &mut ProcessedData, + accessor: &mut ProcessedDataAccessor, + run_name: &str| + -> HashMap { + let keys = accessor.profiler_keys(processed_data, run_name); + keys.into_iter() + .filter_map(|profiler_key| { + let value = accessor.profiler_value_by_key( + processed_data, + run_name, + &profiler_key, + self.key, + )?; + Some((profiler_key, value)) + }) + .collect() }; - // Map base values from all profiles in the first record - let base_values: HashMap = - if let AperfData::Profile(profiling_data) = base_run_data { - profiling_data - .profilers - .iter() - .filter_map(|(key, profiler)| { - profiler - .metadata - .key_value_groups - .get(self.group) - .and_then(|group| group.key_values.get(self.key)) - .map(|value| (key.clone(), value.clone())) - }) - .collect() - } else { - HashMap::new() - }; + let base_values = collect_values(processed_data, processed_data_accessor, &base_run_name); if base_values.is_empty() { if self.should_exist { @@ -109,27 +108,16 @@ impl Analyze for ProfileMetadataComparisonRule { return; } - for (run_name, run_data) in &processed_data.runs { - if *run_name == base_run_name { - continue; - } + let run_names: Vec = processed_data + .runs + .keys() + .filter(|r| **r != base_run_name) + .cloned() + .collect(); - let AperfData::Profile(profiling_data) = run_data else { - continue; - }; - - let comparison_values: HashMap = profiling_data - .profilers - .iter() - .filter_map(|(key, profiler)| { - profiler - .metadata - .key_value_groups - .get(self.group) - .and_then(|group| group.key_values.get(self.key)) - .map(|value| (key.clone(), value.clone())) - }) - .collect(); + for run_name in &run_names { + let comparison_values = + collect_values(processed_data, processed_data_accessor, run_name); // If both runs have exactly one key-value pair, compare values regardless of key match if base_values.len() == 1 && comparison_values.len() == 1 { diff --git a/src/analytics/rule_templates/profile_metadata_expected_rule.rs b/src/analytics/rule_templates/profile_metadata_expected_rule.rs index 4c79febf..a96fa4c0 100644 --- a/src/analytics/rule_templates/profile_metadata_expected_rule.rs +++ b/src/analytics/rule_templates/profile_metadata_expected_rule.rs @@ -1,5 +1,5 @@ use crate::analytics::{AnalyticalFinding, Analyze, DataFindings}; -use crate::data::common::data_formats::{AperfData, ProcessedData}; +use crate::data::common::data_formats::ProcessedData; use crate::data::common::processed_data_accessor::ProcessedDataAccessor; use log::debug; use regex::Regex; @@ -54,8 +54,8 @@ impl Analyze for ProfileMetadataExpectedRule { fn analyze( &self, report_findings: &mut DataFindings, - processed_data: &ProcessedData, - _processed_data_accessor: &mut ProcessedDataAccessor, + processed_data: &mut ProcessedData, + processed_data_accessor: &mut ProcessedDataAccessor, ) { let regex = match Regex::new(self.expected_value) { Ok(re) => Some(re), @@ -68,17 +68,16 @@ impl Analyze for ProfileMetadataExpectedRule { } }; - for (run_name, run_data) in &processed_data.runs { - let AperfData::Profile(profiling_data) = run_data else { - continue; - }; - for (key, profiler) in &profiling_data.profilers { - let metadata_value = profiler - .metadata - .key_value_groups - .get(self.group) - .and_then(|group| group.key_values.get(self.key)) - .cloned(); + let run_names: Vec = processed_data.runs.keys().cloned().collect(); + for run_name in &run_names { + let profiler_keys = processed_data_accessor.profiler_keys(processed_data, run_name); + for key in &profiler_keys { + let metadata_value = processed_data_accessor.profiler_value_by_key( + processed_data, + run_name, + key, + self.key, + ); let field_exists = metadata_value.is_some(); let value_matches = matches!((&metadata_value, ®ex), (Some(value), Some(re)) if re.is_match(value)); diff --git a/src/analytics/rule_templates/profile_stack_frame_threshold_rule.rs b/src/analytics/rule_templates/profile_stack_frame_threshold_rule.rs index 9ce1c50a..22475b35 100644 --- a/src/analytics/rule_templates/profile_stack_frame_threshold_rule.rs +++ b/src/analytics/rule_templates/profile_stack_frame_threshold_rule.rs @@ -1,5 +1,5 @@ use crate::analytics::{AnalyticalFinding, Analyze, DataFindings}; -use crate::data::common::data_formats::{AperfData, ProcessedData}; +use crate::data::common::data_formats::ProcessedData; use crate::data::common::processed_data_accessor::ProcessedDataAccessor; use crate::profiling::{FrameType, ThreadState}; use std::fmt; @@ -66,26 +66,29 @@ impl Analyze for ProfileStackFrameThresholdRule { fn analyze( &self, report_findings: &mut DataFindings, - processed_data: &ProcessedData, - _processed_data_accessor: &mut ProcessedDataAccessor, + processed_data: &mut ProcessedData, + processed_data_accessor: &mut ProcessedDataAccessor, ) { - for (run_name, run_data) in &processed_data.runs { - let AperfData::Profile(profiling_data) = run_data else { - continue; - }; - - for (key, profiler) in &profiling_data.profilers { - if !profiler.profiles.contains_key(self.profile_type) { - continue; - } - let total_samples = - profiler.get_total_samples(self.profile_type, self.thread_states); + let run_names: Vec = processed_data.runs.keys().cloned().collect(); + for run_name in &run_names { + let profiler_keys = processed_data_accessor.profiler_keys(processed_data, run_name); + for key in &profiler_keys { + let total_samples = processed_data_accessor.profiler_total_samples( + processed_data, + run_name, + key, + self.profile_type, + self.thread_states, + ); let (sample_count, matched_pattern) = self.stack_frame .iter() .fold((0u64, None), |(acc, acc_pat), pattern| { - let count = profiler.get_samples( + let count = processed_data_accessor.profiler_samples( + processed_data, + run_name, + key, self.profile_type, pattern, self.frame_type, diff --git a/src/analytics/rule_templates/time_series_data_point_threshold_rule.rs b/src/analytics/rule_templates/time_series_data_point_threshold_rule.rs index f4035f32..e4e1ffaa 100644 --- a/src/analytics/rule_templates/time_series_data_point_threshold_rule.rs +++ b/src/analytics/rule_templates/time_series_data_point_threshold_rule.rs @@ -80,7 +80,7 @@ impl Analyze for TimeSeriesDataPointThresholdRule { fn analyze( &self, data_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { for run_name in processed_data.runs.keys() { diff --git a/src/analytics/rule_templates/time_series_stat_intra_run_comparison_rule.rs b/src/analytics/rule_templates/time_series_stat_intra_run_comparison_rule.rs index 19c5155d..50316d40 100644 --- a/src/analytics/rule_templates/time_series_stat_intra_run_comparison_rule.rs +++ b/src/analytics/rule_templates/time_series_stat_intra_run_comparison_rule.rs @@ -64,7 +64,7 @@ impl Analyze for TimeSeriesStatIntraRunComparisonRule { fn analyze( &self, data_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { for run_name in processed_data.runs.keys() { diff --git a/src/analytics/rule_templates/time_series_stat_run_comparison_rule.rs b/src/analytics/rule_templates/time_series_stat_run_comparison_rule.rs index 2a84dfab..e5288c89 100644 --- a/src/analytics/rule_templates/time_series_stat_run_comparison_rule.rs +++ b/src/analytics/rule_templates/time_series_stat_run_comparison_rule.rs @@ -64,7 +64,7 @@ impl Analyze for TimeSeriesStatRunComparisonRule { fn analyze( &self, data_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { let base_run_name = &get_base_run_name(); diff --git a/src/analytics/rule_templates/time_series_stat_threshold_rule.rs b/src/analytics/rule_templates/time_series_stat_threshold_rule.rs index 0a890d93..6fe63d25 100644 --- a/src/analytics/rule_templates/time_series_stat_threshold_rule.rs +++ b/src/analytics/rule_templates/time_series_stat_threshold_rule.rs @@ -56,7 +56,7 @@ impl Analyze for TimeSeriesStatThresholdRule { fn analyze( &self, data_findings: &mut DataFindings, - processed_data: &ProcessedData, + processed_data: &mut ProcessedData, processed_data_accessor: &mut ProcessedDataAccessor, ) { for run_name in processed_data.runs.keys() { diff --git a/src/data.rs b/src/data.rs index 1268f1a6..8062af23 100644 --- a/src/data.rs +++ b/src/data.rs @@ -140,10 +140,7 @@ impl DataType { } pub fn init_data_type(&mut self, param: &InitParams) -> Result<()> { - let name = format!( - "{}_{}.{}", - self.file_name, param.time_str, APERF_FILE_FORMAT - ); + let name = format!("{}.{}", self.file_name, APERF_FILE_FORMAT); self.file_name = name.clone(); self.full_path = format!("{}/{}", param.dir_name, name); @@ -530,7 +527,7 @@ mod tests { collector_params: CollectorParams::new(), }; - param.dir_name = format!("./performance_data_init_test_{}", param.time_str); + param.dir_name = format!("./performance_data_init_test"); fs::DirBuilder::new() .recursive(true) .create(param.dir_name.clone()) @@ -559,7 +556,7 @@ mod tests { collector_params: CollectorParams::new(), }; - param.dir_name = format!("./performance_data_print_test_{}", param.time_str); + param.dir_name = format!("./performance_data_print_test"); fs::DirBuilder::new() .recursive(true) .create(param.dir_name.clone()) diff --git a/src/data/aperf_stats.rs b/src/data/aperf_stats.rs index 50b4b78a..b7c1b24c 100644 --- a/src/data/aperf_stats.rs +++ b/src/data/aperf_stats.rs @@ -46,7 +46,8 @@ impl ProcessData for AperfStat { params: ReportParams, _raw_data: Vec, ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_sum_aggregate!(); + let mut time_series_data_processor = + time_series_data_processor_with_sum_aggregate!(params.collection_start); time_series_data_processor.set_aggregate_series_name("total"); let mut values = Vec::new(); diff --git a/src/data/common/data_formats.rs b/src/data/common/data_formats.rs index 6b1d3197..c4d4cdc9 100644 --- a/src/data/common/data_formats.rs +++ b/src/data/common/data_formats.rs @@ -1,5 +1,5 @@ use crate::computations::{serialize_f64_vec_fixed2, Statistics}; -use crate::profiling::Profile; +use crate::profiling::{Profile, BUCKET_WIDTH_MS}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use strum_macros::Display; @@ -177,7 +177,7 @@ pub struct ProfilingData { pub profilers: HashMap, } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Profiler { /// Start time of the profile in milliseconds since epoch pub start_time_ms: i64, @@ -189,6 +189,17 @@ pub struct Profiler { pub profiles: HashMap, } +impl Default for Profiler { + fn default() -> Self { + Profiler { + start_time_ms: 0, + block_width_ms: BUCKET_WIDTH_MS, + metadata: KeyValueData::default(), + profiles: HashMap::new(), + } + } +} + impl Profiler { pub fn new(start_time_ms: i64, block_width_ms: u64) -> Self { Profiler { diff --git a/src/data/common/processed_data_accessor.rs b/src/data/common/processed_data_accessor.rs index 9892c847..700cdc7f 100644 --- a/src/data/common/processed_data_accessor.rs +++ b/src/data/common/processed_data_accessor.rs @@ -1,7 +1,10 @@ use crate::computations::{f64_to_fixed_2, Statistics}; use crate::data::common::data_formats::{ - AperfData, DataFormat, KeyValueData, ProcessedData, Series, TimeSeriesData, TimeSeriesMetric, + AperfData, DataFormat, KeyValueData, ProcessedData, Profiler, Series, TimeSeriesData, + TimeSeriesMetric, }; +use crate::data::TimeEnum; +use crate::profiling::{FrameType, ThreadState}; use regex::Regex; use std::collections::HashMap; use std::fmt::Write; @@ -9,16 +12,28 @@ use std::fmt::Write; /// This struct is created to allow for accessing the processed data within the specified /// time range, without modifying the original processed data. Such design allow APerf to /// hold only one copy of the processed data, and they can be accessed for any time range -/// with minimal performance cost. +/// with minimal performance cost. Series time_diffs are calculated relative to +/// `per_run_start_time` (the record start time) when available, falling back to +/// the first sample time for older archives that lack this timestamp. #[derive(Debug)] pub struct ProcessedDataAccessor { /// The start of the time range for every run data per_run_from_time: HashMap, /// The end of the time range for every run data per_run_to_time: HashMap, + /// Wall-clock start of every run's `collect_data_serial`. Used together with + /// `per_run_end_time` to generate `run_duration_seconds` and anchor negative time + /// range bounds and to convert user-supplied relative seconds into absolute wall + /// ms for profile data. A run missing from this map (e.g. archives recorded by + /// older aperf) triggers fallback behavior. + per_run_start_time: HashMap, + /// Wall-clock end of every run's `collect_data_serial`. + per_run_end_time: HashMap, /// The cache for a metric's stat within the time range /// (since stat computation is costly). time_series_metric_stat_cache: HashMap<(String, String), Statistics>, + /// Cache of whether time range has been applied per run per profiler. + profiler_time_range_cache: HashMap>, } impl ProcessedDataAccessor { @@ -26,18 +41,43 @@ impl ProcessedDataAccessor { ProcessedDataAccessor { per_run_from_time: HashMap::new(), per_run_to_time: HashMap::new(), + per_run_start_time: HashMap::new(), + per_run_end_time: HashMap::new(), time_series_metric_stat_cache: HashMap::new(), + profiler_time_range_cache: HashMap::new(), } } pub fn from_time_ranges( per_run_from_time: HashMap, per_run_to_time: HashMap, + per_run_start_time: HashMap, + per_run_end_time: HashMap, ) -> Self { ProcessedDataAccessor { per_run_from_time, per_run_to_time, + per_run_start_time, + per_run_end_time, time_series_metric_stat_cache: HashMap::new(), + profiler_time_range_cache: HashMap::new(), + } + } + + /// Returns the recording duration (seconds, rounded) for a run, derived from + /// the wall-clock start and end times. `None` if either is missing. + fn run_duration_seconds(&self, run_name: &str) -> Option { + let start = self.per_run_start_time.get(run_name).copied()?; + let end = self.per_run_end_time.get(run_name).copied()?; + match end - start { + TimeEnum::TimeDiff(secs) => { + if secs == 0 { + None + } else { + Some(secs) + } + } + _ => None, } } @@ -51,6 +91,7 @@ impl ProcessedDataAccessor { ) -> impl Iterator { let from_time = self.per_run_from_time.get(run_name).copied(); let to_time = self.per_run_to_time.get(run_name).copied(); + let run_duration_seconds = self.run_duration_seconds(run_name); let metric_name = metric_name.to_string(); get_time_series_data(processed_data, run_name) .into_iter() @@ -61,8 +102,12 @@ impl ProcessedDataAccessor { .into_iter() .flat_map(move |metric| { metric.series.iter().flat_map(move |series| { - let (start_idx, end_idx) = - compute_time_diff_index(&series.time_diff, from_time, to_time); + let (start_idx, end_idx) = compute_time_diff_index( + &series.time_diff, + from_time, + to_time, + run_duration_seconds, + ); series.values[start_idx..end_idx].iter() }) }) @@ -116,12 +161,7 @@ impl ProcessedDataAccessor { key: &str, ) -> Option<&'a str> { let key_value_data = get_key_value_data(processed_data, run_name)?; - for key_value_group in key_value_data.key_value_groups.values() { - if let Some(value) = key_value_group.key_values.get(key) { - return Some(value); - } - } - None + lookup_key_value(key_value_data, key) } /// Serializes a processed data into JSON string. @@ -165,6 +205,7 @@ impl ProcessedDataAccessor { if let Some(time_series_data) = get_time_series_data(processed_data, run_name) { let from_time = self.per_run_from_time.get(run_name).copied(); let to_time = self.per_run_to_time.get(run_name).copied(); + let run_duration_seconds = self.run_duration_seconds(run_name); buf.push_str("{\"metrics\":{"); let mut first_metric = true; @@ -182,6 +223,7 @@ impl ProcessedDataAccessor { stats, from_time, to_time, + run_duration_seconds, ); } @@ -214,6 +256,7 @@ impl ProcessedDataAccessor { stats, self.per_run_from_time.get(run_name).copied(), self.per_run_to_time.get(run_name).copied(), + self.run_duration_seconds(run_name), ); buf } @@ -233,6 +276,142 @@ impl ProcessedDataAccessor { serde_json::to_string(processed_data).unwrap() } + /// Returns the profiler keys for a run's profile data, or empty vec if not profile data. + pub fn profiler_keys( + &mut self, + processed_data: &mut ProcessedData, + run_name: &str, + ) -> Vec { + match processed_data.runs.get_mut(run_name) { + Some(AperfData::Profile(profiling_data)) => { + profiling_data.profilers.keys().cloned().collect() + } + _ => Vec::new(), + } + } + + /// Returns total samples for a profiler's profile type within the time range. + pub fn profiler_total_samples( + &mut self, + processed_data: &mut ProcessedData, + run_name: &str, + profiler_key: &str, + profile_type: &str, + thread_states: &[ThreadState], + ) -> u64 { + match processed_data.runs.get_mut(run_name) { + Some(AperfData::Profile(profiling_data)) => { + if let Some(profiler) = profiling_data.profilers.get_mut(profiler_key) { + self.apply_time_range_to_profiler(run_name, profiler_key, profiler); + profiler.get_total_samples(profile_type, thread_states) + } else { + 0 + } + } + _ => 0, + } + } + + /// Returns sample count matching a stack pattern within the time range. + pub fn profiler_samples( + &mut self, + processed_data: &mut ProcessedData, + run_name: &str, + profiler_key: &str, + profile_type: &str, + pattern: &[&str], + frame_type: Option, + thread_states: &[ThreadState], + total_samples: bool, + ) -> u64 { + match processed_data.runs.get_mut(run_name) { + Some(AperfData::Profile(profiling_data)) => { + if let Some(profiler) = profiling_data.profilers.get_mut(profiler_key) { + self.apply_time_range_to_profiler(run_name, profiler_key, profiler); + profiler.get_samples( + profile_type, + pattern, + frame_type, + thread_states, + total_samples, + ) + } else { + 0 + } + } + _ => 0, + } + } + + /// Returns a metadata value from a profiler's KeyValueData, searching all groups. + /// Reuses the same lookup logic as `key_value_value_by_key`. + pub fn profiler_value_by_key( + &self, + processed_data: &mut ProcessedData, + run_name: &str, + profiler_key: &str, + key: &str, + ) -> Option { + match processed_data.runs.get_mut(run_name) { + Some(AperfData::Profile(profiling_data)) => { + lookup_key_value(&profiling_data.profilers.get(profiler_key)?.metadata, key) + .map(|s| s.to_string()) + } + _ => None, + } + } + + /// Applies the run's time range to a specific profiler by calling + /// [`Profiler::set_time_range`]. Negative user bounds anchor at `collection_end` + /// (wall clock), positive at `collection_start`. + /// No-op if either collection timestamp is missing for the run. + pub fn apply_time_range_to_profiler( + &mut self, + run_name: &str, + profiler_key: &str, + profiler: &mut Profiler, + ) { + if self + .profiler_time_range_cache + .get(run_name) + .and_then(|m| m.get(profiler_key)) + == Some(&true) + { + return; + } + + let (Some(TimeEnum::DateTime(start_dt)), Some(TimeEnum::DateTime(end_dt))) = ( + self.per_run_start_time.get(run_name).copied(), + self.per_run_end_time.get(run_name).copied(), + ) else { + return; + }; + + let start_ms = start_dt.timestamp_millis(); + let end_ms = end_dt.timestamp_millis(); + let wall_from_ms = self + .per_run_from_time + .get(run_name) + .copied() + .map(|s| resolve_wall_ms(s, start_ms, end_ms)) + .unwrap_or(start_ms); + let wall_to_ms = self + .per_run_to_time + .get(run_name) + .copied() + .map(|s| resolve_wall_ms(s, start_ms, end_ms)) + .unwrap_or(end_ms); + + let rel_start_ms = (wall_from_ms - profiler.start_time_ms).max(0) as u64; + let rel_end_ms = (wall_to_ms - profiler.start_time_ms).max(0) as u64; + let _ = profiler.set_time_range(rel_start_ms, rel_end_ms); + + self.profiler_time_range_cache + .entry(run_name.to_string()) + .or_default() + .insert(profiler_key.to_string(), true); + } + /// Retrieves the stat of a time-series metric within the time range from the cache, or, /// if it does not exist in the cache, computes the stat and stores it in the cache. fn get_or_compute_time_series_metric_stats( @@ -246,12 +425,17 @@ impl ProcessedDataAccessor { } let from_time = self.per_run_from_time.get(run_name).copied(); let to_time = self.per_run_to_time.get(run_name).copied(); + let run_duration_seconds = self.run_duration_seconds(run_name); let stats = if let Some(stats_series) = time_series_metric .series .get(time_series_metric.stats_series_idx) { - let (start_idx, end_idx) = - compute_time_diff_index(&stats_series.time_diff, from_time, to_time); + let (start_idx, end_idx) = compute_time_diff_index( + &stats_series.time_diff, + from_time, + to_time, + run_duration_seconds, + ); Statistics::from_values(&stats_series.values[start_idx..end_idx]) } else { Statistics::default() @@ -269,6 +453,7 @@ fn write_time_series_metric_json_string( stats: Statistics, from_time: Option, to_time: Option, + run_duration_seconds: Option, ) { buf.push_str("{\"metric_name\":"); write!( @@ -285,7 +470,7 @@ fn write_time_series_metric_json_string( buf.push(','); } first_series = false; - write_series_json_string(buf, series, from_time, to_time); + write_series_json_string(buf, series, from_time, to_time, run_duration_seconds); } buf.push(']'); @@ -309,8 +494,10 @@ fn write_series_json_string( series: &Series, from_time: Option, to_time: Option, + run_duration_seconds: Option, ) { - let (start, end) = compute_time_diff_index(&series.time_diff, from_time, to_time); + let (start, end) = + compute_time_diff_index(&series.time_diff, from_time, to_time, run_duration_seconds); // Uses itoa/ryu for fast int/float number formatting let mut itoa_buf = itoa::Buffer::new(); let mut ryu_buf = ryu::Buffer::new(); @@ -373,19 +560,49 @@ fn get_key_value_data<'a>( } } +/// Looks up a key across all groups in a KeyValueData, returning the first match. +fn lookup_key_value<'a>(key_value_data: &'a KeyValueData, key: &str) -> Option<&'a str> { + for group in key_value_data.key_value_groups.values() { + if let Some(value) = group.key_values.get(key) { + return Some(value); + } + } + None +} + +/// Resolves a user-supplied relative-seconds bound into an absolute wall-clock +/// ms timestamp. A non-negative value anchors at `start_ms`; a negative value +/// anchors at `end_ms` (so `-10` means "10 seconds before end of recording"). +fn resolve_wall_ms(bound_sec: i64, start_ms: i64, end_ms: i64) -> i64 { + if bound_sec >= 0 { + start_ms.saturating_add(bound_sec.saturating_mul(1000)) + } else { + end_ms.saturating_add(bound_sec.saturating_mul(1000)) + } +} + /// Locate the start and end index of a series' time_diff (and values) based on /// the specified time range. +/// +/// Negative `from_time` / `to_time` values are resolved against `run_duration_seconds` (the total +/// recording duration in the same units as `time_diff`, typically seconds). When +/// `run_duration_seconds` is `None`, falls back to the last value of `time_diff`. fn compute_time_diff_index( time_diff: &[u64], from_time: Option, to_time: Option, + run_duration_seconds: Option, ) -> (usize, usize) { + let resolve_negative = |v: i64| -> u64 { + run_duration_seconds + .or_else(|| time_diff.last().copied()) + .map(|a| a.saturating_sub(v.unsigned_abs())) + .unwrap_or(0) + }; + let start_idx = if let Some(from) = from_time { let from_t = if from < 0 { - time_diff - .last() - .map(|&last_time_diff| last_time_diff.saturating_sub(from.abs() as u64)) - .unwrap_or(0) + resolve_negative(from) } else { from as u64 }; @@ -396,10 +613,7 @@ fn compute_time_diff_index( let end_idx = if let Some(to) = to_time { let to_t = if to < 0 { - time_diff - .last() - .map(|&last_time_diff| last_time_diff.saturating_sub(to.abs() as u64)) - .unwrap_or(0) + resolve_negative(to) } else { to as u64 }; @@ -454,7 +668,15 @@ mod tests { #[test] fn test_index_no_bounds() { assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], None, None), + compute_time_diff_index(&[0, 10, 20, 30, 40], None, None, None), + (0, 5) + ); + } + + #[test] + fn test_index_no_bounds_anchored() { + assert_eq!( + compute_time_diff_index(&[0, 10, 20, 30, 40], None, None, Some(45)), (0, 5) ); } @@ -462,7 +684,7 @@ mod tests { #[test] fn test_index_with_from() { assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(15), None), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(15), None, None), (2, 5) ); } @@ -470,7 +692,7 @@ mod tests { #[test] fn test_index_with_to() { assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], None, Some(25)), + compute_time_diff_index(&[0, 10, 20, 30, 40], None, Some(25), None), (0, 3) ); } @@ -478,7 +700,7 @@ mod tests { #[test] fn test_index_with_both() { assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(10), Some(30)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(10), Some(30), None), (1, 4) ); } @@ -486,20 +708,23 @@ mod tests { #[test] fn test_index_exact_match() { assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(20), Some(20)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(20), Some(20), None), (2, 3) ); } #[test] fn test_index_empty() { - assert_eq!(compute_time_diff_index(&[], Some(5), Some(10)), (0, 0)); + assert_eq!( + compute_time_diff_index(&[], Some(5), Some(10), None), + (0, 0) + ); } #[test] fn test_index_from_beyond_end() { assert_eq!( - compute_time_diff_index(&[0, 10, 20], Some(100), None), + compute_time_diff_index(&[0, 10, 20], Some(100), None, None), (3, 3) ); } @@ -507,7 +732,7 @@ mod tests { #[test] fn test_index_to_before_start() { assert_eq!( - compute_time_diff_index(&[10, 20, 30], None, Some(5)), + compute_time_diff_index(&[10, 20, 30], None, Some(5), None), (0, 0) ); } @@ -516,7 +741,7 @@ mod tests { fn test_index_from_equals_first_element() { // from == first element → should include it assert_eq!( - compute_time_diff_index(&[10, 20, 30], Some(10), None), + compute_time_diff_index(&[10, 20, 30], Some(10), None, None), (0, 3) ); } @@ -525,26 +750,32 @@ mod tests { fn test_index_to_equals_last_element() { // to == last element → should include it assert_eq!( - compute_time_diff_index(&[10, 20, 30], None, Some(30)), + compute_time_diff_index(&[10, 20, 30], None, Some(30), None), (0, 3) ); } #[test] fn test_index_single_element_in_range() { - assert_eq!(compute_time_diff_index(&[42], Some(40), Some(50)), (0, 1)); + assert_eq!( + compute_time_diff_index(&[42], Some(40), Some(50), None), + (0, 1) + ); } #[test] fn test_index_single_element_out_of_range() { - assert_eq!(compute_time_diff_index(&[42], Some(50), Some(60)), (1, 1)); + assert_eq!( + compute_time_diff_index(&[42], Some(50), Some(60), None), + (1, 1) + ); } #[test] fn test_index_duplicate_values() { // time_diff with duplicates — all matching values should be included assert_eq!( - compute_time_diff_index(&[10, 10, 20, 20, 30], Some(10), Some(20)), + compute_time_diff_index(&[10, 10, 20, 20, 30], Some(10), Some(20), None), (0, 4) ); } @@ -552,14 +783,17 @@ mod tests { #[test] fn test_index_from_none_to_none() { // Both None → full range (same as test_index_no_bounds but explicit) - assert_eq!(compute_time_diff_index(&[5, 10, 15], None, None), (0, 3)); + assert_eq!( + compute_time_diff_index(&[5, 10, 15], None, None, None), + (0, 3) + ); } #[test] fn test_index_from_equals_to_not_in_array() { // from == to but value doesn't exist in array → empty range assert_eq!( - compute_time_diff_index(&[10, 20, 30], Some(15), Some(15)), + compute_time_diff_index(&[10, 20, 30], Some(15), Some(15), None), (1, 1) ); } @@ -568,7 +802,10 @@ mod tests { fn test_index_contiguous_range() { // Typical real-world case: second-by-second time_diff let td: Vec = (0..100).collect(); - assert_eq!(compute_time_diff_index(&td, Some(25), Some(74)), (25, 75)); + assert_eq!( + compute_time_diff_index(&td, Some(25), Some(74), None), + (25, 75) + ); } // ---- negative index tests ---- @@ -578,7 +815,7 @@ mod tests { // time_diff: [0, 10, 20, 30, 40], last=40 // from=-15 → 40-15=25 → first index where t >= 25 is index 3 (value 30) assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-15), None), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-15), None, None), (3, 5) ); } @@ -588,7 +825,7 @@ mod tests { // time_diff: [0, 10, 20, 30, 40], last=40 // to=-15 → 40-15=25 → first index where t > 25 is index 3 (value 30), so end=3 assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], None, Some(-15)), + compute_time_diff_index(&[0, 10, 20, 30, 40], None, Some(-15), None), (0, 3) ); } @@ -599,7 +836,7 @@ mod tests { // from=-30 → 40-30=10, to=-10 → 40-10=30 // range [10, 30] → indices 1..4 assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-30), Some(-10)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-30), Some(-10), None), (1, 4) ); } @@ -610,7 +847,7 @@ mod tests { // from=-25 → 40-25=15, to=30 // range [15, 30] → indices 2..4 assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-25), Some(30)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-25), Some(30), None), (2, 4) ); } @@ -620,14 +857,17 @@ mod tests { // time_diff: [0, 10, 20], last=20 // from=-100 → 20-100 saturates to 0 → full range from start assert_eq!( - compute_time_diff_index(&[0, 10, 20], Some(-100), None), + compute_time_diff_index(&[0, 10, 20], Some(-100), None, None), (0, 3) ); } #[test] fn test_index_negative_on_empty() { - assert_eq!(compute_time_diff_index(&[], Some(-5), Some(-1)), (0, 0)); + assert_eq!( + compute_time_diff_index(&[], Some(-5), Some(-1), None), + (0, 0) + ); } #[test] @@ -636,7 +876,10 @@ mod tests { let td: Vec = (0..100).collect(); // from=-10 → 99-10=89, to=-1 → 99-1=98 // range [89, 98] → indices 89..99 - assert_eq!(compute_time_diff_index(&td, Some(-10), Some(-1)), (89, 99)); + assert_eq!( + compute_time_diff_index(&td, Some(-10), Some(-1), None), + (89, 99) + ); } #[test] @@ -644,7 +887,7 @@ mod tests { // from=35 (positive), to=-30 → resolves to 40-30=10 // Resolved range [35, 10] is inverted → should return empty (start, start) assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(35), Some(-30)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(35), Some(-30), None), (4, 4) ); } @@ -653,11 +896,89 @@ mod tests { fn test_index_inverted_positive_range_returns_empty() { // Pure positive inversion: from=30, to=10 assert_eq!( - compute_time_diff_index(&[0, 10, 20, 30, 40], Some(30), Some(10)), + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(30), Some(10), None), (3, 3) ); } + // ---- run_duration_seconds anchor tests ---- + + #[test] + fn test_index_anchor_used_for_negative_bounds() { + // run_duration_seconds overrides time_diff.last() when resolving negative bounds. + // time_diff.last()=40 but anchor=100 → from=-10 resolves to 100-10=90, + // which is past every element, so start_idx = len. + assert_eq!( + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-10), None, Some(100)), + (5, 5) + ); + } + + #[test] + fn test_index_anchor_ignored_for_positive_bounds() { + // Positive bounds are interpreted as raw time_diff values; anchor has no effect. + assert_eq!( + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(20), Some(30), Some(9999)), + (2, 4) + ); + } + + #[test] + fn test_index_to_none_ignores_anchor() { + // to_time=None → end_idx = time_diff.len() regardless of anchor. + // Even if anchor is smaller than time_diff.last(), samples past anchor are kept. + assert_eq!( + compute_time_diff_index(&[0, 10, 20, 30, 40], None, None, Some(20)), + (0, 5) + ); + } + + #[test] + fn test_index_anchor_smaller_than_last_with_negative_from() { + // Skew case: time_diff.last()=100 but anchor=60 (nominal collection ended + // before the last recorded sample). from=-20 resolves to 60-20=40, yielding + // a window larger than the user likely expects given that time_diff extends + // past anchor. This test documents the current behavior. + let td: Vec = (0..=100).step_by(10).collect(); // [0,10,...,100] + assert_eq!( + compute_time_diff_index(&td, Some(-20), None, Some(60)), + (4, 11) // from t=40 through t=100 + ); + } + + #[test] + fn test_index_anchor_saturates_on_large_negative() { + // Negative bound whose magnitude exceeds anchor saturates to 0 rather than + // underflowing u64. + assert_eq!( + compute_time_diff_index(&[0, 10, 20, 30, 40], Some(-1000), None, Some(5)), + (0, 5) + ); + } + + // ---- resolve_wall_ms tests ---- + + #[test] + fn test_resolve_wall_ms_nonnegative_anchors_at_start() { + // 10s offset from start + assert_eq!(resolve_wall_ms(10, 1_000, 61_000), 11_000); + // 0s resolves to start itself + assert_eq!(resolve_wall_ms(0, 1_000, 61_000), 1_000); + } + + #[test] + fn test_resolve_wall_ms_negative_anchors_at_end() { + // -5s from end + assert_eq!(resolve_wall_ms(-5, 1_000, 61_000), 56_000); + } + + #[test] + fn test_resolve_wall_ms_saturates_on_overflow() { + // Huge positive seconds value would overflow i64 ms; saturating_mul/add + // keeps the result at i64::MAX instead of panicking. + assert_eq!(resolve_wall_ms(i64::MAX, 1_000, 61_000), i64::MAX); + } + // ---- iterator tests ---- #[test] @@ -690,6 +1011,8 @@ mod tests { let accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 10)]), HashMap::new(), + HashMap::new(), + HashMap::new(), ); let values: Vec<&f64> = accessor .time_series_metric_values_iterator(&pd, "run1", "cpu") @@ -709,6 +1032,8 @@ mod tests { let accessor = ProcessedDataAccessor::from_time_ranges( HashMap::new(), HashMap::from([("run1".to_string(), 10)]), + HashMap::new(), + HashMap::new(), ); let values: Vec<&f64> = accessor .time_series_metric_values_iterator(&pd, "run1", "cpu") @@ -766,6 +1091,8 @@ mod tests { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 10)]), HashMap::from([("run1".to_string(), 20)]), + HashMap::new(), + HashMap::new(), ); let stats = accessor .time_series_metric_stats(&pd, "run1", "cpu") @@ -823,7 +1150,7 @@ mod tests { fn test_write_series_no_range() { let series = make_series("total", vec![0, 10, 20], vec![1.5, 2.7, 3.223]); let mut buf = String::new(); - write_series_json_string(&mut buf, &series, None, None); + write_series_json_string(&mut buf, &series, None, None, None); let v: serde_json::Value = serde_json::from_str(&buf).unwrap(); assert_eq!(v["series_name"], "total"); assert_eq!(v["time_diff"], serde_json::json!([0, 10, 20])); @@ -835,7 +1162,7 @@ mod tests { fn test_write_series_with_time_range() { let series = make_series("s1", vec![0, 10, 20, 30], vec![1.0, 2.0, 3.0, 4.0]); let mut buf = String::new(); - write_series_json_string(&mut buf, &series, Some(10), Some(20)); + write_series_json_string(&mut buf, &series, Some(10), Some(20), None); let v: serde_json::Value = serde_json::from_str(&buf).unwrap(); assert_eq!(v["time_diff"], serde_json::json!([10, 20])); let values: Vec = v["values"] @@ -856,7 +1183,7 @@ mod tests { is_aggregate: true, }; let mut buf = String::new(); - write_series_json_string(&mut buf, &series, None, None); + write_series_json_string(&mut buf, &series, None, None, None); let v: serde_json::Value = serde_json::from_str(&buf).unwrap(); assert_eq!(v["is_aggregate"], true); } @@ -889,6 +1216,8 @@ mod tests { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 10)]), HashMap::from([("run1".to_string(), 20)]), + HashMap::new(), + HashMap::new(), ); let json = accessor.time_series_metric_json_string(&metric, "run1"); let v: serde_json::Value = serde_json::from_str(&json).unwrap(); @@ -943,6 +1272,8 @@ mod tests { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 10)]), HashMap::from([("run1".to_string(), 20)]), + HashMap::new(), + HashMap::new(), ); let json = accessor.time_series_data_json_string(&pd); let v: serde_json::Value = serde_json::from_str(&json).unwrap(); @@ -959,12 +1290,12 @@ mod tests { #[test] fn test_json_string_dispatches_time_series() { - let pd = make_processed_data(vec![( + let mut pd = make_processed_data(vec![( "cpu", vec![make_series("total", vec![0, 10], vec![1.0, 2.0])], )]); let mut accessor = ProcessedDataAccessor::new(); - let json = accessor.json_string(&pd); + let json = accessor.json_string(&mut pd); let v: serde_json::Value = serde_json::from_str(&json).unwrap(); assert!(v["runs"]["run1"]["metrics"]["cpu"].is_object()); } @@ -983,7 +1314,7 @@ mod tests { pd.runs.insert("run1".to_string(), AperfData::KeyValue(kv)); let mut accessor = ProcessedDataAccessor::new(); - let json = accessor.json_string(&pd); + let json = accessor.json_string(&mut pd); let v: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(v["data_format"], "key_value"); assert_eq!( @@ -997,6 +1328,8 @@ mod tests { let accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 10)]), HashMap::from([("run1".to_string(), 30)]), + HashMap::new(), + HashMap::new(), ); assert_eq!(accessor.per_run_from_time.get("run1"), Some(&10)); assert_eq!(accessor.per_run_to_time.get("run1"), Some(&30)); diff --git a/src/data/common/time_series_data_processor.rs b/src/data/common/time_series_data_processor.rs index d6fc5f79..d2308b27 100644 --- a/src/data/common/time_series_data_processor.rs +++ b/src/data/common/time_series_data_processor.rs @@ -52,7 +52,11 @@ pub struct TimeSeriesDataProcessor { } impl TimeSeriesDataProcessor { - pub fn new(data_name: &'static str, aggregate_mode: TimeSeriesDataAggregateMode) -> Self { + pub fn new( + data_name: &'static str, + aggregate_mode: TimeSeriesDataAggregateMode, + time_zero: Option, + ) -> Self { TimeSeriesDataProcessor { data_name, prev_data_points: HashMap::new(), @@ -61,7 +65,7 @@ impl TimeSeriesDataProcessor { aggregate_mode, aggregate_series_name: None, per_metric_sum_count: HashMap::new(), - time_zero: None, + time_zero, cur_time_diff: 0, decreasing_accumulative_data: HashMap::new(), fixed_value_range: None, @@ -401,23 +405,25 @@ fn compress_all_zero_time_series_metric(time_series_metric: &mut TimeSeriesMetri /// The aggregate series of every metric is auto-generated as the average of all series values /// at a timestamp. macro_rules! time_series_data_processor_with_average_aggregate { - () => { + ($time_zero:expr) => {{ crate::data::common::time_series_data_processor::TimeSeriesDataProcessor::new( crate::data::common::utils::get_data_name_from_type::(), crate::data::common::time_series_data_processor::TimeSeriesDataAggregateMode::Average, + $time_zero, ) - }; + }}; } /// The aggregate series of every metric is auto-generated as the sum of all series values /// at a timestamp. macro_rules! time_series_data_processor_with_sum_aggregate { - () => { + ($time_zero:expr) => {{ crate::data::common::time_series_data_processor::TimeSeriesDataProcessor::new( crate::data::common::utils::get_data_name_from_type::(), crate::data::common::time_series_data_processor::TimeSeriesDataAggregateMode::Sum, + $time_zero, ) - }; + }}; } /// The aggregate series of every metric is the series with the maximum average value. @@ -425,22 +431,24 @@ macro_rules! time_series_data_processor_with_sum_aggregate { /// a series is the sum of other series. For example, diskstats metrics include the root /// partition as well as all the subpartitions it contains. macro_rules! time_series_data_processor_with_max_series_aggregate { - () => { + ($time_zero:expr) => {{ crate::data::common::time_series_data_processor::TimeSeriesDataProcessor::new( crate::data::common::utils::get_data_name_from_type::(), crate::data::common::time_series_data_processor::TimeSeriesDataAggregateMode::MaxSeries, + $time_zero, ) - }; + }}; } /// The aggregate series needs to be manually added by calling add_aggregate_data. macro_rules! time_series_data_processor_with_custom_aggregate { - () => { + ($time_zero:expr) => {{ crate::data::common::time_series_data_processor::TimeSeriesDataProcessor::new( crate::data::common::utils::get_data_name_from_type::(), crate::data::common::time_series_data_processor::TimeSeriesDataAggregateMode::Custom, + $time_zero, ) - }; + }}; } pub(crate) use time_series_data_processor_with_average_aggregate; @@ -463,7 +471,7 @@ mod tests { } fn make_processor(mode: TimeSeriesDataAggregateMode) -> TimeSeriesDataProcessor { - TimeSeriesDataProcessor::new(TEST_DATA, mode) + TimeSeriesDataProcessor::new(TEST_DATA, mode, None) } fn data_series<'a>(ts: &'a TimeSeriesData, metric: &str) -> Vec<&'a Series> { diff --git a/src/data/cpu_utilization.rs b/src/data/cpu_utilization.rs index b703907f..3c34fec8 100644 --- a/src/data/cpu_utilization.rs +++ b/src/data/cpu_utilization.rs @@ -126,12 +126,9 @@ fn get_cpu_time(cpu_state: &CpuState, cpu_time: &Vec) -> u64 { } impl ProcessData for CpuUtilization { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_custom_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_custom_aggregate!(params.collection_start); // Override the value ranges - we want every metric graph to show from 0 to 100 time_series_data_processor.set_fixed_value_range((0, 100)); // CPU utils have a dedicated aggregate metric to hold the aggregate of every CPU-state metric, as well as diff --git a/src/data/diskstats.rs b/src/data/diskstats.rs index ffe9968c..ef001f3b 100644 --- a/src/data/diskstats.rs +++ b/src/data/diskstats.rs @@ -134,15 +134,11 @@ impl ProcessData for Diskstats { vec!["disk_stats"] } - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { // For diskstats there is no easy way to compute or find the aggregate metric, so to assign // stats to a metric, we use the stats of the series with the largest average value let mut time_series_data_processor = - time_series_data_processor_with_max_series_aggregate!(); + time_series_data_processor_with_max_series_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/efa_stat.rs b/src/data/efa_stat.rs index c055b446..42b88731 100644 --- a/src/data/efa_stat.rs +++ b/src/data/efa_stat.rs @@ -207,12 +207,9 @@ impl EfaStat { } impl ProcessData for EfaStat { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_average_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_average_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/ena_stat.rs b/src/data/ena_stat.rs index 54fbcd5b..7d255621 100644 --- a/src/data/ena_stat.rs +++ b/src/data/ena_stat.rs @@ -310,12 +310,9 @@ impl EnaStat { } impl ProcessData for EnaStat { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_average_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_average_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/interrupts.rs b/src/data/interrupts.rs index 6858cdd6..10568126 100644 --- a/src/data/interrupts.rs +++ b/src/data/interrupts.rs @@ -140,12 +140,9 @@ fn is_interrupt_name_mis_err(interrupt_name: &String) -> bool { } impl ProcessData for InterruptData { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_average_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_average_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/memalloc.rs b/src/data/memalloc.rs index 0c6e3585..f0ad57f7 100644 --- a/src/data/memalloc.rs +++ b/src/data/memalloc.rs @@ -116,12 +116,9 @@ impl MemallocData { } impl ProcessData for MemallocData { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_average_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_average_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/meminfo.rs b/src/data/meminfo.rs index 65c7d978..baab3387 100644 --- a/src/data/meminfo.rs +++ b/src/data/meminfo.rs @@ -97,12 +97,9 @@ fn parse_meminfo(raw_data: &String) -> IndexMap { } impl ProcessData for MeminfoData { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_custom_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_custom_aggregate!(params.collection_start); let mut metric_name_order: Vec = Vec::new(); diff --git a/src/data/netstat.rs b/src/data/netstat.rs index fb746eb9..f86422be 100644 --- a/src/data/netstat.rs +++ b/src/data/netstat.rs @@ -85,12 +85,9 @@ fn parse_raw_netstat_data(raw_netstat_data: &String) -> Result, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_custom_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_custom_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/numastat.rs b/src/data/numastat.rs index f6f64003..69fa6c5b 100644 --- a/src/data/numastat.rs +++ b/src/data/numastat.rs @@ -99,12 +99,9 @@ impl Numastat { } impl ProcessData for Numastat { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_average_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_average_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/perf_stat.rs b/src/data/perf_stat.rs index ed8848a9..7d240b36 100644 --- a/src/data/perf_stat.rs +++ b/src/data/perf_stat.rs @@ -428,12 +428,9 @@ fn parse_raw_pmu_stat(raw_pmu_stat: &str) -> Result<(usize, String, f64, f64, f6 } impl ProcessData for PerfStat { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_custom_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_custom_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/data/processes.rs b/src/data/processes.rs index 6203adf6..323e3e81 100644 --- a/src/data/processes.rs +++ b/src/data/processes.rs @@ -5,6 +5,7 @@ use crate::data::common::data_formats::{AperfData, Series, TimeSeriesData, TimeS use crate::data::{Data, ProcessData, TimeEnum}; use crate::visualizer::ReportParams; use anyhow::Result; +use chrono::Utc; use core::f64; use log::warn; use serde::{Deserialize, Serialize}; @@ -15,7 +16,6 @@ use strum_macros::{Display, EnumIter}; #[cfg(target_os = "linux")] use { crate::data::{CollectData, CollectorParams}, - chrono::prelude::*, std::fs, std::sync::Mutex, }; @@ -118,18 +118,16 @@ fn get_process_key_stat(process_key: ProcessKey, values: Vec<&str>) -> Option, - ) -> Result { + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { let mut time_series_data = TimeSeriesData::default(); // map process field -> process -> series let mut per_field_per_process_series: HashMap> = HashMap::new(); - let time_zero = if let Some(first_buffer) = raw_data.first() { + let time_zero = if let Some(t) = params.collection_start { + t + } else if let Some(first_buffer) = raw_data.first() { match first_buffer { Data::ProcessesRaw(ref value) => value.time, _ => panic!("Invalid Data type in raw file"), diff --git a/src/data/vmstat.rs b/src/data/vmstat.rs index c75da31f..d25025db 100644 --- a/src/data/vmstat.rs +++ b/src/data/vmstat.rs @@ -47,12 +47,9 @@ impl Vmstat { } impl ProcessData for Vmstat { - fn process_raw_data( - &mut self, - _params: ReportParams, - raw_data: Vec, - ) -> Result { - let mut time_series_data_processor = time_series_data_processor_with_custom_aggregate!(); + fn process_raw_data(&mut self, params: ReportParams, raw_data: Vec) -> Result { + let mut time_series_data_processor = + time_series_data_processor_with_custom_aggregate!(params.collection_start); for buffer in raw_data { let raw_value = match buffer { diff --git a/src/lib.rs b/src/lib.rs index 6f9a79b9..20255b86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod visualizer; use crate::analytics::{AnalyticalEngine, DataFindings}; use crate::data::aperf_runlog::AperfRunlog; use crate::data::common::processed_data_accessor::ProcessedDataAccessor; +use crate::data::TimeEnum; use crate::visualizer::DataVisualizer; use anyhow::Result; use chrono::prelude::*; @@ -29,7 +30,6 @@ use thiserror::Error; #[cfg(target_os = "linux")] use { crate::data::aperf_stats::AperfStat, - data::TimeEnum, flate2::{write::GzEncoder, Compression}, nix::poll::{poll, PollFd, PollFlags, PollTimeout}, nix::sys::{ @@ -190,23 +190,6 @@ impl PerformanceData { pub fn init_collectors(&mut self) -> Result<()> { fs::create_dir(self.init_params.dir_name.clone())?; - /* - * Create a meta_data file to hold the InitParams that was used by the collector. - * This will help when we visualize the data and we don't have to guess these values. - */ - let meta_data_path = format!( - "{}/meta_data.{}", - self.init_params.dir_name.clone(), - APERF_FILE_FORMAT - ); - let meta_data_handle = fs::OpenOptions::new() - .create(true) - .write(true) - .open(meta_data_path.clone()) - .expect("Could not create meta-data file"); - - bincode::serialize_into(meta_data_handle, &self.init_params)?; - self.aperf_stats_path = PathBuf::from(self.init_params.dir_name.clone()).join(format!( "{}.{}", get_data_name_from_type::(), @@ -286,6 +269,7 @@ impl PerformanceData { pub fn collect_data_serial(&mut self) -> Result<()> { let start = time::Instant::now(); + self.init_params.collection_start = Some(TimeEnum::DateTime(Utc::now())); let mut aperf_collect_data = AperfStat::new(); let mut current = time::Instant::now(); let end = current + time::Duration::from_secs(self.init_params.period); @@ -294,7 +278,7 @@ impl PerformanceData { let mut tfd = TimerFd::new()?; tfd.set_state( TimerState::Periodic { - current: time::Duration::from_secs(self.init_params.interval), + current: time::Duration::from_nanos(1), interval: time::Duration::from_secs(self.init_params.interval), }, SetTimeFlags::Default, @@ -372,6 +356,7 @@ impl PerformanceData { } } } + self.init_params.collection_end = Some(TimeEnum::DateTime(Utc::now())); for (_name, datatype) in self.collectors.iter_mut() { datatype.set_signal(datatype_signal); datatype.finish_data_collection()?; @@ -383,10 +368,22 @@ impl PerformanceData { Ok(()) } - pub fn end(&mut self) -> Result<()> { + pub fn create_record_archive(&mut self) -> Result<()> { let dst_path = PathBuf::from(&self.init_params.dir_name).join(*APERF_RUNLOG); fs::copy(&self.init_params.runlog, dst_path)?; + // Persist meta_data once, at the end of the recording. This captures the final + // InitParams including collection_start/collection_end stamped by + // collect_data_serial. + let meta_data_path = PathBuf::from(&self.init_params.dir_name) + .join(format!("meta_data.{}", APERF_FILE_FORMAT)); + let meta_data_handle = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&meta_data_path)?; + bincode::serialize_into(meta_data_handle, &self.init_params)?; + // All activities in the record folder should be complete before this. self.create_data_archive()?; Ok(()) @@ -458,6 +455,7 @@ impl VisualizationData { run_data_dir: PathBuf, tmp_dir: &Path, report_dir: &Path, + collection_start: Option, ) -> Result<()> { let visualizers_len = self.visualizers.len(); let mut error_count = 0; @@ -468,6 +466,7 @@ impl VisualizationData { run_data_dir.clone(), tmp_dir, report_dir, + collection_start, ) { debug!("{:#?}", e); error_count += 1; @@ -504,13 +503,13 @@ impl VisualizationData { processed_data_accessor: &mut ProcessedDataAccessor, ) -> HashMap { let mut analytical_engine = AnalyticalEngine::default(); - for (data_name, data_visualizer) in &self.visualizers { + for (data_name, data_visualizer) in &mut self.visualizers { analytical_engine.add_data_rules( data_name.clone(), data_visualizer.data.get_analytical_rules(), ); analytical_engine - .add_processed_data(data_name.clone(), &data_visualizer.processed_data); + .add_processed_data(data_name.clone(), &mut data_visualizer.processed_data); } info!("Running analytical rules"); @@ -533,8 +532,6 @@ impl VisualizationData { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct InitParams { - pub time_now: DateTime, - pub time_str: String, pub dir_name: String, pub period: u64, pub profile: HashMap, @@ -548,13 +545,22 @@ pub struct InitParams { pub perf_frequency: u32, pub hotline_frequency: u32, pub num_to_report: u32, + /// Wall-clock start of `collect_data_serial`. `None` for archives + /// produced by versions of aperf that did not record this. + #[serde(default)] + pub collection_start: Option, + /// Wall-clock end of `collect_data_serial`. `None` for archives + /// produced by versions of aperf that did not record this. + #[serde(default)] + pub collection_end: Option, } impl InitParams { pub fn new(dir: String) -> Self { - let time_now = Utc::now(); - let time_str = time_now.format("%Y-%m-%d_%H_%M_%S").to_string(); - let mut dir_name = format!("./aperf_{}", time_str); + let mut dir_name = format!( + "./aperf_{}", + Utc::now().format("%Y-%m-%d_%H_%M_%S").to_string() + ); let mut run_name = String::new(); if !dir.is_empty() { dir_name = Path::new(&dir) @@ -575,8 +581,6 @@ impl InitParams { let commit_sha_short = env!("VERGEN_GIT_SHA").to_string(); InitParams { - time_now, - time_str, dir_name, period: 0, profile: HashMap::new(), @@ -590,6 +594,8 @@ impl InitParams { perf_frequency: 99, hotline_frequency: 1000, num_to_report: 5000, + collection_start: None, + collection_end: None, } } } @@ -604,7 +610,8 @@ impl Default for InitParams { mod tests { #[cfg(target_os = "linux")] use { - super::{InitParams, PerformanceData, APERF_FILE_FORMAT}, + super::{InitParams, PerformanceData}, + chrono::Utc, std::fs, std::path::Path, }; @@ -616,7 +623,7 @@ mod tests { let dir_name = format!( "./aperf_{}", - pd.init_params.time_now.format("%Y-%m-%d_%H_%M_%S") + Utc::now().format("%Y-%m-%d_%H_%M_%S").to_string() ); assert!(pd.collectors.is_empty()); assert_eq!(pd.init_params.dir_name, dir_name); @@ -626,17 +633,14 @@ mod tests { #[test] fn test_performance_data_dir_creation() { let mut params = InitParams::new("".to_string()); - params.dir_name = format!("./performance_data_dir_creation_{}", params.time_str); + params.dir_name = format!( + "./performance_data_dir_creation_{}", + Utc::now().format("%Y-%m-%d_%H_%M_%S").to_string() + ); let mut pd = PerformanceData::new(params.clone()); pd.init_collectors().unwrap(); assert!(Path::new(&pd.init_params.dir_name).exists()); - let full_path = format!( - "{}/meta_data.{}", - params.dir_name.clone(), - APERF_FILE_FORMAT - ); - assert!(Path::new(&full_path).exists()); fs::remove_dir_all(pd.init_params.dir_name).unwrap(); } } diff --git a/src/record.rs b/src/record.rs index 26e34b35..2189f66c 100644 --- a/src/record.rs +++ b/src/record.rs @@ -179,7 +179,7 @@ pub fn record(record: &Record, tmp_dir: &Path, runlog: &Path) -> Result<()> { performance_data.collect_data_serial()?; info!("Data collection complete."); - performance_data.end()?; + performance_data.create_record_archive()?; Ok(()) } diff --git a/src/report.rs b/src/report.rs index 493202ec..7e8ddfb1 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,20 +1,27 @@ use crate::analytics::BASE_RUN_NAME; use crate::data::common::processed_data_accessor::ProcessedDataAccessor; use crate::data::common::utils::no_tar_gz_file_name; -use crate::data::JS_DIR; -use crate::{data, PDError, VisualizationData}; +use crate::data::{TimeEnum, JS_DIR}; +use crate::{data, InitParams, PDError, VisualizationData, APERF_FILE_FORMAT}; use anyhow::{Context, Result}; use chrono::Utc; use clap::Args; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use log::{info, warn}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs::File; use std::io::Write; use std::path::PathBuf; use std::{env, fs}; +#[derive(Serialize, Deserialize)] +struct RunMetadata { + name: String, + collection_start_ms: Option, + collection_end_ms: Option, +} + #[derive(Serialize)] struct VersionInfo { version: &'static str, @@ -30,6 +37,17 @@ impl VersionInfo { } } +/// Reads `meta_data.bin` from a run's data directory and returns the run's +/// wall-clock collection start and end times. Returns `None` if the file is +/// missing, fails to deserialize, or was produced by an older aperf that did +/// not stamp collection start/end. +fn read_collection_times(run_dir_path: &PathBuf) -> Option<(TimeEnum, TimeEnum)> { + let meta_data_path = run_dir_path.join(format!("meta_data.{}", APERF_FILE_FORMAT)); + let bytes = fs::read(&meta_data_path).ok()?; + let params: InitParams = bincode::deserialize(&bytes).ok()?; + Some((params.collection_start?, params.collection_end?)) +} + /// Stores the information of all runs to be included in the report #[derive(Default)] struct RunsInfo { @@ -45,6 +63,12 @@ struct RunsInfo { per_run_from_time: HashMap, /// The specified end time of every run's time range per_run_to_time: HashMap, + /// Wall-clock start of `collect_data_serial`, derived from the run's + /// `meta_data.bin`. Missing entry => archive was recorded by an older + /// aperf that did not stamp collection start/end times. + per_run_start_time: HashMap, + /// Wall-clock end of `collect_data_serial`. + per_run_end_time: HashMap, } impl RunsInfo { @@ -62,6 +86,11 @@ impl RunsInfo { self.run_names.push(deduped_run_name.clone()); self.run_archive_paths .insert(deduped_run_name.clone(), run_archive_path); + if let Some((start, end)) = read_collection_times(&run_dir_path) { + self.per_run_start_time + .insert(deduped_run_name.clone(), start); + self.per_run_end_time.insert(deduped_run_name.clone(), end); + } self.run_dir_paths.insert(deduped_run_name, run_dir_path); Ok(()) @@ -366,7 +395,12 @@ pub fn get_report_run_archive_paths(report_path: &PathBuf) -> Option = serde_json::from_str(json_str).unwrap(); + // Try the current structured shape first; fall back to the legacy string-array + // shape emitted by older aperf versions so previously generated reports still open. + let runs: Vec = match serde_json::from_str::>(json_str) { + Ok(metas) => metas.into_iter().map(|m| m.name).collect(), + Err(_) => serde_json::from_str(json_str).unwrap(), + }; let mut run_archive_paths = Vec::new(); for run_name in runs { @@ -397,29 +431,56 @@ fn generate_report_files(report_dir: PathBuf, runs_info: RunsInfo, tmp_dir: &Pat data::add_all_visualization_data(&mut visualization_data); /* Init visualizers */ for (run_name, run_data_dir) in runs_info.run_dir_paths { + let collection_start = runs_info.per_run_start_time.get(&run_name).copied(); visualization_data - .init_visualizers(run_name.clone(), run_data_dir.clone(), tmp_dir, &report_dir) + .init_visualizers( + run_name.clone(), + run_data_dir.clone(), + tmp_dir, + &report_dir, + collection_start, + ) .unwrap(); visualization_data.process_raw_data().unwrap(); } - let mut processed_data_accessor = ProcessedDataAccessor::from_time_ranges( - runs_info.per_run_from_time, - runs_info.per_run_to_time, - ); - - let analytical_findings = visualization_data.run_analytics(&mut processed_data_accessor); - - /* Generate run.js */ + /* Generate run.js, containing run name and metadata (currently only start and end wall time) */ let run_js_path = processed_data_js_dir.join("runs.js"); let mut runs_file = File::create(run_js_path).unwrap(); + let runs_metadata: Vec = runs_info + .run_names + .iter() + .map(|name| RunMetadata { + name: name.clone(), + collection_start_ms: runs_info + .per_run_start_time + .get(name) + .and_then(|t| match t { + TimeEnum::DateTime(dt) => Some(dt.timestamp_millis()), + _ => None, + }), + collection_end_ms: runs_info.per_run_end_time.get(name).and_then(|t| match t { + TimeEnum::DateTime(dt) => Some(dt.timestamp_millis()), + _ => None, + }), + }) + .collect(); write!( runs_file, "runs_raw = {}", - serde_json::to_string(&runs_info.run_names).unwrap() + serde_json::to_string(&runs_metadata).unwrap() ) .unwrap(); + let mut processed_data_accessor = ProcessedDataAccessor::from_time_ranges( + runs_info.per_run_from_time, + runs_info.per_run_to_time, + runs_info.per_run_start_time, + runs_info.per_run_end_time, + ); + + let analytical_findings = visualization_data.run_analytics(&mut processed_data_accessor); + /* Generate version.js */ let version_js_path = processed_data_js_dir.join("version.js"); let mut version_file = File::create(version_js_path).unwrap(); diff --git a/src/report_frontend/src/definitions/data-config.ts b/src/report_frontend/src/definitions/data-config.ts index 69631893..f85f1bdc 100644 --- a/src/report_frontend/src/definitions/data-config.ts +++ b/src/report_frontend/src/definitions/data-config.ts @@ -1,6 +1,6 @@ import { ALL_DATA_TYPES, DataFindings, DataType, ReportData } from "./types"; -declare let runs_raw; +declare let runs_raw: RunInfo[]; declare let version_info; declare let processed_systeminfo_data; declare let processed_cpu_utilization_data; @@ -93,7 +93,16 @@ export const PER_DATA_ANALYTICAL_FINDINGS: { [key in DataType]: DataFindings } = aperf_stats: aperf_stats_findings, }; -export const RUNS: string[] = Array.from(runs_raw); +export interface RunInfo { + readonly name: string; + readonly collection_start_ms: number | null; + readonly collection_end_ms: number | null; +} + +// TODO: Display per run start and end times in frontend, used to annotate profiles +export const RUN_INFO: RunInfo[] = Array.from(runs_raw); + +export const RUNS: string[] = RUN_INFO.map((r) => r.name); export const VERSION_INFO = version_info; diff --git a/src/visualizer.rs b/src/visualizer.rs index 863ee318..d427d641 100644 --- a/src/visualizer.rs +++ b/src/visualizer.rs @@ -1,5 +1,6 @@ use crate::data::common::data_formats::{AperfData, DataFormat, ProcessedData}; use crate::data::common::utils::{combine_value_ranges, topological_sort}; +use crate::data::TimeEnum; use crate::{data::Data, data::ReportData, get_file}; use anyhow::Result; use log::{debug, error}; @@ -14,6 +15,9 @@ pub struct ReportParams { pub report_dir: PathBuf, pub run_name: String, pub data_file_path: PathBuf, + /// Wall-clock start of `collect_data_serial`. Used to anchor time_diff=0 in + /// time-series data to the actual collection start rather than the first sample time. + pub collection_start: Option, } impl ReportParams { @@ -24,6 +28,7 @@ impl ReportParams { report_dir: PathBuf::new(), run_name: String::new(), data_file_path: PathBuf::new(), + collection_start: None, } } } @@ -55,8 +60,10 @@ impl DataVisualizer { run_data_dir: PathBuf, tmp_dir: &Path, report_dir: &Path, + collection_start: Option, ) -> Result<()> { self.report_params.run_name = run_name.clone(); + self.report_params.collection_start = collection_start; let (file_path, file) = get_file(&run_data_dir, self.data_name.to_string()).or_else(|e| { // Backward compatibility: if file is not found using the data's name, diff --git a/tests/analytics/test_key_value_key_expected_rule.rs b/tests/analytics/test_key_value_key_expected_rule.rs index 7357fd6f..1006d0e3 100644 --- a/tests/analytics/test_key_value_key_expected_rule.rs +++ b/tests/analytics/test_key_value_key_expected_rule.rs @@ -8,7 +8,7 @@ use super::test_helpers::{create_key_value_data, create_processed_data, DataFind #[test] fn test_key_matches_expected_value() { let kv_data = create_key_value_data(vec![("test_key", "expected_value")]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::KeyValue(kv_data))]); let rule = KeyValueKeyExpectedRule { @@ -22,7 +22,7 @@ fn test_key_matches_expected_value() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -32,7 +32,7 @@ fn test_key_matches_expected_value() { #[test] fn test_key_does_not_match_expected_value() { let kv_data = create_key_value_data(vec![("test_key", "wrong_value")]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::KeyValue(kv_data))]); let rule = KeyValueKeyExpectedRule { @@ -46,7 +46,7 @@ fn test_key_does_not_match_expected_value() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -57,7 +57,7 @@ fn test_key_does_not_match_expected_value() { #[test] fn test_key_missing() { let kv_data = create_key_value_data(vec![("other_key", "value")]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::KeyValue(kv_data))]); let rule = KeyValueKeyExpectedRule { @@ -71,7 +71,7 @@ fn test_key_missing() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -83,7 +83,7 @@ fn test_key_missing() { fn test_multiple_runs() { let kv_data1 = create_key_value_data(vec![("test_key", "expected_value")]); let kv_data2 = create_key_value_data(vec![("test_key", "wrong_value")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -102,7 +102,7 @@ fn test_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -114,7 +114,7 @@ fn test_multiple_runs() { #[test] fn test_empty_key_value_data() { let kv_data = create_key_value_data(vec![]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::KeyValue(kv_data))]); let rule = KeyValueKeyExpectedRule { @@ -128,7 +128,7 @@ fn test_empty_key_value_data() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); diff --git a/tests/analytics/test_key_value_key_run_comparison_rule.rs b/tests/analytics/test_key_value_key_run_comparison_rule.rs index e1d196dc..c089bd82 100644 --- a/tests/analytics/test_key_value_key_run_comparison_rule.rs +++ b/tests/analytics/test_key_value_key_run_comparison_rule.rs @@ -15,7 +15,7 @@ fn test_values_match_across_runs() { let kv_data1 = create_key_value_data(vec![("test_key", "same_value")]); let kv_data2 = create_key_value_data(vec![("test_key", "same_value")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -33,7 +33,7 @@ fn test_values_match_across_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -46,7 +46,7 @@ fn test_values_differ_across_runs() { let kv_data1 = create_key_value_data(vec![("test_key", "value1")]); let kv_data2 = create_key_value_data(vec![("test_key", "value2")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -64,7 +64,7 @@ fn test_values_differ_across_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -78,7 +78,7 @@ fn test_key_missing_in_non_base_run() { let kv_data1 = create_key_value_data(vec![("test_key", "value1")]); let kv_data2 = create_key_value_data(vec![("other_key", "value2")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -96,7 +96,7 @@ fn test_key_missing_in_non_base_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -110,7 +110,7 @@ fn test_key_missing_in_base_run() { let kv_data1 = create_key_value_data(vec![("other_key", "value1")]); let kv_data2 = create_key_value_data(vec![("test_key", "value2")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -128,7 +128,7 @@ fn test_key_missing_in_base_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -142,7 +142,7 @@ fn test_multiple_non_base_runs() { let kv_data1 = create_key_value_data(vec![("test_key", "base_value")]); let kv_data2 = create_key_value_data(vec![("test_key", "base_value")]); let kv_data3 = create_key_value_data(vec![("test_key", "different_value")]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::KeyValue(kv_data1)), @@ -161,7 +161,7 @@ fn test_multiple_non_base_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); diff --git a/tests/analytics/test_profile_metadata_comparison_rule.rs b/tests/analytics/test_profile_metadata_comparison_rule.rs index 488fe4be..46e79248 100644 --- a/tests/analytics/test_profile_metadata_comparison_rule.rs +++ b/tests/analytics/test_profile_metadata_comparison_rule.rs @@ -46,7 +46,7 @@ fn test_values_match_across_runs() { create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); let profiling_data2 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -66,7 +66,7 @@ fn test_values_match_across_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -81,7 +81,7 @@ fn test_values_differ_across_runs() { create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); let profiling_data2 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("user"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -101,7 +101,7 @@ fn test_values_differ_across_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -116,7 +116,7 @@ fn test_field_missing_in_non_base_run() { let profiling_data1 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); let profiling_data2 = create_profiling_data(vec![create_key_value_data("cpu", "mode", None)]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -136,7 +136,7 @@ fn test_field_missing_in_non_base_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -151,7 +151,7 @@ fn test_field_missing_in_base_run() { let profiling_data1 = create_profiling_data(vec![create_key_value_data("cpu", "mode", None)]); let profiling_data2 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -171,7 +171,7 @@ fn test_field_missing_in_base_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -188,7 +188,7 @@ fn test_multiple_non_base_runs() { create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); let profiling_data3 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("user"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -209,7 +209,7 @@ fn test_multiple_non_base_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); diff --git a/tests/analytics/test_profile_metadata_expected_rule.rs b/tests/analytics/test_profile_metadata_expected_rule.rs index 80987d67..e0917eb1 100644 --- a/tests/analytics/test_profile_metadata_expected_rule.rs +++ b/tests/analytics/test_profile_metadata_expected_rule.rs @@ -38,7 +38,7 @@ fn create_profiling_data(metadata: Vec) -> ProfilingData { fn test_field_matches_expected_value() { let profiling_data = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -56,7 +56,7 @@ fn test_field_matches_expected_value() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -67,7 +67,7 @@ fn test_field_matches_expected_value() { fn test_field_does_not_match_expected_value() { let profiling_data = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("user"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -85,7 +85,7 @@ fn test_field_does_not_match_expected_value() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -96,7 +96,7 @@ fn test_field_does_not_match_expected_value() { #[test] fn test_field_missing_should_exist() { let profiling_data = create_profiling_data(vec![create_key_value_data("cpu", "mode", None)]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -114,7 +114,7 @@ fn test_field_missing_should_exist() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -129,7 +129,7 @@ fn test_regex_pattern_match() { "mode", Some("kernel_mode"), )]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -147,7 +147,7 @@ fn test_regex_pattern_match() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -160,7 +160,7 @@ fn test_multiple_runs() { create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("kernel"))]); let profiling_data2 = create_profiling_data(vec![create_key_value_data("cpu", "mode", Some("user"))]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::Profile(profiling_data1)), @@ -181,7 +181,7 @@ fn test_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); diff --git a/tests/analytics/test_profile_stack_frame_threshold_rule.rs b/tests/analytics/test_profile_stack_frame_threshold_rule.rs index 5ce4619e..2ef2ea22 100644 --- a/tests/analytics/test_profile_stack_frame_threshold_rule.rs +++ b/tests/analytics/test_profile_stack_frame_threshold_rule.rs @@ -58,7 +58,7 @@ fn create_profiler_data(group_name: &str) -> ProfilingData { #[test] fn test_below_threshold() { let profiling_data = create_profiler_data("cpu"); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -79,7 +79,7 @@ fn test_below_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 0); @@ -88,7 +88,7 @@ fn test_below_threshold() { #[test] fn test_above_threshold() { let profiling_data = create_profiler_data("cpu"); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -109,7 +109,7 @@ fn test_above_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -119,7 +119,7 @@ fn test_above_threshold() { #[test] fn test_stack_pattern() { let profiling_data = create_profiler_data("cpu"); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -140,7 +140,7 @@ fn test_stack_pattern() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -152,7 +152,7 @@ fn test_missing_metric() { let profiling_data = ProfilingData { profilers: HashMap::new(), }; - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![("run1", AperfData::Profile(profiling_data))], ); @@ -173,7 +173,7 @@ fn test_missing_metric() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 0); diff --git a/tests/analytics/test_time_series_data_point_threshold_rule.rs b/tests/analytics/test_time_series_data_point_threshold_rule.rs index 524585c9..90a89d80 100644 --- a/tests/analytics/test_time_series_data_point_threshold_rule.rs +++ b/tests/analytics/test_time_series_data_point_threshold_rule.rs @@ -11,7 +11,7 @@ use super::test_helpers::{create_processed_data, create_time_series_data, DataFi #[test] fn test_no_data_points_exceed_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -27,7 +27,7 @@ fn test_no_data_points_exceed_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -37,7 +37,7 @@ fn test_no_data_points_exceed_threshold() { #[test] fn test_one_data_point_exceeds_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 60.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -53,7 +53,7 @@ fn test_one_data_point_exceeds_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -64,7 +64,7 @@ fn test_one_data_point_exceeds_threshold() { #[test] fn test_multiple_data_points_exceed_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![60.0, 70.0, 80.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -80,7 +80,7 @@ fn test_multiple_data_points_exceed_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -90,7 +90,7 @@ fn test_multiple_data_points_exceed_threshold() { #[test] fn test_less_than_comparator() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -106,7 +106,7 @@ fn test_less_than_comparator() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -116,7 +116,7 @@ fn test_less_than_comparator() { #[test] fn test_equal_comparator() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 50.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -132,7 +132,7 @@ fn test_equal_comparator() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -142,7 +142,7 @@ fn test_equal_comparator() { #[test] fn test_metric_not_found() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -158,7 +158,7 @@ fn test_metric_not_found() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -168,7 +168,7 @@ fn test_metric_not_found() { #[test] fn test_empty_values() { let ts_data = create_time_series_data(vec![("metric1", vec![])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -184,7 +184,7 @@ fn test_empty_values() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -195,7 +195,7 @@ fn test_empty_values() { fn test_multiple_runs() { let ts_data1 = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![60.0, 70.0, 80.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -216,7 +216,7 @@ fn test_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -228,7 +228,7 @@ fn test_multiple_runs() { #[test] fn test_negative_score() { let ts_data = create_time_series_data(vec![("metric1", vec![60.0, 70.0, 80.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -244,7 +244,7 @@ fn test_negative_score() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -258,7 +258,7 @@ fn test_time_range_filters_data_points() { "metric1", vec![10.0, 20.0, 30.0, 60.0, 70.0, 80.0, 10.0, 20.0], )]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -275,7 +275,7 @@ fn test_time_range_filters_data_points() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -284,18 +284,22 @@ fn test_time_range_filters_data_points() { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 0)]), HashMap::from([("run1".to_string(), 2)]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 0); // Time range 3:5 → [60,70,80] → finding let mut accessor2 = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 3)]), HashMap::from([("run1".to_string(), 5)]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); } @@ -310,7 +314,7 @@ fn test_time_range_multi_run() { create_time_series_data(vec![("metric1", vec![10.0, 10.0, 10.0, 10.0, 80.0, 90.0])]); let ts_data3 = create_time_series_data(vec![("metric1", vec![60.0, 70.0, 80.0, 10.0, 10.0, 10.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -333,7 +337,7 @@ fn test_time_range_multi_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 3); @@ -350,9 +354,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 1), ("run3".to_string(), 1), ]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 1); assert!(findings2.has_findings_for_run("run3")); @@ -368,9 +374,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 5), ("run3".to_string(), 5), ]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); assert!(findings3.has_findings_for_run("run2")); @@ -386,9 +394,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 5), ("run3".to_string(), 5), ]), + HashMap::new(), + HashMap::new(), ); let mut findings4 = DataFindings::default(); - rule.analyze(&mut findings4, &processed_data, &mut accessor3); + rule.analyze(&mut findings4, &mut processed_data, &mut accessor3); assert_eq!(findings4.num_runs_with_findings(), 2); assert!(findings4.has_findings_for_run("run1")); assert!(findings4.has_findings_for_run("run2")); @@ -405,7 +415,7 @@ fn test_regex_matching_matches_multiple_metrics() { ("cpu_system", vec![10.0, 70.0, 30.0]), ("mem_used", vec![90.0, 90.0, 90.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -421,7 +431,7 @@ fn test_regex_matching_matches_multiple_metrics() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -439,7 +449,7 @@ fn test_regex_matching_no_metrics_match_pattern() { ("mem_used", vec![60.0, 70.0, 80.0]), ("mem_free", vec![60.0, 70.0, 80.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -455,7 +465,7 @@ fn test_regex_matching_no_metrics_match_pattern() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -469,7 +479,7 @@ fn test_regex_matching_only_some_matched_metrics_exceed_threshold() { ("cpu_user", vec![60.0, 70.0, 80.0]), ("cpu_system", vec![10.0, 20.0, 30.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -485,7 +495,7 @@ fn test_regex_matching_only_some_matched_metrics_exceed_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -507,7 +517,7 @@ fn test_regex_matching_multiple_runs() { ("cpu_user", vec![80.0, 90.0]), ("cpu_system", vec![60.0, 70.0]), ]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -528,7 +538,7 @@ fn test_regex_matching_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -543,7 +553,7 @@ fn test_regex_matching_multiple_runs() { #[test] fn test_regex_matching_invalid_regex() { let ts_data = create_time_series_data(vec![("cpu_user", vec![60.0, 70.0, 80.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -559,7 +569,7 @@ fn test_regex_matching_invalid_regex() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -574,7 +584,7 @@ fn test_regex_matching_with_time_range() { ("cpu_user", vec![10.0, 20.0, 30.0, 60.0, 70.0, 10.0]), ("cpu_system", vec![60.0, 70.0, 10.0, 20.0, 30.0, 10.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -591,9 +601,11 @@ fn test_regex_matching_with_time_range() { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 0)]), HashMap::from([("run1".to_string(), 1)]), + HashMap::new(), + HashMap::new(), ); let mut findings = DataFindings::default(); - rule.analyze(&mut findings, &processed_data, &mut accessor); + rule.analyze(&mut findings, &mut processed_data, &mut accessor); assert_eq!(findings.num_runs_with_findings(), 1); assert!(findings.has_findings_for_metric("run1", "cpu_system")); assert!(!findings.has_findings_for_metric("run1", "cpu_user")); @@ -602,9 +614,11 @@ fn test_regex_matching_with_time_range() { let mut accessor2 = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 3)]), HashMap::from([("run1".to_string(), 4)]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor2); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor2); assert_eq!(findings2.num_runs_with_findings(), 1); assert!(findings2.has_findings_for_metric("run1", "cpu_user")); assert!(!findings2.has_findings_for_metric("run1", "cpu_system")); @@ -617,7 +631,7 @@ fn test_regex_matching_false_uses_exact_metric_name() { ("cpu_user", vec![60.0, 70.0, 80.0]), ("cpu_system", vec![60.0, 70.0, 80.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesDataPointThresholdRule { @@ -633,7 +647,7 @@ fn test_regex_matching_false_uses_exact_metric_name() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); diff --git a/tests/analytics/test_time_series_stat_intra_run_comparison_rule.rs b/tests/analytics/test_time_series_stat_intra_run_comparison_rule.rs index 6fc34b06..4c795f51 100644 --- a/tests/analytics/test_time_series_stat_intra_run_comparison_rule.rs +++ b/tests/analytics/test_time_series_stat_intra_run_comparison_rule.rs @@ -14,7 +14,7 @@ fn test_no_significant_delta() { ("baseline_metric", vec![100.0, 100.0, 100.0]), ("comparison_metric", vec![105.0, 105.0, 105.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -32,7 +32,7 @@ fn test_no_significant_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -45,7 +45,7 @@ fn test_significant_positive_delta() { ("baseline_metric", vec![100.0, 100.0, 100.0]), ("comparison_metric", vec![150.0, 150.0, 150.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -63,7 +63,7 @@ fn test_significant_positive_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -77,7 +77,7 @@ fn test_significant_negative_delta() { ("baseline_metric", vec![100.0, 100.0, 100.0]), ("comparison_metric", vec![50.0, 50.0, 50.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -95,7 +95,7 @@ fn test_significant_negative_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -108,7 +108,7 @@ fn test_abs_delta() { ("baseline_metric", vec![100.0, 100.0, 100.0]), ("comparison_metric", vec![50.0, 50.0, 50.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -126,7 +126,7 @@ fn test_abs_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -139,7 +139,7 @@ fn test_baseline_stat_zero() { ("baseline_metric", vec![0.0, 0.0, 0.0]), ("comparison_metric", vec![50.0, 50.0, 50.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -157,7 +157,7 @@ fn test_baseline_stat_zero() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -170,7 +170,7 @@ fn test_equal_zero_stats() { ("baseline_metric", vec![0.0, 0.0, 0.0]), ("comparison_metric", vec![0.0, 0.0, 0.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -188,7 +188,7 @@ fn test_equal_zero_stats() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -201,7 +201,7 @@ fn test_max_stat() { ("baseline_metric", vec![10.0, 20.0, 100.0]), ("comparison_metric", vec![10.0, 20.0, 200.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -219,7 +219,7 @@ fn test_max_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -232,7 +232,7 @@ fn test_min_stat() { ("baseline_metric", vec![10.0, 20.0, 100.0]), ("comparison_metric", vec![5.0, 20.0, 100.0]), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -250,7 +250,7 @@ fn test_min_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -260,7 +260,7 @@ fn test_min_stat() { #[test] fn test_baseline_metric_not_found() { let ts_data = create_time_series_data(vec![("comparison_metric", vec![150.0, 150.0, 150.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -278,7 +278,7 @@ fn test_baseline_metric_not_found() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -288,7 +288,7 @@ fn test_baseline_metric_not_found() { #[test] fn test_comparison_metric_not_found() { let ts_data = create_time_series_data(vec![("baseline_metric", vec![100.0, 100.0, 100.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -306,7 +306,7 @@ fn test_comparison_metric_not_found() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -323,7 +323,7 @@ fn test_multiple_runs() { ("baseline_metric", vec![100.0, 100.0, 100.0]), ("comparison_metric", vec![150.0, 150.0, 150.0]), ]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -346,7 +346,7 @@ fn test_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -369,7 +369,7 @@ fn test_time_range_affects_intra_run_comparison() { vec![100.0, 100.0, 100.0, 200.0, 200.0, 200.0], ), ]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatIntraRunComparisonRule { @@ -388,7 +388,7 @@ fn test_time_range_affects_intra_run_comparison() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -397,18 +397,22 @@ fn test_time_range_affects_intra_run_comparison() { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 0)]), HashMap::from([("run1".to_string(), 2)]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 0); // Time range 3:5 → baseline avg=100, comparison avg=200 → 100% delta → finding let mut accessor2 = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 3)]), HashMap::from([("run1".to_string(), 5)]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); } @@ -447,7 +451,7 @@ fn test_time_range_multi_run() { vec![105.0, 105.0, 105.0, 105.0, 105.0, 105.0], ), ]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -472,7 +476,7 @@ fn test_time_range_multi_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 2); @@ -492,9 +496,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 2), ("run3".to_string(), 2), ]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 1); assert!(findings2.has_findings_for_run("run2")); @@ -510,9 +516,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 5), ("run3".to_string(), 5), ]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); assert!(findings3.has_findings_for_run("run1")); } diff --git a/tests/analytics/test_time_series_stat_run_comparison_rule.rs b/tests/analytics/test_time_series_stat_run_comparison_rule.rs index 971ddabb..ddfdc06b 100644 --- a/tests/analytics/test_time_series_stat_run_comparison_rule.rs +++ b/tests/analytics/test_time_series_stat_run_comparison_rule.rs @@ -18,7 +18,7 @@ fn test_no_significant_delta() { let ts_data1 = create_time_series_data(vec![("metric1", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![105.0, 105.0, 105.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -40,7 +40,7 @@ fn test_no_significant_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -53,7 +53,7 @@ fn test_significant_positive_delta() { let ts_data1 = create_time_series_data(vec![("metric1", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![150.0, 150.0, 150.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -75,7 +75,7 @@ fn test_significant_positive_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -89,7 +89,7 @@ fn test_significant_negative_delta() { let ts_data1 = create_time_series_data(vec![("metric1", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![50.0, 50.0, 50.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -111,7 +111,7 @@ fn test_significant_negative_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -124,7 +124,7 @@ fn test_abs_delta() { let ts_data1 = create_time_series_data(vec![("metric1", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![50.0, 50.0, 50.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -146,7 +146,7 @@ fn test_abs_delta() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -159,7 +159,7 @@ fn test_base_stat_zero() { let ts_data1 = create_time_series_data(vec![("metric1", vec![0.0, 0.0, 0.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![50.0, 50.0, 50.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -181,7 +181,7 @@ fn test_base_stat_zero() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -194,7 +194,7 @@ fn test_equal_zero_stats() { let ts_data1 = create_time_series_data(vec![("metric1", vec![0.0, 0.0, 0.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![0.0, 0.0, 0.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -216,7 +216,7 @@ fn test_equal_zero_stats() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -229,7 +229,7 @@ fn test_max_stat() { let ts_data1 = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 200.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -251,7 +251,7 @@ fn test_max_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -264,7 +264,7 @@ fn test_metric_not_in_base_run() { let ts_data1 = create_time_series_data(vec![("other_metric", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![150.0, 150.0, 150.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -286,7 +286,7 @@ fn test_metric_not_in_base_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -300,7 +300,7 @@ fn test_multiple_non_base_runs() { let ts_data1 = create_time_series_data(vec![("metric1", vec![100.0, 100.0, 100.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![105.0, 105.0, 105.0])]); let ts_data3 = create_time_series_data(vec![("metric1", vec![150.0, 150.0, 150.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -323,7 +323,7 @@ fn test_multiple_non_base_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -347,7 +347,7 @@ fn test_time_range_affects_comparison() { "metric1", vec![100.0, 100.0, 100.0, 200.0, 200.0, 200.0], )]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -370,7 +370,7 @@ fn test_time_range_affects_comparison() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -379,18 +379,22 @@ fn test_time_range_affects_comparison() { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 0), ("run2".to_string(), 0)]), HashMap::from([("run1".to_string(), 2), ("run2".to_string(), 2)]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 0); // Time range 3:5 → run1 avg=100, run2 avg=200 → 100% delta → finding let mut accessor2 = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 3), ("run2".to_string(), 3)]), HashMap::from([("run1".to_string(), 5), ("run2".to_string(), 5)]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); } @@ -413,7 +417,7 @@ fn test_time_range_multi_run() { "metric1", vec![200.0, 200.0, 200.0, 100.0, 100.0, 100.0], )]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -437,7 +441,7 @@ fn test_time_range_multi_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 2); @@ -456,9 +460,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 2), ("run3".to_string(), 2), ]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 1); assert!(findings2.has_findings_for_run("run3")); @@ -474,9 +480,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 5), ("run3".to_string(), 5), ]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); assert!(findings3.has_findings_for_run("run2")); } diff --git a/tests/analytics/test_time_series_stat_threshold_rule.rs b/tests/analytics/test_time_series_stat_threshold_rule.rs index 64d0cddd..77dd0736 100644 --- a/tests/analytics/test_time_series_stat_threshold_rule.rs +++ b/tests/analytics/test_time_series_stat_threshold_rule.rs @@ -14,7 +14,7 @@ use super::test_helpers::{ #[test] fn test_stat_below_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -30,7 +30,7 @@ fn test_stat_below_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -40,7 +40,7 @@ fn test_stat_below_threshold() { #[test] fn test_stat_above_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![50.0, 60.0, 70.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -56,7 +56,7 @@ fn test_stat_above_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -67,7 +67,7 @@ fn test_stat_above_threshold() { #[test] fn test_stat_equal_threshold() { let ts_data = create_time_series_data(vec![("metric1", vec![50.0, 50.0, 50.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -83,7 +83,7 @@ fn test_stat_equal_threshold() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -93,7 +93,7 @@ fn test_stat_equal_threshold() { #[test] fn test_less_than_comparator() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -109,7 +109,7 @@ fn test_less_than_comparator() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -119,7 +119,7 @@ fn test_less_than_comparator() { #[test] fn test_max_stat() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 100.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -135,7 +135,7 @@ fn test_max_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -145,7 +145,7 @@ fn test_max_stat() { #[test] fn test_min_stat() { let ts_data = create_time_series_data(vec![("metric1", vec![5.0, 20.0, 100.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -161,7 +161,7 @@ fn test_min_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -171,7 +171,7 @@ fn test_min_stat() { #[test] fn test_metric_not_found() { let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -187,7 +187,7 @@ fn test_metric_not_found() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -198,7 +198,7 @@ fn test_metric_not_found() { fn test_multiple_runs() { let ts_data1 = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0])]); let ts_data2 = create_time_series_data(vec![("metric1", vec![60.0, 70.0, 80.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -219,7 +219,7 @@ fn test_multiple_runs() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -238,7 +238,7 @@ fn test_multiple_series_uses_aggregate() { (None, vec![60.0, 70.0, 80.0]), // aggregate series ], )]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -254,7 +254,7 @@ fn test_multiple_series_uses_aggregate() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); @@ -266,7 +266,7 @@ fn test_time_range_changes_stat() { // Values: [10, 20, 30, 90, 90, 90] (time_diff: [0,1,2,3,4,5]) let ts_data = create_time_series_data(vec![("metric1", vec![10.0, 20.0, 30.0, 90.0, 90.0, 90.0])]); - let processed_data = + let mut processed_data = create_processed_data("test_data", vec![("run1", AperfData::TimeSeries(ts_data))]); let rule = TimeSeriesStatThresholdRule { @@ -283,7 +283,7 @@ fn test_time_range_changes_stat() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 1); @@ -292,18 +292,22 @@ fn test_time_range_changes_stat() { let mut accessor = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 0)]), HashMap::from([("run1".to_string(), 2)]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 0); // Time range 3:5 → [90,90,90], avg = 90 → finding let mut accessor2 = ProcessedDataAccessor::from_time_ranges( HashMap::from([("run1".to_string(), 3)]), HashMap::from([("run1".to_string(), 5)]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); } @@ -318,7 +322,7 @@ fn test_time_range_multi_run() { create_time_series_data(vec![("metric1", vec![90.0, 90.0, 90.0, 10.0, 20.0, 30.0])]); let ts_data3 = create_time_series_data(vec![("metric1", vec![40.0, 40.0, 40.0, 40.0, 40.0, 40.0])]); - let processed_data = create_processed_data( + let mut processed_data = create_processed_data( "test_data", vec![ ("run1", AperfData::TimeSeries(ts_data1)), @@ -341,7 +345,7 @@ fn test_time_range_multi_run() { let mut findings = DataFindings::default(); rule.analyze( &mut findings, - &processed_data, + &mut processed_data, &mut ProcessedDataAccessor::new(), ); assert_eq!(findings.num_runs_with_findings(), 2); @@ -361,9 +365,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 2), ("run3".to_string(), 2), ]), + HashMap::new(), + HashMap::new(), ); let mut findings2 = DataFindings::default(); - rule.analyze(&mut findings2, &processed_data, &mut accessor); + rule.analyze(&mut findings2, &mut processed_data, &mut accessor); assert_eq!(findings2.num_runs_with_findings(), 1); assert!(findings2.has_findings_for_run("run2")); @@ -379,9 +385,11 @@ fn test_time_range_multi_run() { ("run2".to_string(), 5), ("run3".to_string(), 5), ]), + HashMap::new(), + HashMap::new(), ); let mut findings3 = DataFindings::default(); - rule.analyze(&mut findings3, &processed_data, &mut accessor2); + rule.analyze(&mut findings3, &mut processed_data, &mut accessor2); assert_eq!(findings3.num_runs_with_findings(), 1); assert!(findings3.has_findings_for_run("run1")); } diff --git a/tests/test_aperf.rs b/tests/test_aperf.rs index 67f7b2fd..28950ce6 100644 --- a/tests/test_aperf.rs +++ b/tests/test_aperf.rs @@ -4,7 +4,6 @@ use chrono::Utc; use flate2::read::GzDecoder; #[cfg(feature = "hotline")] use libc::c_int; -use serial_test::serial; use std::path::{Path, PathBuf}; use std::{fs, panic}; use tar::Archive; diff --git a/tests/test_aperf_stats.rs b/tests/test_aperf_stats.rs index 7a00b9e7..02023d33 100644 --- a/tests/test_aperf_stats.rs +++ b/tests/test_aperf_stats.rs @@ -28,6 +28,7 @@ mod aperf_stats_tests { report_dir: PathBuf::new(), run_name: String::new(), data_file_path: PathBuf::from(data_file_path), + collection_start: None, } } diff --git a/tests/test_java_profile.rs b/tests/test_java_profile.rs index 46318332..55d2f18a 100644 --- a/tests/test_java_profile.rs +++ b/tests/test_java_profile.rs @@ -20,6 +20,7 @@ fn setup_test_env() -> (TempDir, PathBuf, PathBuf, ReportParams) { report_dir: report_dir.clone(), run_name: "test_run".to_string(), data_file_path: PathBuf::new(), + collection_start: None, }; (temp_dir, data_dir, report_dir, params)