From b0270417bc72efa15957fa0bae6a7e78453c95ac Mon Sep 17 00:00:00 2001 From: CongkaiTan Date: Thu, 28 May 2026 14:34:42 -0700 Subject: [PATCH] Add backward compatibility for Profile visualization and remove flamegraph data --- .gitignore | 1 + src/analytics/rules.rs | 1 - src/analytics/rules/flamegraphs.rs | 21 -- src/data.rs | 26 +- src/data/common/data_formats.rs | 57 ++++- src/data/common/processed_data_accessor.rs | 1 + src/data/common/utils.rs | 40 ++++ src/data/flamegraphs.rs | 210 ----------------- src/data/hotline.rs | 26 +- src/data/java_profile.rs | 139 +++++------ src/data/perf_profile.rs | 222 +++++++++++++----- src/lib.rs | 8 +- src/profiling/mod.rs | 30 --- src/record.rs | 2 +- src/report_frontend/src/components/Report.tsx | 2 + .../src/components/data/IframeGraph.tsx | 16 +- .../data/profile-panel/ProfilePanel.tsx | 42 +++- .../components/data/profile-panel/utils.ts | 31 ++- .../src/components/pages/GraphDataPage.tsx | 140 +++++++++++ .../components/pages/ProfilingDataPage.tsx | 88 ++----- .../src/definitions/data-config.ts | 9 +- .../src/definitions/data-descriptions.ts | 11 +- src/report_frontend/src/definitions/types.ts | 17 +- src/visualizer.rs | 123 ++++++++++ tests/test_java_profile.rs | 117 ++++++--- 25 files changed, 795 insertions(+), 585 deletions(-) delete mode 100644 src/analytics/rules/flamegraphs.rs delete mode 100644 src/data/flamegraphs.rs create mode 100644 src/report_frontend/src/components/pages/GraphDataPage.tsx diff --git a/.gitignore b/.gitignore index 09d29ad4..917d4164 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /node_modules +/dist .idea pmu_config.json .kiro/ \ No newline at end of file diff --git a/src/analytics/rules.rs b/src/analytics/rules.rs index 9d38a518..89b7fc4b 100644 --- a/src/analytics/rules.rs +++ b/src/analytics/rules.rs @@ -4,7 +4,6 @@ mod cpu_utilization; mod diskstats; mod efa_stat; mod ena_stat; -mod flamegraphs; mod hotline; mod interrupts; mod java_profile; diff --git a/src/analytics/rules/flamegraphs.rs b/src/analytics/rules/flamegraphs.rs deleted file mode 100644 index 8390b1d4..00000000 --- a/src/analytics/rules/flamegraphs.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::analytics::rule_templates::profile_stack_frame_threshold_rule::profile_stack_frame_threshold; -use crate::analytics::{AnalyticalRule, ProfileStackFrameThresholdRule, Score}; -use crate::data::flamegraphs::Flamegraph; -use crate::data::AnalyzeData; - -impl AnalyzeData for Flamegraph { - fn get_analytical_rules(&self) -> Vec { - vec![profile_stack_frame_threshold! { - name: "Place Holder", - profile_type: "default", - stack_frame: [["place_holder_frame"]], - frame_type: None, - thread_states: [], - aggregate_occurences: true, - total_samples: true, - threshold: 100.0, - score: Score::Poor, - message: "Resolution", - }] - } -} diff --git a/src/data.rs b/src/data.rs index 7adddd2c..29416a6e 100644 --- a/src/data.rs +++ b/src/data.rs @@ -5,7 +5,6 @@ pub mod cpu_utilization; pub mod diskstats; pub mod efa_stat; pub mod ena_stat; -pub mod flamegraphs; pub mod hotline; pub mod interrupts; pub mod java_profile; @@ -33,7 +32,6 @@ use cpu_utilization::{CpuUtilization, CpuUtilizationRaw}; use diskstats::{Diskstats, DiskstatsRaw}; use efa_stat::{EfaStat, EfaStatRaw}; use ena_stat::{EnaStat, EnaStatRaw}; -use flamegraphs::{Flamegraph, FlamegraphRaw}; use hotline::{Hotline, HotlineRaw}; use include_directory::{include_directory, Dir}; use interrupts::{InterruptData, InterruptDataRaw}; @@ -43,7 +41,7 @@ use memalloc::{MemallocData, MemallocDataRaw}; use meminfo::{MeminfoData, MeminfoDataRaw}; use netstat::{Netstat, NetstatRaw}; use numastat::{Numastat, NumastatRaw}; -use perf_profile::{PerfProfile, PerfProfileRaw}; +use perf_profile::{FlamegraphRaw, PerfProfile, PerfProfileRaw}; use perf_stat::{PerfStat, PerfStatRaw}; use processes::{Processes, ProcessesRaw}; use serde::{Deserialize, Serialize}; @@ -194,11 +192,6 @@ impl DataType { self.data.finish_data_collection(&self.collector_params)?; Ok(()) } - - pub fn after_data_collection(&mut self) -> Result<()> { - self.data.after_data_collection(&self.collector_params)?; - Ok(()) - } } #[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash)] @@ -292,15 +285,6 @@ macro_rules! data { } Ok(()) } - - fn after_data_collection(&mut self, params: &CollectorParams) -> Result<()> { - match self { - $( - Data::$data(ref mut value) => value.after_data_collection(params)?, - )* - } - Ok(()) - } } #[cfg(target_os = "linux")] @@ -414,7 +398,7 @@ data!( NetstatRaw, NumastatRaw, PerfProfileRaw, - FlamegraphRaw, + FlamegraphRaw, // Dummy one to maintain the order JavaProfileRaw, HotlineRaw, MemallocDataRaw, @@ -437,7 +421,6 @@ report_data!( Numastat, PerfProfile, Hotline, - Flamegraph, AperfStat, AperfRunlog, JavaProfile, @@ -463,11 +446,6 @@ pub trait CollectData { Ok(()) } - fn after_data_collection(&mut self, _params: &CollectorParams) -> Result<()> { - noop!(); - Ok(()) - } - fn is_static() -> bool { false } diff --git a/src/data/common/data_formats.rs b/src/data/common/data_formats.rs index 99068ad7..ff9a27c1 100644 --- a/src/data/common/data_formats.rs +++ b/src/data/common/data_formats.rs @@ -18,6 +18,7 @@ pub enum DataFormat { Text, KeyValue, Profile, + Graph, Unknown, } @@ -46,6 +47,7 @@ pub enum AperfData { Text(TextData), KeyValue(KeyValueData), Profile(ProfilingData), + Graph(GraphData), } impl AperfData { @@ -55,6 +57,7 @@ impl AperfData { AperfData::Text(_) => DataFormat::Text, AperfData::KeyValue(_) => DataFormat::KeyValue, AperfData::Profile(_) => DataFormat::Profile, + AperfData::Graph(_) => DataFormat::Graph, } } } @@ -163,6 +166,57 @@ pub struct TextData { pub lines: Vec, } +// ------------------------------------------ GRAPH DATA ------------------------------------------- +/// TODO: Temporary format only used by hotline to ship pre-rendered HTML/SVG tables to the frontend. +/// +/// Data types falling into this format produce one or more HTML or SVG files at the end of a +/// recording run, which are to be rendered through IFrame in the report. The graphs can be +/// categorized into different groups, so that only one group of graphs are shown in the report +/// at a time. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct GraphData { + pub graph_groups: Vec, +} + +/// Contents of a graph group, which contains all graphs to be displayed together. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct GraphGroup { + /// Name of the graph group. + pub group_name: String, + /// A map from graph names to all graphs within the group. + pub graphs: HashMap, +} + +impl GraphGroup { + pub fn new(graph_group_name: &str) -> Self { + GraphGroup { + group_name: graph_group_name.to_string(), + graphs: HashMap::new(), + } + } +} + +/// Information about a graph. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Graph { + /// The name of the graph. + pub graph_name: String, + /// The relative path to graph (value of the IFrame's src attribute). + pub graph_path: String, + /// The size of the graph, which can be used for graph ordering in the report. + pub graph_size: Option, +} + +impl Graph { + pub fn new(graph_name: String, graph_path: String, graph_size: Option) -> Self { + Graph { + graph_name, + graph_path, + graph_size, + } + } +} + // ---------------------------------------- PROFILING DATA ------------------------------------------ /// Data types falling into this format collect profiling data from one or more profiled /// targets (e.g. JVMs for java profiling, and system for perf profiling). Each target is @@ -170,9 +224,6 @@ pub struct TextData { /// type (e.g. "cpu", "wall", "allocation"). Each [`Profile`] carries: /// - Time bucketed sample data with total counts for the entire profile /// - Calling context tree used to analyze a selected time range -/// - A [`ProfileGraph`](crate::profiling::ProfileGraph) pointing to a pre-rendered HTML/SVG -/// file displayed via IFrame in the report (legacy path, to be removed once native -/// rendering is complete). #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct ProfilingData { /// Map from profiler name to its profiler data diff --git a/src/data/common/processed_data_accessor.rs b/src/data/common/processed_data_accessor.rs index 700cdc7f..7e0ed1ba 100644 --- a/src/data/common/processed_data_accessor.rs +++ b/src/data/common/processed_data_accessor.rs @@ -171,6 +171,7 @@ impl ProcessedDataAccessor { DataFormat::KeyValue => self.key_value_data_json_string(processed_data), DataFormat::Text => self.text_data_json_string(processed_data), DataFormat::Profile => self.profiler_data_json_string(processed_data), + DataFormat::Graph => serde_json::to_string(processed_data).unwrap(), DataFormat::Unknown => serde_json::to_string(processed_data).unwrap(), } } diff --git a/src/data/common/utils.rs b/src/data/common/utils.rs index adda60cb..1e55b550 100644 --- a/src/data/common/utils.rs +++ b/src/data/common/utils.rs @@ -1,10 +1,13 @@ use anyhow::{Error, Result}; +use log::error; use std::collections::{HashMap, HashSet, VecDeque}; use std::fs; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; +use crate::data::common::data_formats::{Graph, GraphData}; + pub fn get_data_name_from_type() -> &'static str { let full_data_module_path = std::any::type_name::(); @@ -79,6 +82,43 @@ pub fn no_tar_gz_file_name(path: &PathBuf) -> Option { Some(file_name_str) } +/// Copy a graph file to the report data dir and update the GraphData with its info. +pub fn copy_graph_and_update_graph_data( + source_dir: &PathBuf, + dest_dir: &PathBuf, + filename: &str, + graph_group_name: &str, + graph_key: &str, + graph_name: String, + graph_data: &mut GraphData, +) { + let source_graph_path = source_dir.join(&filename); + if !source_graph_path.exists() { + return; + } + let relative_graph_path = PathBuf::from("data").join("js").join(&filename); + let dest_graph_path = dest_dir.join(&relative_graph_path); + + if let Err(e) = std::fs::copy(&source_graph_path, &dest_graph_path) { + error!("Failed to copy graph file: {e}"); + } else { + graph_data + .graph_groups + .iter_mut() + .find(|graph_group| graph_group.group_name == graph_group_name) + .map(|graph_group| { + graph_group.graphs.insert( + graph_key.to_string(), + Graph { + graph_name, + graph_path: relative_graph_path.to_string_lossy().into_owned(), + graph_size: None, + }, + ); + }); + } +} + /// Collects the paths of all files in a dir and returns a map from file names to file paths, /// if the file system read was successful pub fn collect_file_paths_in_dir(dir: &PathBuf) -> Result> { diff --git a/src/data/flamegraphs.rs b/src/data/flamegraphs.rs deleted file mode 100644 index 02073102..00000000 --- a/src/data/flamegraphs.rs +++ /dev/null @@ -1,210 +0,0 @@ -use crate::data::common::data_formats::{AperfData, Profiler, ProfilingData}; -use crate::data::{Data, ProcessData}; -use crate::profiling::{Profile, ProfileGraph}; -use crate::visualizer::ReportParams; -use anyhow::Result; -use log::error; -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -#[cfg(target_os = "linux")] -use { - crate::data::{CollectData, CollectorParams}, - crate::{get_file_name, PDError}, - inferno::collapse::{perf::Folder, Collapse}, - inferno::flamegraph::{self, Direction, Options}, - log::{debug, info}, - std::fs, - std::fs::File, - std::io::Write, - std::process::Command, -}; - -#[cfg(target_os = "linux")] -fn write_msg_to_svg(mut file: File, msg: String) -> Result<()> { - write!( - file, - "{}", - msg - )?; - Ok(()) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlamegraphRaw { - pub data: String, -} - -#[cfg(target_os = "linux")] -impl FlamegraphRaw { - pub fn new() -> Self { - FlamegraphRaw { - data: String::new(), - } - } -} - -#[cfg(target_os = "linux")] -impl CollectData for FlamegraphRaw { - fn prepare_data_collector(&mut self, _params: &CollectorParams) -> Result<()> { - match Command::new("perf").args(["--version"]).output() { - Err(e) => Err(PDError::DependencyError(format!("'perf' command failed. {}", e)).into()), - _ => Ok(()), - } - } - - fn after_data_collection(&mut self, params: &CollectorParams) -> Result<()> { - let data_dir = PathBuf::from(¶ms.data_dir); - - let file_pathbuf = data_dir.join(get_file_name( - params.data_dir.display().to_string(), - "perf_profile".to_string(), - )?); - - let perf_jit_loc = data_dir.join("perf.data.jit"); - - debug!("Running Perf inject..."); - let out_jit = Command::new("perf") - .args([ - "inject", - "-j", - "-i", - file_pathbuf.to_str().unwrap(), - "-o", - perf_jit_loc.to_str().unwrap(), - ]) - .status(); - - let fg_out = File::create(data_dir.join(format!("{}-flamegraph.svg", params.run_name)))?; - let reverse_fg_out = - File::create(data_dir.join(format!("{}-reverse-flamegraph.svg", params.run_name)))?; - - match out_jit { - Err(e) => { - let out = format!("Skip processing profiling data due to: {}", e); - error!("{}", out); - write_msg_to_svg(fg_out, out)?; - } - Ok(_) => { - info!("Creating flamegraph..."); - // TODO: extract metadata from perf record and generate script -> ProfilingData - let script_loc = data_dir.join("script.out"); - let out = Command::new("perf") - .stdout(File::create(&script_loc)?) - .args(["script", "-f", "-i", perf_jit_loc.to_str().unwrap()]) - .output(); - match out { - Err(e) => { - let out = format!("Did not process profiling data due to: {}", e); - error!("{}", out); - write_msg_to_svg(fg_out, out)?; - } - Ok(_) => { - let collapse_loc = data_dir.join("collapse.out"); - // TODO: move flamegraph generation to report phase using ProfilingData (so user specifies time range) - Folder::default().collapse_file( - Some(script_loc.clone()), - File::create(&collapse_loc)?, - )?; - - // Generate icicle graph as default - let mut reverse_options = Options::default(); - reverse_options.direction = Direction::Inverted; - reverse_options.reverse_stack_order = false; - flamegraph::from_files( - &mut reverse_options, - &[collapse_loc.to_path_buf()], - fg_out, - )?; - - // Generate reverse icicle graph - reverse_options.reverse_stack_order = true; - flamegraph::from_files( - &mut reverse_options, - &[collapse_loc.to_path_buf()], - reverse_fg_out, - )?; - - // Clean up intermediate files after creating flamegraphs and saving - for file in [&script_loc, &perf_jit_loc, &collapse_loc] { - fs::remove_file(file).ok(); - } - } - } - } - } - Ok(()) - } - - fn is_profile() -> bool { - true - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Flamegraph; - -impl Flamegraph { - pub fn new() -> Self { - Flamegraph - } -} - -impl ProcessData for Flamegraph { - fn compatible_filenames(&self) -> Vec<&str> { - vec!["flamegraph"] - } - - fn process_raw_data( - &mut self, - params: ReportParams, - _raw_data: Vec, - ) -> Result { - fn copy_and_add_to_profiler( - params: &ReportParams, - filename: String, - profiling_data: &mut ProfilingData, - profiler_name: String, - profile_name: String, - ) { - let source_path = params.data_dir.join(&filename); - let relative_dest_path = PathBuf::from("data/js").join(filename); - let dest_path = params.report_dir.join(relative_dest_path.clone()); - - if source_path.exists() { - if let Ok(_) = std::fs::copy(&source_path, &dest_path) { - let profiler = profiling_data - .profilers - .entry(profiler_name.clone()) - .or_insert_with(Profiler::default); - profiler.profiles.insert( - profile_name.clone(), - Profile::with_graph(ProfileGraph::new( - format!("Kernel Profiling Flamegraph ({profiler_name})"), - relative_dest_path.into_os_string().into_string().unwrap(), - None, - )), - ); - } - } - } - - let mut profiling_data = ProfilingData::default(); - - copy_and_add_to_profiler( - ¶ms, - format!("{}-flamegraph.svg", params.run_name), - &mut profiling_data, - String::from("perf"), - String::from("default"), - ); - copy_and_add_to_profiler( - ¶ms, - format!("{}-reverse-flamegraph.svg", params.run_name), - &mut profiling_data, - String::from("perf"), - String::from("reverse"), - ); - - Ok(AperfData::Profile(profiling_data)) - } -} diff --git a/src/data/hotline.rs b/src/data/hotline.rs index 884aa613..12a7dc67 100644 --- a/src/data/hotline.rs +++ b/src/data/hotline.rs @@ -1,8 +1,7 @@ extern crate ctor; -use crate::data::common::data_formats::{AperfData, Profiler, ProfilingData}; +use crate::data::common::data_formats::{AperfData, Graph, GraphData, GraphGroup}; use crate::data::ProcessData; -use crate::profiling::{Profile, ProfileGraph}; use crate::{data::Data, visualizer::ReportParams}; use anyhow::Result; use serde::{Deserialize, Serialize}; @@ -289,7 +288,9 @@ impl ProcessData for Hotline { ) -> Result { use crate::data::hotline::hotline_reports::REPORT_CONFIGS; - let mut profiling_data = ProfilingData::default(); + // TODO: GraphData is a temporary format used here only to ship pre-rendered HTML + // tables to the frontend. + let mut graph_data = GraphData::default(); for config in REPORT_CONFIGS { let csv_string = fs::read_to_string(params.data_dir.join(config.filename))?; @@ -321,21 +322,20 @@ impl ProcessData for Hotline { let mut file = File::create(dest_path)?; file.write_all(full_html.as_bytes())?; - // TODO: Create standard data format for hotline (and similar) data - let profiler = profiling_data - .profilers - .entry(config.table_id.to_string()) - .or_insert_with(Profiler::default); - profiler.profiles.insert( - config.table_id.to_string(), - Profile::with_graph(ProfileGraph::new( + let mut graph_group = GraphGroup::default(); + graph_group.group_name = config.table_id.to_string(); + graph_group.graphs.insert( + String::new(), + Graph::new( format!("{}_table", config.table_id), relative_dest_path.into_os_string().into_string().unwrap(), None, - )), + ), ); + + graph_data.graph_groups.push(graph_group); } - Ok(AperfData::Profile(profiling_data)) + Ok(AperfData::Graph(graph_data)) } } diff --git a/src/data/java_profile.rs b/src/data/java_profile.rs index 836ee352..45efa0d0 100644 --- a/src/data/java_profile.rs +++ b/src/data/java_profile.rs @@ -1,8 +1,11 @@ -use crate::data::common::data_formats::{AperfData, Profiler, ProfilingData}; +use crate::data::common::data_formats::{ + AperfData, GraphData, GraphGroup, Profiler, ProfilingData, +}; +use crate::data::common::utils::copy_graph_and_update_graph_data; #[cfg(target_os = "linux")] use crate::data::common::utils::get_data_name_from_type; use crate::data::{Data, ProcessData}; -use crate::profiling::{jfr, Profile, ProfileGraph}; +use crate::profiling::{jfr, Profile}; use crate::visualizer::ReportParams; use anyhow::Result; use log::error; @@ -10,7 +13,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; use std::fs; -use std::path::PathBuf; #[cfg(target_os = "linux")] use { crate::data::{CollectData, CollectorParams}, @@ -352,15 +354,20 @@ impl CollectData for JavaProfileRaw { None }; - match jfr::build_java_profiler_data(&jfr_path, events_out_path) { - Ok(mut profiler) => { - profiler.metadata = jfr::parse_jfr_metadata(&metadata_json); - if let Ok(json) = serde_json::to_string(&profiler) { - fs::write(&profiler_data_path, json).ok(); + // TODO: Guard the new profile processing logic by the save_profile_events flag, + // so that the new flow is only executed in tests. Remove the guardrail + // after the feature is ready to launch. + if params.save_profile_events { + match jfr::build_java_profiler_data(&jfr_path, events_out_path) { + Ok(mut profiler) => { + profiler.metadata = jfr::parse_jfr_metadata(&metadata_json); + if let Ok(json) = serde_json::to_string(&profiler) { + fs::write(&profiler_data_path, json).ok(); + } + } + Err(e) => { + error!("Failed to build Profiler Data for {}: {}", key, e); } - } - Err(e) => { - error!("Failed to build Profiler Data for {}: {}", key, e); } } @@ -401,6 +408,8 @@ impl ProcessData for JavaProfile { _raw_data: Vec, ) -> Result { let mut profiling_data = ProfilingData::default(); + // For backward compatibility + let mut graph_data = GraphData::default(); let processes_loc = params .data_dir @@ -412,18 +421,32 @@ impl ProcessData for JavaProfile { let mut profile_metrics = Vec::from(PROFILE_METRICS); profile_metrics.push("legacy"); + profile_metrics.iter().for_each(|&metric| { + graph_data.graph_groups.push(GraphGroup::new(metric)); + }); // Track JVMs with same name let mut jvm_name_counts: HashMap = HashMap::new(); - let relative_path = PathBuf::from("data/js"); - let report_dest_dir = params.report_dir.join(&relative_path); + // Stores the deduped JVM name of each PID + let mut deduped_names: HashMap = HashMap::new(); for (process, process_names) in &process_map { - // TODO: Remove when natively rendering profile data. Holds jfrconv-generated - // html graphs for this process, keyed by metric name. - let mut jfrconv_graphs: HashMap = HashMap::new(); - for metric in &profile_metrics { - let filename = if *metric == "legacy" { + let jvm_name = process_names.first().map_or("unknown", |s| s.as_str()); + let deduped_name = deduped_names.entry(process.clone()).or_insert_with(|| { + let jvm_name_count = jvm_name_counts.entry(jvm_name.to_string()).or_insert(0); + *jvm_name_count += 1; + + if *jvm_name_count > 1 { + format!("{} ({})", jvm_name, *jvm_name_count - 1) + } else { + jvm_name.to_string() + } + }); + + // Copy jfrconv-generated HTML graphs for this process to the report data dir + // and build the GraphData (backward compatibility). + for &metric in &profile_metrics { + let filename = if metric == "legacy" { // backward compatibility - previous versions generated a single flamegraph format!("{}-java-flamegraph-{}.html", params.run_name, process) } else { @@ -432,70 +455,50 @@ impl ProcessData for JavaProfile { params.run_name, process, metric ) }; - let Some(file_size) = - copy_file_to_report_data(&filename, ¶ms.data_dir, &report_dest_dir) - else { - continue; - }; - let graph_path = relative_path - .join(&filename) - .into_os_string() - .into_string() - .unwrap(); - jfrconv_graphs.insert( - metric.to_string(), - ProfileGraph::new(String::new(), graph_path, Some(file_size)), + copy_graph_and_update_graph_data( + ¶ms.data_dir, + ¶ms.report_dir, + &filename, + metric, + &deduped_name, + format!("({}) JVM: {}", metric, deduped_name), + &mut graph_data, ); } - if jfrconv_graphs.is_empty() { - continue; - } - // Assign a deduped JVM name for this process - let jvm_name = process_names.first().map_or("unknown", |s| s.as_str()); - let count = jvm_name_counts.entry(jvm_name.to_string()).or_insert(0); - *count += 1; - let deduped_name = if *count > 1 { - format!("{} ({})", jvm_name, *count - 1) - } else { - jvm_name.to_string() - }; - - // Load profiler JSON if present, attach jfrconv graphs. + // Deserialize the ProfilerData generated at the end of recording. let profiler_data_path = params.data_dir.join(format!( "{}-java-profile-{}-profiler-data.json", params.run_name, process )); - let mut profiler = fs::read_to_string(&profiler_data_path) + let mut profiler = match fs::read_to_string(&profiler_data_path) .ok() .and_then(|json| serde_json::from_str::(&json).ok()) - .unwrap_or_default(); - for (metric, mut graph) in jfrconv_graphs { - graph.graph_name = format!("({}) JVM: {}", metric, deduped_name); + { + Some(profiler) => profiler, + None => continue, + }; + + // Ensure every profiling metric has an entry so the frontend renders a + // tab for it even when the JFR contained no events for that metric. + for metric in PROFILE_METRICS { profiler .profiles - .entry(metric) - .or_insert_with(Profile::new) - .profile_graph = graph; + .entry(metric.to_string()) + .or_insert_with(Profile::new); } - profiling_data.profilers.insert(deduped_name, profiler); + + profiling_data + .profilers + .insert(deduped_name.clone(), profiler); + } + + // If no ProfilerData was read, chances are this run was created before the + // introduction of ProfilingData, so fall back to the old GraphData. + if profiling_data.profilers.is_empty() { + return Ok(AperfData::Graph(graph_data)); } Ok(AperfData::Profile(profiling_data)) } } - -fn copy_file_to_report_data( - filename: &String, - src_dir: &PathBuf, - dest_dir: &PathBuf, -) -> Option { - let src_path = src_dir.join(filename); - let file_metadata = fs::metadata(&src_path).ok()?; - let file_size = file_metadata.len(); - let dest_path = dest_dir.join(filename); - - fs::copy(&src_path, &dest_path).ok()?; - - Some(file_size) -} diff --git a/src/data/perf_profile.rs b/src/data/perf_profile.rs index 756654ff..7f3d6028 100644 --- a/src/data/perf_profile.rs +++ b/src/data/perf_profile.rs @@ -1,4 +1,7 @@ -use crate::data::common::data_formats::{AperfData, Profiler, TextData}; +use crate::data::common::data_formats::{ + AperfData, GraphData, GraphGroup, Profiler, ProfilingData, +}; +use crate::data::common::utils::copy_graph_and_update_graph_data; use crate::data::{Data, ProcessData}; use crate::visualizer::ReportParams; use anyhow::Result; @@ -10,14 +13,37 @@ use { crate::profiling::perf::parser::build_perf_profiler_data, crate::PDError, chrono::Utc, + inferno::collapse::{perf::Folder, Collapse}, + inferno::flamegraph::{self, Direction, Options}, log::{debug, error, warn}, nix::{sys::signal, unistd, unistd::Pid}, + std::fs::File, std::io::Write, std::process::{Command, Stdio}, std::{process::Child, sync::Mutex}, }; -pub const PERF_TOP_FUNCTIONS_FILE_NAME: &str = "top_functions"; +// Dummy struct used to maintain the Data enum order after the flamegraph +// data type was removed. This is to avoid deserialization failure and +// maintain backward compatibility. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlamegraphRaw { + pub data: String, +} +#[cfg(target_os = "linux")] +impl FlamegraphRaw { + pub fn new() -> Self { + FlamegraphRaw { + data: String::new(), + } + } +} +#[cfg(target_os = "linux")] +impl CollectData for FlamegraphRaw { + fn is_static() -> bool { + true + } +} #[cfg(target_os = "linux")] lazy_static! { @@ -136,54 +162,109 @@ impl CollectData for PerfProfileRaw { Ok(_) => debug!("'perf record' executed successfully."), } - let event_out_path_buf = params.data_dir.join("parsed_perf_data.out"); - let events_out_path = if params.save_profile_events { - Some(event_out_path_buf.as_path()) - } else { - None - }; - - // Parse raw Perf profile and build ProfilingData - let perf_profiler_data = build_perf_profiler_data( - ¶ms.data_file_path, - *PROFILE_START_TIME_MS.lock().unwrap(), - events_out_path, - ); - let perf_profiler_data_path = params - .data_dir - .join(format!("{}-perf-profiler-data.json", params.run_name)); - if let Ok(json) = serde_json::to_string(&perf_profiler_data) { - fs::write(&perf_profiler_data_path, json)?; - } - - let mut top_functions_file = - fs::File::create(params.data_dir.join(PERF_TOP_FUNCTIONS_FILE_NAME))?; - - let out = Command::new("perf") + debug!("Running Perf inject..."); + let perf_jit_loc = params.data_dir.join("perf.data.jit"); + let out_jit = Command::new("perf") .args([ - "report", - "--stdio", - "--percent-limit", - "1", + "inject", + "-j", "-i", - ¶ms.data_file_path.display().to_string(), + params.data_file_path.to_str().unwrap(), + "-o", + perf_jit_loc.to_str().unwrap(), ]) - .output(); + .status(); - match out { + let fg_out = File::create( + params + .data_dir + .join(format!("{}-flamegraph.svg", params.run_name)), + )?; + let reverse_fg_out = File::create( + params + .data_dir + .join(format!("{}-reverse-flamegraph.svg", params.run_name)), + )?; + + match out_jit { Err(e) => { - let out = format!("Skipped processing profiling data due to : {}", e); + let out = format!("Perf inject failed due to: {e}"); error!("{}", out); - write!(top_functions_file, "{}", out)?; + write_msg_to_svg(fg_out, out)?; } - Ok(v) => { - let mut top_functions = "No data collected"; - if !v.stdout.is_empty() { - top_functions = std::str::from_utf8(&v.stdout)?; + Ok(_) => { + debug!("Creating flamegraph..."); + // TODO: extract metadata from perf record and generate script -> ProfilingData + let script_loc = params.data_dir.join("script.out"); + let out = Command::new("perf") + .stdout(File::create(&script_loc)?) + .args(["script", "-f", "-i", perf_jit_loc.to_str().unwrap()]) + .output(); + match out { + Err(e) => { + let out = format!("Perf script failed due to: {}", e); + error!("{}", out); + write_msg_to_svg(fg_out, out)?; + } + Ok(_) => { + let collapse_loc = params.data_dir.join("collapse.out"); + Folder::default().collapse_file( + Some(script_loc.clone()), + File::create(&collapse_loc)?, + )?; + + // Generate icicle graph as default + let mut reverse_options = Options::default(); + reverse_options.direction = Direction::Inverted; + reverse_options.reverse_stack_order = false; + flamegraph::from_files( + &mut reverse_options, + &[collapse_loc.to_path_buf()], + fg_out, + )?; + + // Generate reverse icicle graph + reverse_options.reverse_stack_order = true; + flamegraph::from_files( + &mut reverse_options, + &[collapse_loc.to_path_buf()], + reverse_fg_out, + )?; + + // Clean up intermediate files after creating flamegraphs and saving + for file in [&script_loc, &perf_jit_loc, &collapse_loc] { + fs::remove_file(file).ok(); + } + } } - write!(top_functions_file, "{}", top_functions)?; } } + + let event_out_path_buf = params.data_dir.join("parsed_perf_data.out"); + let events_out_path = if params.save_profile_events { + Some(event_out_path_buf.as_path()) + } else { + None + }; + + // TODO: Guard the new profile processing logic by the save_profile_events flag, + // so that the new flow is only executed in tests. Remove the guardrail + // after the feature is ready to launch. + if params.save_profile_events { + // Parse raw Perf profile and build ProfilingData + let perf_profiler_data = build_perf_profiler_data( + ¶ms.data_file_path, + *PROFILE_START_TIME_MS.lock().unwrap(), + events_out_path, + ); + let perf_profiler_data_path = params + .data_dir + .join(format!("{}-perf-profiler-data.json", params.run_name)); + if let Ok(json) = serde_json::to_string(&perf_profiler_data) { + fs::write(&perf_profiler_data_path, json)?; + } + } + Ok(()) } @@ -192,6 +273,16 @@ impl CollectData for PerfProfileRaw { } } +#[cfg(target_os = "linux")] +fn write_msg_to_svg(mut file: File, msg: String) -> Result<()> { + write!( + file, + "{}", + msg + )?; + Ok(()) +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PerfProfile; @@ -207,22 +298,47 @@ impl ProcessData for PerfProfile { params: ReportParams, _raw_data: Vec, ) -> Result { + // Still attempt to process perf script + inferno generated flamegraphs for + // backward compatibility + let mut graph_data = GraphData::default(); + graph_data.graph_groups.push(GraphGroup::new("default")); + graph_data.graph_groups.push(GraphGroup::new("reverse")); + [false, true].iter().for_each(|&is_reverse| { + let filename = if is_reverse { + format!("{}-reverse-flamegraph.svg", params.run_name) + } else { + format!("{}-flamegraph.svg", params.run_name) + }; + copy_graph_and_update_graph_data( + ¶ms.data_dir, + ¶ms.report_dir, + &filename, + if is_reverse { "reverse" } else { "default" }, + "cpu", + "Perf CPU Profile".to_string(), + &mut graph_data, + ); + }); + // Deserialize the ProfilerData generated at the end of record. - // TODO: build ProfilingData from the ProfilerData and return it to replace top_function data. let perf_profiler_data_path = params .data_dir .join(format!("{}-perf-profiler-data.json", params.run_name)); - let json_string = fs::read_to_string(perf_profiler_data_path)?; - let _perf_profiler_data = serde_json::from_str::(&json_string)?; - - let mut text_data = TextData::default(); - let file_loc = params.data_dir.join(PERF_TOP_FUNCTIONS_FILE_NAME); - if file_loc.exists() { - text_data.lines = fs::read_to_string(&file_loc)? - .split('\n') - .map(|x| x.to_string()) - .collect(); - } - Ok(AperfData::Text(text_data)) + let perf_profiler_data = match fs::read_to_string(&perf_profiler_data_path) + .ok() + .and_then(|json| serde_json::from_str::(&json).ok()) + { + Some(perf_profiler_data) => perf_profiler_data, + // If ProfilerData could not be read, chaces are this run was created before the + // introduction of ProfilingData, so fall back to the old GraphData. + None => return Ok(AperfData::Graph(graph_data)), + }; + + let mut profiling_data = ProfilingData::default(); + profiling_data + .profilers + .insert(String::from("cpu"), perf_profiler_data); + + Ok(AperfData::Profile(profiling_data)) } } diff --git a/src/lib.rs b/src/lib.rs index a68a3550..a983ce7b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ use thiserror::Error; #[cfg(target_os = "linux")] use { crate::data::aperf_stats::AperfStat, + crate::data::Data, flate2::{write::GzEncoder, Compression}, nix::poll::{poll, PollFd, PollFlags, PollTimeout}, nix::sys::{ @@ -186,6 +187,10 @@ impl PerformanceData { } pub fn add_datatype(&mut self, name: String, dt: data::DataType) { + // Ignore dummy data type. + if matches!(dt.data, Data::FlamegraphRaw(_)) { + return; + } self.collectors.insert(name, dt); } @@ -363,9 +368,6 @@ impl PerformanceData { datatype.set_signal(datatype_signal); datatype.finish_data_collection()?; } - for (_name, datatype) in self.collectors.iter_mut() { - datatype.after_data_collection()?; - } tfd.set_state(TimerState::Disarmed, SetTimeFlags::Default); Ok(()) } diff --git a/src/profiling/mod.rs b/src/profiling/mod.rs index 183b7f53..6bb4e587 100644 --- a/src/profiling/mod.rs +++ b/src/profiling/mod.rs @@ -20,8 +20,6 @@ use std::collections::HashMap; /// A single profiling type's data (e.g., cpu, wall, allocation). #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Profile { - /// Profile graph visualization (to be removed after native visualization is supported) - pub profile_graph: ProfileGraph, /// Time-ordered blocks of sample data, [thread_state_id -> node_id (index into context_tree) -> sample count] pub blocks: Vec>>, /// Block number time range where profile node aggregate counts are calculated @@ -35,27 +33,6 @@ pub struct Profile { total_samples_per_thread_state: OnceCell>, } -/// Information about a graph. TODO: Will remove after native profiler visualization is implemented -#[derive(Serialize, Deserialize, Debug, Default, Clone)] -pub struct ProfileGraph { - /// The name of the graph. - pub graph_name: String, - /// The relative path to graph (value of the IFrame's src attribute). - pub graph_path: String, - /// The size of the graph, which can be used for graph ordering in the report. - pub graph_size: Option, -} - -impl ProfileGraph { - pub fn new(graph_name: String, graph_path: String, graph_size: Option) -> Self { - ProfileGraph { - graph_name, - graph_path, - graph_size, - } - } -} - /// A node in the call tree. Each node represents a unique call path. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CCTreeNode { @@ -216,7 +193,6 @@ impl Profile { let mut frame_map = FrameMap::new(); frame_map.get_or_insert("[root]"); Self { - profile_graph: ProfileGraph::default(), blocks: Vec::new(), time_range: (0, 0), context_tree: vec![CCTreeNode { @@ -230,12 +206,6 @@ impl Profile { } } - pub fn with_graph(profile_graph: ProfileGraph) -> Self { - let mut p = Self::new(); - p.profile_graph = profile_graph; - p - } - /// Returns sum of sample counts for call graph nodes matching a stack pattern. /// Pattern frames must appear in order walking from root to leaf (i.e., parent→child order). /// Example: pattern ["A", "C"] matches nodes named "C" that have an ancestor named "A". diff --git a/src/record.rs b/src/record.rs index ea62205d..5e48e652 100644 --- a/src/record.rs +++ b/src/record.rs @@ -84,7 +84,7 @@ pub struct Record { pub profile_java: Option, /// Save all profile events in the output file. - #[clap(help_heading = "Profiling", long, value_parser)] + #[clap(help_heading = "Profiling", long, value_parser, hide = true)] pub save_profile_events: bool, /// Custom PMU config file to use. diff --git a/src/report_frontend/src/components/Report.tsx b/src/report_frontend/src/components/Report.tsx index 5cff30e4..c39b751a 100644 --- a/src/report_frontend/src/components/Report.tsx +++ b/src/report_frontend/src/components/Report.tsx @@ -10,6 +10,7 @@ import { NumCpusPerRun, SelectedCpusPerRun } from "../definitions/types"; import TimeSeriesDataPage from "./pages/TimeSeriesDataPage"; import KeyValueDataPage from "./pages/KeyValueDataPage"; import ProfilingDataPage from "./pages/ProfilingDataPage"; +import GraphDataPage from "./pages/GraphDataPage"; import TextDataPage from "./pages/TextDataPage"; import ReportHomePage from "./pages/ReportHomePage"; import { MAX_NUM_CPU_SHOW_DEFAULT } from "../definitions/constants"; @@ -113,6 +114,7 @@ export default function () { {!preprocessing && dataFormat == "profile" && ( )} + {!preprocessing && dataFormat == "graph" && } {!preprocessing && dataFormat == "text" && } {!preprocessing && (dataFormat == "unknown" || (dataFormat as string) === "" || dataFormat === undefined) && ( diff --git a/src/report_frontend/src/components/data/IframeGraph.tsx b/src/report_frontend/src/components/data/IframeGraph.tsx index 5dedbb7e..a3f638e7 100644 --- a/src/report_frontend/src/components/data/IframeGraph.tsx +++ b/src/report_frontend/src/components/data/IframeGraph.tsx @@ -1,23 +1,27 @@ import React from "react"; -import { DataType, ProfilingData, GraphInfo } from "../../definitions/types"; +import { DataType, GraphData, GraphInfo } from "../../definitions/types"; import { PROCESSED_DATA } from "../../definitions/data-config"; import { Container, Icon, Link } from "@cloudscape-design/components"; import Header from "@cloudscape-design/components/header"; +// TODO: IframeGraph is a temporary component used only by GraphDataPage (hotline). export interface IframeGraphProps { readonly dataType: DataType; readonly runName: string; - readonly profilerName: string; + readonly graphGroup: string; readonly graphName: string; } export default function (props: IframeGraphProps) { - const profilingData: ProfilingData | undefined = PROCESSED_DATA[props.dataType].runs[props.runName] as ProfilingData; - const graphInfo: GraphInfo | undefined = - profilingData?.profilers?.[props.profilerName]?.profiles?.[props.graphName]?.profile_graph; + const graphData: GraphData | undefined = PROCESSED_DATA[props.dataType]?.runs?.[props.runName] as + | GraphData + | undefined; + const graphInfo: GraphInfo | undefined = graphData?.graph_groups?.find( + (graph_group) => graph_group.group_name == props.graphGroup, + )?.graphs?.[props.graphName]; if (!graphInfo) { - return This graph was not collected in this APerf run.; + return This data was not collected in the APerf run.; } else { return ( - 0 ? validBaseline : undefined} - isBaseRun={runIdx === 0} - startTimeMs={profiler.start_time_ms} - blockWidthMs={profiler.block_width_ms} - /> + {profiler && profile ? ( + 0 ? validBaseline : undefined} + isBaseRun={runIdx === 0} + startTimeMs={profiler.start_time_ms} + blockWidthMs={profiler.block_width_ms} + /> + ) : ( + + )} ); @@ -98,7 +102,7 @@ function ProfilePanelView({ analytics, baseline, isBaseRun, startTimeMs, blockWi const containerWidth = useContainerWidth(containerRef); const alignedStartTimeMs = startTimeMs - (startTimeMs % blockWidthMs); - const numBlocks = analytics.blocks.length; + const numBlocks = analytics.blocks?.length ?? 0; const layout = useHeatmapLayout(numBlocks, blockWidthMs, alignedStartTimeMs, containerWidth); const sel = useHeatmapSelection(numBlocks, layout.groupSize, isBaseRun, setBaseRunSelection); @@ -106,7 +110,7 @@ function ProfilePanelView({ analytics, baseline, isBaseRun, startTimeMs, blockWi const [tooltip, setTooltip] = React.useState<{ x: number; y: number; text: string } | null>(null); const [zoomedNode, setZoomedNode] = React.useState(null); - const blockTotals = React.useMemo(() => analytics.blocks.map(blockTotal), [analytics.blocks]); + const blockTotals = React.useMemo(() => (analytics.blocks ?? []).map(blockTotal), [analytics.blocks]); const cellTotals = React.useMemo(() => { if (layout.groupSize === 1) return blockTotals; @@ -226,7 +230,7 @@ function ProfilePanelView({ analytics, baseline, isBaseRun, startTimeMs, blockWi }, [flatNodes, flamegraphRoot]); if (numBlocks === 0) { - return
No profiler data available.
; + return ; } return ( @@ -398,3 +402,17 @@ function FrameTypeLegend() { ); } + +/** + * Placeholder rendered in place of the heatmap+flamegraph when a run has no + * samples for the selected profile (e.g. allocation profiling on a workload + * that allocates very little). Keeps the per-run column framing so the user + * can still see which run is empty alongside runs that have data. + */ +function EmptyProfileState({ message }: { readonly message: string }) { + return ( + + {message} + + ); +} diff --git a/src/report_frontend/src/components/data/profile-panel/utils.ts b/src/report_frontend/src/components/data/profile-panel/utils.ts index 33ccbc9e..cd42aa1c 100644 --- a/src/report_frontend/src/components/data/profile-panel/utils.ts +++ b/src/report_frontend/src/components/data/profile-panel/utils.ts @@ -17,8 +17,10 @@ export function blockTotal(block: { [ts: string]: { [nodeId: string]: number } } /** Aggregate self_samples per node_id from blocks in [start,end) */ export function computeNodeSelfSamples(analytics: Profile, blockStart: number, blockEnd: number): Map { const nodeSamples = new Map(); + const blocks = analytics.blocks; + if (!blocks) return nodeSamples; for (let bi = blockStart; bi < blockEnd; bi++) { - const block = analytics.blocks[bi]; + const block = blocks[bi]; if (!block) continue; for (const ts in block) { for (const nid in block[ts]) { @@ -35,8 +37,10 @@ export function aggregateByFrame( analytics: Profile, nodeSelf: Map, ): Map { + const byFrameEmpty = new Map(); const tree = analytics.context_tree; - const frames = analytics.frame_map.frame_id_to_frame; + const frames = analytics.frame_map?.frame_id_to_frame; + if (!tree || !frames) return byFrameEmpty; const nodeTotal = new Map(); function total(nid: number): number { @@ -65,8 +69,10 @@ export function aggregateByStack( nodeSelf: Map, reverse?: boolean, ): Map { + const byStackEmpty = new Map(); const tree = analytics.context_tree; - const frames = analytics.frame_map.frame_id_to_frame; + const frames = analytics.frame_map?.frame_id_to_frame; + if (!tree || !frames) return byStackEmpty; const nodeTotal = new Map(); function total(nid: number): number { @@ -156,8 +162,8 @@ export function buildFlamegraph( reverse: boolean, ): FlamegraphNode | null { const tree = analytics.context_tree; - const frames = analytics.frame_map.frame_id_to_frame; - if (tree.length === 0) return null; + const frames = analytics.frame_map?.frame_id_to_frame; + if (!tree || !frames || tree.length === 0) return null; const nodeSelf = computeNodeSelfSamples(analytics, blockStart, blockEnd); @@ -487,11 +493,16 @@ export function useHeatmapSelection( React.useEffect(() => { setSelection(([s, e]) => { - const alignedS = Math.floor(s / groupSize) * groupSize; - const alignedE = Math.min(numBlocks, Math.ceil(e / groupSize) * groupSize); - if (alignedS >= numBlocks) return [0, numBlocks]; - if (alignedS === s && alignedE === e) return [s, e]; - return [alignedS, alignedE]; + if (numBlocks === 0) return [0, 0]; + const alignedStart = Math.floor(s / groupSize) * groupSize; + const alignedEnd = Math.min(numBlocks, Math.ceil(e / groupSize) * groupSize); + // Reset to full range when the previous selection has gone out of bounds + // OR realigning has produced an empty range (e.g. switching from an empty + // profile back to a non-empty one would otherwise leave selection at [0,0] + // and render an empty flamegraph). + if (alignedStart >= numBlocks || alignedEnd <= alignedStart) return [0, numBlocks]; + if (alignedStart === s && alignedEnd === e) return [s, e]; + return [alignedStart, alignedEnd]; }); }, [groupSize, numBlocks]); diff --git a/src/report_frontend/src/components/pages/GraphDataPage.tsx b/src/report_frontend/src/components/pages/GraphDataPage.tsx new file mode 100644 index 00000000..854aca32 --- /dev/null +++ b/src/report_frontend/src/components/pages/GraphDataPage.tsx @@ -0,0 +1,140 @@ +import React from "react"; +import { DataPageProps, DataType, GraphData } from "../../definitions/types"; +import { PROCESSED_DATA, RUNS } from "../../definitions/data-config"; +import { + Box, + Cards, + CardsProps, + Pagination, + SegmentedControl, + SegmentedControlProps, + TextFilter, +} from "@cloudscape-design/components"; +import { useCollection } from "@cloudscape-design/collection-hooks"; +import Header from "@cloudscape-design/components/header"; +import IframeGraph from "../data/IframeGraph"; +import { DATA_DESCRIPTIONS } from "../../definitions/data-descriptions"; +import { RunHeader } from "../data/RunSystemInfo"; +import { ReportHelpPanelLink } from "../misc/ReportHelpPanel"; + +// TODO: GraphDataPage and the GraphData type are temporary, currently used only +// by hotline. They will be replaced by new data formats. + +const NUM_GRAPHS_PER_PAGE = 10; + +/** + * Collect all graph groups across all runs and transform them into the format required + * by SegmentedControl + */ +function getAllGraphGroups(dataType: DataType): SegmentedControlProps.Option[] { + const graphGroupNames: string[] = []; + for (const runName of RUNS) { + const reportData = PROCESSED_DATA[dataType]?.runs?.[runName] as GraphData | undefined; + if (!reportData?.graph_groups) continue; + for (const graphGroup of reportData.graph_groups) { + if (!graphGroupNames.includes(graphGroup.group_name)) { + graphGroupNames.push(graphGroup.group_name); + } + } + } + + const allGraphGroups: SegmentedControlProps.Option[] = []; + for (const groupName of graphGroupNames) { + allGraphGroups.push({ + id: groupName, + text: DATA_DESCRIPTIONS[dataType].fieldDescriptions[groupName]?.readableName || groupName, + }); + } + + return allGraphGroups; +} + +/** + * Compute the list of graph names sorted by size + */ +function getGraphNames(dataType: DataType, graphGroupName: string): string[] { + const graphSizes = new Map(); + for (const runName of RUNS) { + const reportData = PROCESSED_DATA[dataType]?.runs?.[runName] as GraphData | undefined; + const graphGroup = reportData?.graph_groups?.find((graphGroup) => graphGroup.group_name === graphGroupName); + if (graphGroup == undefined || !graphGroup.graphs) continue; + for (const graphName in graphGroup.graphs) { + if (!graphSizes.has(graphName)) { + graphSizes.set(graphName, 0); + } + graphSizes.set(graphName, graphSizes.get(graphName) + (graphGroup.graphs[graphName]?.graph_size || 0)); + } + } + return Array.from(graphSizes.keys()).sort((a, b) => graphSizes.get(b) - graphSizes.get(a)); +} + +/** + * This component renders the page for graph data type, where the graphs are rendered within Iframes + */ +export default function (props: DataPageProps) { + const allGraphGroups = React.useMemo(() => getAllGraphGroups(props.dataType), [props.dataType]); + + const [graphGroupName, setGraphGroupName] = React.useState(allGraphGroups[0]?.id || ""); + + const graphRowPercentage = Math.floor(100 / RUNS.length); + const cardDefinition: CardsProps.CardDefinition = { + header: (graphName: string) =>
{graphName}
, + sections: RUNS.map((runName) => ({ + id: runName, + header: , + content: (graphName) => ( +
+ +
+ ), + width: graphRowPercentage, + })), + }; + + const sortedGraphNames = getGraphNames(props.dataType, graphGroupName); + const { items, filteredItemsCount, collectionProps, filterProps, paginationProps } = useCollection(sortedGraphNames, { + filtering: { + filteringFunction: (item: string, filteringText: string) => + item.toLowerCase().includes(filteringText.toLowerCase()), + empty: No graphs were collected, + noMatch: No graphs found, + }, + pagination: { pageSize: Math.floor(NUM_GRAPHS_PER_PAGE / RUNS.length) }, + }); + + return ( + } + stickyHeader={true} + header={ +
} + actions={ + allGraphGroups.length > 1 && ( + setGraphGroupName(detail.selectedId)} + options={allGraphGroups} + /> + ) + } + > + {DATA_DESCRIPTIONS[props.dataType].readableName} +
+ } + filter={ + + } + variant={"full-page"} + items={items} + cardDefinition={cardDefinition} + /> + ); +} diff --git a/src/report_frontend/src/components/pages/ProfilingDataPage.tsx b/src/report_frontend/src/components/pages/ProfilingDataPage.tsx index 3dbda92b..9fc8057e 100644 --- a/src/report_frontend/src/components/pages/ProfilingDataPage.tsx +++ b/src/report_frontend/src/components/pages/ProfilingDataPage.tsx @@ -11,10 +11,8 @@ import { Table, } from "@cloudscape-design/components"; import Header from "@cloudscape-design/components/header"; -import IframeGraph from "../data/IframeGraph"; import ProfilePanel from "../data/profile-panel/ProfilePanel"; import { DATA_DESCRIPTIONS } from "../../definitions/data-descriptions"; -import { RunHeader } from "../data/RunSystemInfo"; import { ReportHelpPanelLink } from "../misc/ReportHelpPanel"; import { ShowFindingsPanelButton } from "../analytics/FindingsSplitPanel"; import { buildKeyValueTable } from "../data/KeyValueTable"; @@ -22,22 +20,20 @@ import { ProfileAnalyticalFindings } from "../analytics/AnalyticalFindings"; import { useReportState } from "../ReportStateProvider"; /** - * Collect all profiler instance names across all runs for the Select dropdown + * Collect all profiler instance names across all runs for the Select dropdown. + * Sorted alphabetically for stable presentation. */ function getAllInstanceNames(dataType: DataType): SelectProps.Option[] { - const sizes = new Map(); + const names = new Set(); for (const runName of RUNS) { - const reportData = PROCESSED_DATA[dataType].runs[runName] as ProfilingData; - if (reportData == undefined) continue; - for (const [name, profiler] of Object.entries(reportData.profilers)) { - if (!sizes.has(name)) sizes.set(name, 0); - for (const profile of Object.values(profiler.profiles)) { - sizes.set(name, sizes.get(name) + (profile.profile_graph?.graph_size || 0)); - } + const reportData = PROCESSED_DATA[dataType]?.runs?.[runName] as ProfilingData | undefined; + if (!reportData?.profilers) continue; + for (const name of Object.keys(reportData.profilers)) { + names.add(name); } } - return Array.from(sizes.keys()) - .sort((a, b) => sizes.get(b) - sizes.get(a)) + return Array.from(names) + .sort((a, b) => a.localeCompare(b)) .map((name) => ({ label: name, value: name })); } @@ -47,9 +43,9 @@ function getAllInstanceNames(dataType: DataType): SelectProps.Option[] { function getProfileNames(dataType: DataType, instanceName: string): SegmentedControlProps.Option[] { const profileNames = new Set(); for (const runName of RUNS) { - const reportData = PROCESSED_DATA[dataType].runs[runName] as ProfilingData; + const reportData = PROCESSED_DATA[dataType]?.runs?.[runName] as ProfilingData | undefined; const instance = reportData?.profilers?.[instanceName]; - if (instance == undefined) continue; + if (!instance?.profiles) continue; for (const profileName in instance.profiles) { profileNames.add(profileName); } @@ -63,7 +59,8 @@ function getProfileNames(dataType: DataType, instanceName: string): SegmentedCon } /** - * This component renders the page for ProfilingData data type, where the graphs are rendered within Iframes + * This component renders the page for ProfilingData data types using the native ProfilePanel + * (heatmap + flamegraph / methods table). */ export default function (props: DataPageProps) { const { searchKey, setSearchKey } = useReportState(); @@ -81,9 +78,6 @@ export default function (props: DataPageProps) { ); const [selectedProfile, setSelectedProfile] = React.useState(profileOptions[0]?.id || ""); - // Page-level mode toggle: "profile" (native profile) or "iframe" (legacy SVG/HTML) - const [pageMode, setPageMode] = React.useState<"profile" | "iframe">("profile"); - const graphRowPercentage = Math.floor(100 / RUNS.length); // Reset profile selection when Profiler changes @@ -107,16 +101,6 @@ export default function (props: DataPageProps) { } }, [searchKey, instanceOptions]); - // Check if any run has profile data for the current instance/profile - const hasProfileData = React.useMemo(() => { - if (!instanceName || !selectedProfile) return false; - return RUNS.some((runName) => { - const runData = PROCESSED_DATA[props.dataType]?.runs[runName] as ProfilingData | undefined; - const profile = runData?.profilers?.[instanceName]?.profiles?.[selectedProfile]; - return profile?.blocks?.length > 0; - }); - }, [props.dataType, instanceName, selectedProfile]); - // Metadata comes from the profiler instance const { tableItems, tableColumnDefinitions } = React.useMemo(() => { if (!instanceName) return { tableItems: [], tableColumnDefinitions: [] }; @@ -159,52 +143,10 @@ export default function (props: DataPageProps) { filteringType="auto" /> - {/* TODO: remove iframe display logic when profile-panel visualization is finalized */} {selectedInstance && selectedProfile && ( - setPageMode(detail.selectedId as "profile" | "iframe")} - options={[ - { id: "profile", text: "APerf Profile" }, - { id: "iframe", text: "Legacy Visualization" }, - ]} - /> - ) : undefined - } - > - {hasProfileData ? instanceName : "Profiling Data"} - - } - > - {pageMode === "profile" && hasProfileData ? ( - - ) : ( -
- {RUNS.map((runName) => ( -
- - - - -
- ))} -
- )} + {instanceName}}> + {tableItems.length > 0 && ( diff --git a/src/report_frontend/src/definitions/data-config.ts b/src/report_frontend/src/definitions/data-config.ts index f85f1bdc..19834913 100644 --- a/src/report_frontend/src/definitions/data-config.ts +++ b/src/report_frontend/src/definitions/data-config.ts @@ -18,7 +18,6 @@ declare let processed_ena_stat_data; declare let processed_efa_stat_data; declare let processed_numastat_data; declare let processed_perf_profile_data; -declare let processed_flamegraphs_data; declare let processed_aperf_stats_data; declare let processed_java_profile_data; declare let processed_aperf_runlog_data; @@ -39,7 +38,6 @@ declare let ena_stat_findings; declare let efa_stat_findings; declare let numastat_findings; declare let perf_profile_findings; -declare let flamegraphs_findings; declare let aperf_stats_findings; declare let java_profile_findings; declare let aperf_runlog_findings; @@ -61,7 +59,6 @@ export const PROCESSED_DATA: { [key in DataType]: ReportData } = { numastat: processed_numastat_data, kernel_config: processed_kernel_config_data, sysctl: processed_sysctl_data, - flamegraphs: processed_flamegraphs_data, perf_profile: processed_perf_profile_data, java_profile: processed_java_profile_data, hotline: processed_hotline_data, @@ -85,7 +82,6 @@ export const PER_DATA_ANALYTICAL_FINDINGS: { [key in DataType]: DataFindings } = numastat: numastat_findings, kernel_config: kernel_config_findings, sysctl: sysctl_findings, - flamegraphs: flamegraphs_findings, perf_profile: perf_profile_findings, java_profile: java_profile_findings, hotline: hotline_findings, @@ -143,10 +139,7 @@ export const NAVIGATION_SECTIONS: NavigationSection[] = [ { sectionName: "Profiling", items: [ - { - sectionName: "Perf Profile", - items: ["flamegraphs", "perf_profile"], - }, + "perf_profile", "java_profile", "hotline", ], diff --git a/src/report_frontend/src/definitions/data-descriptions.ts b/src/report_frontend/src/definitions/data-descriptions.ts index 1095b2c2..393ea9eb 100644 --- a/src/report_frontend/src/definitions/data-descriptions.ts +++ b/src/report_frontend/src/definitions/data-descriptions.ts @@ -4657,17 +4657,10 @@ export const DATA_DESCRIPTIONS: { [key in DataType]: DataDescription } = { defaultHelpfulLinks: ["https://docs.kernel.org/admin-guide/sysctl/kernel.html"], fieldDescriptions: {}, }, - flamegraphs: { - readableName: "Flamegraphs", - summary: - "Kernel profiling flamegraphs visualize the call stack hierarchies and the amount of CPU time consumed by different functions. It supports viewing in both the normal (bottom-top) and reverse (top-bottom) order.", - defaultHelpfulLinks: ["https://perfwiki.github.io/main/"], - fieldDescriptions: {}, - }, perf_profile: { - readableName: "Top Functions", + readableName: "Perf Profiling", summary: - "Kernel profiling top functions are the text-based version of the flamegraphs and show the percentage of CPU time spent in each function. It only includes functions with at least 1% of CPU time.", + "Perf profiling is system-wide CPU profiling performed through Linux's Perf tool.", defaultHelpfulLinks: ["https://perfwiki.github.io/main/"], fieldDescriptions: {}, }, diff --git a/src/report_frontend/src/definitions/types.ts b/src/report_frontend/src/definitions/types.ts index d7859685..66fc3cbc 100644 --- a/src/report_frontend/src/definitions/types.ts +++ b/src/report_frontend/src/definitions/types.ts @@ -14,7 +14,6 @@ export const ALL_DATA_TYPES = [ "numastat", "kernel_config", "sysctl", - "flamegraphs", "perf_profile", "java_profile", "hotline", @@ -23,10 +22,10 @@ export const ALL_DATA_TYPES = [ ] as const; export type DataType = (typeof ALL_DATA_TYPES)[number]; -export type DataFormat = "time_series" | "key_value" | "text" | "profile" | "unknown"; +export type DataFormat = "time_series" | "key_value" | "text" | "profile" | "graph" | "unknown"; // See src/data/data_formats.rs -export type AperfData = TimeSeriesData | KeyValueData | TextData | ProfilingData; +export type AperfData = TimeSeriesData | KeyValueData | TextData | ProfilingData | GraphData; export interface ReportData { readonly data_name: DataType; @@ -77,13 +76,23 @@ export interface Profiler { } export interface Profile { - readonly profile_graph: GraphInfo; readonly blocks: { [threadStateId: string]: { [nodeId: string]: number } }[]; readonly time_range: [number, number]; readonly context_tree: CCTreeNode[]; readonly frame_map: FrameMap; } +// TODO: GraphData is a temporary format used only by hotline. It will be +// replaced by new data format. +export interface GraphData { + readonly graph_groups: GraphGroup[]; +} + +export interface GraphGroup { + readonly group_name: string; + readonly graphs: { [key in string]: GraphInfo }; +} + export interface CCTreeNode { readonly parent: number | null; readonly frame_id: number; diff --git a/src/visualizer.rs b/src/visualizer.rs index d427d641..12037c7c 100644 --- a/src/visualizer.rs +++ b/src/visualizer.rs @@ -156,6 +156,9 @@ impl DataVisualizer { pub fn post_process_data(&mut self) { match self.processed_data.data_format { DataFormat::TimeSeries => post_process_time_series_data(&mut self.processed_data), + DataFormat::Profile | DataFormat::Graph => { + post_process_profiling_data(&mut self.processed_data) + } _ => return, } } @@ -231,3 +234,123 @@ fn post_process_time_series_data(processed_data: &mut ProcessedData) { } } } + +/// Run post-processing for ProfilingData: +/// - if any run produced ProfilingData, ensure that the data_format of the ProcessedData +/// is profile (the legacy GraphData runs won't be shown in the frontend). +fn post_process_profiling_data(processed_data: &mut ProcessedData) { + let any_profile = processed_data + .runs + .values() + .any(|aperf_data| matches!(aperf_data, AperfData::Profile(_))); + processed_data.data_format = if any_profile { + DataFormat::Profile + } else { + DataFormat::Graph + }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::data::common::data_formats::{ + GraphData, GraphGroup, ProfilingData, Series, TimeSeriesData, TimeSeriesMetric, + }; + + fn make_metric(name: &str, value_range: (u64, u64)) -> TimeSeriesMetric { + let mut metric = TimeSeriesMetric::new(name.to_string()); + metric.value_range = value_range; + metric.series = vec![Series { + series_name: "agg".to_string(), + time_diff: vec![0, 10], + values: vec![value_range.0 as f64, value_range.1 as f64], + is_aggregate: true, + }]; + metric + } + + fn make_time_series_data( + sorted_metric_names: Vec<&str>, + metrics: Vec<(&str, (u64, u64))>, + ) -> TimeSeriesData { + let mut ts = TimeSeriesData::default(); + ts.sorted_metric_names = sorted_metric_names.into_iter().map(String::from).collect(); + for (name, value_range) in metrics { + ts.metrics + .insert(name.to_string(), make_metric(name, value_range)); + } + ts + } + + #[test] + fn test_post_process_time_series_data() { + let mut pd = ProcessedData::new("test".to_string()); + pd.data_format = DataFormat::TimeSeries; + + // run1 has metrics [a, b] with ranges (0,10) and (5,15) + pd.runs.insert( + "run1".to_string(), + AperfData::TimeSeries(make_time_series_data( + vec!["a", "b"], + vec![("a", (0, 10)), ("b", (5, 15))], + )), + ); + // run2 has metrics [b, c] with ranges (3, 12) and (20, 30) + pd.runs.insert( + "run2".to_string(), + AperfData::TimeSeries(make_time_series_data( + vec!["b", "c"], + vec![("b", (3, 12)), ("c", (20, 30))], + )), + ); + + post_process_time_series_data(&mut pd); + + // Both runs end up with the same consolidated sorted_metric_names and value_ranges. + for run_name in &["run1", "run2"] { + let ts = match pd.runs.get(*run_name).unwrap() { + AperfData::TimeSeries(ts) => ts, + _ => panic!("expected TimeSeriesData"), + }; + // Topological sort of [[a,b], [b,c]] yields [a,b,c]. + assert_eq!(ts.sorted_metric_names, vec!["a", "b", "c"]); + // Per-metric ranges combined: min of starts, max of ends across the runs. + // a only in run1: (0,10); b in both: (min(5,3), max(15,12)) = (3,15); + // c only in run2: (20,30). + if let Some(a) = ts.metrics.get("a") { + assert_eq!(a.value_range, (0, 10)); + } + assert_eq!(ts.metrics.get("b").unwrap().value_range, (3, 15)); + if let Some(c) = ts.metrics.get("c") { + assert_eq!(c.value_range, (20, 30)); + } + } + } + + #[test] + fn test_post_process_profiling_data() { + // Branch 1: mixed Profile + Graph -> Profile wins (regardless of insertion order). + let mut pd = ProcessedData::new("java_profile".to_string()); + pd.data_format = DataFormat::Graph; + pd.runs + .insert("run1".to_string(), AperfData::Graph(GraphData::default())); + pd.runs.insert( + "run2".to_string(), + AperfData::Profile(ProfilingData::default()), + ); + post_process_profiling_data(&mut pd); + assert!(matches!(pd.data_format, DataFormat::Profile)); + + // Branch 2: all runs are Graph -> ProcessedData stays Graph. + let mut pd = ProcessedData::new("hotline".to_string()); + pd.data_format = DataFormat::Profile; + let mut graph_data = GraphData::default(); + graph_data.graph_groups.push(GraphGroup::new("table_id_1")); + pd.runs + .insert("run1".to_string(), AperfData::Graph(graph_data)); + pd.runs + .insert("run2".to_string(), AperfData::Graph(GraphData::default())); + post_process_profiling_data(&mut pd); + assert!(matches!(pd.data_format, DataFormat::Graph)); + } +} diff --git a/tests/test_java_profile.rs b/tests/test_java_profile.rs index 55d2f18a..ec88da1d 100644 --- a/tests/test_java_profile.rs +++ b/tests/test_java_profile.rs @@ -50,7 +50,7 @@ fn create_html_file(data_dir: &PathBuf, run_name: &str, pid: &str, metric: &str) #[test] fn test_process_raw_data_with_valid_files() { - let (_temp_dir, data_dir, _report_dir, params) = setup_test_env(); + let (_temp_dir, data_dir, report_dir, params) = setup_test_env(); let mut process_map = HashMap::new(); process_map.insert("12345".to_string(), vec!["TestApp".to_string()]); @@ -69,13 +69,34 @@ fn test_process_raw_data_with_valid_files() { assert!(result.is_ok()); if let Ok(AperfData::Profile(profiling_data)) = result { - // 2 JVMs + // 2 JVMs from the jps map. assert_eq!(profiling_data.profilers.len(), 2); - // Each JVM has 4 metrics + // No JFR profiler-data files were created in this test, but the processor + // seeds an empty Profile for each metric in PROFILE_METRICS so the frontend + // renders a tab for each of them. for (_name, profiler) in &profiling_data.profilers { - assert_eq!(profiler.profiles.len(), 4); + assert_eq!(profiler.profiles.len(), 3); + assert!(profiler.profiles.contains_key("cpu")); + assert!(profiler.profiles.contains_key("wall")); + assert!(profiler.profiles.contains_key("alloc")); } } + + // Verify the jfrconv-generated HTML files were copied to the report data dir. + for metric in &["cpu", "alloc", "wall"] { + for pid in &["12345", "67890"] { + assert!(report_dir + .join("data/js") + .join(format!("test_run-java-profile-{}-{}.html", pid, metric)) + .exists()); + } + } + for pid in &["12345", "67890"] { + assert!(report_dir + .join("data/js") + .join(format!("test_run-java-flamegraph-{}.html", pid)) + .exists()); + } } #[test] @@ -134,13 +155,22 @@ fn test_process_raw_data_with_no_html_files() { assert!(result.is_ok()); if let Ok(AperfData::Profile(profiling_data)) = result { - assert!(profiling_data.profilers.is_empty()); + // Even with no graphs and no profiler-data JSON, every JVM in the jps map + // gets a default Profiler with one empty Profile per metric. Frontend + // ProfilingDataPage will treat each as "no data". + assert_eq!(profiling_data.profilers.len(), 1); + let (_name, profiler) = profiling_data.profilers.iter().next().unwrap(); + assert_eq!(profiler.profiles.len(), 3); + for metric in &["cpu", "wall", "alloc"] { + let profile = profiler.profiles.get(*metric).unwrap(); + assert!(profile.blocks.is_empty()); + } } } #[test] fn test_process_raw_data_with_complex_duplicate_names_and_missing_files() { - let (_temp_dir, data_dir, _report_dir, params) = setup_test_env(); + let (_temp_dir, data_dir, report_dir, params) = setup_test_env(); let mut process_map = HashMap::new(); process_map.insert("1001".to_string(), vec!["App".to_string()]); @@ -179,38 +209,10 @@ fn test_process_raw_data_with_complex_duplicate_names_and_missing_files() { assert!(result.is_ok()); if let Ok(AperfData::Profile(profiling_data)) = result { - // 8 JVMs total (6 App deduped + 2 Service deduped) + // 8 JVMs total (6 App deduped + 2 Service deduped). Each PID had at least + // one jfrconv HTML graph, so all of them are recorded. assert_eq!(profiling_data.profilers.len(), 8); - // Count total profiles per metric across all JVMs - let cpu_count: usize = profiling_data - .profilers - .values() - .filter(|pd| pd.profiles.contains_key("cpu")) - .count(); - assert_eq!(cpu_count, 7); // 6 App + 1 Service - - let alloc_count: usize = profiling_data - .profilers - .values() - .filter(|pd| pd.profiles.contains_key("alloc")) - .count(); - assert_eq!(alloc_count, 5); // 3 App + 2 Service - - let wall_count: usize = profiling_data - .profilers - .values() - .filter(|pd| pd.profiles.contains_key("wall")) - .count(); - assert_eq!(wall_count, 4); // 3 App + 1 Service - - let legacy_count: usize = profiling_data - .profilers - .values() - .filter(|pd| pd.profiles.contains_key("legacy")) - .count(); - assert_eq!(legacy_count, 2); // 2 Service - // Verify deduped App names exist let names: Vec = profiling_data.profilers.keys().cloned().collect(); assert!(names.iter().any(|n| n == "App")); @@ -221,4 +223,47 @@ fn test_process_raw_data_with_complex_duplicate_names_and_missing_files() { assert!(names.iter().any(|n| n == "App (5)")); assert!(names.iter().any(|n| n == "Service")); } + + // Verify the jfrconv-generated HTML files were copied to the report data dir. + // Counts mirror the per-metric file creations above. + let js_dir = report_dir.join("data/js"); + let cpu_files: Vec<_> = ["1001", "1002", "1003", "1004", "1005", "1006", "2002"] + .iter() + .filter(|pid| { + js_dir + .join(format!("test_run-java-profile-{}-cpu.html", pid)) + .exists() + }) + .collect(); + assert_eq!(cpu_files.len(), 7); + + let alloc_files: Vec<_> = ["1001", "1004", "1005", "2001", "2002"] + .iter() + .filter(|pid| { + js_dir + .join(format!("test_run-java-profile-{}-alloc.html", pid)) + .exists() + }) + .collect(); + assert_eq!(alloc_files.len(), 5); + + let wall_files: Vec<_> = ["1003", "1004", "1006", "2002"] + .iter() + .filter(|pid| { + js_dir + .join(format!("test_run-java-profile-{}-wall.html", pid)) + .exists() + }) + .collect(); + assert_eq!(wall_files.len(), 4); + + let legacy_files: Vec<_> = ["2001", "2002"] + .iter() + .filter(|pid| { + js_dir + .join(format!("test_run-java-flamegraph-{}.html", pid)) + .exists() + }) + .collect(); + assert_eq!(legacy_files.len(), 2); }