From 96a014712476b288fccb8cda5275624fad0836c6 Mon Sep 17 00:00:00 2001 From: Hiroo HAYASHI <24754036+hirooih@users.noreply.github.com> Date: Mon, 11 Oct 2021 00:04:04 +0900 Subject: [PATCH 1/2] add --benchmark and --exclude option to the benchmark scripts for debugging --- benchmark_size.py | 17 +++++++++++++++++ benchmark_speed.py | 17 +++++++++++++++++ build_all.py | 17 +++++++++++++++++ pylib/embench_core.py | 7 ++++++- 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/benchmark_size.py b/benchmark_size.py index 66e32909..cc28e886 100755 --- a/benchmark_size.py +++ b/benchmark_size.py @@ -206,6 +206,20 @@ def build_parser(): help='Section categories to include in metric: one or more of "text", "rodata", ' + '"data" or "bss". Default "text"', ) + parser.add_argument( + '--benchmark', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to measure. By default all tests are measure.' + ) + parser.add_argument( + '--exclude', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to exclude.' + ) return parser @@ -257,6 +271,9 @@ def validate_args(args): else: gp['metric'] = ['text'] + gp['benchmark'] = args.benchmark + gp['exclude'] = args.exclude + def benchmark_size(bench, metrics): """Compute the total size of the desired sections in a benchmark. Returns the size in bytes, which may be zero if the section wasn't found.""" diff --git a/benchmark_speed.py b/benchmark_speed.py index d7312066..1555e040 100755 --- a/benchmark_speed.py +++ b/benchmark_speed.py @@ -132,6 +132,20 @@ def get_common_args(): action='store_false', help='Launch all benchmarks in series (the default)' ) + parser.add_argument( + '--benchmark', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to measure. By default all tests are measure.' + ) + parser.add_argument( + '--exclude', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to exclude.' + ) return parser.parse_known_args() @@ -168,6 +182,9 @@ def validate_args(args): gp['timeout'] = args.timeout gp['sim_parallel'] = args.sim_parallel + gp['benchmark'] = args.benchmark + gp['exclude'] = args.exclude + try: newmodule = importlib.import_module(args.target_module) except ImportError as error: diff --git a/build_all.py b/build_all.py index c154a289..3cd44ddc 100755 --- a/build_all.py +++ b/build_all.py @@ -121,6 +121,20 @@ def build_parser(): default=5, help='Timeout used for the compiler and linker invocations' ) + parser.add_argument( + '--benchmark', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to build. By default all tests are build.' + ) + parser.add_argument( + '--exclude', + type=str, + default=[], + nargs='+', + help='Benchmark name(s) to exclude.' + ) return parser @@ -196,6 +210,9 @@ def validate_args(args): var, val = envarg.split('=', 1) gp['env'][var] = val + gp['benchmark'] = args.benchmark + gp['exclude'] = args.exclude + # Other args validated later. diff --git a/pylib/embench_core.py b/pylib/embench_core.py index 442f5122..7a784e8a 100644 --- a/pylib/embench_core.py +++ b/pylib/embench_core.py @@ -130,11 +130,16 @@ def find_benchmarks(): Return the list of benchmarks.""" gp['benchdir'] = os.path.join(gp['rootdir'], 'src') gp['bd_benchdir'] = os.path.join(gp['bd'], 'src') - dirlist = os.listdir(gp['benchdir']) + if gp['benchmark']: + dirlist = gp['benchmark'] + else: + dirlist = os.listdir(gp['benchdir']) benchmarks = [] for bench in dirlist: + if bench in gp['exclude']: + continue abs_b = os.path.join(gp['benchdir'], bench) if os.path.isdir(abs_b): benchmarks.append(bench) From f6e075d26a1e9a2045ed160b1b4ea6782aeed2ab Mon Sep 17 00:00:00 2001 From: Hiroo HAYASHI <24754036+hirooih@users.noreply.github.com> Date: Wed, 18 Jan 2023 00:00:23 +0900 Subject: [PATCH 2/2] Add warning about how subsets of the benchmark suites don't generate valid results. fixes proposed by @PoaloS02 --- benchmark_size.py | 11 +++++++++-- benchmark_speed.py | 11 +++++++++-- build_all.py | 4 ++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/benchmark_size.py b/benchmark_size.py index cc28e886..d091fe4c 100755 --- a/benchmark_size.py +++ b/benchmark_size.py @@ -211,14 +211,14 @@ def build_parser(): type=str, default=[], nargs='+', - help='Benchmark name(s) to measure. By default all tests are measure.' + help='Benchmark name(s) to measure. By default all tests are measured. Results obtained from subsets are not valid Embench scores.' ) parser.add_argument( '--exclude', type=str, default=[], nargs='+', - help='Benchmark name(s) to exclude.' + help='Benchmark name(s) to exclude. Results obtained from subsets are not valid Embench scores.' ) return parser @@ -359,6 +359,8 @@ def collect_data(benchmarks): # Output it if gp['output_format'] == output_format.JSON: + if gp['benchmark'] or gp['exclude']: + log.info('These results are not valid Embench scores as they are taken from a subset of the Embench suite.') log.info('{ "size results" :') log.info(' { "detailed size results" :') for bench in benchmarks: @@ -377,6 +379,8 @@ def collect_data(benchmarks): log.info(' },') elif gp['output_format'] == output_format.TEXT: + if gp['benchmark'] or gp['exclude']: + log.info('These results are not valid Embench scores as they are taken from a subset of the Embench suite.') log.info('Benchmark size') log.info('--------- ----') for bench in benchmarks: @@ -387,6 +391,9 @@ def collect_data(benchmarks): res_output = f' {rel_data[bench]:6.2f}' log.info(f'{bench:15} {res_output:8}') elif gp['output_format'] == output_format.BASELINE: + if gp['benchmark'] or gp['exclude']: + log.info('ERROR: These results are not valid Embench scores as they are taken from a subset of the Embench suite.') + return [], [] log.info('{') for bench in benchmarks: res_output = '' diff --git a/benchmark_speed.py b/benchmark_speed.py index 1555e040..6964fce2 100755 --- a/benchmark_speed.py +++ b/benchmark_speed.py @@ -137,14 +137,14 @@ def get_common_args(): type=str, default=[], nargs='+', - help='Benchmark name(s) to measure. By default all tests are measure.' + help='Benchmark name(s) to measure. By default all tests are measured. Results obtained from subsets are not valid Embench scores.' ) parser.add_argument( '--exclude', type=str, default=[], nargs='+', - help='Benchmark name(s) to exclude.' + help='Benchmark name(s) to exclude. Results obtained from subsets are not valid Embench scores.' ) return parser.parse_known_args() @@ -314,6 +314,8 @@ def collect_data(benchmarks, remnant): # Output it if gp['output_format'] == output_format.JSON: + if gp['benchmark'] or gp['exclude']: + log.info('These results are not valid Embench scores as they are taken from a subset of the Embench suite.') log.info('{ "speed results" :') log.info(' { "detailed speed results" :') for bench in benchmarks: @@ -332,6 +334,8 @@ def collect_data(benchmarks, remnant): log.info(f' "{bench}" : {output},') log.info(' },') elif gp['output_format'] == output_format.TEXT: + if gp['benchmark'] or gp['exclude']: + log.info('These results are not valid Embench scores as they are taken from a subset of the Embench suite.') log.info('Benchmark Speed') log.info('--------- -----') for bench in benchmarks: @@ -344,6 +348,9 @@ def collect_data(benchmarks, remnant): # Want relative results (the default). Only use non-zero values. log.info(f'{bench:15} {output:8}') elif gp['output_format'] == output_format.BASELINE: + if gp['benchmark'] or gp['exclude']: + log.info('ERROR: These results are not valid Embench scores as they are taken from a subset of the Embench suite.') + return [], [] log.info('{') for bench in benchmarks: if bench == benchmarks[-1]: diff --git a/build_all.py b/build_all.py index 3cd44ddc..187d9f26 100755 --- a/build_all.py +++ b/build_all.py @@ -126,14 +126,14 @@ def build_parser(): type=str, default=[], nargs='+', - help='Benchmark name(s) to build. By default all tests are build.' + help='Benchmark name(s) to build. By default all tests are built. Results obtained from subsets are not valid Embench scores.' ) parser.add_argument( '--exclude', type=str, default=[], nargs='+', - help='Benchmark name(s) to exclude.' + help='Benchmark name(s) to exclude. Results obtained from subsets are not valid Embench scores.' ) return parser