diff --git a/.gitignore b/.gitignore index 63fd3d5..0ae5c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ /scripts/.vs/scripts/v17 /scripts/.vs /scripts/cclog -/.vs/c4/v17/DocumentLayout.json +/.vs/ +/.vs/c4/v17/ diff --git a/.vs/VSWorkspaceState.json b/.vs/VSWorkspaceState.json index ff30029..97971c7 100644 --- a/.vs/VSWorkspaceState.json +++ b/.vs/VSWorkspaceState.json @@ -2,6 +2,6 @@ "ExpandedNodes": [ "" ], - "SelectedNode": "\\pico_sim_vs.sln", + "SelectedNode": "\\C:\\Users\\huitema\\GitHub\\c4", "PreviewInSolutionExplorer": false } \ No newline at end of file diff --git a/.vs/c4/v17/.wsuo b/.vs/c4/v17/.wsuo index 338c6fc..83fd56b 100644 Binary files a/.vs/c4/v17/.wsuo and b/.vs/c4/v17/.wsuo differ diff --git a/scripts/test_loop.py b/scripts/test_loop.py index 1fd1445..2e219df 100644 --- a/scripts/test_loop.py +++ b/scripts/test_loop.py @@ -27,7 +27,7 @@ def run_one_test(t_name, sim_path, exe_path, exe_options, bucket_id, stats_dir): ret = -1; else: range_max = 100 - if t_name == "c4_satellite.txt": + if t_name.endswith("_satellite.txt"): range_max = 20 cmd = exe_path + " -N " + str(range_max) + " " + " ".join(exe_options) + " " + test_path #print(cmd) diff --git a/scripts/test_report.py b/scripts/test_report.py new file mode 100644 index 0000000..3ee6c4c --- /dev/null +++ b/scripts/test_report.py @@ -0,0 +1,181 @@ +# Write a test report from the list of results. +# +# The statistics directory (./tmp/stats) has one CSV file per test case. The CSV +# has column for: +# - result: Success or Fail +# - rep: from 1 to N, the repeat number +# - time: the execution time of the test +# - av_latency: for "media" time of tests, the average frame latency. +# - max_latency: for "media" time of tests, the max frame latency. +# - test: the name of the test +# +# we could do graphs, or compute key statistics like media, top 95% and max for +# the time, av_latency and max_latency. We can do that with panda, e.g.: +# q75 = df['Column2'].quantile(q=0.75) +# qmax = df['Column2'].max() +# +# The name of the file is normally the name of the test. That name is +# composed the for: "_" ".csv". +# +# We should group the test by test cases, and possibly the test cases by related +# groups like "single connection", "wi-fi", "Compete", etc. We may see results +# for c4, bbr and cubic. We want to present them in a comparison table. +# Maybe write all that in markdown. +# + + + +import sys +import os +from pathlib import Path +import pandas as pd +import traceback + +# dict of valid algorithms +algo_dict = { + "c4":0, + "bbr":1, + "cubic":2 } + +# groups of tests + +test_groups = [ + [ "simple", ["alone", "low_and_up", "drop_and_back", "blackhole", "satellite"], "time"], + [ "wifi", [ "wifi_bad", "wifi_fade", "wifi_suspension"], "time" ], + [ "media", [ "media", "media10", "media_short_long", "media_wb", "media_wf", "media_ws" ], "media" ], + [ "compete", [ "vs_bbr", "vs_c4", "vs_cubic", "wifi_bad_bbr", "wifi_bad_c4", "wifi_bad_cubic" ], "time" ] +] + + +# Operations on a test report. +class test_report: + def __init__(self, test_case, algo): + self.df = None + self.test_case = test_case + self.algo = algo + + + def load(self, file_name): + self.df = pd.read_csv(file_name) + + def average(self, metric): + x = self.df[metric].mean() + return int(round(x)) + + def top90(self, metric): + x = self.df[metric].quantile(q=0.9, interpolation='linear') + return int(round(x)) + +class test_case_group: + def __init__(self, tc): + self.tc = tc + self.alg_report = [ None, None, None ] + +# Grouping of reports + +class report_list: + def __init__(self): + self.test_cases=dict() + self.reported = set() + + def add_dir(self, dir_path): + for report_name in os.listdir(dir_path): + if report_name.endswith(".csv"): + algo_case = report_name[:-4] + algo_case_part = algo_case.split("_") + if len(algo_case_part) > 1: + algo = algo_case_part[0] + if algo in algo_dict: + tc = algo_case[(len(algo) + 1):] + rp = test_report(algo, tc) + rp.load(os.path.join(dir_path, report_name)) + if not tc in self.test_cases: + self.test_cases[tc] = test_case_group(tc) + self.test_cases[tc].alg_report[algo_dict[algo]] = rp + + + + def do_metric_report(self, F, grp, tl, metric, use_top_90): + if use_top_90: + top = " top 90% " + metric + else: + top = " average " + metric + top += " for " + grp + " tests" + + F.write("### " + top + "\n") + F.write("| " + top + "| c4 | bbr | cubic |\n") + F.write("| --------- | ---:| ---:| ---:|\n") + + for tc in tl: + if not tc in self.test_cases: + continue + tc_data = self.test_cases[tc] + + sm = "| " + tc + " | " + has_metric = False + for i in range(0,3): + x = 0 + if tc_data.alg_report[i] == None: + sm += " |" + else: + if use_top_90: + x = tc_data.alg_report[i].top90(metric) + else: + x = tc_data.alg_report[i].average(metric) + sm += " " + str(x) + " |" + if x > 0: + has_metric = True + if has_metric: + F.write(sm + '\n') + self.reported.add(tc) + + def do_case_metrics(self, F, grp, tl, metric_type): + if metric_type == 'time': + self.do_metric_report(F, grp, tl,'time', False) + self.do_metric_report(F, grp, tl, 'time', True) + else: + self.do_metric_report(F, grp, tl, 'av_latency', False) + self.do_metric_report(F, grp, tl, 'max_latency', True) + + def do_report(self, F): + self.reported = set() + F.write("# Statistics\n") + F.write("Here is a collection of statistics on all test cases.\n\n") + for tg in test_groups: + F.write("## " + tg[0] + "\n") + F.write("Here the statistics for the " + tg[0] + " test cases.\n\n") + self.do_case_metrics(F, tg[0], tg[1], tg[2]) + F.write("\n") + + F.write("## others\n") + F.write("Here the statistics for the other test cases.\n\n") + F.write("| average time | c4 | bbr | cubic |\n") + F.write("|----| ---:| ---:| ---:|\n") + for tc in self.test_cases: + if not tc in self.reported: + sm = "| " + tc + " | " + has_metric = False + for i in range(0,3): + x = 0 + if self.test_cases[tc].alg_report[i] == None: + sm += " |" + else: + x = self.test_cases[tc].alg_report[i].average('time') + sm += " " + str(x) + " |" + if x > 0: + has_metric = True + if has_metric: + F.write(sm + '\n') + + F.write("\n") + + + +# main + +rl = report_list() + +rl.add_dir(sys.argv[1]) + +with open(sys.argv[2], "wt") as F: + rl.do_report(F) diff --git a/sim_specs/bbr_blackhole.txt b/sim_specs/bbr_blackhole.txt new file mode 100644 index 0000000..d4b34c6 --- /dev/null +++ b/sim_specs/bbr_blackhole.txt @@ -0,0 +1,11 @@ +main_cc_algo: bbr +main_start_time: 0 +main_scenario_text: =b1:*1:397:10000000; +nb_connections: 1 +main_target_time: 6100000 +data_rate_in_gbps: 0.025 +latency: 35000 +queue_delay_max: 80000 +icid: ccb1c400 +qlog_dir: cclog +link_scenario: black_hole \ No newline at end of file diff --git a/sim_specs/bbr_drop_and_back.txt b/sim_specs/bbr_drop_and_back.txt new file mode 100644 index 0000000..41d9ac7 --- /dev/null +++ b/sim_specs/bbr_drop_and_back.txt @@ -0,0 +1,11 @@ +main_cc_algo: bbr +main_start_time: 0 +main_scenario_text: =b1:*1:397:7000000; +nb_connections: 1 +main_target_time: 8250000 +data_rate_in_gbps: 0.01 +latency: 50000 +queue_delay_max: 80000 +icid: ccdbc400 +qlog_dir: cclog +link_scenario: drop_and_back \ No newline at end of file diff --git a/sim_specs/bbr_media_wb.txt b/sim_specs/bbr_media_wb.txt index cb18314..ed90fd0 100644 --- a/sim_specs/bbr_media_wb.txt +++ b/sim_specs/bbr_media_wb.txt @@ -12,6 +12,6 @@ icid: ed1abbfb qlog_dir: cclog qperf_log: bbr_media_wb_qperflog.csv media_stats_start: 200000 -media_latency_average: 210000 +media_latency_average: 250000 media_latency_max: 922000 media_excluded: vhigh, vmid, vlast diff --git a/sim_specs/bbr_media_wf.txt b/sim_specs/bbr_media_wf.txt index 0d46853..53f3072 100644 --- a/sim_specs/bbr_media_wf.txt +++ b/sim_specs/bbr_media_wf.txt @@ -10,7 +10,7 @@ icid: ed1abbfa qlog_dir: cclog qperf_log: bbr_media_wf_qperflog.csv media_stats_start: 200000 -media_latency_average: 200000 -media_latency_max: 1273639 +media_latency_average: 250000 +media_latency_max: 1600000 media_excluded: vhigh, vmid, vlast link_scenario: wifi_fade diff --git a/sim_specs/bbr_vs_bbr.txt b/sim_specs/bbr_vs_bbr.txt new file mode 100644 index 0000000..6d6f2bd --- /dev/null +++ b/sim_specs/bbr_vs_bbr.txt @@ -0,0 +1,13 @@ +main_cc_algo: bbr +main_start_time: 0 +main_scenario_text: =b1:*1:397:5000000; +nb_connections: 2 +background_cc_algo: bbr +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 6700000 +data_rate_in_gbps: 0.02 +latency: 40000 +queue_delay_max: 80000 +icid: ccc0c4bb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/bbr_vs_c4.txt b/sim_specs/bbr_vs_c4.txt new file mode 100644 index 0000000..799ab1a --- /dev/null +++ b/sim_specs/bbr_vs_c4.txt @@ -0,0 +1,13 @@ +main_cc_algo: bbr +main_start_time: 0 +main_scenario_text: =b1:*1:397:5000000; +nb_connections: 2 +background_cc_algo: c4 +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 7000000 +data_rate_in_gbps: 0.02 +latency: 40000 +queue_delay_max: 80000 +icid: ccc0c4c4 +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/bbr_vs_cubic.txt b/sim_specs/bbr_vs_cubic.txt index 9407f9b..ebe41fc 100644 --- a/sim_specs/bbr_vs_cubic.txt +++ b/sim_specs/bbr_vs_cubic.txt @@ -1,13 +1,13 @@ -main_cc_algo: bbr +main_cc_algo: c4 main_start_time: 0 main_scenario_text: =b1:*1:397:5000000; nb_connections: 2 background_cc_algo: cubic background_start_time: 0 background_scenario_text: =b1:*1:397:10000000; -main_target_time: 10000000 +main_target_time: 6800000 data_rate_in_gbps: 0.02 latency: 40000 queue_delay_max: 80000 -icid: ccc0bbcb +icid: ccc0c4cb qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/bbr_wifi_bad.txt b/sim_specs/bbr_wifi_bad.txt index 5517707..d8ac001 100644 --- a/sim_specs/bbr_wifi_bad.txt +++ b/sim_specs/bbr_wifi_bad.txt @@ -2,7 +2,7 @@ main_cc_algo: bbr main_start_time: 0 main_scenario_text: =b1:*1:397:4000000; nb_connections: 1 -main_target_time: 9500000 +main_target_time: 9900000 data_rate_in_gbps: 0.01 latency: 1000 jitter: 7000 diff --git a/sim_specs/bbr_wifi_bad_bbr.txt b/sim_specs/bbr_wifi_bad_bbr.txt new file mode 100644 index 0000000..9fbe3cb --- /dev/null +++ b/sim_specs/bbr_wifi_bad_bbr.txt @@ -0,0 +1,16 @@ +main_cc_algo: bbr +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: bbr +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 18500000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4bb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/bbr_wifi_bad_c4.txt b/sim_specs/bbr_wifi_bad_c4.txt new file mode 100644 index 0000000..86ea5be --- /dev/null +++ b/sim_specs/bbr_wifi_bad_c4.txt @@ -0,0 +1,16 @@ +main_cc_algo: bbr +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: c4 +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 18500000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4c4 +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/bbr_wifi_bad_cubic.txt b/sim_specs/bbr_wifi_bad_cubic.txt new file mode 100644 index 0000000..1f21f5e --- /dev/null +++ b/sim_specs/bbr_wifi_bad_cubic.txt @@ -0,0 +1,16 @@ +main_cc_algo: bbr +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: cubic +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 16000000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4cb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubi_wifi_bad_bbr.txt b/sim_specs/cubi_wifi_bad_bbr.txt new file mode 100644 index 0000000..8a0b97d --- /dev/null +++ b/sim_specs/cubi_wifi_bad_bbr.txt @@ -0,0 +1,16 @@ +main_cc_algo: cubic +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: bbr +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 18500000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4bb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_blackhole.txt b/sim_specs/cubic_blackhole.txt new file mode 100644 index 0000000..2e2348d --- /dev/null +++ b/sim_specs/cubic_blackhole.txt @@ -0,0 +1,11 @@ +main_cc_algo: cubic +main_start_time: 0 +main_scenario_text: =b1:*1:397:10000000; +nb_connections: 1 +main_target_time: 6100000 +data_rate_in_gbps: 0.025 +latency: 35000 +queue_delay_max: 80000 +icid: ccb1c400 +qlog_dir: cclog +link_scenario: black_hole \ No newline at end of file diff --git a/sim_specs/cubic_drop_and_back.txt b/sim_specs/cubic_drop_and_back.txt new file mode 100644 index 0000000..3a1504b --- /dev/null +++ b/sim_specs/cubic_drop_and_back.txt @@ -0,0 +1,11 @@ +main_cc_algo: cubic +main_start_time: 0 +main_scenario_text: =b1:*1:397:7000000; +nb_connections: 1 +main_target_time: 8250000 +data_rate_in_gbps: 0.01 +latency: 50000 +queue_delay_max: 80000 +icid: ccdbc400 +qlog_dir: cclog +link_scenario: drop_and_back \ No newline at end of file diff --git a/sim_specs/cubic_media_wb.txt b/sim_specs/cubic_media_wb.txt index 0e2893e..942fcee 100644 --- a/sim_specs/cubic_media_wb.txt +++ b/sim_specs/cubic_media_wb.txt @@ -12,6 +12,6 @@ icid: ed1acbfb qlog_dir: cclog qperf_log: cubic_media_wb_qperflog.csv media_stats_start: 200000 -media_latency_average: 200000 +media_latency_average: 250000 media_latency_max: 922000 media_excluded: vhigh, vmid, vlast diff --git a/sim_specs/cubic_media_wf.txt b/sim_specs/cubic_media_wf.txt index 231c10d..4d9e49c 100644 --- a/sim_specs/cubic_media_wf.txt +++ b/sim_specs/cubic_media_wf.txt @@ -11,6 +11,6 @@ qlog_dir: cclog qperf_log: cubic_media_wf_qperflog.csv media_stats_start: 200000 media_latency_average: 130000 -media_latency_max: 400000 +media_latency_max: 500000 media_excluded: vhigh, vmid, vlast link_scenario: wifi_fade diff --git a/sim_specs/cubic_vs_bbr.txt b/sim_specs/cubic_vs_bbr.txt new file mode 100644 index 0000000..a9c9174 --- /dev/null +++ b/sim_specs/cubic_vs_bbr.txt @@ -0,0 +1,13 @@ +main_cc_algo: cubic +main_start_time: 0 +main_scenario_text: =b1:*1:397:5000000; +nb_connections: 2 +background_cc_algo: bbr +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 6700000 +data_rate_in_gbps: 0.02 +latency: 40000 +queue_delay_max: 80000 +icid: ccc0c4bb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_vs_c4.txt b/sim_specs/cubic_vs_c4.txt new file mode 100644 index 0000000..68aa96b --- /dev/null +++ b/sim_specs/cubic_vs_c4.txt @@ -0,0 +1,13 @@ +main_cc_algo: cubic +main_start_time: 0 +main_scenario_text: =b1:*1:397:5000000; +nb_connections: 2 +background_cc_algo: c4 +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 7500000 +data_rate_in_gbps: 0.02 +latency: 40000 +queue_delay_max: 80000 +icid: ccc0c4c4 +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_vs_cubic.txt b/sim_specs/cubic_vs_cubic.txt index f596e0a..e38c1e2 100644 --- a/sim_specs/cubic_vs_cubic.txt +++ b/sim_specs/cubic_vs_cubic.txt @@ -5,9 +5,9 @@ nb_connections: 2 background_cc_algo: cubic background_start_time: 0 background_scenario_text: =b1:*1:397:10000000; -main_target_time: 10000000 +main_target_time: 6800000 data_rate_in_gbps: 0.02 latency: 40000 queue_delay_max: 80000 -icid: ccc0cbcb +icid: ccc0c4cb qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_wifi_bad_bbr.txt b/sim_specs/cubic_wifi_bad_bbr.txt new file mode 100644 index 0000000..8a0b97d --- /dev/null +++ b/sim_specs/cubic_wifi_bad_bbr.txt @@ -0,0 +1,16 @@ +main_cc_algo: cubic +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: bbr +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 18500000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4bb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_wifi_bad_c4.txt b/sim_specs/cubic_wifi_bad_c4.txt new file mode 100644 index 0000000..0a6d009 --- /dev/null +++ b/sim_specs/cubic_wifi_bad_c4.txt @@ -0,0 +1,16 @@ +main_cc_algo: cubic +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: c4 +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 18500000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4c4 +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_wifi_bad_cubic.txt b/sim_specs/cubic_wifi_bad_cubic.txt new file mode 100644 index 0000000..a5b76ce --- /dev/null +++ b/sim_specs/cubic_wifi_bad_cubic.txt @@ -0,0 +1,16 @@ +main_cc_algo: cubic +main_cc_options: +main_start_time: 1000000 +main_scenario_text: =b1:*1:397:4000000; +nb_connections: 2 +background_cc_algo: cubic +background_start_time: 0 +background_scenario_text: =b1:*1:397:10000000; +main_target_time: 16000000 +data_rate_in_gbps: 0.01 +latency: 1000 +jitter: 7000 +wifi_jitter: 1 +queue_delay_max: 250000 +icid: badfc4cb +qlog_dir: cclog \ No newline at end of file diff --git a/sim_specs/cubic_wifi_suspension.txt b/sim_specs/cubic_wifi_suspension.txt new file mode 100644 index 0000000..adb037c --- /dev/null +++ b/sim_specs/cubic_wifi_suspension.txt @@ -0,0 +1,11 @@ +main_cc_algo: cubic +main_start_time: 0 +main_scenario_text: =b1:*1:397:10000000; +nb_connections: 1 +main_target_time: 10000000 +data_rate_in_gbps: 0.02 +latency: 5000 +queue_delay_max: 20000 +icid: ccc0bb55 +qlog_dir: cclog +link_scenario: wifi_suspension \ No newline at end of file