Develop test result script

.gitignore

@@ -30,4 +30,5 @@
 /scripts/.vs/scripts/v17
 /scripts/.vs
 /scripts/cclog
-/.vs/c4/v17/DocumentLayout.json
+/.vs/
+/.vs/c4/v17/

.vs/VSWorkspaceState.json

@@ -2,6 +2,6 @@
   "ExpandedNodes": [
     ""
   ],
-  "SelectedNode": "\\pico_sim_vs.sln",
+  "SelectedNode": "\\C:\\Users\\huitema\\GitHub\\c4",
   "PreviewInSolutionExplorer": false
 }

scripts/test_loop.py

@@ -27,7 +27,7 @@ def run_one_test(t_name, sim_path, exe_path, exe_options, bucket_id, stats_dir):
         ret = -1;
     else:
         range_max = 100
-        if t_name == "c4_satellite.txt":
+        if t_name.endswith("_satellite.txt"):
             range_max = 20
         cmd = exe_path + " -N " + str(range_max) + " " + " ".join(exe_options) + " " + test_path
         #print(cmd)

scripts/test_report.py

@@ -0,0 +1,181 @@
+# Write a test report from the list of results.
+#
+# The statistics directory (./tmp/stats) has one CSV file per test case. The CSV
+# has column for:
+#  - result: Success or Fail
+#  - rep: from 1 to N, the repeat number
+#  - time: the execution time of the test
+#  - av_latency: for "media" time of tests, the average frame latency.
+#  - max_latency: for "media" time of tests, the max frame latency.
+#  - test: the name of the test
+#
+# we could do graphs, or compute key statistics like media, top 95% and max for
+# the time, av_latency and max_latency. We can do that with panda, e.g.:
+#   q75 = df['Column2'].quantile(q=0.75)
+#   qmax = df['Column2'].max()
+#
+# The name of the file is normally the name of the test. That name is
+# composed the for: <alg_name> "_" <test_case> ".csv". 
+#
+# We should group the test by test cases, and possibly the test cases by related
+# groups like "single connection", "wi-fi", "Compete", etc. We may see results
+# for c4, bbr and cubic. We want to present them in a comparison table.
+# Maybe write all that in markdown.
+#
+
+
+
+import sys
+import os
+from pathlib import Path
+import pandas as pd
+import traceback
+
+# dict of valid algorithms
+algo_dict = {
+    "c4":0,
+    "bbr":1,
+    "cubic":2 }
+
+# groups of tests
+
+test_groups = [
+    [ "simple", ["alone", "low_and_up", "drop_and_back", "blackhole", "satellite"], "time"],
+    [ "wifi", [ "wifi_bad", "wifi_fade", "wifi_suspension"], "time" ],
+    [ "media", [ "media", "media10", "media_short_long", "media_wb", "media_wf", "media_ws" ], "media" ],
+    [ "compete", [ "vs_bbr", "vs_c4", "vs_cubic", "wifi_bad_bbr", "wifi_bad_c4", "wifi_bad_cubic" ], "time" ]
+]
+
+
+# Operations on a test report.
+class test_report:
+    def __init__(self, test_case, algo):
+        self.df = None
+        self.test_case = test_case
+        self.algo = algo
+
+
+    def load(self, file_name):
+        self.df = pd.read_csv(file_name)
+
+    def average(self, metric):
+        x = self.df[metric].mean()
+        return int(round(x))
+
+    def top90(self, metric):
+        x = self.df[metric].quantile(q=0.9, interpolation='linear')
+        return int(round(x))
+
+class test_case_group:
+    def __init__(self, tc):
+        self.tc = tc
+        self.alg_report = [ None, None, None ]
+
+# Grouping of reports
+
+class report_list:
+    def __init__(self):
+        self.test_cases=dict()
+        self.reported = set()
+
+    def add_dir(self, dir_path):
+        for report_name in os.listdir(dir_path):
+            if report_name.endswith(".csv"):
+                algo_case = report_name[:-4]
+                algo_case_part = algo_case.split("_")
+                if len(algo_case_part) > 1:
+                    algo = algo_case_part[0]
+                    if algo in algo_dict:
+                        tc = algo_case[(len(algo) + 1):]
+                        rp = test_report(algo, tc)
+                        rp.load(os.path.join(dir_path, report_name))
+                        if not tc in self.test_cases:
+                            self.test_cases[tc] = test_case_group(tc)
+                        self.test_cases[tc].alg_report[algo_dict[algo]] = rp
+
+
+
+    def do_metric_report(self, F, grp, tl, metric, use_top_90):
+        if use_top_90:
+            top = " top 90% " + metric
+        else:
+            top = " average " + metric
+        top += " for " + grp + " tests"
+
+        F.write("### " + top + "\n")
+        F.write("| " + top + "| c4 | bbr | cubic |\n")
+        F.write("| --------- | ---:| ---:| ---:|\n")
+
+        for tc in tl:
+            if not tc in self.test_cases:
+                continue
+            tc_data = self.test_cases[tc]
+
+            sm = "| " + tc + " | " 
+            has_metric = False
+            for i in range(0,3):
+                x = 0
+                if tc_data.alg_report[i] == None:
+                    sm += " |"
+                else:
+                    if use_top_90:
+                        x = tc_data.alg_report[i].top90(metric)
+                    else:
+                        x = tc_data.alg_report[i].average(metric)
+                    sm += " " + str(x) + " |"
+                    if x > 0:
+                        has_metric = True
+            if has_metric:
+                F.write(sm + '\n')
+                self.reported.add(tc)
+
+    def do_case_metrics(self, F, grp, tl, metric_type):
+        if metric_type == 'time':
+            self.do_metric_report(F, grp, tl,'time', False)
+            self.do_metric_report(F, grp, tl, 'time', True)
+        else:
+            self.do_metric_report(F, grp, tl, 'av_latency', False)
+            self.do_metric_report(F, grp, tl, 'max_latency', True)
+
+    def do_report(self, F):
+        self.reported = set()
+        F.write("# Statistics\n")
+        F.write("Here is a collection of statistics on all test cases.\n\n")
+        for tg in test_groups:
+            F.write("## " + tg[0] + "\n")
+            F.write("Here the statistics for the " + tg[0] + " test cases.\n\n")
+            self.do_case_metrics(F, tg[0], tg[1], tg[2])
+            F.write("\n")
+
+        F.write("## others\n")
+        F.write("Here the statistics for the other test cases.\n\n")
+        F.write("| average time | c4 | bbr | cubic |\n")
+        F.write("|----| ---:| ---:| ---:|\n")
+        for tc in self.test_cases:
+           if not tc in self.reported:
+                sm = "| " + tc + " | " 
+                has_metric = False
+                for i in range(0,3):
+                    x = 0
+                    if self.test_cases[tc].alg_report[i] == None:
+                        sm += " |"
+                    else:
+                        x = self.test_cases[tc].alg_report[i].average('time')
+                        sm += " " + str(x) + " |"
+                    if x > 0:
+                        has_metric = True
+                if has_metric:
+                    F.write(sm + '\n')
+
+        F.write("\n")
+
+
+
+# main
+
+rl = report_list()
+
+rl.add_dir(sys.argv[1])
+
+with open(sys.argv[2], "wt") as F:
+    rl.do_report(F)

sim_specs/bbr_blackhole.txt

@@ -0,0 +1,11 @@
+main_cc_algo: bbr
+main_start_time: 0
+main_scenario_text: =b1:*1:397:10000000;
+nb_connections: 1
+main_target_time: 6100000
+data_rate_in_gbps: 0.025
+latency: 35000
+queue_delay_max: 80000
+icid: ccb1c400
+qlog_dir: cclog
+link_scenario: black_hole

sim_specs/bbr_drop_and_back.txt

@@ -0,0 +1,11 @@
+main_cc_algo: bbr
+main_start_time: 0
+main_scenario_text: =b1:*1:397:7000000;
+nb_connections: 1
+main_target_time: 8250000
+data_rate_in_gbps: 0.01
+latency: 50000
+queue_delay_max: 80000
+icid: ccdbc400
+qlog_dir: cclog
+link_scenario: drop_and_back

sim_specs/bbr_media_wb.txt

@@ -12,6 +12,6 @@ icid: ed1abbfb
 qlog_dir: cclog
 qperf_log: bbr_media_wb_qperflog.csv
 media_stats_start: 200000
-media_latency_average: 210000
+media_latency_average: 250000
 media_latency_max: 922000
 media_excluded: vhigh, vmid, vlast

sim_specs/bbr_media_wf.txt

@@ -10,7 +10,7 @@ icid: ed1abbfa
 qlog_dir: cclog
 qperf_log: bbr_media_wf_qperflog.csv
 media_stats_start: 200000
-media_latency_average: 200000
-media_latency_max: 1273639
+media_latency_average: 250000
+media_latency_max: 1600000
 media_excluded: vhigh, vmid, vlast
 link_scenario: wifi_fade

sim_specs/bbr_vs_bbr.txt

@@ -0,0 +1,13 @@
+main_cc_algo: bbr
+main_start_time: 0
+main_scenario_text: =b1:*1:397:5000000;
+nb_connections: 2
+background_cc_algo: bbr
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 6700000
+data_rate_in_gbps: 0.02
+latency: 40000
+queue_delay_max: 80000
+icid: ccc0c4bb
+qlog_dir: cclog

sim_specs/bbr_vs_c4.txt

@@ -0,0 +1,13 @@
+main_cc_algo: bbr
+main_start_time: 0
+main_scenario_text: =b1:*1:397:5000000;
+nb_connections: 2
+background_cc_algo: c4
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 7000000
+data_rate_in_gbps: 0.02
+latency: 40000
+queue_delay_max: 80000
+icid: ccc0c4c4
+qlog_dir: cclog

sim_specs/bbr_vs_cubic.txt

@@ -1,13 +1,13 @@
-main_cc_algo: bbr
+main_cc_algo: c4
 main_start_time: 0
 main_scenario_text: =b1:*1:397:5000000;
 nb_connections: 2
 background_cc_algo: cubic
 background_start_time: 0
 background_scenario_text: =b1:*1:397:10000000;
-main_target_time: 10000000
+main_target_time: 6800000
 data_rate_in_gbps: 0.02
 latency: 40000
 queue_delay_max: 80000
-icid: ccc0bbcb
+icid: ccc0c4cb
 qlog_dir: cclog

sim_specs/bbr_wifi_bad.txt

@@ -2,7 +2,7 @@ main_cc_algo: bbr
 main_start_time: 0
 main_scenario_text: =b1:*1:397:4000000;
 nb_connections: 1
-main_target_time: 9500000
+main_target_time: 9900000
 data_rate_in_gbps: 0.01
 latency: 1000
 jitter: 7000

sim_specs/bbr_wifi_bad_bbr.txt

@@ -0,0 +1,16 @@
+main_cc_algo: bbr
+main_cc_options:
+main_start_time: 1000000
+main_scenario_text: =b1:*1:397:4000000;
+nb_connections: 2
+background_cc_algo: bbr
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 18500000
+data_rate_in_gbps: 0.01
+latency: 1000
+jitter: 7000
+wifi_jitter: 1
+queue_delay_max: 250000
+icid: badfc4bb
+qlog_dir: cclog

sim_specs/bbr_wifi_bad_c4.txt

@@ -0,0 +1,16 @@
+main_cc_algo: bbr
+main_cc_options:
+main_start_time: 1000000
+main_scenario_text: =b1:*1:397:4000000;
+nb_connections: 2
+background_cc_algo: c4
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 18500000
+data_rate_in_gbps: 0.01
+latency: 1000
+jitter: 7000
+wifi_jitter: 1
+queue_delay_max: 250000
+icid: badfc4c4
+qlog_dir: cclog

sim_specs/bbr_wifi_bad_cubic.txt

@@ -0,0 +1,16 @@
+main_cc_algo: bbr
+main_cc_options:
+main_start_time: 1000000
+main_scenario_text: =b1:*1:397:4000000;
+nb_connections: 2
+background_cc_algo: cubic
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 16000000
+data_rate_in_gbps: 0.01
+latency: 1000
+jitter: 7000
+wifi_jitter: 1
+queue_delay_max: 250000
+icid: badfc4cb
+qlog_dir: cclog

sim_specs/cubi_wifi_bad_bbr.txt

@@ -0,0 +1,16 @@
+main_cc_algo: cubic
+main_cc_options:
+main_start_time: 1000000
+main_scenario_text: =b1:*1:397:4000000;
+nb_connections: 2
+background_cc_algo: bbr
+background_start_time: 0
+background_scenario_text: =b1:*1:397:10000000;
+main_target_time: 18500000
+data_rate_in_gbps: 0.01
+latency: 1000
+jitter: 7000
+wifi_jitter: 1
+queue_delay_max: 250000
+icid: badfc4bb
+qlog_dir: cclog

sim_specs/cubic_blackhole.txt

@@ -0,0 +1,11 @@
+main_cc_algo: cubic
+main_start_time: 0
+main_scenario_text: =b1:*1:397:10000000;
+nb_connections: 1
+main_target_time: 6100000
+data_rate_in_gbps: 0.025
+latency: 35000
+queue_delay_max: 80000
+icid: ccb1c400
+qlog_dir: cclog
+link_scenario: black_hole