Integrate BI tests with Runt & CI

.github/workflows/test.yml

@@ -36,6 +36,8 @@ jobs:
       run: runt runt/graph_interp
     - name: Run Runt tests for monitor
       run: runt runt/monitor
+    - name: Run Runt tests for bi
+      run: runt runt/bi
     - name: Run Runt tests for waveform
       run: runt runt/waveform
     - name: Run Runt tests for fail

bi/src/main.rs

@@ -75,6 +75,11 @@ struct Cli {
     #[arg(long)]
     max_traces: Option<u32>,
 
+    /// To suppress colors in error messages, pass in `--color never`.
+    /// Otherwise, by default, error messages are displayed with colors.
+    #[arg(long, value_name = "COLOR_CHOICE", default_value = "auto")]
+    color: ColorChoice,
+
     /// If enabled, displays integer literals using hexadecimal notation
     #[arg(short, long, value_name = "DISPLAY_IN_HEX")]
     display_hex: bool,
@@ -86,6 +91,14 @@ struct Cli {
 
 #[allow(unused_variables)]
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // By default, a Rust panic error messages prints the OS's thread ID
+    // (e.g. `"thread 'main' (<tid>)`), but this changes every time we run this executable.
+    // To prevent the TID from appearing in error messages that appear in the expected
+    // output files for Runt, we register a custom panic hook that only prints
+    // the error message & source code location associated with the panic
+    // (via the `Display` trait for the `PanicHookInfo` type).
+    std::panic::set_hook(Box::new(|info| eprintln!("{}", info)));
+
     // Parse CLI args
     let cli = Cli::parse();
 
@@ -96,7 +109,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // parse protocol file
     let show_warnings = false;
     let skip_static_step_fork_checks = false;
-    let mut d = DiagnosticHandler::new(ColorChoice::Auto, false, show_warnings, false);
+    let mut d = DiagnosticHandler::new(cli.color, false, show_warnings, false);
     let (st, modules) = frontend(&cli.protocol, &mut d, skip_static_step_fork_checks)?;
 
     // try to find instances that we care about

justfile

@@ -1,8 +1,9 @@
 # Runs the Runt snapshot suites that together cover every test
 runt:
-  cargo build --offline --package protocols-interp --package protocols-monitor --package graph-interp
+  cargo build --offline --package protocols-interp --package protocols-monitor --package graph-interp --package bi
   runt --max-futures 1 runt/interp
   runt --max-futures 1 runt/monitor
+  runt --max-futures 1 runt/bi
   runt --max-futures 1 runt/graph_interp
   runt --max-futures 1 runt/waveform
   runt --max-futures 1 runt/fail

scripts/generate_runt_configs.py

@@ -91,7 +91,7 @@ def expect_name(case: dict, runner: str) -> str:
 
 def expect_dir(case: dict, runner: str) -> str:
     wave = case.get("wave")
-    base = Path(wave if runner == "monitor" and wave else case["path"]).parent
+    base = Path(wave if runner in ("monitor", "bi") and wave else case["path"]).parent
     return f"../../{base.as_posix()}/expects"
 
 
@@ -195,6 +195,23 @@ def monitor_runt_command(case: dict) -> list[tuple[str, str]]:
     return [("", repo_root_command(cmd))]
 
 
+# Same as `monitor_runt_command` above but for the BI test cases
+def bi_runt_command(case: dict) -> list[tuple[str, str]]:
+    # We pass in `--color never` to BI to suppress color in error messages
+    # (so that the .expect files only display plaintext)
+    cmd = [*binary_prefix("bi"), "--color", "never", "--protocol", case["path"]]
+    if case["wave"]:
+        cmd += ["--wave", case["wave"]]
+    if case["instances"]:
+        cmd += ["--instances", *case["instances"]]
+    cmd += case["extra_args"]
+    if case["timeout_secs"] is not None:
+        cmd = timeout_cmd(case["timeout_secs"], cmd)
+    # We redirect stderr to stdout so that the expected output files
+    # contain error messages if BI fails
+    return [("", repo_root_command(cmd, stderr="stdout"))]
+
+
 def waveform_runt_command(case: dict) -> list[tuple[str, str]]:
     ast_cmd = [
         *binary_prefix("protocols-interp"),
@@ -261,6 +278,7 @@ def fail_runt_command(case: dict) -> list[tuple[str, str]]:
     "interp": interp_runt_command,
     "graph_interp": graph_interp_runt_command,
     "monitor": monitor_runt_command,
+    "bi": bi_runt_command,
     "waveform": waveform_runt_command,
     "fail": fail_runt_command,
 }
@@ -388,6 +406,9 @@ def generate_runt_configs() -> None:
     suite_specs = {
         "interp": ("interp", tx),
         "monitor": ("monitor", mon),
+        # Note: bi uses the same cases as the monitor (i.e. MONITOR_CASES)
+        # as the monitor/bi share largely the same CLI args
+        "bi": ("bi", mon),
         "graph_interp": ("graph_interp", graph_interp_cases(tx)),
         "waveform": ("waveform", waveform_cases(tx)),
         "fail": ("fail", fail_cases(tx)),

tests/adders/expects/add_d1.bi.expect

@@ -0,0 +1,5 @@
+// trace 0
+trace {
+    add(1, 2, 3);
+    add(4, 5, 9);
+}

tests/adders/expects/add_d2.bi.expect

@@ -0,0 +1,5 @@
+// trace 0
+trace {
+    add(1, 2, 3);
+    add(4, 5, 9);
+}

tests/adders/expects/add_var_cyc.bi.expect

@@ -0,0 +1,27 @@
+// trace 0
+trace {
+    loop_add(1, 2, 3, 1);
+    loop_add(1, 2, 3, 0);
+    loop_add(10, 20, 30, 0);
+}
+
+// trace 1
+trace {
+    loop_add(1, 2, 3, 2);
+    loop_add(10, 20, 30, 0);
+}
+
+// trace 2
+trace {
+    loop_add(1, 2, 3, 0);
+    loop_add(1, 2, 3, 0);
+    loop_add(1, 2, 3, 0);
+    loop_add(10, 20, 30, 0);
+}
+
+// trace 3
+trace {
+    loop_add(1, 2, 3, 0);
+    loop_add(1, 2, 3, 1);
+    loop_add(10, 20, 30, 0);
+}

tests/adders/expects/busy_wait.bi.expect

@@ -0,0 +1,4 @@
+panicked at bi/src/bi.rs:628:21:
+[add_busy_wait@00!] Cannot fork at step zero!
+---CODE---
+101

tests/adders/expects/loop_with_assigns.bi.expect

@@ -0,0 +1,4 @@
+panicked at bi/src/bi.rs:628:21:
+[loop_add@00!] Cannot fork at step zero!
+---CODE---
+101

tests/adders/expects/nested_busy_wait.bi.expect

@@ -0,0 +1,4 @@
+panicked at bi/src/bi.rs:628:21:
+[nested_busy_wait@00!] Cannot fork at step zero!
+---CODE---
+101

tests/alus/expects/alu_d1.bi.expect

@@ -0,0 +1,8 @@
+// trace 0
+trace {
+    add(1, 2, 3);
+    add(123, 245, 368);
+    sub(200, 200, 0);
+    and(100, 100, 100);
+    or(0, 230, 230);
+}

tests/alus/expects/alu_d2.bi.expect

@@ -0,0 +1,8 @@
+// trace 0
+trace {
+    add(1, 2, 3);
+    add(123, 245, 368);
+    sub(200, 200, 0);
+    and(100, 100, 100);
+    or(0, 230, 230);
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_1.bi.expect

@@ -0,0 +1,11 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 3511065769, 0);  // [time: 110ns -> 130ns]
+    reset();  // [time: 170.001ns -> 180.001ns]
+    reset();  // [time: 180.001ns -> 190.001ns]
+    reset();  // [time: 190.001ns -> 200.001ns]
+    write(15, 872415232, 2839611496, 0);  // [time: 240.001ns -> 260.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_2.bi.expect

@@ -0,0 +1,13 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 3511065769, 0);  // [time: 130ns -> 150ns]
+    read(15, 872415232, 2839611496, 0);  // [time: 150ns -> 170ns]
+    reset();  // [time: 210.001ns -> 220.001ns]
+    reset();  // [time: 220.001ns -> 230.001ns]
+    reset();  // [time: 230.001ns -> 240.001ns]
+    write(15, 872415232, 4255599317, 0);  // [time: 280.001ns -> 300.001ns]
+    write(15, 872415232, 2978480611, 0);  // [time: 300.001ns -> 320.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_3.bi.expect

@@ -0,0 +1,15 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 3511065769, 0);  // [time: 150ns -> 170ns]
+    read(15, 872415232, 2839611496, 0);  // [time: 170ns -> 190ns]
+    read(15, 872415232, 4255599317, 0);  // [time: 190ns -> 210ns]
+    reset();  // [time: 250.001ns -> 260.001ns]
+    reset();  // [time: 260.001ns -> 270.001ns]
+    reset();  // [time: 270.001ns -> 280.001ns]
+    write(15, 872415232, 2978480611, 0);  // [time: 320.001ns -> 340.001ns]
+    write(15, 872415232, 2227458782, 0);  // [time: 340.001ns -> 360.001ns]
+    write(15, 872415232, 2594436392, 0);  // [time: 360.001ns -> 380.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_4.bi.expect

@@ -0,0 +1,17 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 3511065769, 0);  // [time: 170ns -> 190ns]
+    read(15, 872415232, 2839611496, 0);  // [time: 190ns -> 210ns]
+    read(15, 872415232, 4255599317, 0);  // [time: 210ns -> 230ns]
+    read(15, 872415232, 2978480611, 0);  // [time: 230ns -> 250ns]
+    reset();  // [time: 290.001ns -> 300.001ns]
+    reset();  // [time: 300.001ns -> 310.001ns]
+    reset();  // [time: 310.001ns -> 320.001ns]
+    write(15, 872415232, 2227458782, 0);  // [time: 360.001ns -> 380.001ns]
+    write(15, 872415232, 2594436392, 0);  // [time: 380.001ns -> 400.001ns]
+    write(15, 872415232, 2879904594, 0);  // [time: 400.001ns -> 420.001ns]
+    write(15, 872415232, 2502756630, 0);  // [time: 420.001ns -> 440.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_5.bi.expect

@@ -0,0 +1,19 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 3511065769, 0);  // [time: 190ns -> 210ns]
+    read(15, 872415232, 2839611496, 0);  // [time: 210ns -> 230ns]
+    read(15, 872415232, 4255599317, 0);  // [time: 230ns -> 250ns]
+    read(15, 872415232, 2978480611, 0);  // [time: 250ns -> 270ns]
+    read(15, 872415232, 2227458782, 0);  // [time: 270ns -> 290ns]
+    reset();  // [time: 330.001ns -> 340.001ns]
+    reset();  // [time: 340.001ns -> 350.001ns]
+    reset();  // [time: 350.001ns -> 360.001ns]
+    write(15, 872415232, 2594436392, 0);  // [time: 400.001ns -> 420.001ns]
+    write(15, 872415232, 2879904594, 0);  // [time: 420.001ns -> 440.001ns]
+    write(15, 872415232, 2502756630, 0);  // [time: 440.001ns -> 460.001ns]
+    write(15, 872415232, 3888695605, 0);  // [time: 460.001ns -> 480.001ns]
+    write(15, 872415232, 4181798306, 0);  // [time: 480.001ns -> 500.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_6.bi.expect

@@ -0,0 +1,21 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 2215956531, 0);  // [time: 210ns -> 230ns]
+    read(15, 872415232, 2174594346, 0);  // [time: 230ns -> 250ns]
+    read(15, 872415232, 3706263408, 0);  // [time: 250ns -> 270ns]
+    read(15, 872415232, 3375769081, 0);  // [time: 270ns -> 290ns]
+    read(15, 872415232, 3915557832, 0);  // [time: 290ns -> 310ns]
+    read(15, 872415232, 2529678886, 0);  // [time: 310ns -> 330ns]
+    reset();  // [time: 370.001ns -> 380.001ns]
+    reset();  // [time: 380.001ns -> 390.001ns]
+    reset();  // [time: 390.001ns -> 400.001ns]
+    write(15, 872415232, 2550371957, 0);  // [time: 440.001ns -> 460.001ns]
+    write(15, 872415232, 2973140726, 0);  // [time: 460.001ns -> 480.001ns]
+    write(15, 872415232, 3658948496, 0);  // [time: 480.001ns -> 500.001ns]
+    write(15, 872415232, 4239119767, 0);  // [time: 500.001ns -> 520.001ns]
+    write(15, 872415232, 2305440508, 0);  // [time: 520.001ns -> 540.001ns]
+    write(15, 872415232, 2674771663, 0);  // [time: 540.001ns -> 560.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_7.bi.expect

@@ -0,0 +1,23 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 2215956531, 0);  // [time: 230ns -> 250ns]
+    read(15, 872415232, 2174594346, 0);  // [time: 250ns -> 270ns]
+    read(15, 872415232, 3706263408, 0);  // [time: 270ns -> 290ns]
+    read(15, 872415232, 3375769081, 0);  // [time: 290ns -> 310ns]
+    read(15, 872415232, 3915557832, 0);  // [time: 310ns -> 330ns]
+    read(15, 872415232, 2529678886, 0);  // [time: 330ns -> 350ns]
+    read(15, 872415232, 2550371957, 0);  // [time: 350ns -> 370ns]
+    reset();  // [time: 410.001ns -> 420.001ns]
+    reset();  // [time: 420.001ns -> 430.001ns]
+    reset();  // [time: 430.001ns -> 440.001ns]
+    write(15, 872415232, 2973140726, 0);  // [time: 480.001ns -> 500.001ns]
+    write(15, 872415232, 3658948496, 0);  // [time: 500.001ns -> 520.001ns]
+    write(15, 872415232, 4239119767, 0);  // [time: 520.001ns -> 540.001ns]
+    write(15, 872415232, 2305440508, 0);  // [time: 540.001ns -> 560.001ns]
+    write(15, 872415232, 2674771663, 0);  // [time: 560.001ns -> 580.001ns]
+    write(15, 872415232, 4005297405, 0);  // [time: 580.001ns -> 600.001ns]
+    write(15, 872415232, 2923162757, 0);  // [time: 600.001ns -> 620.001ns]
+}

tests/antmicro/fifo_classic/expects/test_fifo_classic_8.bi.expect

@@ -0,0 +1,25 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 2215956531, 0);  // [time: 250ns -> 270ns]
+    read(15, 872415232, 2174594346, 0);  // [time: 270ns -> 290ns]
+    read(15, 872415232, 3706263408, 0);  // [time: 290ns -> 310ns]
+    read(15, 872415232, 3375769081, 0);  // [time: 310ns -> 330ns]
+    read(15, 872415232, 3915557832, 0);  // [time: 330ns -> 350ns]
+    read(15, 872415232, 2529678886, 0);  // [time: 350ns -> 370ns]
+    read(15, 872415232, 2550371957, 0);  // [time: 370ns -> 390ns]
+    read(15, 872415232, 2973140726, 0);  // [time: 390ns -> 410ns]
+    reset();  // [time: 450.001ns -> 460.001ns]
+    reset();  // [time: 460.001ns -> 470.001ns]
+    reset();  // [time: 470.001ns -> 480.001ns]
+    write(15, 872415232, 3658948496, 0);  // [time: 520.001ns -> 540.001ns]
+    write(15, 872415232, 4239119767, 0);  // [time: 540.001ns -> 560.001ns]
+    write(15, 872415232, 2305440508, 0);  // [time: 560.001ns -> 580.001ns]
+    write(15, 872415232, 2674771663, 0);  // [time: 580.001ns -> 600.001ns]
+    write(15, 872415232, 4005297405, 0);  // [time: 600.001ns -> 620.001ns]
+    write(15, 872415232, 2923162757, 0);  // [time: 620.001ns -> 640.001ns]
+    write(15, 872415232, 2212602639, 0);  // [time: 640.001ns -> 660.001ns]
+    write(15, 872415232, 3940387843, 0);  // [time: 660.001ns -> 680.001ns]
+}

tests/antmicro/fifo_constant/expects/test_fifo_constant_1.bi.expect

@@ -0,0 +1,16 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 136945767, 0);  // [time: 110ns -> 130ns]
+    reset();  // [time: 170.001ns -> 180.001ns]
+    reset();  // [time: 180.001ns -> 190.001ns]
+    reset();  // [time: 190.001ns -> 200.001ns]
+    write(15, 872415232, 54221397, 0);  // [time: 240.001ns -> 260.001ns]
+    reset();  // [time: 320.002ns -> 330.002ns]
+    reset();  // [time: 330.002ns -> 340.002ns]
+    reset();  // [time: 340.002ns -> 350.002ns]
+    write(15, 872415232, 3117559521, 0);  // [time: 410.002ns -> 430.002ns]
+    read(15, 872415232, 6, 0);  // [time: 430.002ns -> 450.002ns]
+}

tests/antmicro/fifo_constant/expects/test_fifo_constant_2.bi.expect

@@ -0,0 +1,19 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 136945767, 0);  // [time: 130ns -> 150ns]
+    read(15, 872415232, 54221397, 0);  // [time: 150ns -> 160ns]
+    reset();  // [time: 200.001ns -> 210.001ns]
+    reset();  // [time: 210.001ns -> 220.001ns]
+    reset();  // [time: 220.001ns -> 230.001ns]
+    write(15, 872415232, 3117559521, 0);  // [time: 270.001ns -> 290.001ns]
+    write(15, 872415232, 2456570867, 0);  // [time: 290.001ns -> 300.001ns]
+    reset();  // [time: 380.002ns -> 390.002ns]
+    reset();  // [time: 390.002ns -> 400.002ns]
+    reset();  // [time: 400.002ns -> 410.002ns]
+    write(15, 872415232, 3536148369, 0);  // [time: 470.002ns -> 490.002ns]
+    write(15, 872415232, 764390477, 0);  // [time: 490.002ns -> 500.002ns]
+    read(15, 872415232, 6, 0);  // [time: 500.002ns -> 520.002ns]
+}

tests/antmicro/fifo_constant/expects/test_fifo_constant_3.bi.expect

@@ -0,0 +1,22 @@
+// trace 0
+trace {
+    reset();  // [time: 20ns -> 30ns]
+    reset();  // [time: 30ns -> 40ns]
+    reset();  // [time: 40ns -> 50ns]
+    read(15, 872415232, 136945767, 0);  // [time: 150ns -> 170ns]
+    read(15, 872415232, 54221397, 0);  // [time: 170ns -> 180ns]
+    read(15, 872415232, 3117559521, 0);  // [time: 180ns -> 190ns]
+    reset();  // [time: 230.001ns -> 240.001ns]
+    reset();  // [time: 240.001ns -> 250.001ns]
+    reset();  // [time: 250.001ns -> 260.001ns]
+    write(15, 872415232, 2456570867, 0);  // [time: 300.001ns -> 320.001ns]
+    write(15, 872415232, 3536148369, 0);  // [time: 320.001ns -> 330.001ns]
+    write(15, 872415232, 764390477, 0);  // [time: 330.001ns -> 340.001ns]
+    reset();  // [time: 440.002ns -> 450.002ns]
+    reset();  // [time: 450.002ns -> 460.002ns]
+    reset();  // [time: 460.002ns -> 470.002ns]
+    write(15, 872415232, 805776619, 0);  // [time: 530.002ns -> 550.002ns]
+    write(15, 872415232, 1651314158, 0);  // [time: 550.002ns -> 560.002ns]
+    write(15, 872415232, 3022929697, 0);  // [time: 560.002ns -> 570.002ns]
+    read(15, 872415232, 6, 0);  // [time: 570.002ns -> 590.002ns]
+}