From 20956e5fab249e8dec93ae1304e140700cf7ed6c Mon Sep 17 00:00:00 2001 From: Hungyu Lin Date: Sun, 1 Nov 2026 20:01:27 -0800 Subject: [PATCH 1/3] fix: distinguish actual and expected watchdog exits --- src/userspace/scx_lertick.c | 9 ++-- tests/README.md | 10 +++++ tests/validate_watchdog_flags.py | 76 ++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/validate_watchdog_flags.py diff --git a/src/userspace/scx_lertick.c b/src/userspace/scx_lertick.c index 11165b0..d4cffbe 100644 --- a/src/userspace/scx_lertick.c +++ b/src/userspace/scx_lertick.c @@ -450,7 +450,7 @@ int main(int argc, char **argv) unsigned fixed_us = 2000, min_us = 500, max_us = 5000, tracked_tgid = 0, stall_tgid = 0; bool check = false, partial = false, fault = false, danger = false, expect_watchdog = false, profile = false; bool track_all = false; - bool fallback, restored, watchdog_observed = false; + bool fallback, restored, watchdog_observed = false, stall_kind_known; __u64 start, next_sample, exit_code; u64 stall_kind = 0, unreg_kind = 0; int opt, status = 0, final_kind; @@ -622,7 +622,8 @@ int main(int argc, char **argv) if (!flag) { fprintf(stderr, "BLOCKER: kernel lacks partial switch flag.\n"); scx_lertick__destroy(skel); return 1; } skel->struct_ops.lertick_ops->flags |= flag; } - if (expect_watchdog && !__COMPAT_read_enum("scx_exit_kind", "SCX_EXIT_ERROR_STALL", &stall_kind)) { + stall_kind_known = __COMPAT_read_enum("scx_exit_kind", "SCX_EXIT_ERROR_STALL", &stall_kind); + if (expect_watchdog && !stall_kind_known) { fprintf(stderr, "BLOCKER: cannot identify watchdog exit kind in running kernel BTF.\n"); scx_lertick__destroy(skel); return 1; @@ -729,8 +730,10 @@ int main(int argc, char **argv) lerwatt_detach(skel, pinned, demand_mem, demand_len); free(cpu_buf); exit_code = UEI_REPORT(skel, uei); - fprintf(out, "{\"event\":\"exit\",\"unexpected_fallback\":%s,\"watchdog_observed\":%s,\"exit_kind\":%d,\"exit_code\":%lld,\"reason\":", + fprintf(out, "{\"event\":\"exit\",\"unexpected_fallback\":%s,\"watchdog_observed\":%s,\"watchdog_expected\":%s,\"watchdog_triggered\":%s,\"exit_kind\":%d,\"exit_code\":%lld,\"reason\":", fallback && !watchdog_observed ? "true" : "false", watchdog_observed ? "true" : "false", + expect_watchdog ? "true" : "false", + !stall_kind_known ? "null" : (__u64)final_kind == stall_kind ? "true" : "false", skel->data->uei.kind, (long long)exit_code); json_string(out, skel->data->uei.reason); fprintf(out, ",\"message\":"); json_string(out, skel->data->uei.msg); diff --git a/tests/README.md b/tests/README.md index b5275e0..d0495c9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -42,6 +42,16 @@ QEMU guests with an independent host control path. Map capacity, extended affinity/migration stress, fairness and starvation require broader validation beyond the small Phase 1 deployment check. +Exit telemetry reports `watchdog_triggered` whenever the kernel reports a +runnable-task stall, and `watchdog_expected` when the loader was explicitly +started with `--expect-watchdog`. The legacy `watchdog_observed` field means +that an **expected** watchdog was observed. An unexpected stall therefore +has `watchdog_triggered: true`, `watchdog_observed: false`, +`unexpected_fallback: true`, and a nonzero loader status. Existing status, +recovery and legacy-field behavior are preserved. +`watchdog_triggered` is null if the running kernel does not expose the stall +exit enum; this optional classification does not prevent an ordinary run. + Phase 2 adds independent service arithmetic and actual Linux lifecycle tests: ```sh diff --git a/tests/validate_watchdog_flags.py b/tests/validate_watchdog_flags.py new file mode 100644 index 0000000..4ae135d --- /dev/null +++ b/tests/validate_watchdog_flags.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Check normal, expected-stall and unexpected-stall exit telemetry in QEMU.""" +import argparse +import os +from pathlib import Path +import shlex +import signal +import subprocess +import time + +from validate_linux import Validation, read_state, rows, wait_state + + +def stalled(validation, expected): + name = "expected" if expected else "unexpected" + command = shlex.join([str(validation.workload), "--role", "burst", "--duration", "1", + "--policy-ext", "--cpu", "0", "--burst-us", "1000", "--sleep-us", "2000"]) + worker = validation.launch(["/bin/bash", "-c", f"kill -STOP $$; exec {command}"], name + "-worker") + deadline = time.monotonic() + 3 + while "State:\tT" not in Path(f"/proc/{worker.pid}/status").read_text(): + if time.monotonic() > deadline: + raise RuntimeError("fault target did not stop") + time.sleep(.01) + extra = ["--partial", "--fault-stall", "--stall-tgid", str(worker.pid), "--dangerous-vm-test"] + if expected: + extra.append("--expect-watchdog") + loader = validation.scheduler(name, mode="fixed", duration=10, extra=extra) + worker.send_signal(signal.SIGCONT) + wait_state("disabled", timeout=12) + validation.finish(loader, expected=0 if expected else 1) + validation.finish(worker) + events = rows(validation.output / f"{name}.jsonl") + end = events[-1] + assert end["watchdog_expected"] is expected + assert end["watchdog_triggered"] is True + assert end["watchdog_observed"] is expected + assert end["unexpected_fallback"] is (not expected) + assert end["default_restored"] and end["status"] == (0 if expected else 1) + final = next(row for row in reversed(events) if row.get("phase") == "final") + assert final["stats"]["fault_stalls"] > 0 + validation.result["tests"][name] = {"passed": True, "intentional_fault": True, "exit": end} + validation.save() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + assert os.uname().sysname == "Linux" and os.geteuid() == 0 + assert subprocess.check_output(["systemd-detect-virt", "--vm"], text=True).strip() in ("qemu", "kvm") + assert read_state() == "disabled", "refusing an existing scheduler" + args.output.mkdir(parents=True, exist_ok=False) + validation = Validation(args) + try: + loader = validation.scheduler("normal", mode="fixed", duration=2) + validation.finish(loader) + end = rows(validation.output / "normal.jsonl")[-1] + assert all(end[field] is False for field in ("watchdog_expected", "watchdog_triggered", "watchdog_observed", "unexpected_fallback")) + assert end["default_restored"] and end["status"] == 0 + validation.result["tests"]["normal"] = {"passed": True, "exit": end} + stalled(validation, True) + stalled(validation, False) + validation.result.update(completed=True, final_state=read_state()) + validation.save() + print("PASS normal/expected/unexpected watchdog flags and unchanged statuses", flush=True) + except BaseException as error: + validation.result.update(completed=False, failure=repr(error)) + validation.save() + raise + finally: + validation.cleanup() + + +if __name__ == "__main__": + main() From ed04d6ef98a6a390ad1f9862585dfcfac363da17 Mon Sep 17 00:00:00 2001 From: Hungyu Lin Date: Sun, 1 Nov 2026 20:03:48 -0800 Subject: [PATCH 2/3] test: document and check watchdog telemetry validation --- .github/workflows/docs.yml | 2 +- tests/README.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2426c7e..2abb078 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,4 +39,4 @@ jobs: - name: Check script syntax run: | bash -n scripts/preflight.sh scripts/fetch_reference.sh scripts/build_libbpf.sh - python3 -m py_compile tests/validate_linux.py tests/validate_phase2.py benchmarks/run_phase2.py benchmarks/run_model_confirmation.py benchmarks/compact_confirmation.py benchmarks/analyze_phase2.py benchmarks/analyze_model_init.py benchmarks/analyze_online_utility.py scripts/qemu_vm.py + python3 -m py_compile tests/validate_linux.py tests/validate_phase2.py tests/validate_watchdog_flags.py benchmarks/run_phase2.py benchmarks/run_model_confirmation.py benchmarks/compact_confirmation.py benchmarks/analyze_phase2.py benchmarks/analyze_model_init.py benchmarks/analyze_online_utility.py scripts/qemu_vm.py diff --git a/tests/README.md b/tests/README.md index d0495c9..6bf35b3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -52,6 +52,11 @@ recovery and legacy-field behavior are preserved. `watchdog_triggered` is null if the running kernel does not expose the stall exit enum; this optional classification does not prevent an ordinary run. +In an isolated QEMU/KVM guest, `sudo python3 tests/validate_watchdog_flags.py +--output results/local/watchdog-flags-new` checks normal shutdown and both +expected/unexpected intentional stalls, including their distinct loader +statuses. It refuses an existing scheduler and retains every log. + Phase 2 adds independent service arithmetic and actual Linux lifecycle tests: ```sh From 45521336c248cf607733b4ad300c9fa5cb8b8eba Mon Sep 17 00:00:00 2001 From: Hungyu Lin Date: Sun, 1 Nov 2026 21:46:04 -0800 Subject: [PATCH 3/3] test: record watchdog classification and three-cluster integration --- results/watchdog-status/README.md | 93 +++++++ results/watchdog-status/archive.json | 5 + .../build-and-first-integration.log | 233 ++++++++++++++++++ results/watchdog-status/integration-retry.log | 1 + .../environment.json | 10 + .../standalone-after/summary.json | 18 ++ .../standalone-before/summary.json | 18 ++ .../summary.json | 193 +++++++++++++++ .../three-cluster/summary.json | 189 ++++++++++++++ .../environment.json | 10 + .../standalone-before/summary.json | 18 ++ .../three-cluster/lw_contract-0.json | 1 + .../three-cluster/summary.json | 11 + .../commands.txt | 5 + .../expected.log | 54 ++++ .../lertick-watchdog-flags-ed04d6e/normal.log | 1 + .../unexpected.log | 54 ++++ .../validation.json | 65 +++++ results/watchdog-status/provenance.json | 16 ++ ...rify-new-model-plugin-key-array-failure.py | 121 +++++++++ .../verify-new-model-plugin.py | 123 +++++++++ tests/README.md | 2 + 22 files changed, 1241 insertions(+) create mode 100644 results/watchdog-status/README.md create mode 100644 results/watchdog-status/archive.json create mode 100644 results/watchdog-status/build-and-first-integration.log create mode 100644 results/watchdog-status/integration-retry.log create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/environment.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-after/summary.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-before/summary.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/summary.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/three-cluster/summary.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e/environment.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e/standalone-before/summary.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/lw_contract-0.json create mode 100644 results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/summary.json create mode 100644 results/watchdog-status/lertick-watchdog-flags-ed04d6e/commands.txt create mode 100644 results/watchdog-status/lertick-watchdog-flags-ed04d6e/expected.log create mode 100644 results/watchdog-status/lertick-watchdog-flags-ed04d6e/normal.log create mode 100644 results/watchdog-status/lertick-watchdog-flags-ed04d6e/unexpected.log create mode 100644 results/watchdog-status/lertick-watchdog-flags-ed04d6e/validation.json create mode 100644 results/watchdog-status/provenance.json create mode 100644 results/watchdog-status/verify-new-model-plugin-key-array-failure.py create mode 100644 results/watchdog-status/verify-new-model-plugin.py diff --git a/results/watchdog-status/README.md b/results/watchdog-status/README.md new file mode 100644 index 0000000..6e80e09 --- /dev/null +++ b/results/watchdog-status/README.md @@ -0,0 +1,93 @@ +# Watchdog exit classification and optional plug-in integration + +Linux 7.0.0-38-generic, aarch64 QEMU/HVF, eight vCPUs. The tested scheduler +source is `ed04d6e`; [provenance](provenance.json) and the integration environment +record source and binary hashes. This change clarifies exit telemetry. It does +**not** fix or explain the natural stall in the +[model confirmation](../model-confirmation/README.md). + +## Exit classification + +The existing `watchdog_observed` field only identifies an **expected** watchdog. +It could therefore be false on a real unexpected runnable-task stall. Preserve +that legacy field and all exit-status decisions; add `watchdog_expected` and +`watchdog_triggered` so consumers can distinguish the cases. + +Actual [VM results](lertick-watchdog-flags-ed04d6e/validation.json): + +| Case | Expected | Triggered | Legacy observed | Unexpected fallback | Loader status | +| --- | --- | --- | --- | --- | --- | +| Normal shutdown | false | false | false | false | 0 | +| Intentional stall with expectation flag | true | true | true | false | 0 | +| Intentional stall without expectation flag | false | true | false | true | 1 | + +All three test cases passed and restored the default Linux scheduler. The last +loader's status 1 is the required result, not a successful loader exit. The +fault tests use the existing guarded partial-switch mechanism in a disposable +VM. Run `sudo python3 tests/validate_watchdog_flags.py --output FRESH_DIRECTORY` +after building the scheduler and workload in an isolated QEMU/KVM Linux guest. + +If kernel BTF lacks the stall enum, `watchdog_triggered` is `null`; normal runs +remain allowed and an explicitly expected watchdog still requires the enum. +That compatibility branch was reviewed but not exercised on this kernel. +The BPF program, model, slices, defaults and fallback gate are unchanged. +Compilation succeeded; existing signedness warnings in vendored `compat.h` +are retained in [the build log](build-and-first-integration.log). + +## Eight-vCPU integration + +The complete [fresh rerun](lertick-temporal-integration-ed04d6e-decoded/summary.json) +used the real scheduler and Lerwatt daemon with the RK3588 file-interface +simulator. All eight pinned workers alternated 30% and 90% duty cycles. +The scheduler ran 240 seconds with `--lerwatt --track-all --model-init ewma +--model-update carry`; the daemon ran 250 seconds with `--policy mpc --forecast +temporal --cross-cluster`. Both standalone checks before and after used the +ordinary scheduler defaults without Lerwatt. + +| Check | Recorded result | +| --- | --- | +| Cluster CPU sets | 0–3, 4–5, 6–7 | +| Maximum observed CPU utilization | 1022–1024 out of 1024 on every CPU | +| All three clusters simultaneously above 50% busy | 1,470 telemetry rows | +| Rows with joint planning active | 1,595 | +| Cluster forecasts selecting temporal prediction | 152 | +| Contract applications | 1,545,310 | +| Contract/demand map | Three consistent version-1, nonzero even-sequence entries each | +| Simulator peak temperatures, big0 / big1 / little | 49.9 / 49.9 / 48.1 °C | +| Throttle steps / bad writes / daemon write errors | 0 / 0 / 0 | +| Actuator restoration | All 30 integer knob values restored; journal removed | +| Scheduler cleanup | Default Linux restored; both pinned maps removed | +| Standalone before and after | Both passed | + +These are activation and cleanup observations from one simulated board, not +hardware validation or evidence of improved prediction accuracy, throughput, +energy efficiency or controller superiority. The default control policy stays +`rule`; scheduler defaults stay `bias` / `trunc`. + +## Failed attempt and retained evidence + +The [first integration attempt](lertick-temporal-integration-ed04d6e/three-cluster/summary.json) +remains `completed: false`. At the map audit, the collector treated bpftool's +raw byte-array key as an integer and raised `TypeError: unhashable type: 'list'`. +The [actual dump](lertick-temporal-integration-ed04d6e/three-cluster/lw_contract-0.json) +also contains the decoded integer/dictionary fields under `formatted`. +All three programs exited 0, all 30 knob values were restored and the journal +was removed; the after-standalone check was not reached on that attempt. + +Only the test decoder changed: use `formatted` when present and validate the +semantic key/value types. The actual nested representation and the older +direct representation both passed local decoder fixtures. The full 250-second +integration then restarted from a fresh output directory and passed, including +both standalone checks. Binary hashes match across attempts. No model, +parameters, temperature limits or runtime gates changed to obtain the pass. +Both [original](verify-new-model-plugin-key-array-failure.py) and +[corrected](verify-new-model-plugin.py) collectors are preserved. Their paths +target this disposable VM; reproduction elsewhere requires adjusting them. + +The complete 20,272,308-byte raw archive is retained locally in +`results/local/watchdog-status-ed04d6e/lertick-watchdog-integration-ed04d6e.tgz` +in the main checkout and validation worktree. It includes all three output +roots, not just the passing rerun. [The archive manifest](archive.json) records +SHA-256 `84bed19679efb78dab7c79b787a1318f625d441cb021e212d632696bc4426b1b`. +Small summaries and the build/failure log are committed here; the full raw +archive is not uploaded to Git. diff --git a/results/watchdog-status/archive.json b/results/watchdog-status/archive.json new file mode 100644 index 0000000..955100f --- /dev/null +++ b/results/watchdog-status/archive.json @@ -0,0 +1,5 @@ +{ + "file": "lertick-watchdog-integration-ed04d6e.tgz", + "bytes": 20272308, + "sha256": "84bed19679efb78dab7c79b787a1318f625d441cb021e212d632696bc4426b1b" +} diff --git a/results/watchdog-status/build-and-first-integration.log b/results/watchdog-status/build-and-first-integration.log new file mode 100644 index 0000000..b249ad7 --- /dev/null +++ b/results/watchdog-status/build-and-first-integration.log @@ -0,0 +1,233 @@ +Linux build and three exit-classification cases +From /var/tmp/lertick-watchdog-status-source.bundle + * branch HEAD -> FETCH_HEAD +Switched to a new branch 'verify-watchdog-ed04d6e' +Updating 0b23633..ed04d6e +Fast-forward + .github/workflows/docs.yml | 4 +- + benchmarks/analyze_online_utility.py | 89 + + results/model-confirmation/README.md | 55 + + .../analysis/stable-full--full-prefault.json | 1587 +++++++++++++++ + .../stable-full--full-prefault/compact.json | 12 + + .../stable-full--full-prefault/environment.json | 48 + + .../compact/stable-full--full-prefault/plan.json | 202 ++ + .../counter-boundaries.json | 107 + + .../r00-mixed-shadow-ewma-carry/trial.json | 401 ++++ + .../r00-mixed-shadow/counter-boundaries.json | 107 + + .../r00-mixed-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r00-multicore-shadow-ewma-carry/trial.json | 540 +++++ + .../r00-multicore-shadow/counter-boundaries.json | 107 + + .../r00-multicore-shadow/trial.json | 540 +++++ + .../counter-boundaries.json | 107 + + .../r00-shortburst-shadow-ewma-carry/trial.json | 351 ++++ + .../r00-shortburst-shadow/counter-boundaries.json | 107 + + .../r00-shortburst-shadow/trial.json | 351 ++++ + .../counter-boundaries.json | 107 + + .../r00-wakeheavy-shadow-ewma-carry/trial.json | 401 ++++ + .../r00-wakeheavy-shadow/counter-boundaries.json | 107 + + .../r00-wakeheavy-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r01-mixed-shadow-ewma-carry/trial.json | 401 ++++ + .../r01-mixed-shadow/counter-boundaries.json | 107 + + .../r01-mixed-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r01-multicore-shadow-ewma-carry/trial.json | 540 +++++ + .../r01-multicore-shadow/counter-boundaries.json | 107 + + .../r01-multicore-shadow/trial.json | 540 +++++ + .../counter-boundaries.json | 107 + + .../r01-shortburst-shadow-ewma-carry/trial.json | 351 ++++ + .../r01-shortburst-shadow/counter-boundaries.json | 107 + + .../r01-shortburst-shadow/trial.json | 351 ++++ + .../counter-boundaries.json | 107 + + .../r01-wakeheavy-shadow-ewma-carry/trial.json | 401 ++++ + .../r01-wakeheavy-shadow/counter-boundaries.json | 107 + + .../r01-wakeheavy-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r02-mixed-shadow-ewma-carry/trial.json | 401 ++++ + .../r02-mixed-shadow/counter-boundaries.json | 107 + + .../r02-mixed-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r02-multicore-shadow-ewma-carry/trial.json | 540 +++++ + .../r02-multicore-shadow/counter-boundaries.json | 107 + + .../r02-multicore-shadow/trial.json | 540 +++++ + .../counter-boundaries.json | 107 + + .../r02-shortburst-shadow-ewma-carry/trial.json | 351 ++++ + .../r02-shortburst-shadow/counter-boundaries.json | 107 + + .../r02-shortburst-shadow/trial.json | 351 ++++ + .../counter-boundaries.json | 107 + + .../r02-wakeheavy-shadow-ewma-carry/trial.json | 401 ++++ + .../r02-wakeheavy-shadow/counter-boundaries.json | 107 + + .../r02-wakeheavy-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r03-mixed-shadow-ewma-carry/trial.json | 401 ++++ + .../r03-mixed-shadow/counter-boundaries.json | 107 + + .../r03-mixed-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r03-multicore-shadow-ewma-carry/trial.json | 540 +++++ + .../r03-multicore-shadow/counter-boundaries.json | 107 + + .../r03-multicore-shadow/trial.json | 540 +++++ + .../counter-boundaries.json | 107 + + .../r03-shortburst-shadow-ewma-carry/trial.json | 351 ++++ + .../r03-shortburst-shadow/counter-boundaries.json | 107 + + .../r03-shortburst-shadow/trial.json | 351 ++++ + .../counter-boundaries.json | 107 + + .../r03-wakeheavy-shadow-ewma-carry/trial.json | 401 ++++ + .../r03-wakeheavy-shadow/counter-boundaries.json | 107 + + .../r03-wakeheavy-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r04-mixed-shadow-ewma-carry/trial.json | 401 ++++ + .../r04-mixed-shadow/counter-boundaries.json | 107 + + .../r04-mixed-shadow/trial.json | 401 ++++ + .../counter-boundaries.json | 107 + + .../r04-multicore-shadow-ewma-carry/trial.json | 540 +++++ + .../r04-multicore-shadow/counter-boundaries.json | 107 + + .../r04-multicore-shadow/trial.json | 540 +++++ + .../counter-boundaries.json | 107 + + .../r04-shortburst-shadow-ewma-carry/trial.json | 351 ++++ + .../r04-shortburst-shadow/counter-boundaries.json | 107 + + .../r04-shortburst-shadow/trial.json | 351 ++++ + .../counter-boundaries.json | 107 + + .../r04-wakeheavy-shadow-ewma-carry/trial.json | 401 ++++ + .../r04-wakeheavy-shadow/counter-boundaries.json | 107 + + .../r04-wakeheavy-shadow/trial.json | 401 ++++ + .../stable-full--full-prefault/summary.json | 2083 ++++++++++++++++++++ + .../failures/stable-lazy-initial/archive.json | 8 + + .../failures/stable-lazy-initial/confirmation.json | 44 + + .../failures/stable-lazy-initial/diagnosis.json | 15 + + .../failures/stable-lazy-initial/failure.json | 6 + + .../failures/stable-lazy-initial/scheduler.log | 134 ++ + src/userspace/scx_lertick.c | 9 +- + tests/README.md | 15 + + tests/test_online_utility.py | 66 + + tests/validate_watchdog_flags.py | 76 + + 97 files changed, 25659 insertions(+), 4 deletions(-) + create mode 100644 benchmarks/analyze_online_utility.py + create mode 100644 results/model-confirmation/README.md + create mode 100644 results/model-confirmation/analysis/stable-full--full-prefault.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/compact.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/environment.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/plan.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-mixed-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-mixed-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-mixed-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-mixed-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-multicore-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-multicore-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-multicore-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-multicore-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-shortburst-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-shortburst-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-shortburst-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-shortburst-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-wakeheavy-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-wakeheavy-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-wakeheavy-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r00-wakeheavy-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-mixed-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-mixed-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-mixed-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-mixed-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-multicore-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-multicore-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-multicore-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-multicore-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-shortburst-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-shortburst-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-shortburst-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-shortburst-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-wakeheavy-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-wakeheavy-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-wakeheavy-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r01-wakeheavy-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-mixed-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-mixed-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-mixed-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-mixed-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-multicore-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-multicore-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-multicore-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-multicore-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-shortburst-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-shortburst-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-shortburst-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-shortburst-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-wakeheavy-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-wakeheavy-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-wakeheavy-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r02-wakeheavy-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-mixed-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-mixed-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-mixed-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-mixed-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-multicore-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-multicore-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-multicore-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-multicore-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-shortburst-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-shortburst-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-shortburst-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-shortburst-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-wakeheavy-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-wakeheavy-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-wakeheavy-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r03-wakeheavy-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-mixed-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-mixed-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-mixed-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-mixed-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-multicore-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-multicore-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-multicore-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-multicore-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-shortburst-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-shortburst-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-shortburst-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-shortburst-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-wakeheavy-shadow-ewma-carry/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-wakeheavy-shadow-ewma-carry/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-wakeheavy-shadow/counter-boundaries.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/r04-wakeheavy-shadow/trial.json + create mode 100644 results/model-confirmation/compact/stable-full--full-prefault/summary.json + create mode 100644 results/model-confirmation/failures/stable-lazy-initial/archive.json + create mode 100644 results/model-confirmation/failures/stable-lazy-initial/confirmation.json + create mode 100644 results/model-confirmation/failures/stable-lazy-initial/diagnosis.json + create mode 100644 results/model-confirmation/failures/stable-lazy-initial/failure.json + create mode 100644 results/model-confirmation/failures/stable-lazy-initial/scheduler.log + create mode 100644 tests/test_online_utility.py + create mode 100644 tests/validate_watchdog_flags.py +cc -Ivendor -Isrc -Isrc/shared -Ibuild -I/home/ubuntu/lertick-pr/build/deps/install/include -Isrc/userspace -O2 -g -std=gnu11 -Wall -Wextra src/userspace/scx_lertick.c src/userspace/lerwatt_plugin.c -o build/scx_lertick -L/home/ubuntu/lertick-pr/build/deps/install/lib -lbpf -lelf -lzstd -pthread -lz -lelf -lz +make: 'build/burst_workload' is up to date. +In file included from vendor/scx/common.h:74, + from src/userspace/scx_lertick.c:27: +vendor/scx/compat.h: In function ‘__COMPAT_read_enum’: +vendor/scx/compat.h:44:31: warning: comparison of integer expressions of different signedness: ‘int’ and ‘__u32’ {aka ‘unsigned int’} [-Wsign-compare] + 44 | for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + | ^ +vendor/scx/compat.h:55:31: warning: comparison of integer expressions of different signedness: ‘int’ and ‘__u32’ {aka ‘unsigned int’} [-Wsign-compare] + 55 | for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + | ^ +vendor/scx/compat.h: In function ‘__COMPAT_struct_has_field’: +vendor/scx/compat.h:99:23: warning: comparison of integer expressions of different signedness: ‘int’ and ‘__u32’ {aka ‘unsigned int’} [-Wsign-compare] + 99 | for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + | ^ +PASS normal/expected/unexpected watchdog flags and unchanged statuses +Traceback (most recent call last): + File "/var/tmp/verify-new-model-plugin.py", line 121, in + if __name__=='__main__':main() + ^^^^^^ + File "/var/tmp/verify-new-model-plugin.py", line 120, in main + standalone(a.output,'standalone-before');r=integration(a.output);standalone(a.output,'standalone-after');save(a.output/'summary.json',{'passed':True,'integration':r,'standalone_before_and_after':True});print('PASS three-cluster model/plugin integration and both default standalone runs',flush=True) + ^^^^^^^^^^^^^^^^^^^^^ + File "/var/tmp/verify-new-model-plugin.py", line 67, in integration + records={r['key']:r['value'] for r in json.loads(result)} + ^^^^^^^^^^^^^^^^^^^ +TypeError: unhashable type: 'list' +Traceback (most recent call last): + File "/private/tmp/lertick-postbenchmark-validation.py", line 50, in + subprocess.run(SSH+['bash -s'],input=script,text=True,check=True) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/homebrew/Cellar/python@3.14/3.14.6/Frameworks/Python.framework/Versions/3.14/lib/python3.14/subprocess.py", line 578, in run + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) +subprocess.CalledProcessError: Command '['ssh', '-i', '/Users/hungyu_lin/Desktop/Github/lertick/build/vm/id_ed25519', '-p', '22222', '-o', 'BatchMode=yes', '-o', 'UserKnownHostsFile=/Users/hungyu_lin/Desktop/Github/lertick/build/vm/known_hosts', 'ubuntu@127.0.0.1', 'bash -s']' returned non-zero exit status 1. diff --git a/results/watchdog-status/integration-retry.log b/results/watchdog-status/integration-retry.log new file mode 100644 index 0000000..c15fef8 --- /dev/null +++ b/results/watchdog-status/integration-retry.log @@ -0,0 +1 @@ +PASS three-cluster model/plugin integration and both default standalone runs diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/environment.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/environment.json new file mode 100644 index 0000000..a75051e --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/environment.json @@ -0,0 +1,10 @@ +{ + "kernel": "7.0.0-38-generic", + "cpus": 8, + "sha256": { + "/home/ubuntu/lertick-burst-init/build/scx_lertick": "a49bc8af07226b8f7782b9c161546ad07914fd1f92fc3b1277c6db1ff6fcc36a", + "/home/ubuntu/lerwatt-temporal-forecast/build/lerwattd": "00bbd2b6161ae9070e6bd4bfb2362df257370cd6f4e70c232e44dbb0b4727ad9", + "/home/ubuntu/lerwatt-temporal-forecast/build/lerwatt-sim": "ddb14f5de44f961d2a4dca46d961581c9e01448bdbf29eef3c85896a61d8b9a8", + "/var/tmp/verify-new-model-plugin.py": "bf6116dcdb1ba1ed5866c8be86c58e59ce43a476122015f62667a20a0dd510d3" + } +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-after/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-after/summary.json new file mode 100644 index 0000000..d16449e --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-after/summary.json @@ -0,0 +1,18 @@ +{ + "passed": true, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + }, + "defaults_used": true +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-before/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-before/summary.json new file mode 100644 index 0000000..d16449e --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/standalone-before/summary.json @@ -0,0 +1,18 @@ +{ + "passed": true, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + }, + "defaults_used": true +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/summary.json new file mode 100644 index 0000000..570a43f --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/summary.json @@ -0,0 +1,193 @@ +{ + "passed": true, + "integration": { + "completed": true, + "duration_s": 250, + "scope": "One simulated board and real BPF/daemon integration; no accuracy or throughput claim.", + "restored_knobs": true, + "journal_removed": true, + "topology": { + "source": "sim", + "root": "/var/tmp/lertick-temporal-integration-ed04d6e-decoded/three-cluster/root", + "policies": 3, + "clusters": [ + [ + 0, + 1, + 2, + 3 + ], + [ + 4, + 5 + ], + [ + 6, + 7 + ] + ], + "capacity_override": [ + 397, + 397, + 397, + 397, + 1024, + 1024, + 1024, + 1024 + ] + }, + "cpu_util_max": { + "0": 1024, + "1": 1023, + "2": 1023, + "3": 1023, + "4": 1022, + "5": 1024, + "6": 1023, + "7": 1023 + }, + "simultaneous_busy_periods": 1470, + "joint_periods": 1595, + "temporal_points": 152, + "maps": { + "lw_contract": [ + { + "key": 0, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 397, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 33770, + "valid_until_ns": 32190149211615 + } + }, + { + "key": 1, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 1024, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 31924, + "valid_until_ns": 32190149211615 + } + }, + { + "key": 2, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 1024, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 31000, + "valid_until_ns": 32190149211615 + } + } + ], + "lt_demand": [ + { + "key": 0, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 395, + "cpuperf_max": 395, + "cpuperf_mean": 395, + "lat_crit_milli": 3, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + }, + { + "key": 1, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 1021, + "cpuperf_max": 1021, + "cpuperf_mean": 1021, + "lat_crit_milli": 4, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + }, + { + "key": 2, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 1021, + "cpuperf_max": 1021, + "cpuperf_mean": 1021, + "lat_crit_milli": 7, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + } + ] + }, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + }, + "final_stats": { + "local_enqueue": 522550, + "global_enqueue": 146102, + "runs": 313742, + "runtime_ns": 1167310086584, + "completed_bursts": 252195, + "censored_bursts": 79, + "training_updates": 252195, + "inferences": 252199, + "adaptive_slices": 251614, + "slice_changes": 22817, + "fault_stalls": 0, + "counter_errors": 0, + "alloc_failures": 0, + "snapshot_failures": 0, + "prediction_ns_sum": 42730485339, + "selected_slice_ns_sum": 982957956863, + "matching_dispatch_slices": 313396, + "continuation_slices": 355255, + "continuation_pending_ns": 6567611156723, + "init_tasks": 198, + "exit_tasks": 198, + "training_time_ns": 0, + "inference_time_ns": 0, + "prediction_samples": 251786, + "linear_abs_error_ns": 7654161662, + "ewma_abs_error_ns": 7715026562, + "wake_samples": 252199, + "wake_to_run_ns": 36079760558, + "wake_to_run_max_ns": 7171180, + "cpuperf_sets": 1196781, + "cpuperf_idle_sets": 35497, + "contract_applied": 1545310, + "contract_stale": 4129, + "live_tasks": 0 + }, + "simulator_summary": "lerwatt-sim: 250.1 s simulated in 250.1 s: energy 1743.1 J (mean 6.97 W), work served 65.8%, 0 throttle steps, peak 49.9 / 49.9 / 48.1 C (big0 / big1 / little), 0 bad writes", + "daemon_exit": 0, + "scx_exit": 0, + "simulator_exit": 0 + }, + "standalone_before_and_after": true +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/three-cluster/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/three-cluster/summary.json new file mode 100644 index 0000000..d785b9d --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e-decoded/three-cluster/summary.json @@ -0,0 +1,189 @@ +{ + "completed": true, + "duration_s": 250, + "scope": "One simulated board and real BPF/daemon integration; no accuracy or throughput claim.", + "restored_knobs": true, + "journal_removed": true, + "topology": { + "source": "sim", + "root": "/var/tmp/lertick-temporal-integration-ed04d6e-decoded/three-cluster/root", + "policies": 3, + "clusters": [ + [ + 0, + 1, + 2, + 3 + ], + [ + 4, + 5 + ], + [ + 6, + 7 + ] + ], + "capacity_override": [ + 397, + 397, + 397, + 397, + 1024, + 1024, + 1024, + 1024 + ] + }, + "cpu_util_max": { + "0": 1024, + "1": 1023, + "2": 1023, + "3": 1023, + "4": 1022, + "5": 1024, + "6": 1023, + "7": 1023 + }, + "simultaneous_busy_periods": 1470, + "joint_periods": 1595, + "temporal_points": 152, + "maps": { + "lw_contract": [ + { + "key": 0, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 397, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 33770, + "valid_until_ns": 32190149211615 + } + }, + { + "key": 1, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 1024, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 31924, + "valid_until_ns": 32190149211615 + } + }, + { + "key": 2, + "value": { + "seq": 4592, + "version": 1, + "ceiling_cap": 1024, + "floor_cap": 0, + "energy_price_milli": 0, + "thermal_headroom_mc": 31000, + "valid_until_ns": 32190149211615 + } + } + ], + "lt_demand": [ + { + "key": 0, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 395, + "cpuperf_max": 395, + "cpuperf_mean": 395, + "lat_crit_milli": 3, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + }, + { + "key": 1, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 1021, + "cpuperf_max": 1021, + "cpuperf_mean": 1021, + "lat_crit_milli": 4, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + }, + { + "key": 2, + "value": { + "seq": 4402, + "version": 1, + "pred_util": 1021, + "cpuperf_max": 1021, + "cpuperf_mean": 1021, + "lat_crit_milli": 7, + "p99_wake_ns": 1048576, + "updated_ns": 32189859605860, + "reserved": 0 + } + } + ] + }, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + }, + "final_stats": { + "local_enqueue": 522550, + "global_enqueue": 146102, + "runs": 313742, + "runtime_ns": 1167310086584, + "completed_bursts": 252195, + "censored_bursts": 79, + "training_updates": 252195, + "inferences": 252199, + "adaptive_slices": 251614, + "slice_changes": 22817, + "fault_stalls": 0, + "counter_errors": 0, + "alloc_failures": 0, + "snapshot_failures": 0, + "prediction_ns_sum": 42730485339, + "selected_slice_ns_sum": 982957956863, + "matching_dispatch_slices": 313396, + "continuation_slices": 355255, + "continuation_pending_ns": 6567611156723, + "init_tasks": 198, + "exit_tasks": 198, + "training_time_ns": 0, + "inference_time_ns": 0, + "prediction_samples": 251786, + "linear_abs_error_ns": 7654161662, + "ewma_abs_error_ns": 7715026562, + "wake_samples": 252199, + "wake_to_run_ns": 36079760558, + "wake_to_run_max_ns": 7171180, + "cpuperf_sets": 1196781, + "cpuperf_idle_sets": 35497, + "contract_applied": 1545310, + "contract_stale": 4129, + "live_tasks": 0 + }, + "simulator_summary": "lerwatt-sim: 250.1 s simulated in 250.1 s: energy 1743.1 J (mean 6.97 W), work served 65.8%, 0 throttle steps, peak 49.9 / 49.9 / 48.1 C (big0 / big1 / little), 0 bad writes", + "daemon_exit": 0, + "scx_exit": 0, + "simulator_exit": 0 +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e/environment.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e/environment.json new file mode 100644 index 0000000..6f24438 --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e/environment.json @@ -0,0 +1,10 @@ +{ + "kernel": "7.0.0-38-generic", + "cpus": 8, + "sha256": { + "/home/ubuntu/lertick-burst-init/build/scx_lertick": "a49bc8af07226b8f7782b9c161546ad07914fd1f92fc3b1277c6db1ff6fcc36a", + "/home/ubuntu/lerwatt-temporal-forecast/build/lerwattd": "00bbd2b6161ae9070e6bd4bfb2362df257370cd6f4e70c232e44dbb0b4727ad9", + "/home/ubuntu/lerwatt-temporal-forecast/build/lerwatt-sim": "ddb14f5de44f961d2a4dca46d961581c9e01448bdbf29eef3c85896a61d8b9a8", + "/var/tmp/verify-new-model-plugin.py": "376d63f3b745fe2533b0d88c9ccdcef8856054de7f0f4c1003b3438fd0b77834" + } +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e/standalone-before/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e/standalone-before/summary.json new file mode 100644 index 0000000..d16449e --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e/standalone-before/summary.json @@ -0,0 +1,18 @@ +{ + "passed": true, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + }, + "defaults_used": true +} diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/lw_contract-0.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/lw_contract-0.json new file mode 100644 index 0000000..075f148 --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/lw_contract-0.json @@ -0,0 +1 @@ +[{"key":["0x00","0x00","0x00","0x00"],"value":["0xf0","0x11","0x00","0x00","0x01","0x00","0x00","0x00","0x8d","0x01","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0xea","0x83","0x00","0x00","0x0f","0x3a","0xd3","0xf3","0xba","0x1c","0x00","0x00"],"formatted":{"key":0,"value":{"seq":4592,"version":1,"ceiling_cap":397,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":33770,"valid_until_ns":31589280201231}}},{"key":["0x01","0x00","0x00","0x00"],"value":["0xf0","0x11","0x00","0x00","0x01","0x00","0x00","0x00","0x00","0x04","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0xb4","0x7c","0x00","0x00","0x0f","0x3a","0xd3","0xf3","0xba","0x1c","0x00","0x00"],"formatted":{"key":1,"value":{"seq":4592,"version":1,"ceiling_cap":1024,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":31924,"valid_until_ns":31589280201231}}},{"key":["0x02","0x00","0x00","0x00"],"value":["0xf0","0x11","0x00","0x00","0x01","0x00","0x00","0x00","0x00","0x04","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x18","0x79","0x00","0x00","0x0f","0x3a","0xd3","0xf3","0xba","0x1c","0x00","0x00"],"formatted":{"key":2,"value":{"seq":4592,"version":1,"ceiling_cap":1024,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":31000,"valid_until_ns":31589280201231}}},{"key":["0x03","0x00","0x00","0x00"],"value":["0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00"],"formatted":{"key":3,"value":{"seq":0,"version":0,"ceiling_cap":0,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":0,"valid_until_ns":0}}},{"key":["0x04","0x00","0x00","0x00"],"value":["0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00"],"formatted":{"key":4,"value":{"seq":0,"version":0,"ceiling_cap":0,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":0,"valid_until_ns":0}}},{"key":["0x05","0x00","0x00","0x00"],"value":["0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00"],"formatted":{"key":5,"value":{"seq":0,"version":0,"ceiling_cap":0,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":0,"valid_until_ns":0}}},{"key":["0x06","0x00","0x00","0x00"],"value":["0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00"],"formatted":{"key":6,"value":{"seq":0,"version":0,"ceiling_cap":0,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":0,"valid_until_ns":0}}},{"key":["0x07","0x00","0x00","0x00"],"value":["0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00","0x00"],"formatted":{"key":7,"value":{"seq":0,"version":0,"ceiling_cap":0,"floor_cap":0,"energy_price_milli":0,"thermal_headroom_mc":0,"valid_until_ns":0}}}] diff --git a/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/summary.json b/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/summary.json new file mode 100644 index 0000000..823d370 --- /dev/null +++ b/results/watchdog-status/lertick-temporal-integration-ed04d6e/three-cluster/summary.json @@ -0,0 +1,11 @@ +{ + "completed": false, + "duration_s": 250, + "scope": "One simulated board and real BPF/daemon integration; no accuracy or throughput claim.", + "failure": "TypeError(\"unhashable type: 'list'\")", + "restored_knobs": true, + "journal_removed": true, + "daemon_exit": 0, + "scx_exit": 0, + "simulator_exit": 0 +} diff --git a/results/watchdog-status/lertick-watchdog-flags-ed04d6e/commands.txt b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/commands.txt new file mode 100644 index 0000000..d9162ac --- /dev/null +++ b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/commands.txt @@ -0,0 +1,5 @@ +/home/ubuntu/lertick-burst-init/build/scx_lertick --mode fixed --duration 2 --sample-ms 250 --stats-json /var/tmp/lertick-watchdog-flags-ed04d6e/normal.jsonl +/bin/bash -c 'kill -STOP $$; exec /home/ubuntu/lertick-burst-init/build/burst_workload --role burst --duration 1 --policy-ext --cpu 0 --burst-us 1000 --sleep-us 2000' +/home/ubuntu/lertick-burst-init/build/scx_lertick --mode fixed --duration 10 --sample-ms 250 --stats-json /var/tmp/lertick-watchdog-flags-ed04d6e/expected.jsonl --partial --fault-stall --stall-tgid 107889 --dangerous-vm-test --expect-watchdog +/bin/bash -c 'kill -STOP $$; exec /home/ubuntu/lertick-burst-init/build/burst_workload --role burst --duration 1 --policy-ext --cpu 0 --burst-us 1000 --sleep-us 2000' +/home/ubuntu/lertick-burst-init/build/scx_lertick --mode fixed --duration 10 --sample-ms 250 --stats-json /var/tmp/lertick-watchdog-flags-ed04d6e/unexpected.jsonl --partial --fault-stall --stall-tgid 107892 --dangerous-vm-test diff --git a/results/watchdog-status/lertick-watchdog-flags-ed04d6e/expected.log b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/expected.log new file mode 100644 index 0000000..6230a7c --- /dev/null +++ b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/expected.log @@ -0,0 +1,54 @@ + +DEBUG DUMP +================================================================================ + +kworker/u32:2[104334] triggered exit kind 1026: + runnable task stall (lertick_burst[107889] failed to run for 1.505s) + +Backtrace: + scx_exit+0x64/0xc8 + scx_watchdog_workfn+0x178/0x280 + process_one_work+0x1a0/0x578 + worker_thread+0x19c/0x350 + kthread+0x114/0x130 + ret_from_fork+0x10/0x20 + +CPU states +---------- + +CPU 0 : nr_run=1 flags=0x1 cpu_rel=0 ops_qseq=12748482 ksync=26571153 + curr=swapper/0[0] class=idle_sched_class + + R lertick_burst[107889] -1505ms + scx_state/flags=3/0x9 dsq_flags=0x0 ops_state/qseq=2/12748481 + sticky/holding_cpu=-1/-1 dsq_id=(n/a) + dsq_vtime=2029625 slice=2970375 weight=100 + cpus=01 no_mig=0 + + common_nsleep_timens+0x58/0xc0 + __arm64_sys_clock_nanosleep+0xf0/0x1b0 + invoke_syscall+0x64/0xf8 + el0_svc_common.constprop.0+0x4c/0x140 + do_el0_svc+0x28/0x58 + el0_svc+0x40/0x1d8 + el0t_64_sync_handler+0xc0/0x108 + el0t_64_sync+0x1b8/0x1c0 + +CPU 7 : nr_run=0 flags=0x1 cpu_rel=0 ops_qseq=83443 ksync=927374 + curr=kworker/u32:2[104334] class=fair_sched_class + +Event counters +-------------- + SCX_EV_SELECT_CPU_FALLBACK: 0 + SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE: 0 + SCX_EV_DISPATCH_KEEP_LAST: 0 + SCX_EV_ENQ_SKIP_EXITING: 0 + SCX_EV_ENQ_SKIP_MIGRATION_DISABLED: 0 + SCX_EV_REFILL_SLICE_DFL: 0 + SCX_EV_BYPASS_DURATION: 18125881 + SCX_EV_BYPASS_DISPATCH: 0 + SCX_EV_BYPASS_ACTIVATE: 1 + +================================================================================ + +EXIT: runnable task stall (lertick_burst[107889] failed to run for 1.505s) diff --git a/results/watchdog-status/lertick-watchdog-flags-ed04d6e/normal.log b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/normal.log new file mode 100644 index 0000000..a157f00 --- /dev/null +++ b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/normal.log @@ -0,0 +1 @@ +EXIT: unregistered from user space diff --git a/results/watchdog-status/lertick-watchdog-flags-ed04d6e/unexpected.log b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/unexpected.log new file mode 100644 index 0000000..4a2812e --- /dev/null +++ b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/unexpected.log @@ -0,0 +1,54 @@ + +DEBUG DUMP +================================================================================ + +kworker/u32:0[106638] triggered exit kind 1026: + runnable task stall (lertick_burst[107892] failed to run for 1.489s) + +Backtrace: + scx_exit+0x64/0xc8 + scx_watchdog_workfn+0x178/0x280 + process_one_work+0x1a0/0x578 + worker_thread+0x19c/0x350 + kthread+0x114/0x130 + ret_from_fork+0x10/0x20 + +CPU states +---------- + +CPU 0 : nr_run=1 flags=0x1 cpu_rel=0 ops_qseq=12748483 ksync=26571174 + curr=swapper/0[0] class=idle_sched_class + + R lertick_burst[107892] -1489ms + scx_state/flags=3/0x9 dsq_flags=0x0 ops_state/qseq=2/12748482 + sticky/holding_cpu=-1/-1 dsq_id=(n/a) + dsq_vtime=2027292 slice=2972708 weight=100 + cpus=01 no_mig=0 + + common_nsleep_timens+0x58/0xc0 + __arm64_sys_clock_nanosleep+0xf0/0x1b0 + invoke_syscall+0x64/0xf8 + el0_svc_common.constprop.0+0x4c/0x140 + do_el0_svc+0x28/0x58 + el0_svc+0x40/0x1d8 + el0t_64_sync_handler+0xc0/0x108 + el0t_64_sync+0x1b8/0x1c0 + +CPU 3 : nr_run=0 flags=0x1 cpu_rel=0 ops_qseq=2940938 ksync=6958500 + curr=kworker/u32:0[106638] class=fair_sched_class + +Event counters +-------------- + SCX_EV_SELECT_CPU_FALLBACK: 0 + SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE: 0 + SCX_EV_DISPATCH_KEEP_LAST: 0 + SCX_EV_ENQ_SKIP_EXITING: 0 + SCX_EV_ENQ_SKIP_MIGRATION_DISABLED: 0 + SCX_EV_REFILL_SLICE_DFL: 0 + SCX_EV_BYPASS_DURATION: 10672310 + SCX_EV_BYPASS_DISPATCH: 0 + SCX_EV_BYPASS_ACTIVATE: 1 + +================================================================================ + +EXIT: runnable task stall (lertick_burst[107892] failed to run for 1.489s) diff --git a/results/watchdog-status/lertick-watchdog-flags-ed04d6e/validation.json b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/validation.json new file mode 100644 index 0000000..6a473dd --- /dev/null +++ b/results/watchdog-status/lertick-watchdog-flags-ed04d6e/validation.json @@ -0,0 +1,65 @@ +{ + "kernel": "7.0.0-38-generic", + "architecture": "aarch64", + "tests": { + "normal": { + "passed": true, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": false, + "exit_kind": 64, + "exit_code": 0, + "reason": "unregistered from user space", + "message": "", + "default_restored": true, + "signal": 0, + "status": 0 + } + }, + "expected": { + "passed": true, + "intentional_fault": true, + "exit": { + "event": "exit", + "unexpected_fallback": false, + "watchdog_observed": true, + "watchdog_expected": true, + "watchdog_triggered": true, + "exit_kind": 1026, + "exit_code": 0, + "reason": "runnable task stall", + "message": "lertick_burst[107889] failed to run for 1.505s", + "default_restored": true, + "signal": 0, + "status": 0 + } + }, + "unexpected": { + "passed": true, + "intentional_fault": true, + "exit": { + "event": "exit", + "unexpected_fallback": true, + "watchdog_observed": false, + "watchdog_expected": false, + "watchdog_triggered": true, + "exit_kind": 1026, + "exit_code": 0, + "reason": "runnable task stall", + "message": "lertick_burst[107892] failed to run for 1.489s", + "default_restored": true, + "signal": 0, + "status": 1 + } + } + }, + "comparison": [], + "performance_claim": false, + "model_init": "bias", + "model_update": "trunc", + "completed": true, + "final_state": "disabled" +} diff --git a/results/watchdog-status/provenance.json b/results/watchdog-status/provenance.json new file mode 100644 index 0000000..82936c2 --- /dev/null +++ b/results/watchdog-status/provenance.json @@ -0,0 +1,16 @@ +{ + "linux_tested_source": "ed04d6ef98a6a390ad1f9862585dfcfac363da17", + "kernel_model_reference": "6beaba2", + "sources": { + "src/userspace/scx_lertick.c": "a54e77488434a240c9daa4470edc43ed43563967f4005fb5c5be25fd619ee602", + "src/shared/model.h": "9ce54cb889d29d2532c4ea465c728143e2cb44dfd43dd27c32b24d3641a3c639", + "src/bpf/scx_lertick.bpf.c": "9a2ae2499edf4c2bb891ea51607f823f1960755f0aaa42b08e8a7984c9abda9b", + "tests/validate_watchdog_flags.py": "3d874d394eee6ae8273285e96546dbc902a937b8bc1371eea3f2c9fd95737797" + }, + "raw_archive": { + "file": "lertick-watchdog-integration-ed04d6e.tgz", + "bytes": 20272308, + "sha256": "84bed19679efb78dab7c79b787a1318f625d441cb021e212d632696bc4426b1b" + }, + "note": "Raw archive is retained in results/local/watchdog-status-ed04d6e in the main checkout and this worktree. Three output roots include the original failed collector attempt. This is a telemetry and integration check, not a performance comparison." +} diff --git a/results/watchdog-status/verify-new-model-plugin-key-array-failure.py b/results/watchdog-status/verify-new-model-plugin-key-array-failure.py new file mode 100644 index 0000000..5a92f84 --- /dev/null +++ b/results/watchdog-status/verify-new-model-plugin-key-array-failure.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Isolated QEMU integration, not an accuracy or throughput benchmark.""" +import argparse,hashlib,json,os,pathlib,re,signal,subprocess,sys,time +SCX=pathlib.Path('/home/ubuntu/lertick-burst-init/build/scx_lertick') +LW=pathlib.Path('/home/ubuntu/lerwatt-temporal-forecast') +STATE=pathlib.Path('/sys/kernel/sched_ext/state') +sys.path.insert(0,str(LW/'tests')) +from verify_runtime_forecast import knobs,stop + +def save(p,a):p.write_text(json.dumps(a,indent=2)+'\n') +def rows(p):return [json.loads(v) for v in p.read_text().splitlines() if v.startswith('{')] +def wait_for(predicate,process): + end=time.monotonic()+15 + while not predicate(): + if process.poll() is not None or time.monotonic()>end:raise RuntimeError('readiness failed') + time.sleep(.05) +def final_exit(p): + r=[r for r in rows(p) if r.get('event')=='exit'] + assert len(r)==1 and r[0]['default_restored'] and not r[0]['unexpected_fallback'],r + assert r[0]['watchdog_expected'] is False and r[0]['watchdog_triggered'] is False + return r[0] +def standalone(out,name): + d=out/name;d.mkdir() + with (d/'stderr.log').open('w') as f: + subprocess.run([str(SCX),'--duration','3','--stats-json',str(d/'stats.jsonl')],stderr=f,stdout=f,check=True,timeout=30) + assert STATE.read_text().strip()=='disabled' + assert not pathlib.Path('/sys/fs/bpf/lertick/lw_contract').exists() + save(d/'summary.json',{'passed':True,'exit':final_exit(d/'stats.jsonl'),'defaults_used':True}) + +def integration(out): + d=out/'three-cluster';d.mkdir();root=d/'root' + evidence={'completed':False,'duration_s':250,'scope':'One simulated board and real BPF/daemon integration; no accuracy or throughput claim.'} + logs=[];workers=[];sim=daemon=scx=None;original=None + env=dict(os.environ,LERWATT_SIM_ROOT=str(root)) + def launch(args,name): + f=(d/name).open('w');logs.append(f) + return subprocess.Popen([str(x) for x in args],stdout=f,stderr=f,env=env) + try: + sim=launch([LW/'build/lerwatt-sim','--root',root,'--load','procstat','--cooling','passive','--ambient','25','--initial-temp','25','--duration','280','--log',d/'truth.jsonl'],'sim.log') + wait_for(lambda:(root/'.lerwatt-sim').exists(),sim) + original=knobs(root);assert len(original)==30;save(d/'original-knobs.json',original) + daemon=launch([LW/'build/lerwattd','--sysroot',root,'--policy','mpc','--forecast','temporal','--cross-cluster','--duration','250','--telemetry',d/'telemetry.jsonl'],'daemon.log') + wait_for(lambda:(d/'telemetry.jsonl').exists(),daemon) + start=time.monotonic()+.3 + code='''import sys,time +start=float(sys.argv[1]);x=1 +while time.monotonic()0 and records[k]['seq']%2==0 for k in range(3)): + maps[name]=[{'key':k,'value':records[k]} for k in range(3)];break + time.sleep(.05) + else:raise RuntimeError('no consistent published map snapshot') + assert all(0.5 for c in row['clusters']) for row in t) + assert simultaneous>0 and all(v>100 for v in cpuutil.values()) + assert snapshots[-1]['stats']['contract_applied']>0 + assert STATE.read_text().strip()=='disabled' and not pathlib.Path('/sys/fs/bpf/lertick/lw_contract').exists() and not pathlib.Path('/sys/fs/bpf/lertick/lt_demand').exists() + stderr=(d/'sim.log').read_text();throttle=re.search(r'(\d+) throttle steps',stderr);bad=re.search(r'(\d+) bad writes',stderr) + peaks=re.search(r'peak ([\d.]+) / ([\d.]+) / ([\d.]+) C',stderr) + assert sim.returncode==0 and throttle and int(throttle[1])==0 and bad and int(bad[1])==0 and peaks and max(map(float,peaks.groups()))<=84 + writes=re.search(r'(\d+) write errors',(d/'daemon.log').read_text()) + assert writes and int(writes[1])==0 + assert evidence['restored_knobs'] and evidence['journal_removed'] + evidence.update(completed=True,topology=topology,cpu_util_max=cpuutil,simultaneous_busy_periods=simultaneous,joint_periods=sum(any(c['joint'] for c in row['clusters']) for row in t),temporal_points=sum(c['forecast_source']=='temporal' for row in t for c in row['clusters']),maps=maps,exit=final_exit(d/'scx.jsonl'),final_stats=snapshots[-1]['stats'],simulator_summary=stderr.splitlines()[-1]) + except BaseException as error: + evidence['failure']=repr(error);raise + finally: + stop(daemon);stop(scx) + for w in workers:stop(w) + stop(sim) + if original is not None: + evidence['restored_knobs']=knobs(root)==original;evidence['journal_removed']=not (root/'run/lerwatt/restore.journal').exists() + if not evidence['restored_knobs'] or not evidence['journal_removed']:evidence['completed']=False + evidence.update(daemon_exit=daemon.returncode if daemon else None,scx_exit=scx.returncode if scx else None,simulator_exit=sim.returncode if sim else None) + save(d/'summary.json',evidence) + for f in logs:f.close() + return evidence + +def main(): + p=argparse.ArgumentParser();p.add_argument('--output',type=pathlib.Path,required=True);a=p.parse_args() + assert os.geteuid()==0 and os.cpu_count()==8 and STATE.read_text().strip()=='disabled' + assert subprocess.check_output(['systemd-detect-virt','--vm'],text=True).strip()=='qemu' + for d in pathlib.Path('/proc').iterdir(): + if not d.name.isdigit():continue + try:name=pathlib.Path((d/'cmdline').read_bytes().split(b'\0')[0].decode()).name + except (OSError,UnicodeError):continue + assert name not in ('lerwattd','lerwatt-sim') + a.output.mkdir(parents=True,exist_ok=False) + save(a.output/'environment.json',{'kernel':os.uname().release,'cpus':8,'sha256':{str(f):hashlib.sha256(f.read_bytes()).hexdigest() for f in [SCX,LW/'build/lerwattd',LW/'build/lerwatt-sim',pathlib.Path(__file__)]}}) + standalone(a.output,'standalone-before');r=integration(a.output);standalone(a.output,'standalone-after');save(a.output/'summary.json',{'passed':True,'integration':r,'standalone_before_and_after':True});print('PASS three-cluster model/plugin integration and both default standalone runs',flush=True) +if __name__=='__main__':main() diff --git a/results/watchdog-status/verify-new-model-plugin.py b/results/watchdog-status/verify-new-model-plugin.py new file mode 100644 index 0000000..0a570a4 --- /dev/null +++ b/results/watchdog-status/verify-new-model-plugin.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Isolated QEMU integration, not an accuracy or throughput benchmark.""" +import argparse,hashlib,json,os,pathlib,re,signal,subprocess,sys,time +SCX=pathlib.Path('/home/ubuntu/lertick-burst-init/build/scx_lertick') +LW=pathlib.Path('/home/ubuntu/lerwatt-temporal-forecast') +STATE=pathlib.Path('/sys/kernel/sched_ext/state') +sys.path.insert(0,str(LW/'tests')) +from verify_runtime_forecast import knobs,stop + +def save(p,a):p.write_text(json.dumps(a,indent=2)+'\n') +def rows(p):return [json.loads(v) for v in p.read_text().splitlines() if v.startswith('{')] +def wait_for(predicate,process): + end=time.monotonic()+15 + while not predicate(): + if process.poll() is not None or time.monotonic()>end:raise RuntimeError('readiness failed') + time.sleep(.05) +def final_exit(p): + r=[r for r in rows(p) if r.get('event')=='exit'] + assert len(r)==1 and r[0]['default_restored'] and not r[0]['unexpected_fallback'],r + assert r[0]['watchdog_expected'] is False and r[0]['watchdog_triggered'] is False + return r[0] +def standalone(out,name): + d=out/name;d.mkdir() + with (d/'stderr.log').open('w') as f: + subprocess.run([str(SCX),'--duration','3','--stats-json',str(d/'stats.jsonl')],stderr=f,stdout=f,check=True,timeout=30) + assert STATE.read_text().strip()=='disabled' + assert not pathlib.Path('/sys/fs/bpf/lertick/lw_contract').exists() + save(d/'summary.json',{'passed':True,'exit':final_exit(d/'stats.jsonl'),'defaults_used':True}) + +def integration(out): + d=out/'three-cluster';d.mkdir();root=d/'root' + evidence={'completed':False,'duration_s':250,'scope':'One simulated board and real BPF/daemon integration; no accuracy or throughput claim.'} + logs=[];workers=[];sim=daemon=scx=None;original=None + env=dict(os.environ,LERWATT_SIM_ROOT=str(root)) + def launch(args,name): + f=(d/name).open('w');logs.append(f) + return subprocess.Popen([str(x) for x in args],stdout=f,stderr=f,env=env) + try: + sim=launch([LW/'build/lerwatt-sim','--root',root,'--load','procstat','--cooling','passive','--ambient','25','--initial-temp','25','--duration','280','--log',d/'truth.jsonl'],'sim.log') + wait_for(lambda:(root/'.lerwatt-sim').exists(),sim) + original=knobs(root);assert len(original)==30;save(d/'original-knobs.json',original) + daemon=launch([LW/'build/lerwattd','--sysroot',root,'--policy','mpc','--forecast','temporal','--cross-cluster','--duration','250','--telemetry',d/'telemetry.jsonl'],'daemon.log') + wait_for(lambda:(d/'telemetry.jsonl').exists(),daemon) + start=time.monotonic()+.3 + code='''import sys,time +start=float(sys.argv[1]);x=1 +while time.monotonic()0 and records[k]['seq']%2==0 for k in range(3)): + maps[name]=[{'key':k,'value':records[k]} for k in range(3)];break + time.sleep(.05) + else:raise RuntimeError('no consistent published map snapshot') + assert all(0.5 for c in row['clusters']) for row in t) + assert simultaneous>0 and all(v>100 for v in cpuutil.values()) + assert snapshots[-1]['stats']['contract_applied']>0 + assert STATE.read_text().strip()=='disabled' and not pathlib.Path('/sys/fs/bpf/lertick/lw_contract').exists() and not pathlib.Path('/sys/fs/bpf/lertick/lt_demand').exists() + stderr=(d/'sim.log').read_text();throttle=re.search(r'(\d+) throttle steps',stderr);bad=re.search(r'(\d+) bad writes',stderr) + peaks=re.search(r'peak ([\d.]+) / ([\d.]+) / ([\d.]+) C',stderr) + assert sim.returncode==0 and throttle and int(throttle[1])==0 and bad and int(bad[1])==0 and peaks and max(map(float,peaks.groups()))<=84 + writes=re.search(r'(\d+) write errors',(d/'daemon.log').read_text()) + assert writes and int(writes[1])==0 + assert evidence['restored_knobs'] and evidence['journal_removed'] + evidence.update(completed=True,topology=topology,cpu_util_max=cpuutil,simultaneous_busy_periods=simultaneous,joint_periods=sum(any(c['joint'] for c in row['clusters']) for row in t),temporal_points=sum(c['forecast_source']=='temporal' for row in t for c in row['clusters']),maps=maps,exit=final_exit(d/'scx.jsonl'),final_stats=snapshots[-1]['stats'],simulator_summary=stderr.splitlines()[-1]) + except BaseException as error: + evidence['failure']=repr(error);raise + finally: + stop(daemon);stop(scx) + for w in workers:stop(w) + stop(sim) + if original is not None: + evidence['restored_knobs']=knobs(root)==original;evidence['journal_removed']=not (root/'run/lerwatt/restore.journal').exists() + if not evidence['restored_knobs'] or not evidence['journal_removed']:evidence['completed']=False + evidence.update(daemon_exit=daemon.returncode if daemon else None,scx_exit=scx.returncode if scx else None,simulator_exit=sim.returncode if sim else None) + save(d/'summary.json',evidence) + for f in logs:f.close() + return evidence + +def main(): + p=argparse.ArgumentParser();p.add_argument('--output',type=pathlib.Path,required=True);a=p.parse_args() + assert os.geteuid()==0 and os.cpu_count()==8 and STATE.read_text().strip()=='disabled' + assert subprocess.check_output(['systemd-detect-virt','--vm'],text=True).strip()=='qemu' + for d in pathlib.Path('/proc').iterdir(): + if not d.name.isdigit():continue + try:name=pathlib.Path((d/'cmdline').read_bytes().split(b'\0')[0].decode()).name + except (OSError,UnicodeError):continue + assert name not in ('lerwattd','lerwatt-sim') + a.output.mkdir(parents=True,exist_ok=False) + save(a.output/'environment.json',{'kernel':os.uname().release,'cpus':8,'sha256':{str(f):hashlib.sha256(f.read_bytes()).hexdigest() for f in [SCX,LW/'build/lerwattd',LW/'build/lerwatt-sim',pathlib.Path(__file__)]}}) + standalone(a.output,'standalone-before');r=integration(a.output);standalone(a.output,'standalone-after');save(a.output/'summary.json',{'passed':True,'integration':r,'standalone_before_and_after':True});print('PASS three-cluster model/plugin integration and both default standalone runs',flush=True) +if __name__=='__main__':main() diff --git a/tests/README.md b/tests/README.md index 6bf35b3..02eb659 100644 --- a/tests/README.md +++ b/tests/README.md @@ -56,6 +56,8 @@ In an isolated QEMU/KVM guest, `sudo python3 tests/validate_watchdog_flags.py --output results/local/watchdog-flags-new` checks normal shutdown and both expected/unexpected intentional stalls, including their distinct loader statuses. It refuses an existing scheduler and retains every log. +See the [recorded exit and eight-vCPU plug-in checks](../results/watchdog-status/README.md) +for actual results, including the failed collector attempt and complete rerun. Phase 2 adds independent service arithmetic and actual Linux lifecycle tests: