diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2738a0d24..1839d18a5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,13 +23,30 @@ on: description: 'Release fast path already published the tag images' type: boolean default: false + sqlite_benchmark: + description: 'Run only the native SQLite benchmark (never deploy or publish)' + type: boolean + default: false + sqlite_profile: + description: 'Smoke only, five B pairs, or separate B profiling' + type: choice + options: [smoke, primary, diagnostic] + default: smoke + baseline_sha: + description: 'Exact baseline commit (empty uses the reviewed SQLite baseline)' + type: string + candidate_sha: + description: 'Exact application candidate commit (empty uses this workflow ref)' + type: string + harness_sha: + description: 'Exact harness commit (empty uses this workflow ref)' + type: string permissions: contents: read - packages: write concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ${{ inputs.sqlite_benchmark && format('sqlite-benchmark-dispatch-{0}', github.run_id) || format('ci-{0}', github.event.pull_request.number || github.ref) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: @@ -47,6 +64,17 @@ env: # PRs stop after build-and-publish (no GHCR push). Master continues to deploy + badges. jobs: + sqlite-benchmark: + if: github.event_name == 'workflow_dispatch' && inputs.sqlite_benchmark + permissions: + contents: read + uses: ./.github/workflows/sqlite-benchmark.yml + with: + profile: ${{ inputs.sqlite_profile }} + baseline_sha: ${{ inputs.baseline_sha }} + candidate_sha: ${{ inputs.candidate_sha }} + harness_sha: ${{ inputs.harness_sha }} + # ─────────────────────────────────────────────────────────────── # 0. Change scope — decides whether the expensive jobs below run. # ─────────────────────────────────────────────────────────────── @@ -58,15 +86,24 @@ jobs: ingestor: ${{ steps.scope.outputs.ingestor }} steps: - name: Checkout code + if: ${{ !inputs.sqlite_benchmark }} uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Decide whether anything but documentation changed id: scope + env: + BENCHMARK_ONLY: ${{ inputs.sqlite_benchmark }} run: | set -euo pipefail - # Only pull requests are scoped. Pushes and dispatches always count as + if [ "$BENCHMARK_ONLY" = "true" ]; then + echo "code=false" >> "$GITHUB_OUTPUT" + echo "ingestor=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Only pull requests are scoped. Pushes and other dispatches count as # code, so master keeps producing an :edge image and tag builds are # never skipped. if [ "${{ github.event_name }}" != "pull_request" ]; then @@ -1001,7 +1038,10 @@ jobs: name: "🏗️ Build & Publish Docker Image" needs: [go-test, race-test, e2e-test, image-check, changes] runs-on: ubuntu-latest - if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} + permissions: + contents: read + packages: write + if: ${{ !inputs.sqlite_benchmark && !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} steps: - name: Require every test job and the image check to have passed run: | @@ -1078,7 +1118,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - if: startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true' + if: ${{ !inputs.sqlite_benchmark && startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true' }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -1178,7 +1218,8 @@ jobs: # variable is set while the runner is still missing the job fails in ten # minutes rather than blocking the branch again. if: | - vars.ENABLE_STAGING_DEPLOY == 'true' + !inputs.sqlite_benchmark + && vars.ENABLE_STAGING_DEPLOY == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' needs: [build-and-publish] @@ -1269,7 +1310,7 @@ jobs: # ─────────────────────────────────────────────────────────────── publish: name: "📝 Publish Badges & Summary" - if: github.event_name == 'push' + if: github.event_name == 'push' && !inputs.sqlite_benchmark # Was needs: [deploy]. The badges are built from the go-test and e2e-test # artifacts and never needed the deploy step; depending on it meant the # coverage badges stopped updating whenever the self-hosted runner was diff --git a/.github/workflows/sqlite-benchmark.yml b/.github/workflows/sqlite-benchmark.yml new file mode 100644 index 000000000..34c3b2f12 --- /dev/null +++ b/.github/workflows/sqlite-benchmark.yml @@ -0,0 +1,251 @@ +name: Native SQLite benchmark + +# Harness PRs run S smoke automatically. To request a larger run, add one line: +# - [x] Run SQLite benchmark (primary) +# Or select (diagnostic) for separate profiling. Optional exact ref markers: +# sqlite-baseline-sha: <40 lowercase hex characters> +# sqlite-candidate-sha: <40 lowercase hex characters> +# sqlite-harness-sha: <40 lowercase hex characters> +# Defaults are the reviewed baseline below and the PR head for candidate/harness. +on: + pull_request: + branches: [master] + types: [opened, synchronize, reopened, edited] + paths: + - 'scripts/sqlite-benchmark/**' + - '.github/workflows/sqlite-benchmark.yml' + - '.github/workflows/deploy.yml' + - 'tests/unit/test-sqlite-benchmark-ci.js' + # Use the already registered deploy.yml dispatcher before this workflow is + # on the default branch. Application and harness commits stay independent. + workflow_call: + inputs: + profile: + type: string + default: smoke + baseline_sha: + type: string + default: '' + candidate_sha: + type: string + default: '' + harness_sha: + type: string + default: '' + +permissions: + contents: read + +jobs: + request: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + profile: ${{ steps.request.outputs.profile }} + baseline_sha: ${{ steps.request.outputs.baseline_sha }} + candidate_sha: ${{ steps.request.outputs.candidate_sha }} + harness_sha: ${{ steps.request.outputs.harness_sha }} + repository: ${{ steps.request.outputs.repository }} + steps: + - name: Parse benchmark request + id: request + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + SOURCE_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + INPUT_PROFILE: ${{ inputs.profile }} + INPUT_BASELINE: ${{ inputs.baseline_sha }} + INPUT_CANDIDATE: ${{ inputs.candidate_sha }} + INPUT_HARNESS: ${{ inputs.harness_sha }} + run: | + python3 - <<'PY' + import json, os, re, sys + + def body_request(body): + lines, fence = [], None + for line in (body or '').splitlines(): + line = line.strip() + if fence: + if re.fullmatch(re.escape(fence[0]) + '{' + str(len(fence)) + ',}', line): + fence = None + continue + marker = re.match(r'(`{3,}|~{3,})', line) + if marker: + fence = marker[1] + else: + lines.append(line) + modes = [m[1] for line in lines if (m := re.fullmatch(r'- \[[xX]\] Run SQLite benchmark \(([^)]+)\)', line))] + if not modes: + return None + if len(modes) != 1 or modes[0] not in ('primary', 'diagnostic'): + raise ValueError('Select exactly one SQLite benchmark profile: primary or diagnostic') + refs = {} + for line in lines: + for name in ('baseline', 'candidate', 'harness'): + if line.startswith('sqlite-' + name + '-sha'): + match = re.fullmatch('sqlite-' + name + r'-sha:\s*([0-9a-f]{40})', line) + if not match or name in refs: + raise ValueError('Each SQLite SHA marker must occur once and contain a full lowercase commit SHA') + refs[name] = match[1] + return modes[0], refs + + try: + with open(os.environ['GITHUB_EVENT_PATH'], encoding='utf-8') as source: + event = json.load(source) + refs = {} + if os.environ['EVENT_NAME'] == 'pull_request': + selected = body_request(event['pull_request'].get('body')) + profile = selected[0] if selected else 'smoke' + refs = selected[1] if selected else {} + if event['action'] == 'edited': + change = event.get('changes', {}).get('body') + try: + previous = body_request(change.get('from')) if change else None + except ValueError: + previous = None + if not change or not selected or selected == previous: + profile = 'off' + elif os.environ['EVENT_NAME'] == 'workflow_dispatch': + profile = os.environ['INPUT_PROFILE'] or 'smoke' + if profile not in ('smoke', 'primary', 'diagnostic'): + raise ValueError('Unknown SQLite benchmark profile') + refs = {name: os.environ['INPUT_' + name.upper()] for name in ('baseline', 'candidate', 'harness')} + else: + raise ValueError('Unsupported benchmark event') + defaults = {'baseline': '7fa86fe1c1304b88e0f2a44052465dfb86abb8a6', + 'candidate': os.environ['EVENT_SHA'], 'harness': os.environ['EVENT_SHA']} + values = {'profile': profile} + for name, default in defaults.items(): + sha = refs.get(name) or default + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError(name + '_sha must be a full lowercase commit SHA') + values[name + '_sha'] = sha + repository = os.environ['SOURCE_REPOSITORY'] + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repository): + raise ValueError('Invalid source repository') + values['repository'] = repository + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: + for key, value in values.items(): + print(key + '=' + value, file=output) + except (ValueError, KeyError) as error: + sys.exit(str(error)) + PY + + benchmark: + needs: [request] + if: needs.request.outputs.profile != 'off' && needs.request.outputs.profile != '' + runs-on: ubuntu-24.04 + timeout-minutes: ${{ needs.request.outputs.profile == 'smoke' && 30 || 180 }} + # Ignored body edits never enter this job or cancel an active measurement. + concurrency: + group: sqlite-benchmark-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: false + env: + BASELINE_SHA: ${{ needs.request.outputs.baseline_sha }} + CANDIDATE_SHA: ${{ needs.request.outputs.candidate_sha }} + HARNESS_SHA: ${{ needs.request.outputs.harness_sha }} + PROFILE: ${{ needs.request.outputs.profile }} + CGO_ENABLED: '1' + GOMAXPROCS: '3' + steps: + - name: Validate runner capacity + run: | + set -euo pipefail + [[ $(nproc) -ge 4 ]] || { echo 'At least four CPUs are required' >&2; exit 1; } + awk '/MemTotal:/ {exit !($2 >= 8*1024*1024)}' /proc/meminfo || { echo 'At least 8 GiB RAM is required' >&2; exit 1; } + test -f /sys/fs/cgroup/cgroup.controllers + echo "SMOKE_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-smoke" >> "$GITHUB_ENV" + echo "BENCH_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$PROFILE" >> "$GITHUB_ENV" + + - name: Checkout the exact harness + uses: actions/checkout@v5 + with: + repository: ${{ needs.request.outputs.repository }} + ref: ${{ needs.request.outputs.harness_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact refs and workflow source + env: + EVENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + for sha in "$BASELINE_SHA" "$CANDIDATE_SHA" "$HARNESS_SHA" "$EVENT_SHA" "$WORKFLOW_SHA"; do + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo 'All source refs must be full lowercase commit SHAs' >&2; exit 1; } + done + test "$(git rev-parse HEAD)" = "$HARNESS_SHA" + git fetch --no-tags origin "$BASELINE_SHA" "$CANDIDATE_SHA" "$HARNESS_SHA" "$EVENT_SHA" + git fetch --no-tags --depth=1 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$WORKFLOW_SHA" + for sha in "$BASELINE_SHA" "$CANDIDATE_SHA" "$HARNESS_SHA" "$EVENT_SHA" "$WORKFLOW_SHA"; do + test "$(git rev-parse "$sha^{commit}")" = "$sha" + done + # A PR executes its merge workflow, but measures independent exact + # application/harness refs. Refuse workflow changes introduced by the merge. + git diff --quiet "$WORKFLOW_SHA" "$EVENT_SHA" -- \ + .github/workflows/deploy.yml .github/workflows/sqlite-benchmark.yml \ + || { echo 'Workflow source differs from the selected event commit; update the PR branch' >&2; exit 1; } + + - name: Test portable benchmark controls + run: python3 -m unittest discover -s scripts/sqlite-benchmark -p 'test_*.py' + + - name: Use the same native compiler for both revisions + uses: actions/setup-go@v6 + with: + go-version: '1.27.2' + cache-dependency-path: '**/go.sum' + + - name: Run bounded smoke and selected profile + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + test ! -e "$delegated" + sudo mkdir "$delegated" + for controller in cpu memory io; do + grep -qw "$controller" "$delegated/cgroup.controllers" || { echo "Required cgroup controller unavailable: $controller" >&2; exit 1; } + done + printf '+cpu +memory +io\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null + sudo mkdir "$delegated/coordinator" + sudo chown "$(id -u):$(id -g)" "$delegated" "$delegated/cgroup.procs" "$delegated/cgroup.subtree_control" "$delegated/coordinator" "$delegated/coordinator/cgroup.procs" + printf '%s\n' "$$" | sudo tee "$delegated/coordinator/cgroup.procs" >/dev/null + # Each build runs the native overlay controls before corpus preparation. + common=(--repo . --baseline-sha "$BASELINE_SHA" --candidate-sha "$CANDIDATE_SHA" --harness-sha "$HARNESS_SHA" --cgroup "$delegated" --go-version 1.27.2 --ingest-rate 50 --http-rate 20) + python3 scripts/sqlite-benchmark/run.py "${common[@]}" \ + --corpus S --profile smoke --pairs 1 --warmup 2 --seconds 10 --output "$SMOKE_OUTPUT" + case "$PROFILE" in + primary) + python3 scripts/sqlite-benchmark/run.py "${common[@]}" \ + --corpus B --profile primary --pairs 5 --output "$BENCH_OUTPUT" ;; + diagnostic) + python3 scripts/sqlite-benchmark/run.py "${common[@]}" \ + --corpus B --profile diagnostic --pairs 1 --warmup 10 --seconds 60 --output "$BENCH_OUTPUT" ;; + smoke) ;; + *) echo 'Unknown benchmark profile' >&2; exit 1 ;; + esac + + - name: Upload only sanitized public evidence + if: always() && env.SMOKE_OUTPUT != '' + uses: actions/upload-artifact@v4 + with: + name: sqlite-benchmark-${{ needs.request.outputs.profile }}-${{ needs.request.outputs.candidate_sha }} + path: | + ${{ env.SMOKE_OUTPUT }}/public/ + ${{ env.BENCH_OUTPUT }}/public/ + if-no-files-found: error + retention-days: 30 + + - name: Remove only the owned cgroup + if: always() + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + if [ -d "$delegated" ]; then + test "$(realpath "$delegated")" = "$delegated" + if [ -f "$delegated/cgroup.kill" ]; then echo 1 | sudo tee "$delegated/cgroup.kill" >/dev/null; fi + for attempt in $(seq 1 10); do + sudo find "$delegated" -depth -type d -exec rmdir {} \; 2>/dev/null || true + test ! -d "$delegated" && break + sleep 1 + done + test ! -d "$delegated" + fi diff --git a/scripts/sqlite-benchmark/.gitignore b/scripts/sqlite-benchmark/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/scripts/sqlite-benchmark/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/sqlite-benchmark/README.md b/scripts/sqlite-benchmark/README.md new file mode 100644 index 000000000..caedae1a2 --- /dev/null +++ b/scripts/sqlite-benchmark/README.md @@ -0,0 +1,242 @@ +# CoreScope native SQLite before/after benchmark + +This standalone harness measures two exact application revisions using native +CGO SQLite. It contains no production application changes or database-conversion +path. A separate exact harness revision supplies the same test-only overlays to +both archived application trees; neither application branch needs to contain +the harness. All three revisions are recorded and verified. An A/A run qualifies +the controller and supplies no performance-improvement claim. + +This first slice has portable control and native overlay checks. A reviewed, +committed harness must pass the Linux A/A pilot before a candidate matrix. +No new B performance result or L capacity result is claimed here. + +The automatic S smoke pins its baseline to upstream +`7fa86fe1c1304b88e0f2a44052465dfb86abb8a6` (v3.15.0), with the PR head as candidate and +harness. Since this PR changes only benchmark tooling, that is a contemporary +A/A qualification. Explicit baseline, candidate and harness selections remain +independent for requested runs. Inactive historical request markers do not +select a comparison or override the automatic smoke. + +## Sources and prerequisites + +- Pass full lowercase 40-character commits with `--baseline-sha`, + `--candidate-sha` and `--harness-sha`. Mutable refs are rejected. The executable + harness files must match `--harness-sha` (UTF-8 text with LF normalization); + dirty, missing, unexpected and symlinked harness files fail verification. +- Use one isolated Linux host, Python 3.12+, Go 1.27.2, a C compiler, Git and + coreutils. Both revisions use the same compiler and `-trimpath` build flags. + Actual SQLite versions and PRAGMAs are captured through real connections. +- The host needs at least four CPUs and 8 GiB physical RAM. The *available* + memory guard requires 8 GiB before preparation: the 6 GiB measured envelope + plus coordinator/generator and host headroom. Nominal installed RAM is not + evidence that this much is available. +- Primary and diagnostic runs require an owned cgroup-v2 subtree with `cpu`, + `memory` and `io` delegated. The coordinator belongs in a sibling of the + measured child. Every measured application child enters the common + **3-CPU / 6-GiB** envelope before execution. Required counters and effective + limits are checked before corpus generation. No service or system-wide + cgroup changes are performed by this script. +- Use a fresh output directory whose name begins `corescope-bench-`. Corpus + disk guards are S:4, B:12 and L:128 GiB currently free, then at least eight + times the actual generated database size before each restore. These are + conservative guards, not measured corpus sizes or capacity assurances. + +Do not run alongside another heavy benchmark. Keep all data synthetic and use +only the owned local application processes. No MQTT broker or production +endpoint is contacted. The server keeps its native read-only SQLite connection; +the ingestor alone performs application writes and retention. + +## Commands + +Use the runner from the reviewed harness checkout. `REPO` must have all three +commit objects; it may be a different checkout. Parent CI can invoke these +commands through the existing opt-in workflow without putting harness files on +the application candidate branch. + +```sh +python3 "$HARNESS/scripts/sqlite-benchmark/run.py" --repo "$REPO" \ + --harness-sha "$HARNESS_SHA" --baseline-sha "$BASELINE_SHA" \ + --candidate-sha "$BASELINE_SHA" --corpus S --profile smoke --pairs 1 \ + --warmup 2 --seconds 10 --ingest-rate 50 --http-rate 20 \ + --cgroup "$CORESCOPE_BENCH_CGROUP" --output /tmp/corescope-bench-aa-pilot +``` + +After the A/A pilot and a reviewed application fix, run five alternating B pairs: + +```sh +python3 "$HARNESS/scripts/sqlite-benchmark/run.py" --repo "$REPO" \ + --harness-sha "$HARNESS_SHA" --baseline-sha "$BASELINE_SHA" \ + --candidate-sha "$CANDIDATE_SHA" --corpus B --profile primary --pairs 5 \ + --ingest-rate 50 --http-rate 20 --cgroup "$CORESCOPE_BENCH_CGROUP" \ + --output /tmp/corescope-bench-sqlite-primary +``` + +Primary defaults to 60 seconds of warmup and 180 measured seconds. Both revisions +receive the same generated event file, rate and durations. Five pairs contain +40 minutes of load windows plus preparation, validation and startup. Do not run +this matrix until the harness and application candidate have been reviewed. + +Profiling is a **separate diagnostic invocation**, never a headline elapsed run: + +```sh +python3 "$HARNESS/scripts/sqlite-benchmark/run.py" --repo "$REPO" \ + --harness-sha "$HARNESS_SHA" --baseline-sha "$BASELINE_SHA" \ + --candidate-sha "$CANDIDATE_SHA" --corpus B --profile diagnostic --pairs 1 \ + --warmup 10 --seconds 60 --ingest-rate 50 --http-rate 20 \ + --cgroup "$CORESCOPE_BENCH_CGROUP" --output /tmp/corescope-bench-sqlite-profiles +``` + +## Workload and correctness + +S contains 30,000 transmissions / 90,000 observations. B contains 128,000 / +2,048,000, 2,000 nodes and 128 observers over eight days. The generator uses the +baseline's actual schema, decoder, ingestion APIs and SQLite adapter. The +firmware reference is MeshCore `a366955cb2f67b8e6842d4f00d2b6a554dddd88a`. +Protocol construction and encryption reuse the existing tested helpers. + +The paths request uses a separately recorded `paths_node`: the first synthetic +repeater with retained canonical longest paths under both ascending and +descending observation-ID tie orders. Small native S/B controls verify this +coverage against the generated SQL data. Reach and RX coverage retain their +original node target. This avoids measuring empty path responses when fanout +and loading order select a different representative observation. + +B/L fanouts are 50%×4, 20%×22, 20%×23 and 10%×50 (mean 16). Nullable observers, +fractional signal values, integral scores, and one-/two-/three-byte hop hashes +are included. Observation identities are unique under both SQLite and the +unchanged in-memory loader's observer/path dedup rule; bounded deterministic +path retries preserve NULL/fanout counts and fail on exhaustion. Integer scores +retain common baseline behavior; this harness does not patch production code. + +One recorded offset freshens receipt times for each pair. Signed wire content +and hashes stay unchanged. The recorded input is 50% new transmissions, 30% +additional observations, 15% duplicates and 5% late observations. The ingestion +queue remains bounded at 1024; HTTP concurrency at 64. **Any dropped schedule or +unexpected request/write error, including warmup, fails the run.** No rate is +automatically lowered. An overloaded run remains failed if a later common rate +is chosen for a separately labeled experiment. + +The concurrent HTTP mix has 100 slots: nodes 20, channels/messages 15, observers +5, observer metrics 10, paths/reach/neighbors/RX coverage 20, packets/details 15, +analytics 10, and stats/healthz 5. All traffic goes through the real handlers. +Queries must return meaningful data; fixed pages must have their expected size, +and healthz must be ready with loaded rows. Expired detail requests become an +explicit expected-404 class after retention starts. Retention uses the actual +application method, followed by a verified empty pass. A separate 200-envelope +handler control proves durable writes from the real callback. + +Both full and hot startup must load the complete retained S/B corpus, computed +independently from SQLite. B expects 112,000 transmissions / 1,792,000 observations +under the common seven-day retention, 2048 MiB store and 3072 MiB server limits. +S/B coverage overrides are refused. Every application table undergoes ordered +logical digest checks before and after replay, including the current +`tx_rewrite_feed`, `key_backfill_state`, `key_backfill_done` and `client_node_names` +tables (22 application tables on v3.15.0). The original +18 canonical tables remain required; additional tables are discovered from +SQLite, and added, removed or changed column inventories fail validation. +SQLite's internal bookkeeping tables are excluded. The source and recorded +input must remain unchanged. Operational receipt times alone are excluded from post-run parity. +The harness never rewrites production sources or issues its own schema migrations. + +The same manifest covers both restored copies and is checked again after replay. +Index-only candidate changes are allowed; table/column drift within either leg +or between candidates is unsupported and fails. Observer rowids are recorded +once whether implicit or explicit, preserving the supported layout adoption +below. Freshening applies the single pair offset to rewrite-feed `created_at` +and backfill `started_at`/`done_at` as lifecycle times; their digests remain fully +checked, and cursor, fingerprint and rewritten-count values are unchanged. +Client-name `observed_at` and `ingested_at` receipt times receive the same offset; +the reported `repeater_clock` remains unchanged. Certain-liveness timestamps are +derived from the freshened packet receipt times, with no additional persisted +column to shift on this upstream revision. + +Revisions whose archived server explicitly opens a persisted storage selection +also build their native migrator once. Before either timed server startup, it +adopts that leg's disposable SQLite copy with accounts disabled and no PostgreSQL +service or credentials. Setup runs inside the same resource budget, in the +separate `storage_setup` phase; startup still measures a new process with warm OS +cache. The baseline-generated corpus and paired source files stay unchanged. + +Native adoption may add the required `observers_identity_autoincrement_v1` +physical-layout marker. `storage-setup-validation.json` records adoption and +full-row verification time, plus full before/after counts and digests. Every +original migration entry and every +application row (including observer rowids) must remain identical; unknown or +missing ledger entries fail. After replay, `validation-after.json` retains full +native digests, while `validation-after-comparable.json` accounts only for that +verified marker. The ledger is checked again before paired comparison. No table +is excluded, and adoption failures stop the run before server startup. + +## Diagnostics and artifacts + +The diagnostic test overlay calls the real server `main` and the real ingestor +workload. It writes CPU, heap, mutex and block profiles directly to private files +during the diagnostic measured window. Heap is captured after GC; mutex sampling +uses fraction 5 and block sampling 10,000 ns. `ENABLE_PPROF` is forced off, so the +application's optional all-interface profiling listener is never opened. +Both app and test binaries are built with `-trimpath`. + +Raw profiles stay under each run's `.profiles/` directory and are not uploaded. +`go tool pprof -top` produces public summaries with local mapping paths removed. +Diagnostic runs are comparison-ineligible because sampling adds overhead. +The format, source provenance and file capture helpers can be reused by another +native adapter; this harness intentionally has only the SQLite adapter. + +`db-pool.json` records actual ingestor `database/sql` WaitCount and WaitDuration +deltas across the measured window, with sample bounds and pool limit. They are +separate from request latency and writer-lock statistics. The existing server +does not expose its pool handle to the test launcher; server DBStats are +**unobserved**, not assumed zero or inferred by subtracting latency percentiles. +Server CPU/heap/mutex/block profiles still capture its actual execution. + +The public bundle includes: + +| File | Evidence | +|---|---| +| `manifest.json` | Three exact refs, harness/production/binary hashes, shared settings, workload, A/A or before/after scope | +| `ingest.jsonl`, `requests.jsonl`, `websocket.jsonl` | Raw event timings and outcomes; no packet bodies | +| `summary.csv` | Separate queue, service and schedule-to-complete distributions; p99 only at ≥1000 samples | +| `paired-summary.json` | Per-pair candidate/baseline ratios, without pooling requests or dividing by nonpositive baselines | +| `startup*.json`, `*validation*.json`, `retention*.json` | Loaded-count, durable-effect, full-table parity and retention checks | +| `resources.csv`, `processes.csv`, `resource-final.json` | Cgroup CPU/memory/I/O, RSS/PSS, database/WAL bytes; native peak reset per leg | +| `db-pool.json`, `profiles/*-top.txt`, `profile-capture.json` | Pool wait deltas and separately labeled diagnostic profiles | +| `failure-*.json`, `resource-error.json` | Bounded sanitized failures; arbitrary logs remain private | + +`.runtime/`, databases, input event bodies, credentials, raw profiles and raw logs +are private. Upload only `public/`. Warm OS caches are explicit; cgroup I/O is +accounting, not physical disk traffic. WAL size is retained file footprint, +not generated write volume. Five pairs provide observed spread, not an +engine-wide speed guarantee. No OS-cold or long-soak result is implied. + +## Planned 18-million-observation qualification + +L defines 1,125,000 transmissions / 18,000,000 observations, 5,000 nodes and 512 +observers over 30 days. Its complete repeating path-key period is covered by +generator controls. The actual corpus/restore/profile run has **not** been +qualified at this scale. B must pass first. + +An L run refuses to start without both `--expected-loaded-transmissions` and +`--expected-loaded-observations`. Those must be reviewed fixed cache targets +declared before comparing candidates, not copied afterward from an incomplete +load. Both full/hot startup and both revisions must meet them exactly under +identical limits. The report labels this as bounded cache coverage and never +claims that all retained L rows were loaded. If upstream cannot sustain those +targets, the qualification fails; the count gate is not relaxed. + +Before authorizing L, review actual free RAM/disk, measured B database size and +resource use, then the proposed cache/retention targets and maintenance duration. +The script enforces the 128-GiB disk floor, 8-GiB currently available RAM, cgroup +limit/counter checks, and the later eight-times-actual-file-size guard. These do +not prove sufficient capacity or authorize a large run on a busy workstation. + +## Focused controls + +```sh +python3 -m unittest discover -s scripts/sqlite-benchmark -p 'test_*.py' +``` + +Builds execute the actual generator and shared profile/endpoint controls in both +compiled variants before preparation. No production source file is changed by +the overlays; the source hashes are checked again at completion. CI wiring and +the first committed-harness A/A pilot are separate review steps. diff --git a/scripts/sqlite-benchmark/overlays/clock_linux_test.go b/scripts/sqlite-benchmark/overlays/clock_linux_test.go new file mode 100644 index 000000000..0a75fd57c --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/clock_linux_test.go @@ -0,0 +1,18 @@ +//go:build linux + +package main + +import ( + "syscall" + "unsafe" +) + +// CLOCK_MONOTONIC is shared across processes on this one Linux host. +func benchMono() int64 { + var ts syscall.Timespec + _, _, e := syscall.RawSyscall(syscall.SYS_CLOCK_GETTIME, 1, uintptr(unsafe.Pointer(&ts)), 0) + if e != 0 { + panic(e) + } + return ts.Sec*1e9 + ts.Nsec +} diff --git a/scripts/sqlite-benchmark/overlays/clock_other_test.go b/scripts/sqlite-benchmark/overlays/clock_other_test.go new file mode 100644 index 000000000..8ebc3054f --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/clock_other_test.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// Portable protocol-only tests compile here; measured runs are Linux-only. +func benchMono() int64 { panic("measurement requires Linux CLOCK_MONOTONIC") } diff --git a/scripts/sqlite-benchmark/overlays/common_test.go b/scripts/sqlite-benchmark/overlays/common_test.go new file mode 100644 index 000000000..9e0887f11 --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/common_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type benchShape struct { + Transmissions int `json:"transmissions"` + Observations int `json:"observations"` + Nodes int `json:"nodes"` + Observers int `json:"observers"` + Days int `json:"days"` +} +type benchConfig struct { + ProfileDir string `json:"profile_dir"` + ServerArgs []string `json:"server_args"` + EventsFile string `json:"events_file"` + CorpusInfo string `json:"corpus_info"` + Mode string `json:"mode"` + Corpus string `json:"corpus"` + Shape benchShape `json:"shape"` + Seed int64 `json:"seed"` + WireEpoch int64 `json:"wire_epoch"` + Epoch int64 `json:"epoch"` + SQLite string `json:"sqlite"` + Output string `json:"output"` + StateDir string `json:"state_dir"` + BaseURL string `json:"base_url"` + StartFile string `json:"start_file"` + Warmup int `json:"warmup"` + Seconds int `json:"seconds"` + IngestRate int `json:"ingest_rate"` + HTTPRate int `json:"http_rate"` +} + +func benchRead(t *testing.T) benchConfig { + t.Helper() + path := os.Getenv("CORESCOPE_BENCH_CONFIG") + if path == "" { + t.Skip("opt-in benchmark overlay") + } + b, e := os.ReadFile(path) + if e != nil { + t.Fatal(e) + } + var c benchConfig + if e = json.Unmarshal(b, &c); e != nil { + t.Fatal(e) + } + if !strings.Contains(filepath.Clean(c.Output), "corescope-bench-") { + t.Fatal("output is not an allowlisted disposable benchmark directory") + } + if c.Output == "" || c.Shape.Transmissions <= 0 || c.Shape.Nodes <= 0 || c.Shape.Observers < 1 { + t.Fatal("incomplete benchmark configuration") + } + return c +} +func benchJSON(t *testing.T, path string, value any) { + t.Helper() + b, e := json.MarshalIndent(value, "", " ") + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(path, append(b, '\n'), 0600); e != nil { + t.Fatal(e) + } +} +func benchOutput(t *testing.T, c benchConfig, name string) *os.File { + t.Helper() + f, e := os.OpenFile(filepath.Join(c.Output, name), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { f.Close() }) + return f +} +func benchStart(c benchConfig) (int64, error) { + for i := 0; i < 2400; i++ { + if b, e := os.ReadFile(c.StartFile); e == nil { + var start struct { + Mono int64 `json:"mono_ns"` + } + if e = json.Unmarshal(b, &start); e != nil { + return 0, e + } + return start.Mono, nil + } + time.Sleep(50 * time.Millisecond) + } + return 0, fmt.Errorf("benchmark start barrier timed out") +} +func benchSleep(until int64) { + if delay := until - benchMono(); delay > 0 { + time.Sleep(time.Duration(delay)) + } +} +func benchFanout(c benchConfig, i int) int { + if c.Corpus == "S" { + return i%5 + 1 + } + switch { + case i%10 < 5: + return 4 + case i%10 < 7: + return 22 + case i%10 < 9: + return 23 + default: + return 50 + } +} +func benchStamp(c benchConfig, i int) int64 { + day := (i / 10) % c.Shape.Days + if day == 0 { + return c.Epoch - 2700 + int64(i%1800) + } + return c.Epoch - int64(day)*86400 - 43200 + int64(i%3600) +} diff --git a/scripts/sqlite-benchmark/overlays/ingestor_test.go b/scripts/sqlite-benchmark/overlays/ingestor_test.go new file mode 100644 index 000000000..eb261dcfa --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/ingestor_test.go @@ -0,0 +1,631 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// Wire format is pinned to MeshCore a366955: Packet.cpp and Mesh.cpp::createAdvert; +// group encryption reuses this revision's firmware-derived test helper. +type benchFixture struct { + c benchConfig + public []string + private []ed25519.PrivateKey + channels map[string]string +} + +func benchFixtureFor(c benchConfig) *benchFixture { + f := &benchFixture{c: c, channels: map[string]string{}} + for i := 0; i < c.Shape.Nodes; i++ { + seed := sha256.Sum256([]byte(fmt.Sprintf("corescope-bench/%d/node/%d", c.Seed, i))) + key := ed25519.NewKeyFromSeed(seed[:]) + f.private = append(f.private, key) + f.public = append(f.public, hex.EncodeToString(key.Public().(ed25519.PublicKey))) + } + for i := 0; i < 20; i++ { + name := fmt.Sprintf("#bench-%02d", i) + f.channels[name] = deriveHashtagChannelKey(name) + } + return f +} +func (f *benchFixture) packet(index, variant int, stamp int64) (*PacketData, error) { + node := index % len(f.public) + payloadType := 5 + var payload []byte + if index%10 == 0 { + payloadType = 4 + public := f.private[node].Public().(ed25519.PublicKey) + app := []byte{0x92} + location := make([]byte, 8) + binary.LittleEndian.PutUint32(location, uint32(20_000_000+node*100)) + binary.LittleEndian.PutUint32(location[4:], uint32(30_000_000+node*100)) + app = append(app, location...) + app = append(app, []byte(fmt.Sprintf("Bench%04d-%07d", node, index))...) + signed := append([]byte{}, public...) + clock := make([]byte, 4) + binary.LittleEndian.PutUint32(clock, uint32(stamp)) + signed = append(signed, clock...) + signed = append(signed, app...) + payload = append(payload, public...) + payload = append(payload, clock...) + payload = append(payload, ed25519.Sign(f.private[node], signed)...) + payload = append(payload, app...) + } else { + channelIndex := 0 + if index%10 >= 5 { + channelIndex = 1 + index%19 + } + channel := fmt.Sprintf("#bench-%02d", channelIndex) + key := f.channels[channel] + inner := make([]byte, 5) + binary.LittleEndian.PutUint32(inner, uint32(stamp)) + inner = append(inner, []byte(fmt.Sprintf("Bench%04d: synthetic sequence %d seed %d %s", node, index, f.c.Seed, strings.Repeat("x", index%40)))...) + cipher, mac := buildChannelEncrypted(key, inner) + keyBytes, _ := hex.DecodeString(key) + sum := sha256.Sum256(keyBytes) + macBytes, _ := hex.DecodeString(mac) + cipherBytes, _ := hex.DecodeString(cipher) + payload = append([]byte{sum[0]}, macBytes...) + payload = append(payload, cipherBytes...) + } + hashSize := 1 + index%3 + hopCount := 1 + index%3 + route := 1 + if index%5 == 0 && variant == 0 { + hopCount = 0 + } + if payloadType == 4 && variant == 1 { + route = 2 + hopCount = 0 + } + var path []byte + for hop := 0; hop < hopCount; hop++ { + key, _ := hex.DecodeString(f.public[(index+variant*7+hop+1)%len(f.public)]) + path = append(path, key[:hashSize]...) + } + if variant >= 1000 { + hashSize = 3 + hopCount = 2 + route = 1 + path = []byte{0xfa, byte(variant >> 8), byte(variant), 0xfb, byte(index >> 8), byte(index)} + } + wire := append([]byte{byte(payloadType<<2 | route), byte((hashSize-1)<<6 | hopCount)}, path...) + wire = append(wire, payload...) + raw := hex.EncodeToString(wire) + decoded, err := DecodePacket(raw, f.channels, true) + if err != nil { + return nil, err + } + score, snr, rssi := float64(index%5), float64(index%20)-10.5, -100.5+float64(index%25) + msg := &MQTTPacketMessage{Raw: raw, SNR: &snr, RSSI: &rssi, Score: &score, Timestamp: time.Unix(stamp, 0).UTC().Format(time.RFC3339)} + p := BuildPacketData(msg, decoded, f.public[0], "AAA", nil) + p.Timestamp = msg.Timestamp + if p.Hash == "" || p.PayloadType != payloadType { + return nil, fmt.Errorf("generated frame failed native decoder") + } + return p, nil +} + +type benchGeneratedObservation struct { + Packet *PacketData + ObserverIndex any + PathVariant int +} + +func (f *benchFixture) observation(index, ordinal int, stamp int64, used map[string]bool) (benchGeneratedObservation, error) { + observerIndex := ordinal%f.c.Shape.Observers + 1 + result := benchGeneratedObservation{ObserverIndex: observerIndex} + observerID := f.public[observerIndex-1] + if ordinal%17 == 16 { + result.ObserverIndex, observerID = nil, "" + } + // Keep every observation and NULL identity. A path-prefix collision for + // the same identity needs a different valid wire path, not different JSON + // spelling. The bound stays below packet()'s special replay variants. + for attempt := 0; attempt < 64; attempt++ { + result.PathVariant = ordinal + attempt + packet, err := f.packet(index, result.PathVariant, stamp) + if err != nil { + return result, err + } + key := observerID + "|" + packet.PathJSON + if !used[key] { + used[key] = true + result.Packet = packet + return result, nil + } + } + return result, fmt.Errorf("no unique observer/path identity after 64 valid path variants") +} + +func TestCoreScopeBenchmarkObservationKeyspace(t *testing.T) { + for _, shape := range []struct { + name string + nodes, observers int + }{{"B", 2000, 128}, {"L", 5000, 512}} { + t.Run(shape.name, func(t *testing.T) { + c := benchConfig{Corpus: shape.name, Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: shape.nodes, Observers: shape.observers, Days: 8}} + f := benchFixtureFor(c) + // The path recipe depends on index modulo Nodes and modulo 3; + // route/fanout periods 5/10 divide both node counts. Thus these + // 6,000/15,000 positions cover the complete B/L path key space. + corrections, missing := 0, 0 + for index := 0; index < 3*shape.nodes; index++ { + used := make(map[string]bool) + count := benchFanout(c, index) + if count > shape.observers { + t.Fatal("known observer identities would repeat within a transmission") + } + // Known observer IDs are distinct. Only NULL identities can + // share a key, so exercise every such pair with real decoding. + for ordinal := 16; ordinal < count; ordinal += 17 { + original, err := f.packet(index, ordinal, c.Epoch) + if err != nil { + t.Fatal(err) + } + got, err := f.observation(index, ordinal, c.Epoch, used) + if err != nil { + t.Fatal(err) + } + if got.ObserverIndex != nil || got.Packet.Hash != original.Hash || got.Packet.RawHex[:4] != original.RawHex[:4] { + t.Fatal("path correction changed observer, payload identity, route or hop/hash shape") + } + if got.PathVariant != ordinal { + corrections++ + if got.Packet.PathJSON == original.PathJSON { + t.Fatal("retry did not change the real decoded path") + } + } + missing++ + } + } + if corrections == 0 || missing != 3*shape.nodes*6/10 { + t.Fatalf("NULL collision coverage: corrections=%d NULLs=%d", corrections, missing) + } + }) + } +} + +func TestCoreScopeBenchmarkObservationRetryBound(t *testing.T) { + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 2000, Observers: 128}} + f := benchFixtureFor(c) + used := make(map[string]bool) + for prefix := 0; prefix < 256; prefix++ { + used[fmt.Sprintf(`|["%02X"]`, prefix)] = true + } + if _, err := f.observation(9, 16, c.Epoch, used); err == nil { + t.Fatal("exhausted path space must fail rather than drop an observation or loop forever") + } +} + +func TestCoreScopeBenchmarkProtocol(t *testing.T) { + c := benchConfig{Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 80, Observers: 8, Transmissions: 100, Days: 8}} + f := benchFixtureFor(c) + a, _ := f.packet(0, 0, c.Epoch) + repeated, _ := f.packet(18000, 0, c.Epoch) + if a.Hash == repeated.Hash { + t.Fatal("repeated advertiser and emission time collapsed distinct corpus transmissions") + } + seen := map[string]bool{} + for i := 0; i < 100; i++ { + a, e := f.packet(i, 0, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + b, e := f.packet(i, 1000, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + if a.Hash != b.Hash { + t.Fatal("path change altered content identity") + } + if seen[a.Hash] { + t.Fatal("generator accidentally deduplicated distinct transmissions") + } + if a.Score == nil || *a.Score != math.Trunc(*a.Score) { + t.Fatal("timed score corpus must stay integral so the unmodified baseline loads the same rows") + } + seen[a.Hash] = true + if i%10 != 0 && !strings.HasPrefix(a.ChannelHash, "#bench-") { + t.Fatal("generated channel message did not decrypt") + } + } + for _, name := range []string{"S", "B", "L"} { + c.Corpus = name + total := 0 + for i := 0; i < 100; i++ { + total += benchFanout(c, i) + } + want := 1600 + if name == "S" { + want = 300 + } + if total != want { + t.Fatalf("fanout %s=%d", name, total) + } + } +} + +func TestCoreScopeBenchmarkEventStream(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791454800, WireEpoch: 1791451200, Shape: benchShape{Nodes: 200, Observers: 16, Transmissions: 100, Observations: 300, Days: 2}, Output: dir, EventsFile: filepath.Join(dir, "one.jsonl"), Seconds: 4, IngestRate: 10} + benchMakeEvents(t, c) + first, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + c.EventsFile = filepath.Join(dir, "two.jsonl") + benchMakeEvents(t, c) + second, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(first, second) { + t.Fatal("same seed/epochs produced different event streams") + } + dec := json.NewDecoder(bytes.NewReader(first)) + counts := map[string]int{} + var additional *PacketData + for i := 0; i < 40; i++ { + var input benchInput + if e := dec.Decode(&input); e != nil { + t.Fatal(e) + } + if input.Sequence != i { + t.Fatal("sequence drift") + } + counts[input.Class]++ + if i == 10 { + additional = input.Packet + } + if i == 16 && (input.Packet.Hash != additional.Hash || input.Packet.PathJSON != additional.PathJSON) { + t.Fatal("duplicate event does not hit the same observation key") + } + } + for class, want := range map[string]int{"new": 20, "additional": 12, "duplicate": 6, "late": 2} { + if counts[class] != want { + t.Fatalf("event class %s=%d want %d", class, counts[class], want) + } + } +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + if c.Mode == "prepare" { + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + return + } + if c.Mode == "events" { + benchMakeEvents(t, c) + return + } + s, err := benchOpen(c) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + if c.Mode == "replay" { + profiles := benchProfileWindow(c) + pool := benchPoolWindow(t, c, s.db) + defer func() { + if err := <-profiles; err != nil { + t.Error(err) + } + <-pool + }() + } + settings, err := benchSettings(s) + if err != nil { + t.Fatal(err) + } + benchJSON(t, filepath.Join(c.Output, "database-settings.json"), settings) + if err = s.RefreshPrefixIndex(); err != nil { + t.Fatal(err) + } + switch c.Mode { + case "replay": + benchReplay(t, c, s) + case "retention": + benchRetention(t, c, s) + case "handler": + benchHandler(t, c, s, benchFixtureFor(c)) + case "plans": + if e := os.MkdirAll(filepath.Join(c.Output, "plans"), 0700); e != nil { + t.Fatal(e) + } + for _, query := range []struct { + name, sql string + args []any + }{ + {"retention-empty", pruneAgedTransmissionIDs, []any{time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339), pruneBatchTransmissions}}, + {"advert-preservation", legacyAdvertObservationSQL, []any{1, 1, "[]"}}, + } { + plan, e := benchExplain(s, query.sql, query.args...) + if e != nil { + t.Fatal(e) + } + benchJSON(t, filepath.Join(c.Output, "plans", query.name+".json"), map[string]any{"sql": query.sql, "args": query.args, "plan": plan, "context": "separate diagnostic after replay and retention; excluded from headline timings"}) + } + default: + t.Fatal("unsupported benchmark mode") + } +} + +type benchEvent struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Hash string `json:"hash"` + Scheduled int64 `json:"scheduled_ns"` + Started int64 `json:"started_ns"` + Completed int64 `json:"completed_ns"` + Measured bool `json:"measured"` + Dropped bool `json:"dropped"` + New bool `json:"new_transmission"` + Error string `json:"error,omitempty"` +} + +type benchInput struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Packet *PacketData `json:"packet"` +} + +func benchMakeEvents(t *testing.T, c benchConfig) { + f := benchFixtureFor(c) + file, e := os.OpenFile(c.EventsFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + defer file.Close() + enc := json.NewEncoder(file) + for sequence := 0; sequence < (c.Warmup+c.Seconds)*c.IngestRate; sequence++ { + class, index, variant := "new", c.Shape.Transmissions+sequence, 0 + part, block := sequence%20, sequence/20 + stamp := c.Epoch + int64(sequence/c.IngestRate) + if part >= 10 { + stride := c.Shape.Days * 10 + blocks := c.Shape.Transmissions / stride + if blocks < 1 { + blocks = 1 + } + index = (block%blocks)*stride + block%10 + wire := c + if c.WireEpoch != 0 { + wire.Epoch = c.WireEpoch + } + stamp = benchStamp(wire, index) + class = "additional" + variant = 1000 + part + if part >= 16 && part < 19 { + class = "duplicate" + variant = 1010 + } + if part == 19 { + class = "late" + variant = 1019 + } + } + packet, e := f.packet(index, variant, stamp) + if e != nil { + t.Fatal(e) + } + packet.ObserverID = f.public[0] + packet.Timestamp = time.Unix(c.Epoch+int64(sequence/c.IngestRate), 0).UTC().Format(time.RFC3339) + if class == "late" { + packet.Timestamp = time.Unix(benchStamp(c, index)-60, 0).UTC().Format(time.RFC3339) + } + if e := enc.Encode(benchInput{sequence, class, packet}); e != nil { + t.Fatal(e) + } + } +} + +func benchReplay(t *testing.T, c benchConfig, s *Store) { + var initialTx, initialObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&initialTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&initialObs); err != nil { + t.Fatal(err) + } + file := benchOutput(t, c, "ingest.jsonl") + encoder := json.NewEncoder(file) + var outputMu sync.Mutex + emit := func(e benchEvent) { + outputMu.Lock() + defer outputMu.Unlock() + if err := encoder.Encode(e); err != nil { + t.Error(err) + } + } + if err := os.WriteFile(filepath.Join(c.Output, "replay-ready"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + start, err := benchStart(c) + if err != nil { + t.Fatal(err) + } + benchSleep(start) + queue := make(chan benchInput, 1024) + done := make(chan struct{}) + var successful, newTx, newObs, dropped int64 + seenReplay := make(map[string]bool) + var firstError error + go func() { + defer close(done) + for input := range queue { + sequence, class, p := input.Sequence, input.Class, input.Packet + var e error + event := benchEvent{Sequence: sequence, Class: class, Scheduled: start + int64(sequence)*int64(time.Second)/int64(c.IngestRate), Measured: sequence >= c.Warmup*c.IngestRate} + if p != nil { + event.Hash = p.Hash + event.Started = benchMono() + beforeErrors := s.Stats.WriteErrors.Load() + event.New, e = s.InsertTransmission(p) + if e == nil && s.Stats.WriteErrors.Load() != beforeErrors { + e = fmt.Errorf("write-error counter advanced despite handler return") + } + event.Completed = benchMono() + if e == nil && event.New != (class == "new") { + e = fmt.Errorf("event class %s had unexpected transmission creation", class) + } + } + if e != nil { + event.Error = e.Error() + if firstError == nil { + firstError = e + } + } else { + successful++ + if event.New { + newTx++ + } + key := p.Hash + "/" + p.ObserverID + "/" + p.PathJSON + if !seenReplay[key] { + newObs++ + seenReplay[key] = true + } + } + emit(event) + } + }() + retained := make(chan benchPruneResult, 1) + go func() { + benchSleep(start + int64(c.Warmup+c.Seconds*3/4)*int64(time.Second)) + _ = os.WriteFile(filepath.Join(c.Output, "retention-started"), []byte("started"), 0600) + retained <- benchPrune(s) + }() + events := (c.Warmup + c.Seconds) * c.IngestRate + inputFile, err := os.Open(c.EventsFile) + if err != nil { + close(queue) + <-done + t.Fatal(err) + } + defer inputFile.Close() + inputDecoder := json.NewDecoder(inputFile) + var producerError error + for sequence := 0; sequence < events; sequence++ { + var input benchInput + if err := inputDecoder.Decode(&input); err != nil { + producerError = err + break + } + if input.Sequence != sequence || input.Packet == nil { + producerError = fmt.Errorf("invalid pre-generated event sequence") + break + } + scheduled := start + int64(sequence)*int64(time.Second)/int64(c.IngestRate) + benchSleep(scheduled) + select { + case queue <- input: + default: + dropped++ + emit(benchEvent{Sequence: sequence, Class: "schedule", Scheduled: scheduled, Dropped: true, Measured: sequence >= c.Warmup*c.IngestRate}) + } + } + close(queue) + select { + case <-done: + case <-time.After(120 * time.Second): + t.Fatal("bounded ingestion queue did not drain in 120 seconds") + } + retention := <-retained + benchJSON(t, filepath.Join(c.Output, "retention.json"), retention) + if retention.Error != "" { + t.Fatal(retention.Error) + } + var actualTx, actualObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&actualTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&actualObs); err != nil { + t.Fatal(err) + } + verified := dropped == 0 && successful == int64(events) && producerError == nil && firstError == nil && s.Stats.WriteErrors.Load() == 0 && actualTx == initialTx+newTx-retention.Transmissions && actualObs == initialObs+newObs-retention.Observations + benchJSON(t, filepath.Join(c.Output, "ingest-validation.json"), map[string]any{"verified": verified, "scheduled": events, "completed": successful, "dropped_schedules": dropped, "expected_transmissions": initialTx + newTx - retention.Transmissions, "actual_transmissions": actualTx, "expected_observations": initialObs + newObs - retention.Observations, "actual_observations": actualObs, "write_errors": s.Stats.WriteErrors.Load(), "writer_wait_hold": s.WriterStatsSnapshot(), "finished_ns": benchMono()}) + if !verified { + t.Fatalf("durable ingest effects did not match: dropped=%d completed=%d/%d tx=%d/%d obs=%d/%d first error=%v", dropped, successful, events, actualTx, initialTx+newTx-retention.Transmissions, actualObs, initialObs+newObs-retention.Observations, firstError) + } +} + +type benchPruneResult struct { + Elapsed int64 `json:"elapsed_ns"` + Transmissions int64 `json:"transmissions_deleted"` + Observations int64 `json:"observations_deleted"` + Error string `json:"error,omitempty"` +} + +func benchPrune(s *Store) benchPruneResult { + result := benchPruneResult{} + cutoff := time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339) + q := fmt.Sprintf("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.first_seen < '%s'", cutoff) + if err := s.db.QueryRow(q).Scan(&result.Observations); err != nil { + result.Error = err.Error() + return result + } + started := benchMono() + n, err := s.PruneOldPackets(7) + result.Elapsed = benchMono() - started + result.Transmissions = n + if err != nil { + result.Error = err.Error() + } + return result +} +func benchRetention(t *testing.T, c benchConfig, s *Store) { + r := benchPrune(s) + benchJSON(t, filepath.Join(c.Output, "retention-empty.json"), r) + if r.Error != "" || r.Transmissions != 0 || r.Observations != 0 { + t.Fatal("empty retention pass found unexpected work or failed") + } +} + +func benchHandler(t *testing.T, c benchConfig, s *Store, f *benchFixture) { + var before, after int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&before); err != nil { + t.Fatal(err) + } + samples := make([]int64, 0, 200) + for i := 0; i < 200; i++ { + p, err := f.packet(c.Shape.Transmissions+10000000+i, 0, c.Epoch+int64(i)) + if err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(map[string]any{"raw": p.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + if err != nil { + t.Fatal(err) + } + msg := &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: payload} + started := benchMono() + handleMessage(s, "bench", MQTTSource{Name: "bench"}, msg, f.channels, nil, &Config{}) + samples = append(samples, benchMono()-started) + } + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&after); err != nil { + t.Fatal(err) + } + valid := after-before == 200 && s.Stats.WriteErrors.Load() == 0 + benchJSON(t, filepath.Join(c.Output, "handler.json"), map[string]any{"verified": valid, "samples_ns": samples, "new_transmissions": after - before, "classification": "separate 200-envelope mechanism control; not mixed-load throughput"}) + if !valid { + t.Fatal("real MQTT handler dropped valid synthetic envelopes") + } +} diff --git a/scripts/sqlite-benchmark/overlays/profiles_test.go b/scripts/sqlite-benchmark/overlays/profiles_test.go new file mode 100644 index 000000000..c73b29b1c --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/profiles_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "testing" + "time" +) + +// File capture only. The application's optional HTTP profiler stays disabled. +// This helper is used exclusively by separate diagnostic invocations. +func benchBeginProfiles(directory string) (func() error, error) { + if err := os.MkdirAll(directory, 0700); err != nil { + return nil, err + } + cpu, err := os.Create(filepath.Join(directory, "cpu.pprof")) + if err != nil { + return nil, err + } + if err := pprof.StartCPUProfile(cpu); err != nil { + cpu.Close() + return nil, err + } + previous := runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(10000) + return func() error { + pprof.StopCPUProfile() + runtime.SetMutexProfileFraction(previous) + runtime.SetBlockProfileRate(0) + if err := cpu.Close(); err != nil { + return err + } + runtime.GC() + for _, kind := range []string{"heap", "mutex", "block"} { + file, err := os.Create(filepath.Join(directory, kind+".pprof")) + if err != nil { + return err + } + err = pprof.Lookup(kind).WriteTo(file, 0) + closeErr := file.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + } + return nil + }, nil +} + +func benchProfileWindow(c benchConfig) <-chan error { + done := make(chan error, 1) + if c.ProfileDir == "" { + done <- nil + close(done) + return done + } + go func() { + start, err := benchStart(c) + var began, stopRequested int64 + if err == nil { + benchSleep(start + int64(c.Warmup)*int64(time.Second)) + var finish func() error + finish, err = benchBeginProfiles(c.ProfileDir) + if err == nil { + began = benchMono() + benchSleep(start + int64(c.Warmup+c.Seconds)*int64(time.Second)) + stopRequested = benchMono() + err = finish() + } + } + result := struct { + Complete bool `json:"complete"` + Error string `json:"error,omitempty"` + RequestedStart int64 `json:"requested_start_ns"` + RequestedEnd int64 `json:"requested_end_ns"` + CaptureStart int64 `json:"capture_start_ns"` + StopRequested int64 `json:"stop_requested_ns"` + Finished int64 `json:"finished_ns"` + }{Complete: err == nil, RequestedStart: start + int64(c.Warmup)*int64(time.Second), RequestedEnd: start + int64(c.Warmup+c.Seconds)*int64(time.Second), CaptureStart: began, StopRequested: stopRequested, Finished: benchMono()} + if err != nil { + result.Error = err.Error() + } + data, _ := json.Marshal(result) + if writeErr := os.WriteFile(filepath.Join(c.ProfileDir, "done.json"), data, 0600); err == nil { + err = writeErr + } + done <- err + close(done) + }() + return done +} + +// database/sql exposes cumulative pool counters. Report their window deltas, +// separately from request and writer-lock distributions; never subtract p95s. +func benchPoolWindow(t *testing.T, c benchConfig, db *sql.DB) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + start, err := benchStart(c) + if err != nil { + t.Error(err) + return + } + benchSleep(start + int64(c.Warmup)*int64(time.Second)) + before, first := db.Stats(), benchMono() + benchSleep(start + int64(c.Warmup+c.Seconds)*int64(time.Second)) + after, last := db.Stats(), benchMono() + result := struct { + Scope string `json:"scope"` + First int64 `json:"first_ns"` + Last int64 `json:"last_ns"` + WaitCount int64 `json:"wait_count_delta"` + WaitDuration int64 `json:"wait_duration_ns_delta"` + MaxOpen int `json:"max_open_connections"` + }{"ingestor database/sql pool", first, last, after.WaitCount - before.WaitCount, int64(after.WaitDuration - before.WaitDuration), after.MaxOpenConnections} + benchJSON(t, filepath.Join(c.Output, "db-pool.json"), result) + }() + return done +} + +func TestCoreScopeBenchmarkControlProfilesWritePrivateFiles(t *testing.T) { + dir := t.TempDir() + finish, err := benchBeginProfiles(dir) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Millisecond) + for time.Now().Before(deadline) { + _ = fmt.Sprint(time.Now().UnixNano()) + } + if err := finish(); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"cpu", "heap", "mutex", "block"} { + info, err := os.Stat(filepath.Join(dir, kind+".pprof")) + if err != nil || info.Size() == 0 { + t.Fatalf("missing %s profile: %v", kind, err) + } + } + if err := <-benchProfileWindow(benchConfig{}); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/sqlite-benchmark/overlays/server_test.go b/scripts/sqlite-benchmark/overlays/server_test.go new file mode 100644 index 000000000..704d3b703 --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/server_test.go @@ -0,0 +1,410 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +type benchCorpusInfo struct { + Node string `json:"node"` + PathsNode string `json:"paths_node"` + Dense string `json:"dense_observer"` + Sparse string `json:"sparse_observer"` + NewPacket string `json:"new_packet"` + OldPacket string `json:"old_packet"` +} + +func benchInfo(c benchConfig) (benchCorpusInfo, error) { + var info benchCorpusInfo + b, e := os.ReadFile(c.CorpusInfo) + if e == nil { + e = json.Unmarshal(b, &info) + } + return info, e +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + switch c.Mode { + case "queries": + benchQueries(t, c) + case "client": + benchClient(t, c) + default: + t.Fatal("unsupported server benchmark mode") + } +} + +// Diagnostic-only launcher: execute the real application main without exposing +// its optional all-interface pprof listener. Production source is not replaced. +func TestCoreScopeBenchmarkServer(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" || c.Mode != "server" || c.ProfileDir == "" || len(c.ServerArgs) == 0 { + t.Fatal("invalid diagnostic server controls") + } + if err := os.Setenv("ENABLE_PPROF", "false"); err != nil { + t.Fatal(err) + } + benchProfileWindow(c) + os.Args = append([]string{os.Args[0]}, c.ServerArgs...) + main() +} + +func TestCoreScopeBenchmarkControlConcurrentEndpoints(t *testing.T) { + info := benchCorpusInfo{Node: strings.Repeat("a", 64), PathsNode: strings.Repeat("b", 64), Dense: "dense", Sparse: "sparse", NewPacket: "new", OldPacket: "old"} + endpoints := benchEndpoints(benchConfig{Epoch: 1791451200}, info) + if len(endpoints) != 100 { + t.Fatalf("workload weight=%d", len(endpoints)) + } + for _, want := range []string{"paths", "reach", "neighbors", "healthz", "stats", "packets_memory"} { + found := false + for _, endpoint := range endpoints { + if endpoint.Class == want { + if want == "paths" && endpoint.Path != "/api/nodes/"+info.PathsNode+"/paths" { + t.Fatal("paths workload did not use the verified corpus target") + } + found = true + } + } + if !found { + t.Fatalf("concurrent workload lacks %s", want) + } + } + if benchHealthy(map[string]any{"ready": false, "loadedTx": 100.0, "loadedObs": 100.0}) || benchHealthy(map[string]any{"ready": true, "loadedTx": 0.0, "loadedObs": 0.0}) { + t.Fatal("incomplete health must fail") + } + if !benchHealthy(map[string]any{"ready": true, "loadedTx": 100.0, "loadedObs": 200.0}) { + t.Fatal("complete health rejected") + } +} + +type benchRequest struct { + Class string `json:"class"` + Scheduled int64 `json:"scheduled_ns"` + Dispatched int64 `json:"dispatched_ns"` + Completed int64 `json:"completed_ns"` + Status int `json:"status"` + ExpectedStatus int `json:"expected_status"` + Bytes int `json:"bytes"` + SHA256 string `json:"response_sha256,omitempty"` + Error string `json:"error,omitempty"` + Measured bool `json:"measured"` + Cache string `json:"cache"` +} + +func benchQueries(t *testing.T, c benchConfig) { + db, e := OpenDB(c.SQLite) + if e != nil { + t.Fatal(e) + } + defer db.Close() + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + out := json.NewEncoder(benchOutput(t, c, "sql-queries.jsonl")) + for i := 0; i < 200; i++ { + sample := benchRequest{Measured: true, Status: 200, Cache: "explicit SQL miss"} + var value any + if i%2 == 0 { + db.msgCacheMu.Lock() + db.msgCache = nil + db.msgCacheMu.Unlock() + sample.Class = "channel_messages_sql" + sample.Dispatched = benchMono() + rows, total, err := db.GetChannelMessages("#bench-00", 50, (i%4)*500) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && (len(rows) == 0 || total < 50) { + e = fmt.Errorf("channel SQL query returned no meaningful data") + } + } else { + sample.Class = "observer_metrics_sql" + sample.Dispatched = benchMono() + rows, _, err := db.GetObserverMetrics(info.Dense, time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339), time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339), "5m", 300) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && len(rows) == 0 { + e = fmt.Errorf("metrics SQL query returned no meaningful data") + } + } + if e != nil { + sample.Error = e.Error() + } + b, _ := json.Marshal(value) + sum := sha256.Sum256(b) + sample.SHA256 = hex.EncodeToString(sum[:]) + sample.Bytes = len(b) + if e := out.Encode(sample); e != nil { + t.Fatal(e) + } + if sample.Error != "" { + t.Fatal(sample.Error) + } + } +} + +type benchEndpoint struct { + Class string `json:"class"` + Path string `json:"path"` +} + +func benchEndpoints(c benchConfig, info benchCorpusInfo) []benchEndpoint { + since24 := url.QueryEscape(time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339)) + since7 := url.QueryEscape(time.Unix(c.Epoch-7*86400, 0).UTC().Format(time.RFC3339)) + until := url.QueryEscape(time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339)) + groups := []struct { + weight int + items []benchEndpoint + }{ + {20, []benchEndpoint{{"nodes_50", "/api/nodes?limit=50"}, {"nodes_2000", "/api/nodes?limit=2000"}, {"nodes_region", "/api/nodes?limit=50®ion=AAA&search=Synthetic"}}}, + {15, []benchEndpoint{{"channels", "/api/channels"}, {"messages_0", "/api/channels/%23bench-00/messages?limit=50&offset=0"}, {"messages_1000", "/api/channels/%23bench-00/messages?limit=50&offset=1000"}}}, + {5, []benchEndpoint{{"observers", "/api/observers"}}}, + {10, []benchEndpoint{{"metrics_dense_24h", "/api/observers/" + info.Dense + "/metrics?since=" + since24 + "&until=" + until + "&resolution=5m"}, {"metrics_sparse_7d", "/api/observers/" + info.Sparse + "/metrics?since=" + since7 + "&until=" + until + "&resolution=1h"}}}, + {20, []benchEndpoint{{"paths", "/api/nodes/" + info.PathsNode + "/paths"}, {"reach", "/api/nodes/" + info.Node + "/reach"}, {"neighbors", "/api/nodes/" + info.Node + "/neighbors"}, {"rx_coverage", "/api/nodes/" + info.Node + "/rx-coverage?bbox=19,29,22,32&z=14"}}}, + {15, []benchEndpoint{{"packets_memory", "/api/packets?limit=50"}, {"packet_detail_memory", "/api/packets/" + info.NewPacket}, {"packet_detail_sql", "/api/packets/" + info.OldPacket}}}, + {10, []benchEndpoint{{"analytics_rf", "/api/analytics/rf"}, {"analytics_topology", "/api/analytics/topology"}, {"analytics_channels", "/api/analytics/channels"}, {"analytics_rf_filtered", "/api/analytics/rf?region=AAA"}}}, + {5, []benchEndpoint{{"stats", "/api/stats"}, {"healthz", "/api/healthz"}}}, + } + var result []benchEndpoint + for _, group := range groups { + for i := 0; i < group.weight; i++ { + result = append(result, group.items[i%len(group.items)]) + } + } + return result +} + +func benchHealthy(value any) bool { + obj, ok := value.(map[string]any) + if !ok || obj["ready"] != true { + return false + } + tx, _ := obj["loadedTx"].(float64) + obs, _ := obj["loadedObs"].(float64) + return tx > 0 && obs > 0 +} + +// Ignore timestamp/version metadata when deciding whether a response carries +// useful data. A successful empty response must not masquerade as fast SQL. +func benchNonempty(value any) bool { + switch v := value.(type) { + case []any: + return len(v) > 0 + case map[string]any: + if _, bad := v["error"]; bad { + return false + } + if hash, ok := v["hash"].(string); ok && hash != "" { + return true + } + for key, child := range v { + switch key { + case "total", "totalNodes", "nodeCount", "packetCount", "totalPackets", "totalTransmissions", "totalMessages", "count", "mobile_receptions", "totalLoaded": + if n, ok := child.(float64); ok && n > 0 { + return true + } + } + switch child.(type) { + case []any, map[string]any: + if benchNonempty(child) { + return true + } + } + } + } + return false +} + +func benchArrayCount(v any) int { + if rows, ok := v.([]any); ok { + return len(rows) + } + if obj, ok := v.(map[string]any); ok { + for _, key := range []string{"nodes", "packets", "messages", "metrics", "samples", "observers", "features", "data"} { + if child, found := obj[key]; found { + if n := benchArrayCount(child); n > 0 { + return n + } + } + } + } + return 0 +} + +func benchClient(t *testing.T, c benchConfig) { + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + endpoints := benchEndpoints(c, info) + benchJSON(t, filepath.Join(c.Output, "http-workload.json"), endpoints) + requests := json.NewEncoder(benchOutput(t, c, "requests.jsonl")) + messages := json.NewEncoder(benchOutput(t, c, "websocket.jsonl")) + conn, _, e := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(c.BaseURL, "http")+"/ws", nil) + if e != nil { + t.Fatal(e) + } + defer conn.Close() + conn.SetReadLimit(8 << 20) + var closing atomic.Bool + var wsErr atomic.Pointer[string] + wsDone := make(chan struct{}) + go func() { + defer close(wsDone) + for { + _, raw, err := conn.ReadMessage() + if err != nil { + if !closing.Load() { + value := err.Error() + wsErr.Store(&value) + } + return + } + var frame struct { + Type string `json:"type"` + Data struct { + Hash string `json:"hash"` + ID int64 `json:"id"` + } `json:"data"` + } + if err = json.Unmarshal(raw, &frame); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + if frame.Type == "packet" && frame.Data.Hash != "" { + if err := messages.Encode(map[string]any{"hash": frame.Data.Hash, "id": frame.Data.ID, "received_ns": benchMono(), "bytes": len(raw)}); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + } + } + }() + if e := os.WriteFile(filepath.Join(c.Output, "client-ready"), []byte("ready"), 0600); e != nil { + t.Fatal(e) + } + start, e := benchStart(c) + if e != nil { + t.Fatal(e) + } + client := &http.Client{Timeout: 15 * time.Second} + defer client.CloseIdleConnections() + limit := make(chan struct{}, 64) + var wg sync.WaitGroup + var mu sync.Mutex + var measuredErrors, allErrors int64 + emit := func(sample benchRequest) { + mu.Lock() + defer mu.Unlock() + if sample.Error != "" { + allErrors++ + } + if sample.Measured && sample.Error != "" { + measuredErrors++ + } + if e := requests.Encode(sample); e != nil { + t.Error(e) + } + } + for i := 0; i < (c.Warmup+c.Seconds)*c.HTTPRate; i++ { + endpoint := endpoints[i%len(endpoints)] + sample := benchRequest{Class: endpoint.Class, Scheduled: start + int64(i)*int64(time.Second)/int64(c.HTTPRate), Measured: i >= c.Warmup*c.HTTPRate, Cache: "production cache; individual hit/miss unobserved"} + benchSleep(sample.Scheduled) + select { + case limit <- struct{}{}: + default: + sample.Error = "load generator concurrency cap reached" + sample.Completed = benchMono() + emit(sample) + continue + } + wg.Add(1) + go func(endpoint benchEndpoint, sample benchRequest) { + defer wg.Done() + defer func() { <-limit }() + sample.Dispatched = benchMono() + response, err := client.Get(c.BaseURL + endpoint.Path) + if err != nil { + sample.Error = err.Error() + } else { + sample.Status = response.StatusCode + raw, readErr := io.ReadAll(io.LimitReader(response.Body, (8<<20)+1)) + response.Body.Close() + sample.Bytes = len(raw) + sum := sha256.Sum256(raw) + sample.SHA256 = hex.EncodeToString(sum[:]) + var value any + switch { + case readErr != nil: + sample.Error = readErr.Error() + case len(raw) > 8<<20: + sample.Error = "response exceeded bounded client limit" + case sample.Status == 404 && endpoint.Class == "packet_detail_sql": + if _, err := os.Stat(filepath.Join(c.Output, "retention-started")); err != nil { + sample.Error = "history packet missing before retention" + } else { + sample.Class = "packet_detail_pruned" + sample.ExpectedStatus = 404 + } + case sample.Status != 200: + sample.Error = fmt.Sprintf("HTTP %d", sample.Status) + case json.Unmarshal(raw, &value) != nil: + sample.Error = "invalid JSON response" + case (endpoint.Class == "nodes_50" || endpoint.Class == "messages_0" || endpoint.Class == "messages_1000" || endpoint.Class == "packets_memory") && benchArrayCount(value) != 50: + sample.Error = "bounded page did not return the expected 50 real records" + case endpoint.Class == "nodes_2000" && benchArrayCount(value) != 2000: + sample.Error = "node directory did not return the expected 2000 records" + case endpoint.Class == "healthz" && !benchHealthy(value): + sample.Error = "health endpoint is not fully ready with loaded data" + case endpoint.Class != "healthz" && !benchNonempty(value): + sample.Error = "empty or invalid benchmark result" + } + } + sample.Completed = benchMono() + emit(sample) + }(endpoint, sample) + } + wg.Wait() + // The writer may still be draining its bounded queue. The completion marker + // is authoritative; allow the unchanged poller to catch up afterward. + deadline := time.Now().Add(150 * time.Second) + for time.Now().Before(deadline) { + if _, e := os.Stat(filepath.Join(c.Output, "ingest-validation.json")); e == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + time.Sleep(10 * time.Second) + closing.Store(true) + conn.Close() + <-wsDone + benchJSON(t, filepath.Join(c.Output, "client-validation.json"), map[string]any{"verified": allErrors == 0 && wsErr.Load() == nil, "errors_including_warmup": allErrors, "measured_errors": measuredErrors, "websocket_error": wsErr.Load(), "finished_ns": benchMono()}) + if allErrors > 0 { + t.Fatalf("%d HTTP requests failed validation, including warmup", allErrors) + } + if e := wsErr.Load(); e != nil { + t.Fatal(*e) + } +} diff --git a/scripts/sqlite-benchmark/overlays/sqlite_adapter_test.go b/scripts/sqlite-benchmark/overlays/sqlite_adapter_test.go new file mode 100644 index 000000000..58cebdb78 --- /dev/null +++ b/scripts/sqlite-benchmark/overlays/sqlite_adapter_test.go @@ -0,0 +1,407 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +func TestCoreScopeBenchmarkSmallCorpus(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, Shape: benchShape{Transmissions: 100, Observations: 300, Nodes: 200, Observers: 16, Days: 2}} + if e := benchPrepare(c); e != nil { + t.Fatal(e) + } + db, e := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if e != nil { + t.Fatal(e) + } + defer db.Close() + for table, want := range map[string]int{"transmissions": 100, "observations": 300, "nodes": 200, "observers": 16} { + var n int + if e := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); e != nil { + t.Fatal(e) + } + if n != want { + t.Fatalf("%s=%d want %d", table, n, want) + } + } + var invalid int + if e := db.QueryRow(`SELECT count(*) FROM observations o LEFT JOIN transmissions t ON t.id=o.transmission_id WHERE t.id IS NULL`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatal("generator created orphans") + } + if e := db.QueryRow(`SELECT count(*) FROM observations WHERE score IS NOT NULL AND typeof(score) != 'integer'`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatalf("%d fractional scores would make the unchanged baseline silently discard corpus rows", invalid) + } + assertBenchPathCoverage(t, c, db) + // The handler returns void, so prove its durable effects rather than + // treating delivery to the callback as a successful ingestion. + db.Close() + store, e := OpenStore(c.SQLite) + if e != nil { + t.Fatal(e) + } + defer store.Close() + store.WaitForAsyncMigrations() + f := benchFixtureFor(c) + packet, e := f.packet(1_000_001, 0, c.Epoch) + if e != nil { + t.Fatal(e) + } + body, _ := json.Marshal(map[string]any{"raw": packet.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + handleMessage(store, "bench", MQTTSource{Name: "bench"}, &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: body}, f.channels, nil, &Config{}) + var inserted int + if e := store.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash=?`, packet.Hash).Scan(&inserted); e != nil { + t.Fatal(e) + } + if inserted != 1 || store.Stats.WriteErrors.Load() != 0 { + t.Fatal("handler callback did not produce a durable synthetic transmission") + } +} + +func TestCoreScopeBenchmarkNullableObservationIdentity(t *testing.T) { + dir := t.TempDir() + // This includes the first collision in the real B key space without + // requiring a two-million-row fixture for the regression itself. + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, + Shape: benchShape{Transmissions: 610, Observations: 9760, Nodes: 2000, Observers: 128, Days: 8}} + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var rows, missing, duplicateKeys int + if err := db.QueryRow(`SELECT count(*),sum(observer_idx IS NULL) FROM observations`).Scan(&rows, &missing); err != nil { + t.Fatal(err) + } + if rows != 9760 || missing != 366 { + t.Fatalf("generator changed fanout/NULL coverage: observations=%d, NULL observers=%d", rows, missing) + } + if err := db.QueryRow(`SELECT count(*) FROM ( + SELECT o.transmission_id,coalesce(obs.id,''),o.path_json FROM observations o + LEFT JOIN observers obs ON obs.rowid=o.observer_idx + GROUP BY o.transmission_id,coalesce(obs.id,''),o.path_json HAVING count(*)>1 + )`).Scan(&duplicateKeys); err != nil { + t.Fatal(err) + } + if duplicateKeys != 0 { + t.Fatalf("%d generated observer/path keys would be dropped by the unchanged upstream loader", duplicateKeys) + } + assertBenchPathCoverage(t, c, db) +} + +// Check the generated data independently of the generator's counters. Full and +// chunked loading can visit equally long observations in opposite ID orders. +// Both canonical choices need actual retained routes for the requested node. +func assertBenchPathCoverage(t *testing.T, c benchConfig, db *sql.DB) { + t.Helper() + data, err := os.ReadFile(filepath.Join(c.Output, "corpus.json")) + if err != nil { + t.Fatal(err) + } + var info struct { + Node string `json:"node"` + PathsNode string `json:"paths_node"` + } + if err := json.Unmarshal(data, &info); err != nil { + t.Fatal(err) + } + if info.PathsNode == "" { + info.PathsNode = info.Node + } + var first, last int + err = db.QueryRow(`WITH ranked AS ( + SELECT json_extract(o.resolved_path,'$[0]') AS pubkey, + row_number() OVER (PARTITION BY t.id ORDER BY json_array_length(o.path_json) DESC,o.id ASC) AS first_rank, + row_number() OVER (PARTITION BY t.id ORDER BY json_array_length(o.path_json) DESC,o.id DESC) AS last_rank + FROM observations o JOIN transmissions t ON t.id=o.transmission_id + WHERE o.resolved_path IS NOT NULL AND json_array_length(o.path_json)>0 AND t.first_seen>? + ) SELECT COALESCE(SUM(first_rank=1),0),COALESCE(SUM(last_rank=1),0) FROM ranked WHERE pubkey=?`, + time.Unix(c.Epoch-7*86400, 0).UTC().Format(time.RFC3339), info.PathsNode).Scan(&first, &last) + if err != nil || first == 0 || last == 0 { + t.Fatalf("paths target has no retained canonical route for a load order: first=%d last=%d err=%v", first, last, err) + } +} + +func benchOpen(c benchConfig) (*Store, error) { return OpenStore(c.SQLite) } +func benchSettings(s *Store) (map[string]string, error) { + out := map[string]string{} + for _, name := range []string{"journal_mode", "synchronous", "foreign_keys", "busy_timeout", "cache_size", "page_size", "mmap_size", "cache_spill", "journal_size_limit", "wal_autocheckpoint"} { + var v string + if e := s.db.QueryRow("PRAGMA " + name).Scan(&v); e != nil { + return nil, e + } + out[name] = v + } + var version string + if e := s.db.QueryRow(`SELECT sqlite_version()`).Scan(&version); e != nil { + return nil, e + } + out["sqlite_version"] = version + if out["journal_mode"] != "wal" || out["synchronous"] != "2" || out["foreign_keys"] != "1" { + return nil, fmt.Errorf("SQLite durability differs from WAL/FULL/FK contract") + } + return out, nil +} + +func benchExplain(s *Store, query string, args ...any) (any, error) { + rows, e := s.db.Query("EXPLAIN QUERY PLAN "+query, args...) + if e != nil { + return nil, e + } + defer rows.Close() + type step struct { + ID int `json:"id"` + Parent int `json:"parent"` + Detail string `json:"detail"` + } + var result []step + for rows.Next() { + var item step + var unused int + if e := rows.Scan(&item.ID, &item.Parent, &unused, &item.Detail); e != nil { + return nil, e + } + result = append(result, item) + } + return result, rows.Err() +} + +func benchNodeRole(index int) string { + return [...]string{"repeater", "companion", "room", "sensor"}[index%4] +} + +func benchPrepare(c benchConfig) error { + if _, e := os.Stat(c.SQLite); !os.IsNotExist(e) { + return fmt.Errorf("corpus path must be new") + } + s, e := OpenStore(c.SQLite) + if e != nil { + return e + } + defer s.Close() + s.WaitForAsyncMigrations() + f := benchFixtureFor(c) + tx, e := s.db.Begin() + if e != nil { + return e + } + defer tx.Rollback() + stamp := time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339) + exec := func(q string, args ...any) error { _, e := tx.Exec(q, args...); return e } + for i, pk := range f.public { + role := benchNodeRole(i) + if e := exec(`INSERT INTO nodes(public_key,name,role,lat,lon,first_seen,last_seen,advert_count) VALUES(?,?,?,?,?,?,?,0)`, pk, fmt.Sprintf("Synthetic %04d", i), role, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, stamp, stamp); e != nil { + return e + } + } + for i := 0; i < c.Shape.Observers; i++ { + if e := exec(`INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count,noise_floor,can_relay,can_relay_seen) VALUES(?,?,?,?,?,?,0,-110.5,1,1)`, i+1, f.public[i], fmt.Sprintf("Synthetic observer %d", i), []string{"AAA", "BBB", "CCC", "DDD"}[i%4], stamp, stamp); e != nil { + return e + } + } + transmission, e := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey,last_seen,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer transmission.Close() + observation, e := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,raw_hex,resolved_path) VALUES(?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer observation.Close() + evidence, e := tx.Prepare(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(?,?)`) + if e != nil { + return e + } + defer evidence.Close() + var observationCount int64 + firstPaths, lastPaths := make(map[string]int), make(map[string]int) + countsByDay := make([]int, c.Shape.Days) + var newest, oldest string + for i := 0; i < c.Shape.Transmissions; i++ { + at := benchStamp(c, i) + p, e := f.packet(i, 0, at) + if e != nil { + return e + } + count := benchFanout(c, i) + if _, e = transmission.Exec(i+1, p.RawHex, p.Hash, p.Timestamp, p.RouteType, p.PayloadType, p.PayloadVersion, p.DecodedJSON, p.ChannelHash, nilIfEmpty(p.FromPubkey), at+int64(count-1), stamp); e != nil { + return e + } + bits := packetpath.AdvertRouteEvidence(p.RawHex) + observationKeys := make(map[string]bool, count) + longestBytes, firstPath, lastPath := -1, "", "" + for o := 0; o < count; o++ { + generated, e := f.observation(i, o, at, observationKeys) + if e != nil { + return e + } + seen, observer := generated.Packet, generated.ObserverIndex + var snr any = float64((i+o)%20) - 10.5 + if o%11 == 10 { + snr = nil + } + var resolved any + var resolvedPK string + if i%3 != 0 && seen.PathJSON != "[]" { + resolvedPK = f.public[(i+generated.PathVariant*7+1)%len(f.public)] + b, _ := json.Marshal([]string{resolvedPK}) + resolved = string(b) + } + // Corpus variants keep one prefix width per transmission. JSON + // length therefore orders hop counts without decoding the path again. + if resolvedPK != "" { + if size := len(seen.PathJSON); size > longestBytes { + longestBytes, firstPath, lastPath = size, resolvedPK, resolvedPK + } else if size == longestBytes { + lastPath = resolvedPK + } + } + if _, e = observation.Exec(i+1, observer, "rx", snr, -100.5+float64(o%25), p.Score, seen.PathJSON, at+int64(o), seen.RawHex, resolved); e != nil { + return e + } + observationCount++ + bits |= packetpath.AdvertRouteEvidence(seen.RawHex) + } + if at > c.Epoch-7*86400 && firstPath != "" { + firstPaths[firstPath]++ + lastPaths[lastPath]++ + } + if p.PayloadType == 4 { + for _, bit := range []uint8{1, 2} { + if bits&bit != 0 { + if _, e = evidence.Exec(i+1, bit); e != nil { + return e + } + } + } + } + countsByDay[(i/10)%c.Shape.Days]++ + if i == 0 { + newest = p.Hash + } + if i == 70 { + oldest = p.Hash + } + } + if observationCount != int64(c.Shape.Observations) { + return fmt.Errorf("fanout generated %d observations, expected %d", observationCount, c.Shape.Observations) + } + if e := exec(`UPDATE nodes SET advert_count=(SELECT count(*) FROM transmissions t WHERE t.payload_type=4 AND t.from_pubkey=nodes.public_key)`); e != nil { + return e + } + for i := 0; i < c.Shape.Nodes; i++ { + if e := exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES(?,?,?,?)`, f.public[i], f.public[(i+1)%len(f.public)], 1+i%100, stamp); e != nil { + return e + } + } + metric, e := tx.Prepare(`INSERT INTO observer_metrics(observer_id,timestamp,noise_floor,tx_air_secs,rx_air_secs,recv_errors,battery_mv,packets_sent,packets_recv) VALUES(?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer metric.Close() + for observer := 0; observer < c.Shape.Observers; observer++ { + for sample := 0; sample < c.Shape.Days*288; sample++ { + if observer == c.Shape.Observers-1 && sample%48 != 0 { + continue + } + at := time.Unix(c.Epoch-int64(c.Shape.Days*86400)+int64(sample*300), 0).UTC().Format(time.RFC3339) + if _, e := metric.Exec(f.public[observer], at, -110.5+float64(observer%5), sample*2, sample*9, sample%3, 3700+observer%200, sample*3, sample*20); e != nil { + return e + } + } + } + // Every enabled mobile feature has deterministic data; this is the base + // corpus extension, not a claim to have run the optional million-RX corpus. + reception, e := tx.Prepare(`INSERT INTO client_receptions(rx_pubkey,heard_key,heard_keylen,rssi,snr,lat,lon,pos_acc_m,rx_at,ingested_at,src) VALUES(?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer reception.Close() + for i := 0; i < c.Shape.Transmissions; i++ { + key := f.public[i%len(f.public)] + size := []int{32, 8, 3, 2}[i%4] + at := time.Unix(benchStamp(c, i), int64(i)).UTC().Format("2006-01-02T15:04:05.000000000Z") + if _, e := reception.Exec(f.public[(i+100)%len(f.public)], key[:size*2], size, -95, 6.5, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, 5, at, stamp, "synthetic"); e != nil { + return e + } + } + for i := 0; i < 100; i++ { + pk := f.public[i%len(f.public)] + at := time.Unix(c.Epoch-int64(i*300), 0).UTC().Format(time.RFC3339) + if e := exec(`INSERT INTO client_observers(pubkey,name,last_seen) VALUES(?,?,?)`, pk, fmt.Sprintf("Synthetic mobile %d", i), stamp); e != nil { + return e + } + if e := exec(`INSERT INTO node_declared_regions(target,rx_pubkey,observed_at,ingested_at,regions_csv,truncated,lat,lon) VALUES(?,?,?,?,?,0,20,30)`, pk, pk, at, stamp, "#bench"); e != nil { + return e + } + if e := exec(`INSERT INTO client_rf_samples(rx_pubkey,sampled_at,ingested_at,lat,lon,stationary,uptime_secs,noise_floor,last_snr) VALUES(?,?,?,20,30,0,1000,-110,6.5)`, pk, at, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO dropped_packets(hash,raw_hex,reason,observer_id,dropped_at) VALUES(?,?,?,?,?)`, fmt.Sprintf("synthetic-drop-%d", i), "", "synthetic-rejection", pk, stamp); e != nil { + return e + } + } + if e := exec(`INSERT INTO client_rx_observations(rx_pubkey,rx_at,ingested_at,pkt_hash,route_type,payload_type,hash_size,hop_count,lat,lon) SELECT ?,first_seen,?,hash,route_type,payload_type,1,0,20,30 FROM transmissions WHERE id<=100`, f.public[0], stamp); e != nil { + return e + } + if e := exec(`INSERT INTO inactive_nodes(public_key,name,role,first_seen,last_seen,advert_count) VALUES('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff','Synthetic inactive','repeater',?,?,0)`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO advert_evidence_backfill(id,tx_cursor,obs_cursor) VALUES(1,?,?)`, c.Shape.Transmissions, c.Shape.Observations); e != nil { + return e + } + if e := exec(`UPDATE _async_migrations SET started_at=?,ended_at=?`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT OR REPLACE INTO _async_migrations(name,status,started_at,ended_at) VALUES('advert_route_evidence_v1','done',?,?)`, stamp, stamp); e != nil { + return e + } + if e := exec(`UPDATE scope_match_totals SET since_unix=?,updated_unix=?`, c.Epoch, c.Epoch); e != nil { + return e + } + if e := tx.Commit(); e != nil { + return e + } + if _, e := s.db.Exec(`ANALYZE`); e != nil { + return e + } + // The B fanout recipe can leave node zero absent from every canonical + // best observation even though it occurs in other stored observations. + // Select a deterministic repeater with retained routes under both tie + // orders used by full/chunked loading. Existing reach/RX targets stay fixed. + pathsNode := "" + for i, pk := range f.public { + if benchNodeRole(i) == "repeater" && firstPaths[pk] > 0 && lastPaths[pk] > 0 { + pathsNode = pk + break + } + } + if pathsNode == "" { + return fmt.Errorf("corpus has no retained repeater paths under both observation tie orders") + } + summary := map[string]any{"transmissions": c.Shape.Transmissions, "observations": observationCount, "transmissions_by_day": countsByDay, "nodes": c.Shape.Nodes, "observers": c.Shape.Observers, "channels": 20, "rx_rows": c.Shape.Transmissions, "dense_observer": f.public[0], "sparse_observer": f.public[c.Shape.Observers-1], "node": f.public[0], "paths_node": pathsNode, "paths_first_order_count": firstPaths[pathsNode], "paths_last_order_count": lastPaths[pathsNode], "new_packet": newest, "old_packet": oldest, "epoch": c.Epoch, "protocol_source": "meshcore-dev/MeshCore@a366955cb2f67b8e6842d4f00d2b6a554dddd88a"} + b, e := json.MarshalIndent(summary, "", " ") + if e != nil { + return e + } + return os.WriteFile(filepath.Join(c.Output, "corpus.json"), b, 0600) +} + +var _ *sql.DB // database/sql remains the same application boundary on both sides. diff --git a/scripts/sqlite-benchmark/run.py b/scripts/sqlite-benchmark/run.py new file mode 100644 index 000000000..c5e5f35ed --- /dev/null +++ b/scripts/sqlite-benchmark/run.py @@ -0,0 +1,1171 @@ +#!/usr/bin/env python3 +"""Isolated, serial, source-pinned CoreScope application comparison. + +Both archived revisions use native CGO SQLite. A successful invocation means +validation completed; it does not establish a general speed or capacity claim. +""" +import argparse +from contextlib import closing +import csv +import decimal +import datetime +import errno +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import socket +import sqlite3 +import subprocess +import sys +import tarfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +UPSTREAM = "7fa86fe1c1304b88e0f2a44052465dfb86abb8a6" +FIRMWARE = "a366955cb2f67b8e6842d4f00d2b6a554dddd88a" +HERE = Path(__file__).resolve().parent +PROCESSES = [] +RESOURCE_COUNTERS = ("cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat") +# Common to both engines: B itself accounts for about 1,222 MiB before replay. +SERVER_MEMORY_MIB = 3072 +PACKET_STORE_MIB = 2048 +SQLITE_IDENTITY_MARKER = "observers_identity_autoincrement_v1" +TABLES = ["nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"] +# These are operational receipt/creation times, not packet timestamps. Full +# pre-run parity includes them; only post-replay logical parity excludes them. +VOLATILE = {"transmissions": {"created_at"}, "observers": {"last_seen", "last_packet_at", "first_seen"}, "_async_migrations": {"started_at", "ended_at"}, "scope_match_totals": {"updated_unix", "since_unix"}} +TEXT_TIMES = {"first_seen", "last_seen", "created_at", "last_packet_at", "clock_last_naive_at", "configured_scope_at", "timestamp", "dropped_at", "rx_at", "ingested_at", "sampled_at", "observed_at", "started_at", "ended_at", "done_at"} + + +def shift_timestamp(value, delta): + if value is None or value == "": + return value + if not re.fullmatch(r"\d{4}-\d\d-\d\d[T ]\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)?", value): + raise ValueError("unsupported synthetic timestamp") + head = datetime.datetime.strptime(value[:19].replace("T", " "), "%Y-%m-%d %H:%M:%S") + datetime.timedelta(seconds=delta) + return head.strftime("%Y-%m-%d" + value[10] + "%H:%M:%S") + value[19:] + + +def freshen(path, delta): + """One recorded receipt-time offset; signed wire bytes remain unchanged.""" + tables = table_manifest(path) + db = sqlite3.connect(path) + try: + db.create_function("bench_shift", 1, lambda value: shift_timestamp(value, delta)) + for entry in tables: + table = entry["name"] + columns = list(db.execute(f"PRAGMA table_info({quote_identifier(table)})")) + updates = [] + for _, name, kind, *_ in columns: + if name not in TEXT_TIMES and not (table == "scope_match_totals" and name in ("since_unix", "updated_unix")): + continue + if kind.upper() in ("INTEGER", "INT", "BIGINT"): + updates.append(f'"{name}"="{name}"+{int(delta)}') + elif kind.upper() in ("TEXT", "DATETIME"): + updates.append(f'"{name}"=bench_shift("{name}")') + if updates: + db.execute(f"UPDATE {quote_identifier(table)} SET " + ",".join(updates)) + db.commit() + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + finally: + db.close() + + +def corpus(name): + return {"S": dict(transmissions=30_000, observations=90_000, nodes=2000, observers=32, days=8), "B": dict(transmissions=128_000, observations=2_048_000, nodes=2000, observers=128, days=8), "L": dict(transmissions=1_125_000, observations=18_000_000, nodes=5000, observers=512, days=30)}[name].copy() + + +def fanout(name, index): + return index % 5 + 1 if name == "S" else (4 if index % 10 < 5 else 22 if index % 10 < 7 else 23 if index % 10 < 9 else 50) + + +def pair_order(pairs): + return [(i, variant) for i in range(pairs) for variant in (("baseline", "candidate") if i % 2 == 0 else ("candidate", "baseline"))] + + +def percentiles(values): + values = sorted(values) + def q(p): + return values[max(0, math.ceil(p * len(values)) - 1)] if values else None + return dict(samples=len(values), p50=q(.50), p95=q(.95), p99=q(.99) if len(values) >= 1000 else None, maximum=q(1)) + + +def visibility(ingest, messages): + sent = {e["hash"]: e for e in ingest if e.get("new_transmission") and not e.get("error") and not e.get("dropped")} + measured = {key: event for key, event in sent.items() if event.get("measured")} + received, duplicate_receipts, duplicate_receipts_all = {}, 0, 0 + for message in messages: + key = message.get("hash") + if key not in sent: + continue + if key in received: + duplicate_receipts_all += 1 + duplicate_receipts += int(key in measured) + received[key] = min(received.get(key, message["received_ns"]), message["received_ns"]) + missing = sorted(set(sent) - set(received)) + lags = [received[key] - event["completed_ns"] for key, event in measured.items() if key in received] + return dict(verified=not missing, verification_scope="full stream including warmup", sample_scope="measured events only", + sentinels=len(sent), received=len(received), missing=len(missing), missing_hashes=missing[:100], + measured_sentinels=len(measured), measured_received=len(measured.keys() & received.keys()), measured_missing=len(measured.keys() - received.keys()), + duplicate_hash_receipts=duplicate_receipts, duplicate_hash_receipts_all=duplicate_receipts_all, + visible_before_ingest_return=sum(x < 0 for x in lags), + visible_before_ingest_return_all=sum(received[key] < event["completed_ns"] for key, event in sent.items() if key in received), + lag_ns=percentiles(lags), samples_ns=lags) + + +def json_lines(path): + with open(path) as stream: + for line in stream: + yield json.loads(line) + + +def canonical_row(row): + out = [] + for value in row: + if value is None: + out.append(["null"]) + elif isinstance(value, (int, float, decimal.Decimal)) and not isinstance(value, bool): + number = decimal.Decimal(str(value)) + if not number.is_finite(): + raise ValueError("non-finite number in corpus") + out.append(["number", str(number.normalize()) if number else "0"]) + elif isinstance(value, str): + out.append(["text", value]) + else: + raise ValueError("unsupported corpus cell type") + return (json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n").encode() + + +def safe_output(path): + path = Path(path).absolute() + if not path.name.startswith("corescope-bench-") or path.exists() or path.is_symlink(): + raise ValueError("output must be a new directory named corescope-bench-") + if not path.parent.is_dir(): + raise ValueError("output parent must already exist") + return path.resolve() + + +def check_revisions(baseline, candidate): + if not all(re.fullmatch(r"[0-9a-f]{40}", ref) for ref in (baseline, candidate)): + raise ValueError("baseline and candidate must be complete lowercase 40-character commit SHAs") + + +def validate_profile(profile, pairs, cgroup): + if profile == "primary" and (pairs != 5 or not cgroup): + raise ValueError("primary requires five paired runs and a delegated CPU/memory cgroup") + if profile == "diagnostic" and (pairs != 1 or not cgroup): + raise ValueError("diagnostic requires one pair and the same delegated CPU/memory cgroup") + if not 1 <= pairs <= 10: + raise ValueError("pairs must be between one and ten") + + +def write_json(path, value): + Path(path).parent.mkdir(parents=True, exist_ok=True) + temporary = Path(str(path) + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def safe_diagnostics(text): + """Keep diagnostic lines only; never publish arbitrary child log tails.""" + out = [] + for line in text.splitlines(): + line = line.strip() + if line.startswith("[migrate] "): + line = "error: " + line.removeprefix("[migrate] ") + if not re.match(r"(?:--- FAIL:|FAIL\b|panic:|fatal error:|[^\s]*\.go:\d+(?::\d+)?:|ERROR:|FATAL:|error:|benchmark:)", line): + codes = re.findall(r"SQLSTATE [0-9A-Z]{5}", line) + if codes: + out.extend(codes) + continue + if line.startswith("panic:") and not line.startswith("panic: runtime error:"): + line = "panic: " + elif re.search(r"\b(?:host|hostaddr|user|dbname|password|sslmode)\s*=", line, re.I): + line = "database connection error: " + else: + line = re.sub(r"[a-zA-Z][a-zA-Z0-9+.-]*://\S+", "", line) + line = re.sub(r"\b[A-Za-z]:[\\/]\S+|(?", line) + line = re.sub(r"\{.*\}|\[.*\]", "", line) + line = re.sub(r'"[^"\n]*"|\x27[^\x27\n]*\x27', "", line) + line = re.sub(r"\b(?:raw|payload|packet|message|body|authorization|cookie|password|passwd|secret|token|dsn|database_url|credentials?)\s*[:=].*", "", line, flags=re.I) + line = re.sub(r"\b[0-9a-fA-F]{16,}\b|(?", line) + out.append(line[:240]) + return out[-10:] + + + + + +def record_failure(log, exit_code): + log = Path(log) + with open(log, "rb") as stream: + stream.seek(max(0, log.stat().st_size - 8192)) + tail = stream.read(8192).decode("utf-8", errors="replace") + result = dict(log=log.name, exit_code=exit_code, diagnostics=safe_diagnostics(tail), policy="At most 10 sanitized diagnostic lines from the final 8 KiB; raw logs and event bodies excluded") + write_json(log.with_name("failure-" + log.stem + ".json"), result) + return result + + +def record_resource_failure(path, error, **context): + filename = getattr(error, "filename", None) + name = Path(filename).name if filename else None + known = (*RESOURCE_COUNTERS, "memory.peak", "smaps_rollup", "comm", "resources.csv", "processes.csv", "resource-final.json", "resource-error.json") + detail = dict(error_type=type(error).__name__, errno=getattr(error, "errno", None), + counter=name if name in known else None, **context) + try: + write_json(path, detail) + except OSError as write_error: + # A full/unwritable output filesystem must not replace the original + # workload exception with another cleanup exception. + detail["evidence_write_error_type"] = type(write_error).__name__ + detail["evidence_write_errno"] = write_error.errno + return detail + + +def public_bundle(output, repository=None): + """Explicit whitelist: never export databases, event bodies or controls.""" + output = Path(output).resolve() + public = output / "public" + public.mkdir(exist_ok=True) + patterns = ["manifest.json", "resource-preflight.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/resource-error.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*", "pair-*/*/db-pool.json", "pair-*/*/profiles/*-top.txt", "pair-*/*/profile-capture.json"] + for pattern in patterns: + for source in sorted(output.glob(pattern)): + if not source.is_file() or source.is_symlink(): + continue + destination = public / source.relative_to(output) + destination.parent.mkdir(parents=True, exist_ok=True) + text = source.read_text() + text = text.replace(str(output), "") + if repository: + text = text.replace(str(Path(repository).resolve()), "") + destination.write_text(text) + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def remove_working_directory(path, runtime_root): + path, runtime_root = Path(path).resolve(), Path(runtime_root).resolve() + if runtime_root.name != ".runtime" or not path.is_relative_to(runtime_root) or not re.fullmatch(r"config-\d+-(baseline|candidate)", path.name): + raise ValueError("refusing cleanup outside an owned benchmark run") + shutil.rmtree(path) + + +def command(args, *, cwd=None, env=None, log=None, input=None, timeout=900, budget=None): + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdin=subprocess.PIPE, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + try: + result, _ = proc.communicate(input, timeout=timeout) + except BaseException: + stop(proc) + raise + if log: + Path(log).write_text(result) + if proc.returncode: + if not log: + diagnostic="; ".join(safe_diagnostics(result[-8192:])) or "no recognized safe diagnostic" + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}): {diagnostic}") + record_failure(log, proc.returncode) + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}); see failure-{Path(log).stem}.json") + return result + + +def production_hash(root): + h = hashlib.sha256() + for folder in ("cmd", "internal", "public"): + for path in sorted((Path(root) / folder).rglob("*")): + if path.is_file() and not path.name.endswith("_test.go"): + h.update(path.relative_to(root).as_posix().encode() + b"\0") + h.update(bytes.fromhex(sha256(path))) + return h.hexdigest() + + +def archive(repo, revision, destination): + actual = command(["git", "rev-parse", revision + "^{commit}"], cwd=repo).strip() + if actual != revision: + raise ValueError("revision did not resolve to the exact requested commit") + destination.mkdir() + proc = subprocess.Popen(["git", "archive", "--format=tar", revision], cwd=repo, stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=proc.stdout, mode="r|") as archive_file: + archive_file.extractall(destination, filter="data") + finally: + proc.stdout.close() + if proc.wait() != 0: + raise RuntimeError("git archive failed") + + +class Budget: + def __init__(self, parent, required): + self.path = None + self._memory_peak_file = None + self.required = required + self.info = {"enforced": False, "cpu_limit": None, "memory_limit_bytes": None} + if not parent: + return + parent = Path(parent).resolve() + if not str(parent).startswith("/sys/fs/cgroup/") or parent == Path("/sys/fs/cgroup"): + raise ValueError("cgroup must be a delegated subtree, not the global cgroup root") + self.path = parent / ("corescope-bench-" + str(os.getpid())) + self.path.mkdir() + try: + (self.path / "cpu.max").write_text("300000 100000") + (self.path / "memory.max").write_text(str(6 * 1024 ** 3)) + if (self.path / "memory.swap.max").exists(): + (self.path / "memory.swap.max").write_text("0") + cpu, memory = 3.0, 6 * 1024 ** 3 + for ancestor in (self.path, *self.path.parents): + if ancestor == Path("/sys/fs"): + break + if (ancestor / "cpu.max").exists(): + q, period = (ancestor / "cpu.max").read_text().split() + if q != "max": + cpu = min(cpu, int(q) / int(period)) + if (ancestor / "memory.max").exists(): + value = (ancestor / "memory.max").read_text().strip() + if value != "max": + memory = min(memory, int(value)) + if cpu < 3 or memory < 6 * 1024 ** 3: + raise ValueError("ancestor cgroup limits are below the shared 3-CPU/6-GiB budget") + cpuset = self.path / "cpuset.cpus.effective" + allowed = set(os.sched_getaffinity(0)) + if cpuset.exists(): + effective = set() + for part in cpuset.read_text().strip().split(","): + bounds = [int(x) for x in part.split("-")] + effective.update(range(bounds[0], bounds[-1] + 1)) + allowed &= effective + if len(allowed) < 3: + raise ValueError("effective CPU affinity provides fewer than three application CPUs") + self.info = dict(enforced=True, cpu_limit=cpu, memory_limit_bytes=memory, allowed_cpus=sorted(allowed)) + except Exception: + self.path.rmdir() + self.path = None + raise + + def preflight(self): + if self.path: + for name in RESOURCE_COUNTERS: + (self.path / name).read_text() + self.info["resource_counters"] = list(RESOURCE_COUNTERS) + + def arguments(self, args): + if not self.path: + return args + # No Python preexec_fn: the resource sampler is threaded. Arguments are + # positional shell parameters, never interpolated into shell source. + return ["/bin/sh", "-c", 'printf "%s" "$$" > "$1/cgroup.procs" || exit 125; shift; exec "$@"', "corescope-bench", str(self.path), *args] + + def verify(self, pid): + if self.path: + for _ in range(100): + if str(pid) in (self.path / "cgroup.procs").read_text().split(): + return + time.sleep(.01) + raise RuntimeError("application/database process failed to enter the benchmark cgroup") + + def reset(self): + self.close_memory_peak() + self.info["memory_peak_reset"] = False + if not self.path: + return + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("previous variant is still active; serial comparison refused") + reclaim = self.path / "memory.reclaim" + if not reclaim.exists(): + raise RuntimeError("primary cgroup needs memory.reclaim to remove inactive prior-variant charges") + amount = int((self.path / "memory.current").read_text()) + if amount: + try: + reclaim.write_text(str(amount)) + except OSError as error: + if error.errno != errno.EAGAIN: + raise + remaining = int((self.path / "memory.current").read_text()) + if remaining > 64 * 1024 ** 2: + raise RuntimeError("more than 64 MiB of inactive prior-variant cgroup memory remains") + self.info["inactive_charge_reset_bytes"] = remaining + try: + # Linux resets memory.peak only for subsequent reads through this + # same open file description, not later opens of the counter. + self._memory_peak_file = (self.path / "memory.peak").open("r+b", buffering=0) + self._memory_peak_file.write(b"0") + self.info["memory_peak_reset"] = True + except OSError: + self.close_memory_peak() + + def memory_peak(self): + if self._memory_peak_file is None: + return None + self._memory_peak_file.seek(0) + return int(self._memory_peak_file.read()) + + def close_memory_peak(self): + if self._memory_peak_file is not None: + self._memory_peak_file.close() + self._memory_peak_file = None + + def close(self): + self.close_memory_peak() + if self.path: + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("owned benchmark processes remain; refusing cgroup removal") + self.path.rmdir() + + +def spawn(args, *, cwd, env, log, budget=None): + stream = open(log, "w") + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + proc.bench_log = stream + if budget: + budget.verify(proc.pid) + return proc + + +def finish(proc, timeout=900): + try: + result = proc.wait(timeout=timeout) + if result: + record_failure(proc.bench_log.name, result) + raise RuntimeError(f"benchmark child exited {result}; see failure-{Path(proc.bench_log.name).stem}.json") + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + proc.bench_log.close() + + +def stop(proc): + if proc and proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + if proc and getattr(proc, "bench_log", None): + proc.bench_log.close() + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + + + + +def quote_identifier(name): + return '"' + name.replace('"', '""') + '"' + + +def table_manifest(source, required=None): + with closing(sqlite3.connect(f"file:{source}?mode=ro", uri=True)) as db: + tables = [row[0] for row in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT GLOB 'sqlite_*' ORDER BY name")] + for table in TABLES if required is None else required: + if table not in tables: + raise ValueError("corpus lacks canonical table " + table) + manifest = [] + for table in tables: + columns = [row[1] for row in db.execute(f"PRAGMA table_info({quote_identifier(table)})")] + if not columns: + raise ValueError("corpus lacks canonical table " + table) + if table == "observers" and "rowid" not in columns: + columns.insert(0, "rowid") + manifest.append(dict(name=table, columns=columns)) + return manifest + + +def validate(source, manifest, logical=False): + if table_manifest(source, required=()) != sorted(manifest, key=lambda table: table["name"]): + raise RuntimeError("application schema differs from the complete corpus manifest") + result = {} + with closing(sqlite3.connect(f"file:{source}?mode=ro", uri=True)) as db: + for table in manifest: + name = table["name"] + columns = [c for c in table["columns"] if not logical or c not in VOLATILE.get(name, set())] + select = ",".join(quote_identifier(c) for c in columns) + digest, count = hashlib.sha256(), 0 + for row in db.execute(f"SELECT {select} FROM {quote_identifier(name)} ORDER BY {select}"): + digest.update(canonical_row(row)) + count += 1 + result[name] = dict(rows=count, sha256=digest.hexdigest()) + return result + + +def requires_storage_selection(root): + source = root / "cmd/server/storage_config.go" + return source.is_file() and "dbconfig.OpenSelection(" in source.read_text(encoding="utf-8") + + +def migration_names(source): + db = sqlite3.connect(source.resolve().as_uri() + "?mode=ro", uri=True) + try: + if [row[1] for row in db.execute("PRAGMA table_info(_migrations)")] != ["name"]: + raise RuntimeError("unsupported benchmark migration ledger schema") + names = [row[0] for row in db.execute("SELECT name FROM _migrations ORDER BY name")] + if any(not isinstance(name, str) for name in names): + raise RuntimeError("unsupported benchmark migration ledger value") + return names + finally: + db.close() + + +def comparable_validation(source, full, setup): + # Only this exact physical-layout marker may differ. Every original entry, + # extra entry and full application-table digest remains checked. + expected = list(setup["migrations_before"]) + if setup["required"] and SQLITE_IDENTITY_MARKER not in expected: + expected.append(SQLITE_IDENTITY_MARKER) + if migration_names(source) != sorted(expected): + raise RuntimeError("unexpected migration ledger change during storage setup or replay") + return dict(full, _migrations=setup["before"]["_migrations"]) + + +def prepare_storage(root, binary, config_dir, tables, before, env, log, budget=None): + source = config_dir / "telemetry.sqlite" + required = requires_storage_selection(root) + proof = dict(required=required, before=before, migrations_before=migration_names(source), elapsed_ns=0) + started = time.monotonic_ns() + if required: + command([binary, "-storage-action=adopt", "-backend=sqlite", "-offline", + "-selection-file", str(config_dir / "state/storage-selection.json"), + "-config-dir", str(config_dir), "-sqlite-path", str(source)], + cwd=root, env=env, log=log, budget=budget) + proof["after"] = validate(source, tables) + proof["elapsed_ns"] = time.monotonic_ns() - started + else: + proof["after"] = before + if comparable_validation(source, proof["after"], proof) != before: + raise RuntimeError("storage setup changed canonical application data") + return proof + + +class Resources: + def __init__(self, path, budget, data_paths): + self.path, self.budget, self.data_paths = path, budget, data_paths + self.phase="prepare" + self.done = threading.Event() + self.error = None + self.thread = threading.Thread(target=self.sample, daemon=True) + + def sample(self): + try: + self.sample_rows() + except BaseException as error: + self.error = error + + def sample_rows(self): + with open(self.path, "w", newline="") as stream, open(self.path.parent/"processes.csv","w",newline="") as processes: + process_writer=csv.DictWriter(processes,fieldnames=["mono_ns","phase","pid","command","rss_bytes","pss_bytes"]) + process_writer.writeheader() + writer = csv.DictWriter(stream, fieldnames=["mono_ns", "phase", "filesystem_free_bytes", "cgroup_memory_bytes", "cgroup_memory_peak_bytes", "cgroup_cpu_usec", "rss_bytes", "pss_bytes", "storage_bytes", "wal_bytes", "io_stat", "memory_events"]) + writer.writeheader() + while not self.done.is_set(): + row = dict(mono_ns=time.monotonic_ns(), phase=self.phase, filesystem_free_bytes=shutil.disk_usage(self.path.parent).free, cgroup_memory_bytes="not measured", cgroup_memory_peak_bytes="not measured", cgroup_cpu_usec="not measured", rss_bytes="not measured", pss_bytes="not measured", storage_bytes=0, wal_bytes=0, io_stat="not measured", memory_events="not measured") + if self.budget.path: + c = self.budget.path + row["cgroup_memory_bytes"] = (c / "memory.current").read_text().strip() + peak = self.budget.memory_peak() + if peak is not None: + row["cgroup_memory_peak_bytes"] = peak + cpu = dict(line.split() for line in (c / "cpu.stat").read_text().splitlines()) + row["cgroup_cpu_usec"] = cpu.get("usage_usec", "not measured") + row["io_stat"] = (c / "io.stat").read_text().strip() + row["memory_events"] = (c / "memory.events").read_text().strip() + rss = pss = 0 + for pid in (c / "cgroup.procs").read_text().split(): + try: + process_rss=process_pss=0 + for line in Path("/proc", pid, "smaps_rollup").read_text().splitlines(): + if line.startswith("Rss:"): + process_rss += int(line.split()[1]) * 1024 + if line.startswith("Pss:"): + process_pss += int(line.split()[1]) * 1024 + rss+=process_rss;pss+=process_pss + process_writer.writerow(dict(mono_ns=row["mono_ns"],phase=self.phase,pid=pid,command=Path("/proc",pid,"comm").read_text().strip(),rss_bytes=process_rss,pss_bytes=process_pss)) + except (FileNotFoundError, PermissionError, ProcessLookupError): + pass + row.update(rss_bytes=rss, pss_bytes=pss) + for root in self.data_paths: + for file in root.rglob("*") if root.is_dir() else [root, Path(str(root) + "-wal")]: + try: + if file.is_file(): + size = file.stat().st_size + row["storage_bytes"] += size + if file.name.endswith("-wal"): + row["wal_bytes"] += size + except FileNotFoundError: + pass + writer.writerow(row) + stream.flush() + processes.flush() + self.done.wait(1) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + self.done.set() + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise TimeoutError("resource sampler failed to stop") + if self.error: + raise self.error + if self.budget.path: + c=self.budget.path + write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=self.budget.memory_peak(),cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + except BaseException as error: + detail = record_resource_failure(self.path.parent/"resource-error.json", error, phase=self.phase, + workload_error_type=exc_type.__name__ if exc_type else None) + if exc_value is None: + raise RuntimeError("resource sampler failed: " + json.dumps(detail, sort_keys=True)) from error + finally: + self.budget.close_memory_peak() + return False + + +def wait_file(path, processes, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if path.exists(): + return + if any(p.poll() is not None for p in processes): + for proc in processes: + if proc.poll() is not None and getattr(proc, "bench_log", None): + record_failure(proc.bench_log.name, proc.returncode) + raise RuntimeError("benchmark worker exited before the start barrier") + time.sleep(.05) + raise TimeoutError("benchmark start barrier timeout") + + +def server_settings(variant, sqlite_path, state_dir, run_env): + config = dict(hashChannels=[f"#bench-{i:02d}" for i in range(20)], dbPath=str(sqlite_path), stateDir=str(state_dir), port=free_port(), packetStore=dict(retentionHours=168, maxMemoryMB=PACKET_STORE_MIB, hotStartupHours=0), runtime=dict(maxMemoryMB=SERVER_MEMORY_MIB), clientRxCoverage=dict(enabled=True), userManagement=dict(enabled=False)) + env = dict(run_env, GOMEMLIMIT=f"{SERVER_MEMORY_MIB}MiB", ENABLE_PPROF="false") + return config, env + + +def retained_counts(source, since_epoch): + db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) + try: + tx = db.execute("SELECT count(*) FROM transmissions WHERE last_seen >= ?", (since_epoch,)).fetchone()[0] + obs = db.execute("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.last_seen >= ?", (since_epoch,)).fetchone()[0] + return dict(transmissions=tx, observations=obs) + finally: + db.close() + + +def check_loaded_rows(health, expected): + if health.get("loadedTx") != expected["transmissions"] or health.get("loadedObs") != expected["observations"]: + raise RuntimeError("startup did not load the complete retained corpus: " + json.dumps(dict(actual=health, expected=expected))) + + +def startup(server, base_url, output, expected, launched_ns, hot_hours, retained_expected=None, coverage_scope="complete retained corpus"): + start, first, ready = launched_ns, None, None + deadline = time.monotonic() + 600 + last = None + while time.monotonic() < deadline: + if server.poll() is not None: + record_failure(server.bench_log.name, server.returncode) + raise RuntimeError("server exited during startup") + try: + with urllib.request.urlopen(base_url + "/api/healthz", timeout=5) as response: + body = json.load(response) + first = first or time.monotonic_ns() + last = body + if response.status == 200 and response.headers.get("X-CoreScope-Load-Status") == "ready": + ready = time.monotonic_ns() + break + except urllib.error.HTTPError as error: + first = first or time.monotonic_ns() + last = {"ready":False,"status":error.code} + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): + pass + time.sleep(.1) + if ready is None: + raise TimeoutError("server did not reach full health/readiness") + full = ready + perf = None + while time.monotonic() < deadline: + with urllib.request.urlopen(base_url + "/api/perf", timeout=30) as response: + perf = json.load(response) + store = perf.get("packetStore", {}) + if store.get("backgroundLoadFailed"): + raise RuntimeError("background startup load failed") + if not hot_hours or store.get("backgroundLoadComplete") is True: + full = time.monotonic_ns() + break + time.sleep(.1) + else: + raise TimeoutError("background startup did not complete") + initial_health=last + with urllib.request.urlopen(base_url+"/api/healthz",timeout=30) as response: + last=json.load(response) + if not last or last.get("loadedTx",0) <= 0 or last.get("loadedObs",0) <= 0: + raise RuntimeError("ready server did not load a meaningful corpus") + if retained_expected is not None: + check_loaded_rows(last, retained_expected) + with urllib.request.urlopen(base_url + "/api/stats", timeout=30) as response: + stats = json.load(response) + result = dict(first_http_ns=first-start, ready_ns=ready-start, full_ready_ns=full-start, hot_startup_hours=hot_hours, initial_health=initial_health, health=last, stats=stats, perf=perf, durable_expected=expected, loaded_expected=retained_expected, retained_expected=retained_expected if coverage_scope == "complete retained corpus" else None, loaded_count_status=coverage_scope + " verified") + write_json(output, result) + return result + + +def verify_completed_pair(output, pair, events, event_hash): + if sha256(events) != event_hash: + raise RuntimeError("paired event stream changed during replay") + for filename in ("startup.json", "startup-hot1.json"): + coverage = [] + for variant in ("baseline", "candidate"): + health = json.loads((output / f"pair-{pair:02d}" / variant / filename).read_text())["health"] + coverage.append((health["loadedTx"], health["loadedObs"])) + if coverage[0] != coverage[1]: + raise RuntimeError("startup memory coverage differs between paired revisions") + + +def summarize(output): + rows=[] + def add(pair,variant,scenario,name,values,errors=0,drops=0,throughput=None,timing="service"): + stats=percentiles(values) + rows.append(dict(pair=pair,variant=variant,scenario=scenario,event_class=name,timing=timing,samples=stats["samples"],p50_ns=stats["p50"],p95_ns=stats["p95"],p99_ns=stats["p99"],maximum_ns=stats["maximum"],errors=errors,drops=drops,completed_per_second_including_drain=throughput)) + for directory in sorted(output.glob("pair-*/*")): + if not directory.is_dir():continue + pair,variant=directory.parent.name,directory.name + for file,scenario in (("requests.jsonl","mixed-http"),("sql-queries.jsonl","sql-miss"),("ingest.jsonl","durable-ingest")): + path=directory/file + if not path.exists():continue + groups={} + for item in json_lines(path): + if not item.get("measured"):continue + group=groups.setdefault(item["class"],dict(values=[],queue=[],total=[],errors=0,drops=0,first=None,last=None)) + if item.get("dropped"): + group["drops"]+=1;continue + if item.get("error") or item.get("status",200)!=(item.get("expected_status") or 200): + group["errors"]+=1;continue + begun=item.get("dispatched_ns",item.get("started_ns")) + group["values"].append(item["completed_ns"]-begun) + scheduled=item.get("scheduled_ns") or begun + if item.get("scheduled_ns"): + group["queue"].append(begun-scheduled) + group["total"].append(item["completed_ns"]-scheduled) + group["first"]=min(group["first"] or scheduled,scheduled) + group["last"]=max(group["last"] or item["completed_ns"],item["completed_ns"]) + for name,group in sorted(groups.items()): + elapsed=(group["last"] or 0)-(group["first"] or 0) + rate=len(group["values"])/(elapsed/1e9) if elapsed>0 else None + add(pair,variant,scenario,name,group["values"],group["errors"],group["drops"],rate) + if group["queue"]: + add(pair,variant,scenario,name,group["queue"],group["errors"],group["drops"],rate,timing="queue") + add(pair,variant,scenario,name,group["total"],group["errors"],group["drops"],rate,timing="end_to_end") + for filename in ("startup.json","startup-hot1.json"): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text()) + for phase in ("first_http_ns","ready_ns","full_ready_ns"): + add(pair,variant,filename.removesuffix(".json"),phase,[obj[phase]]) + for filename,key in (("retention.json","elapsed_ns"),("retention-empty.json","elapsed_ns")): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text());add(pair,variant,filename.removesuffix(".json"),key,[obj[key]]) + if (directory/"handler.json").exists(): + obj=json.loads((directory/"handler.json").read_text());add(pair,variant,"handler-control","200-valid-envelopes",obj["samples_ns"]) + if (directory/"visibility.json").exists(): + obj=json.loads((directory/"visibility.json").read_text());add(pair,variant,"websocket","new-transmission-sentinels",obj["samples_ns"],obj["measured_missing"]) + if rows: + with open(output/"summary.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows) + resource_rows=[] + for path in sorted(output.glob("pair-*/*/resources.csv")): + groups={} + with open(path,newline="") as stream: + for row in csv.DictReader(stream):groups.setdefault(row["phase"],[]).append(row) + for phase,samples in groups.items(): + def numbers(key):return [int(r[key]) for r in samples if str(r.get(key,"")).isdigit()] + resource_rows.append(dict(pair=path.parts[-3],variant=path.parts[-2],phase=phase,samples=len(samples),observed_cgroup_memory_peak_bytes=max(numbers("cgroup_memory_bytes"),default=None),observed_pss_peak_bytes=max(numbers("pss_bytes"),default=None),observed_storage_peak_bytes=max(numbers("storage_bytes"),default=None),observed_wal_peak_bytes=max(numbers("wal_bytes"),default=None),minimum_filesystem_free_bytes=min(numbers("filesystem_free_bytes"),default=None))) + if resource_rows: + with open(output/"summary-resources.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(resource_rows[0]));writer.writeheader();writer.writerows(resource_rows) + paired=[] + keys=sorted({(r["scenario"],r["event_class"],r["timing"]) for r in rows}) + for scenario,name,timing in keys: + for metric in ("p50_ns","p95_ns","p99_ns","completed_per_second_including_drain"): + ratios=[];absolute=[] + for pair in sorted({r["pair"] for r in rows}): + values={r["variant"]:r[metric] for r in rows if (r["pair"],r["scenario"],r["event_class"],r["timing"])==(pair,scenario,name,timing)} + if set(values)=={"baseline","candidate"} and values["baseline"] is not None and values["baseline"] > 0 and values["candidate"] is not None and values["candidate"] >= 0: + ratios.append(values["candidate"]/values["baseline"]);absolute.append(dict(pair=pair,**values)) + if ratios: + paired.append(dict(scenario=scenario,event_class=name,timing=timing,metric=metric,pairs=len(ratios),candidate_over_baseline_median=percentiles(ratios)["p50"],minimum=min(ratios),maximum=max(ratios),absolute_runs=absolute)) + write_json(output/"paired-summary.json",paired) + return rows + + +def build(root, variant, binaries, env, logs): + before = production_hash(root) + replacements = {} + for package in ("ingestor", "server"): + templates = ["common_test.go", "profiles_test.go", "clock_linux_test.go", "clock_other_test.go", package + "_test.go"] + if package == "ingestor": + templates.append("sqlite_adapter_test.go") + for template in templates: + replacements[str(root / "cmd" / package / ("zz_benchmark_" + template))] = str(HERE / "overlays" / template) + overlay = binaries / (variant + "-overlay.json") + write_json(overlay, {"Replace": replacements}) + for package in ("ingestor", "server"): + command(["go", "test", "-trimpath", "-overlay", overlay, "-c", "-o", binaries / (variant + "-" + package + ".test"), "."], cwd=root / "cmd" / package, env=env, log=logs / (variant + "-" + package + "-build.log")) + command([binaries / (variant + "-" + package + ".test"), "-test.run=^TestCoreScopeBenchmarkControl", "-test.count=1", "-test.timeout=3m"], cwd=root / "cmd" / package, env=env, log=logs / (variant + "-" + package + "-controls.log")) + if package == "ingestor": + command([binaries / (variant + "-ingestor.test"), "-test.run=^TestCoreScopeBenchmark.+", "-test.count=1", "-test.timeout=3m"], cwd=root / "cmd" / package, env=env, log=logs / (variant + "-generator-controls.log")) + command(["go", "build", "-trimpath", "-o", binaries / (variant + "-server"), "."], cwd=root / "cmd/server", env=env, log=logs / (variant + "-server-binary.log")) + if requires_storage_selection(root): + command(["go", "build", "-trimpath", "-o", binaries / (variant + "-migrate"), "."], cwd=root / "cmd/migrate", env=env, log=logs / (variant + "-migrate-binary.log")) + if production_hash(root) != before: + raise RuntimeError("build changed production sources") + return before + + +def worker(binary, config, mode, env, log, budget=None): + cfg = dict(config, mode=mode) + path = Path(config["control_dir"]) / (mode + "-config.json") + write_json(path, cfg) + test = "TestCoreScopeBenchmarkServer" if mode == "server" else "TestCoreScopeBenchmark" + return spawn([binary, "-test.run=^" + test + "$", "-test.count=1", "-test.timeout=30m", "-test.v"], cwd=path.parent, env=dict(env, CORESCOPE_BENCH_CONFIG=str(path), ENABLE_PPROF="false"), log=log, budget=budget) + + +def verify_harness(repo, revision, here=None): + """The executable harness may live on a different branch than either app.""" + here = Path(here or HERE) + check_revisions(revision, revision) + prefix = "scripts/sqlite-benchmark/" + if command(["git", "rev-parse", revision + "^{commit}"], cwd=repo).strip() != revision: + raise ValueError("harness revision did not resolve exactly") + names = command(["git", "ls-tree", "-r", "--name-only", revision, "--", prefix], cwd=repo).splitlines() + if any(p.is_symlink() for p in here.rglob("*")): + raise ValueError("harness symlinks are not supported") + expected = {name.removeprefix(prefix) for name in names} + actual = {p.relative_to(here).as_posix() for p in here.rglob("*") if p.is_file() and "__pycache__" not in p.parts} + if not expected or actual != expected: + raise ValueError("executing harness files differ from the requested harness revision") + digest = hashlib.sha256() + for name in sorted(expected): + text = command(["git", "show", revision + ":" + prefix + name], cwd=repo) + if (here / name).read_text(encoding="utf-8") != text: + raise ValueError("executing harness content differs from the requested harness revision: " + name) + digest.update(name.encode() + b"\0" + text.encode()) + return digest.hexdigest() + + +def declared_coverage(name, transmissions, observations): + if name != "L": + if transmissions is not None or observations is not None: + raise ValueError("S/B require complete retained counts; overrides are not accepted") + return None + if not transmissions or not observations or transmissions < 1 or observations < 1: + raise ValueError("L needs predeclared positive loaded-transmission and loaded-observation counts") + if transmissions > corpus(name)["transmissions"] or observations > corpus(name)["observations"]: + raise ValueError("declared cache coverage exceeds the corpus") + return dict(transmissions=transmissions, observations=observations) + + +def capacity_preflight(name, destination): + minimum_disk = {"S": 4, "B": 12, "L": 128}[name] * 1024**3 + if shutil.disk_usage(destination).free < minimum_disk: + raise ValueError("insufficient available disk for corpus safety guard (S:4/B:12/L:128 GiB)") + memory = dict(line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines()) + available = int(memory["MemAvailable"].split()[0]) * 1024 + # Six GiB measured envelope plus coordinator/generator and host headroom. + required = 8 * 1024**3 + if available < required: + raise ValueError("insufficient currently available RAM for corpus guard") + return dict(available_ram_bytes=available, available_disk_bytes=shutil.disk_usage(destination).free, + minimum_ram_bytes=required, minimum_disk_bytes=minimum_disk) + + +def export_profile_tops(directory, binaries, variant, env, logs): + captures = {} + for process, binary in (("server", binaries / (variant + "-server.test")), ("ingestor", binaries / (variant + "-ingestor.test"))): + status = directory / ".profiles" / process / "done.json" + if not status.exists() or not json.loads(status.read_text()).get("complete"): + raise RuntimeError("diagnostic profile capture failed: " + process) + captures[process] = json.loads(status.read_text()) + for kind in ("cpu", "heap", "mutex", "block"): + raw = directory / ".profiles" / process / (kind + ".pprof") + if not raw.is_file() or raw.stat().st_size == 0: + raise RuntimeError("diagnostic profile missing: " + process + "-" + kind) + top = command(["go", "tool", "pprof", "-top", "-nodecount=40", binary, raw], env=env, log=logs / (variant + "-" + process + "-" + kind + "-pprof.log")) + # Mapping filenames can retain the execution directory despite + # -trimpath symbols. Only public top text is exported; raw profiles + # stay private alongside the synthetic database and event file. + for private in (str(directory.resolve()), str(binaries.resolve())): + top = top.replace(private, "") + top = re.sub(r"\b[A-Za-z]:[\\/]\S+|(?", top) + output = directory / "profiles" / (process + "-" + kind + "-top.txt") + output.parent.mkdir(exist_ok=True) + output.write_text(top) + write_json(directory / "profile-capture.json", dict(diagnostic_only=True, scope="measured window of separate diagnostic invocation", captures=captures, mutex_fraction=5, block_rate_ns=10000, heap_after_gc=True, raw_profiles_public=False, profiling_listener=False)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--baseline-sha", required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--harness-sha", required=True) + parser.add_argument("--corpus", choices=("S", "B", "L"), default="B") + parser.add_argument("--pairs", type=int, default=5) + parser.add_argument("--profile", choices=("primary", "smoke", "diagnostic"), default="primary") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cgroup", default=os.environ.get("CORESCOPE_BENCH_CGROUP")) + parser.add_argument("--go-version", default="1.27.2") + parser.add_argument("--seed", type=int, default=20261008) + parser.add_argument("--warmup", type=int) + parser.add_argument("--seconds", type=int) + parser.add_argument("--ingest-rate", type=int, default=50) + parser.add_argument("--http-rate", type=int, default=20) + parser.add_argument("--expected-loaded-transmissions", type=int) + parser.add_argument("--expected-loaded-observations", type=int) + args = parser.parse_args(argv) + check_revisions(args.baseline_sha, args.candidate_sha) + check_revisions(args.harness_sha, args.harness_sha) + validate_profile(args.profile, args.pairs, args.cgroup) + declared = declared_coverage(args.corpus, args.expected_loaded_transmissions, args.expected_loaded_observations) + warmup = args.warmup if args.warmup is not None else (2 if args.profile == "smoke" else 60) + seconds = args.seconds if args.seconds is not None else (10 if args.profile == "smoke" else 180) + if warmup < 0 or seconds < 1 or args.ingest_rate < 1 or args.http_rate < 1 or (warmup + seconds) * max(args.ingest_rate, args.http_rate) > 1_000_000: + raise ValueError("invalid workload duration or offered rate") + if sys.version_info < (3, 12) or sys.platform != "linux": + raise ValueError("measurement requires Linux and Python 3.12+; portable controls can run separately") + if os.geteuid() == 0: + raise ValueError("run as an unprivileged account with delegated cgroup permissions") + if len(os.sched_getaffinity(0)) < 4 or os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") < 8 * 1024**3: + raise ValueError("one host with at least four CPUs and 8 GiB RAM is required") + for executable in ("git", "go", "cc"): + if not shutil.which(executable): + raise ValueError("missing prerequisite: " + executable) + go_version = command(["go", "version"]).strip() + if not re.search(r"\bgo" + re.escape(args.go_version) + r"\b", go_version): + raise ValueError("Go version differs from common toolchain pin") + harness_hash = verify_harness(args.repo.resolve(), args.harness_sha) + output = safe_output(args.output) + guards = capacity_preflight(args.corpus, output.parent) + output.mkdir(mode=0o700) + runtime, logs, binaries = output / ".runtime", output / "logs", output / ".runtime/binaries" + for path in (runtime, logs, binaries): + path.mkdir(exist_ok=True) + budget = Budget(args.cgroup, args.profile in ("primary", "diagnostic") or args.corpus == "L") + env = {k: v for k, v in os.environ.items() if not k.startswith(("CORESCOPE_", "PG", "PPROF_")) and k not in ("DB_PATH", "MQTT_BROKER", "MQTT_TOPIC", "GOFLAGS", "ENABLE_PPROF")} + env.update(CGO_ENABLED="1", GOMAXPROCS="3", GOMEMLIMIT="512MiB", GOTOOLCHAIN="local", GOFLAGS="", ENABLE_PPROF="false") + server = None + manifest = dict(status="incomplete", engine="native CGO SQLite", baseline_sha=args.baseline_sha, candidate_sha=args.candidate_sha, + harness_source_sha=args.harness_sha, harness_sha256=harness_hash, harness_text_normalization="UTF-8 with LF line endings", + profile=args.profile, seed=args.seed, corpus=args.corpus, shape=corpus(args.corpus), firmware_reference=FIRMWARE, + host=dict(system=platform.system(), kernel=platform.release(), architecture=platform.machine(), cpus=os.cpu_count(), ram_bytes=os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")), + go_version=go_version, compiler=command(["cc", "--version"]).splitlines()[0], resource_budget=budget.info, capacity_guards=guards, + cache_policy=dict(startup="new process, OS cache warm", mixed="application warmup before measured requests", sql="channel application cache explicitly cleared; OS cache warm"), + workload=dict(warmup_seconds=warmup, measured_seconds=seconds, ingest_rate=args.ingest_rate, http_rate=args.http_rate, queue_limit=1024, http_concurrency=64, server_memory_mib=SERVER_MEMORY_MIB, ingestor_memory_mib=512, packet_store_mib=PACKET_STORE_MIB, retention_hours=168), + declared_loaded_counts=declared, profiling=args.profile == "diagnostic", comparison_eligible=False, order=pair_order(args.pairs)) + write_json(output / "manifest.json", manifest) + try: + try: + budget.preflight() + except OSError as error: + detail = record_resource_failure(output / "resource-preflight.json", error, status="failed") + raise RuntimeError("resource preflight failed: " + json.dumps(detail, sort_keys=True)) from error + write_json(output / "resource-preflight.json", dict(status="passed", enforced=bool(budget.path), counters=list(RESOURCE_COUNTERS) if budget.path else [])) + roots = {} + for variant, revision in (("baseline", args.baseline_sha), ("candidate", args.candidate_sha)): + roots[variant] = runtime / variant + archive(args.repo.resolve(), revision, roots[variant]) + manifest[variant + "_production_hash_before"] = build(roots[variant], variant, binaries, env, logs) + manifest[variant + "_storage_selection_required"] = requires_storage_selection(roots[variant]) + manifest["comparison_kind"] = "A/A controller qualification" if manifest["baseline_production_hash_before"] == manifest["candidate_production_hash_before"] else "source-pinned SQLite before/after" + epoch = int(time.time()) + canonical = runtime / "corpus.sqlite" + prepare_dir = output / "corpus" + prepare_dir.mkdir() + base_config = dict(control_dir=str(runtime / "prepare-controls"), corpus_info=str(prepare_dir / "corpus.json"), corpus=args.corpus, shape=corpus(args.corpus), seed=args.seed, epoch=epoch, sqlite=str(canonical), output=str(prepare_dir), warmup=0, seconds=0, ingest_rate=args.ingest_rate, http_rate=args.http_rate) + finish(worker(binaries / "baseline-ingestor.test", base_config, "prepare", env, logs / "prepare.log"), timeout=1800) + tables = table_manifest(canonical) + canonical_validation = validate(canonical, tables) + for name in ("transmissions", "observations"): + if canonical_validation[name]["rows"] != corpus(args.corpus)[name]: + raise RuntimeError("canonical corpus counts differ from declared shape") + write_json(prepare_dir / "validation.json", canonical_validation) + manifest.update(epoch=epoch, canonical_file_sha256=sha256(canonical), table_manifest=tables) + manifest["binary_sha256"] = {p.name: sha256(p) for p in binaries.iterdir() if p.is_file() and not p.name.endswith(".json")} + write_json(output / "manifest.json", manifest) + after_by_pair, paired_sources = {}, {} + for pair, variant in pair_order(args.pairs): + budget.reset() + run_dir = output / f"pair-{pair:02d}" / variant + run_dir.mkdir(parents=True) + if shutil.disk_usage(output).free < max(2 * 1024**3, canonical.stat().st_size * 8): + raise RuntimeError("insufficient measured-corpus disk headroom before restore") + config_dir = runtime / f"config-{pair:02d}-{variant}" + state_dir = config_dir / "state" + state_dir.mkdir(parents=True) + sqlite_path = config_dir / "telemetry.sqlite" + if pair not in paired_sources: + pair_source = runtime / f"pair-{pair:02d}.sqlite" + shutil.copy2(canonical, pair_source) + delta = int(time.time()) - epoch + freshen(pair_source, delta) + paired_validation = validate(pair_source, tables) + paired_sources[pair] = pair_source, delta, paired_validation + write_json(output / f"pair-{pair:02d}" / "corpus-offset.json", dict(receipt_epoch_offset_seconds=delta, wire_epoch_unchanged=epoch, sha256=sha256(pair_source), validation=paired_validation)) + pair_source, delta, paired_validation = paired_sources[pair] + source_record = json.loads((output / f"pair-{pair:02d}" / "corpus-offset.json").read_text()) + if sha256(pair_source) != source_record["sha256"]: + raise RuntimeError("paired source database changed before restore") + command(["cp", "--reflink=never", pair_source, sqlite_path], budget=budget, log=run_dir / "restore-copy.log") + config = dict(base_config, control_dir=str(config_dir / "controls"), events_file=str(runtime / f"pair-{pair:02d}-events.jsonl"), epoch=epoch + delta, wire_epoch=epoch, output=str(run_dir), sqlite=str(sqlite_path), state_dir=str(state_dir), warmup=warmup, seconds=seconds, start_file=str(run_dir / "start.json")) + if not Path(config["events_file"]).exists(): + finish(worker(binaries / "baseline-ingestor.test", config, "events", env, run_dir / "event-generation.log"), timeout=600) + write_json(output / f"pair-{pair:02d}" / "events.json", dict(sha256=sha256(config["events_file"]), rows=(warmup + seconds) * args.ingest_rate, source="one baseline-generated stream used unchanged by both revisions")) + event_hash = json.loads((output / f"pair-{pair:02d}" / "events.json").read_text())["sha256"] + if sha256(config["events_file"]) != event_hash: + raise RuntimeError("paired event stream changed before replay") + run_env = dict(env, CORESCOPE_INGESTOR_STATS=str(state_dir / "ingestor-stats.json")) + with Resources(run_dir / "resources.csv", budget, [sqlite_path]) as resources: + with closing(sqlite3.connect(sqlite_path)) as db: + db.execute("ANALYZE") + db.commit() + pre = validate(sqlite_path, tables) + if pre != paired_validation: + raise RuntimeError("initial revision state differs from canonical corpus") + write_json(run_dir / "validation-before.json", pre) + expected = declared or retained_counts(sqlite_path, int(time.time()) - 168 * 3600) + coverage_scope = "predeclared bounded cache" if declared else "complete retained corpus" + app_config, server_env = server_settings(variant, sqlite_path, state_dir, run_env) + write_json(config_dir / "config.json", app_config) + resources.phase = "storage_setup" + setup = prepare_storage(roots[variant], binaries / (variant + "-migrate"), config_dir, tables, pre, + run_env, run_dir / "storage-setup.log", budget) + write_json(run_dir / "storage-setup-validation.json", setup) + server_args = ["-config-dir", str(config_dir), "-public", str(roots[variant] / "public"), "-port", str(app_config["port"]), "-poll-ms", "1000"] + config["base_url"] = f"http://127.0.0.1:{app_config['port']}" + resources.phase = "startup_full" + launched = time.monotonic_ns() + server = spawn([binaries / (variant + "-server"), *server_args], cwd=roots[variant], env=server_env, log=run_dir / "server.log", budget=budget) + startup(server, config["base_url"], run_dir / "startup.json", pre, launched, 0, expected, coverage_scope) + stop(server) + app_config["packetStore"]["hotStartupHours"] = 1 + write_json(config_dir / "config.json", app_config) + resources.phase = "startup_hot" + launched = time.monotonic_ns() + if args.profile == "diagnostic": + server_config = dict(config, server_args=server_args, profile_dir=str(run_dir / ".profiles/server")) + server = worker(binaries / (variant + "-server.test"), server_config, "server", server_env, run_dir / "server-hot.log", budget) + else: + server = spawn([binaries / (variant + "-server"), *server_args], cwd=roots[variant], env=server_env, log=run_dir / "server-hot.log", budget=budget) + startup(server, config["base_url"], run_dir / "startup-hot1.json", pre, launched, 1, expected, coverage_scope) + resources.phase = "sql_queries" + finish(worker(binaries / (variant + "-server.test"), config, "queries", server_env, run_dir / "queries.log", budget), timeout=180) + resources.phase = "diagnostic_mixed" if args.profile == "diagnostic" else "mixed_with_retention" + client = worker(binaries / (variant + "-server.test"), config, "client", server_env, run_dir / "client.log") + replay_config = dict(config, profile_dir=str(run_dir / ".profiles/ingestor") if args.profile == "diagnostic" else "") + replay = worker(binaries / (variant + "-ingestor.test"), replay_config, "replay", run_env, run_dir / "replay.log", budget) + try: + wait_file(run_dir / "client-ready", [client, replay, server]) + wait_file(run_dir / "replay-ready", [client, replay, server]) + write_json(run_dir / "start.json", dict(mono_ns=time.monotonic_ns() + 2_000_000_000)) + finish(replay, timeout=warmup + seconds + 180) + finish(client, timeout=warmup + seconds + 180) + finally: + stop(replay); stop(client) + requests = list(json_lines(run_dir / "requests.jsonl")) + if len(requests) != (warmup + seconds) * args.http_rate or any(item.get("error") for item in requests): + raise RuntimeError("HTTP schedules/errors failed full-stream verification, including warmup") + observed = visibility(json_lines(run_dir / "ingest.jsonl"), json_lines(run_dir / "websocket.jsonl")) + write_json(run_dir / "visibility.json", observed) + if not observed["verified"]: + raise RuntimeError("durable sentinels missing from real WebSocket") + if args.profile == "diagnostic": + wait_file(run_dir / ".profiles/server/done.json", [server]) + export_profile_tops(run_dir, binaries, variant, env, logs) + resources.phase = "retention_empty" + finish(worker(binaries / (variant + "-ingestor.test"), config, "retention", run_env, run_dir / "retention.log", budget), timeout=600) + stop(server); server = None + resources.phase = "plans" + finish(worker(binaries / (variant + "-ingestor.test"), config, "plans", run_env, run_dir / "plans.log", budget), timeout=180) + resources.phase = "post_validation" + post = validate(sqlite_path, tables, logical=True) + write_json(run_dir / "validation-after.json", post) + comparable_post = comparable_validation(sqlite_path, post, setup) + write_json(run_dir / "validation-after-comparable.json", comparable_post) + finish(worker(binaries / (variant + "-ingestor.test"), config, "handler", run_env, run_dir / "handler.log", budget), timeout=180) + after_by_pair.setdefault(pair, {})[variant] = comparable_post + if len(after_by_pair[pair]) == 2: + verify_completed_pair(output, pair, Path(config["events_file"]), event_hash) + if len(after_by_pair[pair]) == 2 and after_by_pair[pair]["baseline"] != after_by_pair[pair]["candidate"]: + raise RuntimeError("post-replay logical parity differs between revisions") + remove_working_directory(config_dir, runtime) + for variant, root in roots.items(): + manifest[variant + "_production_hash_after"] = production_hash(root) + if manifest[variant + "_production_hash_after"] != manifest[variant + "_production_hash_before"]: + raise RuntimeError("production sources changed during measurement") + if sha256(canonical) != manifest["canonical_file_sha256"]: + raise RuntimeError("canonical source database changed during measurement") + for pair, (pair_source, _, _) in paired_sources.items(): + source_record = json.loads((output / f"pair-{pair:02d}" / "corpus-offset.json").read_text()) + if sha256(pair_source) != source_record["sha256"]: + raise RuntimeError("paired source database changed during measurement") + manifest["source_databases_unchanged"] = True + if verify_harness(args.repo.resolve(), args.harness_sha) != harness_hash: + raise RuntimeError("harness changed during measurement") + summary = summarize(output) + manifest.update(status="completed", comparison_eligible=args.profile == "primary" and manifest["comparison_kind"] != "A/A controller qualification", measured_rows=len(summary), completed_pairs=args.pairs) + write_json(output / "manifest.json", manifest) + (output / "report.md").write_text("# CoreScope native SQLite before/after\n\nSee the exact source/harness refs, corpus, common settings, correctness gates and raw evidence in manifest.json. Diagnostic profiling runs and A/A controls establish no performance improvement. OS-cold behavior and long soak were not measured.\n") + except BaseException as error: + for child in reversed(PROCESSES): stop(child) + try: summarize(output) + except (OSError, ValueError, KeyError) as summary_error: manifest["partial_summary_error"] = str(summary_error) + manifest.update(status="failed", comparison_eligible=False, error=str(error)) + write_json(output / "manifest.json", manifest) + raise + finally: + for child in reversed(PROCESSES): stop(child) + PROCESSES.clear(); stop(server) + pending_error = sys.exc_info()[1] + try: + budget.close() + except Exception as cleanup_error: + manifest.update(status="failed", comparison_eligible=False, cleanup_error_type=type(cleanup_error).__name__, cleanup_errno=getattr(cleanup_error,"errno",None)) + write_json(output / "manifest.json", manifest) + if pending_error is None: + raise + finally: + public_bundle(output, args.repo) + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print("benchmark:", error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/sqlite-benchmark/test_harness.py b/scripts/sqlite-benchmark/test_harness.py new file mode 100644 index 000000000..4b6776f5b --- /dev/null +++ b/scripts/sqlite-benchmark/test_harness.py @@ -0,0 +1,674 @@ +import importlib.util +from contextlib import ExitStack +import csv +import errno +import io +import json +import pathlib +import re +import tempfile +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("bench", HERE / "run.py") +bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench) + + +class MemoryPeakFile(io.BytesIO): + """Model cgroup v2's reset scope: one open file description.""" + def __init__(self, state, mode): + super().__init__() + self.state, self.binary, self.peak = state, "b" in mode, state.lifetime + state.handles.append(self) + + def write(self, value): + if self.state.reject_reset: + raise OSError(errno.EINVAL, "reset unsupported") + self.peak = self.state.current + return len(value) + + def read(self, size=-1): + value = b"" if self.tell() else str(self.peak).encode() + b"\n" + self.seek(len(value), io.SEEK_CUR) + return value if self.binary else value.decode() + + +class HarnessTests(unittest.TestCase): + def test_default_common_rate_and_emitted_worker_configs(self): + parse = bench.argparse.ArgumentParser.parse_args + for extra, expected_rate in (([], 50), (["--ingest-rate", "100"], 100)): + parsed = [] + def capture(parser, argv): + parsed.append(parse(parser, argv)) + raise RuntimeError("stop after parsing") + with mock.patch.object(bench.argparse.ArgumentParser, "parse_args", capture): + with self.assertRaisesRegex(RuntimeError, "stop after parsing"): + bench.main(["--baseline-sha", "2" * 40, "--harness-sha", "3" * 40, "--candidate-sha", "1" * 40, "--output", "corescope-bench-test", *extra]) + args = parsed[0] + self.assertEqual((args.ingest_rate, args.http_rate, args.pairs), (expected_rate, 20, 5)) + self.assertEqual((args.warmup, args.seconds), (None, None)) + # Exercise the actual JSON controls consumed by both compiled workers. + emitted = [] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + for backend in ("baseline", "candidate"): + controls = root / backend + controls.mkdir() + config = dict(control_dir=str(controls), warmup=60, seconds=180, + ingest_rate=args.ingest_rate, http_rate=args.http_rate) + with mock.patch.object(bench, "spawn") as spawn: + bench.worker(root / (backend + "-ingestor.test"), config, "replay", {}, root / (backend + ".log")) + written = json.loads(pathlib.Path(spawn.call_args.kwargs["env"]["CORESCOPE_BENCH_CONFIG"]).read_text()) + written.pop("control_dir") + emitted.append(written) + self.assertEqual(emitted[0], emitted[1]) + self.assertEqual(emitted[0], dict(mode="replay", warmup=60, seconds=180, + ingest_rate=expected_rate, http_rate=20)) + self.assertEqual((emitted[0]["warmup"] + emitted[0]["seconds"]) * emitted[0]["ingest_rate"], + 12000 if expected_rate == 50 else 24000) + + + def test_common_server_memory_profile_retains_b_and_replay_headroom(self): + # Native baseline B qualification tracked 1221.7 MiB. The original + # 100/s envelope conservatively bounds the current 50/s experiment too. + required_store_mib = 1221.7 + 24000 * 8279 / 1048576 + settings = [] + for backend in ("baseline", "candidate"): + with mock.patch.object(bench, "free_port", return_value=12345): + config, env = bench.server_settings(backend, pathlib.Path("corpus.sqlite"), pathlib.Path("state"), {}) + self.assertGreater(config["packetStore"]["maxMemoryMB"], required_store_mib) + self.assertEqual(config["packetStore"], {"retentionHours": 168, "maxMemoryMB": 2048, "hotStartupHours": 0}) + self.assertEqual(config["runtime"]["maxMemoryMB"], 3072) + self.assertEqual(env["GOMEMLIMIT"], "3072MiB") + self.assertEqual("dbPath" in config, True) + config.pop("dbPath", None) + settings.append((config, env)) + self.assertEqual(settings[0], settings[1], "capacity settings must be common to both engines") + + def test_build_runs_compiled_generator_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + binaries, logs = root / "binaries", root / "logs" + binaries.mkdir(); logs.mkdir() + with mock.patch.object(bench, "production_hash", return_value="stable"), mock.patch.object(bench, "command") as command: + bench.build(root, "baseline", binaries, {}, logs) + controls = [call.args[0] for call in command.call_args_list if pathlib.Path(call.args[0][0]).name == "baseline-ingestor.test"] + self.assertTrue(any("-test.run=^TestCoreScopeBenchmark.+" in args for args in controls), "new Go generator regressions must execute, not only compile") + + def selection_source(self, root): + path = root / "cmd/server/storage_config.go" + path.parent.mkdir(parents=True) + path.write_text("package main\nfunc bind() { dbconfig.OpenSelection(path) }\n") + + def test_builds_migrator_once_only_when_archived_server_requires_selection(self): + for selected in (False, True): + with self.subTest(selected=selected), tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + binaries, logs = root / "binaries", root / "logs" + binaries.mkdir(); logs.mkdir() + if selected: + self.selection_source(root) + # A legacy migrator alone is not the persisted-selection capability. + (root / "cmd/migrate").mkdir(parents=True) + with mock.patch.object(bench, "production_hash", return_value="stable"), mock.patch.object(bench, "command") as command: + bench.build(root, "candidate", binaries, {}, logs) + builds = [call for call in command.call_args_list if call.kwargs.get("cwd") == root / "cmd/migrate"] + self.assertEqual(len(builds), int(selected), "only the selection-aware revision needs one native setup binary") + if selected: + self.assertEqual(builds[0].args[0], ["go", "build", "-trimpath", "-o", binaries / "candidate-migrate", "."]) + + def test_server_uses_the_same_private_state_directory_as_selection_setup(self): + state = pathlib.Path("private/state") + config, _ = bench.server_settings("candidate", pathlib.Path("private/telemetry.sqlite"), state, {}) + self.assertEqual(config.get("stateDir"), str(state)) + + def selection_fixture(self, root): + config = root / "config" + config.mkdir() + source = config / "telemetry.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE _migrations(name TEXT PRIMARY KEY);" + "INSERT INTO _migrations VALUES('existing_a'),('existing_b');" + "CREATE TABLE observers(id TEXT PRIMARY KEY, last_seen TEXT);" + "INSERT INTO observers(rowid,id,last_seen) VALUES(4,'one','2026-10-09'),(19,'two',NULL);" + "CREATE TABLE tx_rewrite_feed(id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER);" + "INSERT INTO tx_rewrite_feed VALUES(1,100);" + "CREATE TABLE key_backfill_state(kind TEXT, started_at INTEGER, cursor INTEGER);" + "INSERT INTO key_backfill_state VALUES('channel',100,7);" + "CREATE TABLE key_backfill_done(kind TEXT, done_at INTEGER);" + "INSERT INTO key_backfill_done VALUES('channel',110);") + db.close() + with mock.patch.object(bench, "TABLES", ["_migrations", "observers"]): + tables = bench.table_manifest(source) + return config, source, tables, bench.validate(source, tables) + + def test_native_observer_layout_preserves_complete_logical_manifest(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + _, source, tables, before = self.selection_fixture(root) + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE observers_new(rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT UNIQUE, last_seen TEXT);" + "INSERT INTO observers_new SELECT rowid,id,last_seen FROM observers;" + "DROP TABLE observers; ALTER TABLE observers_new RENAME TO observers;" + "INSERT INTO _migrations VALUES('observers_identity_autoincrement_v1');" + "CREATE INDEX observers_last_seen ON observers(last_seen);") + db.close() + with mock.patch.object(bench, "TABLES", ["_migrations", "observers"]): + self.assertEqual(bench.table_manifest(source), tables, + "supported observer identity layout or an ordinary index changed the logical manifest") + proof = dict(required=True, before=before, migrations_before=["existing_a", "existing_b"]) + self.assertEqual(bench.comparable_validation(source, bench.validate(source, tables), proof), before) + + def test_setup_rejects_new_table_row_insert_delete_and_update(self): + for table in ("tx_rewrite_feed", "key_backfill_state", "key_backfill_done"): + changes = [f"DELETE FROM {table}", f"INSERT INTO {table} SELECT * FROM {table}"] + # The feed's integer primary key needs a distinct inserted identity. + if table == "tx_rewrite_feed": + changes[1] = "INSERT INTO tx_rewrite_feed VALUES(2,200)" + column = {"tx_rewrite_feed": "created_at", "key_backfill_state": "cursor", "key_backfill_done": "done_at"}[table] + changes.append(f"UPDATE {table} SET {column}={column}+1") + for change in changes: + with self.subTest(change=change), tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + self.selection_source(root) + config, source, tables, before = self.selection_fixture(root) + def migrate(*args, **kwargs): + with bench.sqlite3.connect(source) as db: + db.execute("INSERT INTO _migrations VALUES('observers_identity_autoincrement_v1')") + db.execute(change) + db.close() + with mock.patch.object(bench, "command", side_effect=migrate): + with self.assertRaisesRegex(RuntimeError, "setup changed canonical application data"): + bench.prepare_storage(root, root / "migrate", config, tables, before, {}, root / "setup.log") + + def test_selection_setup_preserves_all_rows_and_only_adds_known_marker(self): + marker = "observers_identity_autoincrement_v1" + for selected in (False, True): + with self.subTest(selected=selected), tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + config, source, tables, before = self.selection_fixture(root) + if selected: + self.selection_source(root) + def migrate(args, **kwargs): + self.assertEqual(args, [root / "candidate-migrate", "-storage-action=adopt", "-backend=sqlite", "-offline", + "-selection-file", str(config / "state/storage-selection.json"), + "-config-dir", str(config), "-sqlite-path", str(source)]) + self.assertEqual(kwargs["env"], {"CGO_ENABLED": "1"}) + self.assertEqual(kwargs["budget"], "owned-budget") + with bench.sqlite3.connect(source) as db: + db.execute("INSERT INTO _migrations VALUES (?)", (marker,)) + db.close() + with mock.patch.object(bench, "command", side_effect=migrate) as command: + proof = bench.prepare_storage(root, root / "candidate-migrate", config, tables, before, + {"CGO_ENABLED": "1"}, root / "setup.log", "owned-budget") + self.assertEqual(command.call_count, int(selected)) + self.assertEqual(proof["before"], before) + self.assertEqual(proof["after"], bench.validate(source, tables)) + self.assertEqual(proof["migrations_before"], ["existing_a", "existing_b"]) + self.assertEqual(proof["required"], selected) + self.assertEqual(bench.comparable_validation(source, proof["after"], proof), before) + + def test_setup_and_post_replay_reject_data_or_ledger_changes(self): + for extra in ("DELETE FROM _migrations WHERE name='existing_a'", + "INSERT INTO _migrations VALUES('unknown_extra')", + "DELETE FROM _migrations WHERE name='observers_identity_autoincrement_v1'", + "UPDATE observers SET rowid=20 WHERE id='two'", + "UPDATE observers SET last_seen='changed' WHERE id='one'"): + with self.subTest(extra=extra), tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + self.selection_source(root) + config, source, tables, before = self.selection_fixture(root) + def migrate(*args, **kwargs): + with bench.sqlite3.connect(source) as db: + db.execute("INSERT INTO _migrations VALUES('observers_identity_autoincrement_v1')") + db.execute(extra) + db.close() + with mock.patch.object(bench, "command", side_effect=migrate): + with self.assertRaisesRegex(RuntimeError, "setup changed|migration ledger"): + bench.prepare_storage(root, root / "candidate-migrate", config, tables, before, {}, root / "setup.log") + # The approved marker is not a license for later unaccounted ledger changes. + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + config, source, tables, before = self.selection_fixture(root) + proof = dict(required=False, migrations_before=["existing_a", "existing_b"], before=before) + with bench.sqlite3.connect(source) as db: + db.execute("INSERT INTO _migrations VALUES('late_unexpected')") + db.close() + with self.assertRaisesRegex(RuntimeError, "migration ledger"): + bench.comparable_validation(source, bench.validate(source, tables), proof) + + def test_failed_adoption_cannot_be_treated_as_prepared(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + self.selection_source(root) + config, _, tables, before = self.selection_fixture(root) + with mock.patch.object(bench, "command", side_effect=RuntimeError("native adoption failed")): + with self.assertRaisesRegex(RuntimeError, "native adoption failed"): + bench.prepare_storage(root, root / "candidate-migrate", config, tables, before, {}, root / "setup.log") + + def test_controller_adopts_before_startup_clock_and_never_starts_after_failure(self): + for failure in (False, True): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory, ExitStack() as patches: + root = pathlib.Path(directory) + output = root / "corescope-bench-routing" + order = [] + def command(args, **kwargs): + if args[0] == "go": return "go version go1.27.2" + if args[0] == "cc": return "native compiler" + if args[0] == "cp": + bench.shutil.copy2(args[2], args[3]) + return "" + self.assertEqual(pathlib.Path(args[0]).name, "candidate-migrate") + order.append("adopt") + if failure: raise RuntimeError("native adoption failed") + source = pathlib.Path(args[args.index("-sqlite-path") + 1]) + with bench.sqlite3.connect(source) as db: + db.execute("INSERT INTO _migrations VALUES('observers_identity_autoincrement_v1')") + db.close() + return "" + def worker(binary, config, mode, *args, **kwargs): + if mode == "prepare": + with bench.sqlite3.connect(config["sqlite"]) as db: + db.executescript("CREATE TABLE transmissions(id INTEGER,last_seen INTEGER);" + "INSERT INTO transmissions VALUES(1,100);" + "CREATE TABLE observations(transmission_id INTEGER);" + "INSERT INTO observations VALUES(1);" + "CREATE TABLE _migrations(name TEXT PRIMARY KEY);" + "INSERT INTO _migrations VALUES('existing');") + db.close() + elif mode == "events": + pathlib.Path(config["events_file"]).write_text('{"unchanged":"stream"}\n') + else: self.fail("worker started before storage preparation was checked") + def clock(): + order.append("clock") + return len(order) + def spawn(*args, **kwargs): + order.append("server") + self.assertIn("adopt", order) + return mock.Mock(poll=mock.Mock(return_value=0), bench_log=None) + def startup(server, url, output, expected, launched, *args): + # Check the actual timestamp supplied to startup, not the + # setup helper's independent duration clock. + self.assertGreater(launched, order.index("adopt") + 1) + raise RuntimeError("stop at timed server startup") + budget = mock.Mock(info={}, path=None) + patches.enter_context(mock.patch.object(bench.sys, "platform", "linux")) + for name, value in (("geteuid", 1000), ("sched_getaffinity", {0,1,2,3}), ("sysconf", 1048576)): + patches.enter_context(mock.patch.object(bench.os, name, return_value=value, create=True)) + patches.enter_context(mock.patch.object(bench.shutil, "which", return_value="tool")) + patches.enter_context(mock.patch.object(bench.time, "monotonic_ns", side_effect=clock)) + for name, value in (("capacity_preflight", {}), ("verify_harness", "verified"), ("build", "stable"), + ("corpus", dict(transmissions=1, observations=1)), ("Budget", budget), + ("pair_order", [(0, "candidate")]), ("retained_counts", dict(transmissions=1, observations=1))): + patches.enter_context(mock.patch.object(bench, name, return_value=value)) + patches.enter_context(mock.patch.object(bench, "TABLES", ["transmissions", "observations", "_migrations"])) + patches.enter_context(mock.patch.object(bench, "archive", side_effect=lambda repo, revision, target: self.selection_source(target))) + patches.enter_context(mock.patch.object(bench, "Resources")) + patches.enter_context(mock.patch.object(bench, "finish")) + patches.enter_context(mock.patch.object(bench, "worker", side_effect=worker)) + patches.enter_context(mock.patch.object(bench, "command", side_effect=command)) + launch = patches.enter_context(mock.patch.object(bench, "spawn", side_effect=spawn)) + timed_start = patches.enter_context(mock.patch.object(bench, "startup", side_effect=startup)) + with self.assertRaisesRegex(RuntimeError, "native adoption failed" if failure else "stop at timed server startup"): + bench.main(["--baseline-sha", "2" * 40, "--candidate-sha", "1" * 40, "--harness-sha", "3" * 40, + "--profile", "smoke", "--pairs", "1", "--corpus", "S", "--output", str(output)]) + self.assertEqual(launch.call_count, 0 if failure else 1) + self.assertEqual(timed_start.call_count, 0 if failure else 1) + result = json.loads((output / "public/manifest.json").read_text()) + self.assertEqual((result["status"], result["comparison_eligible"]), ("failed", False)) + + def test_native_setup_failure_is_visible_without_private_details(self): + diagnostic = bench.safe_diagnostics('[migrate] storage selection is pending; resume or abort\n' + '[migrate] failed at /tmp/private/config.json token=private-value\n') + self.assertTrue(any("storage selection is pending" in line for line in diagnostic)) + self.assertNotIn("/tmp/private", str(diagnostic)) + self.assertNotIn("private-value", str(diagnostic)) + + def resource_fixture(self, root, missing=None): + group = root / "private-cgroup-path" + group.mkdir() + for name, value in {"memory.current": "0", "memory.events": "oom 0\n", "cpu.stat": "usage_usec 0\n", + "io.stat": "8:0 rbytes=1024 wbytes=2048\n", "cgroup.procs": ""}.items(): + if name != missing: + (group / name).write_text(value) + budget = bench.Budget(None, False) + budget.path = group + budget.info = {"enforced": True, "memory_peak_reset": False} + budget.close = mock.Mock() + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + return budget, run + + def memory_peak_counter(self, budget, reject_reset=False): + (budget.path / "memory.reclaim").touch() + state = types.SimpleNamespace(current=0, lifetime=9000, handles=[], reject_reset=reject_reset) + real_open = pathlib.Path.open + def open_file(path, mode="r", *args, **kwargs): + if path == budget.path / "memory.peak": + return MemoryPeakFile(state, mode) + return real_open(path, mode, *args, **kwargs) + return state, mock.patch.object(pathlib.Path, "open", open_file) + + def test_memory_peak_is_reset_per_leg_for_samples_and_final_report(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + def sample_peak(value): + state.current = value + state.lifetime = max(state.lifetime, value) + for handle in state.handles: + if not handle.closed: + handle.peak = max(handle.peak, value) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + row = next(csv.DictReader(stream)) + # Use the real final-write path after the sampler has stopped. + with mock.patch.object(resource.thread, "join"): + resource.__exit__(None, None, None) + final = json.loads((run / "resource-final.json").read_text()) + self.assertTrue(final["budget"]["memory_peak_reset"]) + self.assertEqual(int(row["cgroup_memory_peak_bytes"]), value) + self.assertEqual(final["memory_peak_bytes"], value) + self.assertTrue(all(handle.closed for handle in state.handles)) + with counter: + budget.reset() + sample_peak(3000) + state.current = 0 + budget.reset() + sample_peak(1000) + + def test_memory_peak_descriptor_closes_on_failure_and_unsupported_reset(self): + for unsupported in (False, True): + with self.subTest(unsupported=unsupported), tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget, reject_reset=unsupported) + with counter: + budget.reset() + self.assertEqual(budget.info["memory_peak_reset"], not unsupported) + with self.assertRaisesRegex(ValueError, "workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("workload failure") + self.assertTrue(all(handle.closed for handle in state.handles)) + final = json.loads((run / "resource-final.json").read_text()) + self.assertEqual(final["memory_peak_bytes"], None if unsupported else 0) + + def test_memory_peak_descriptor_closes_if_no_resource_context_is_entered(self): + with tempfile.TemporaryDirectory() as directory: + budget, _ = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + with counter, mock.patch.object(pathlib.Path, "rmdir"): + budget.reset() + first = state.handles[0] + budget.reset() + self.assertTrue(first.closed) + bench.Budget.close(budget) + self.assertTrue(all(handle.closed for handle in state.handles)) + + def test_sampler_failure_identifies_counter_without_private_path(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(RuntimeError, "io.stat"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + evidence = json.loads((run / "resource-error.json").read_text()) + self.assertEqual(evidence["error_type"], "FileNotFoundError") + self.assertEqual(evidence["errno"], errno.ENOENT) + self.assertEqual(evidence["counter"], "io.stat") + self.assertNotIn("private-cgroup-path", json.dumps(evidence)) + bench.public_bundle(root) + self.assertTrue((root / "public/pair-00/sqlite/resource-error.json").is_file()) + + def test_sampler_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + raise ValueError("independent workload failure") + self.assertEqual(json.loads((run / "resource-error.json").read_text())["counter"], "io.stat") + + def test_successful_resource_sample_keeps_io_metrics(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["io_stat"], "8:0 rbytes=1024 wbytes=2048") + + def test_final_resource_write_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + with mock.patch.object(bench, "write_json", side_effect=PermissionError(errno.EACCES, "private detail", str(run / "resource-final.json"))): + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("independent workload failure") + + def test_missing_counter_fails_before_archiving_or_building(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, _ = self.resource_fixture(root, missing="io.stat") + output = root / "corescope-bench-preflight" + def version(args, **kwargs): + return "go version go1.27.2" if args[0] == "go" else "compiler" + with mock.patch.object(bench.sys, "platform", "linux"), \ + mock.patch.object(bench.os, "geteuid", return_value=1000, create=True), \ + mock.patch.object(bench.os, "sched_getaffinity", return_value={0,1,2,3}, create=True), \ + mock.patch.object(bench.os, "sysconf", return_value=1048576, create=True), \ + mock.patch.object(bench.shutil, "which", return_value="tool"), \ + mock.patch.object(bench, "capacity_preflight", return_value={}), \ + mock.patch.object(bench, "verify_harness", return_value="verified"), \ + mock.patch.object(bench, "command", side_effect=version), \ + mock.patch.object(bench, "Budget", return_value=budget), \ + mock.patch.object(bench, "archive") as archive, \ + mock.patch.object(bench, "build") as build: + with self.assertRaisesRegex(RuntimeError, "io.stat"): + bench.main(["--baseline-sha", "2" * 40, "--candidate-sha", "1" * 40, "--harness-sha", "3" * 40, + "--profile", "smoke", "--pairs", "1", "--corpus", "S", "--output", str(output)]) + archive.assert_not_called() + build.assert_not_called() + evidence = json.loads((output / "public/resource-preflight.json").read_text()) + self.assertEqual(evidence["counter"], "io.stat") + self.assertFalse(json.loads((output / "public/manifest.json").read_text())["comparison_eligible"]) + + + + + + def test_corpus_counts_are_exact(self): + for name, transmissions, observations in [("S", 30_000, 90_000), ("B", 128_000, 2_048_000), ("L", 1_125_000, 18_000_000)]: + shape = bench.corpus(name) + self.assertEqual(shape["transmissions"], transmissions) + self.assertEqual(sum(bench.fanout(name, i) for i in range(transmissions)), observations) + + def test_pair_order_is_serial_and_alternates(self): + self.assertEqual(bench.pair_order(3), [(0, "baseline"), (0, "candidate"), (1, "candidate"), (1, "baseline"), (2, "baseline"), (2, "candidate")]) + + def test_sparse_p99_is_not_reported_as_zero(self): + self.assertIsNone(bench.percentiles([1, 2, 3])["p99"]) + self.assertIsNone(bench.percentiles([])["p50"]) + self.assertEqual(bench.percentiles(range(1, 1001))["p99"], 990) + + def test_logical_digest_distinguishes_null_and_empty_and_normalizes_numbers(self): + self.assertNotEqual(bench.canonical_row([None]), bench.canonical_row([""])) + self.assertEqual(bench.canonical_row([1]), bench.canonical_row([1.0])) + self.assertNotEqual(bench.canonical_row(["1"]), bench.canonical_row([1])) + + def test_destinations_fail_closed(self): + with self.assertRaises(ValueError): + bench.safe_output(pathlib.Path("/tmp")) + with tempfile.TemporaryDirectory() as directory: + target = pathlib.Path(directory) / "corescope-bench-fixture" + self.assertEqual(bench.safe_output(target), target.resolve()) + target.mkdir() + with self.assertRaises(ValueError): + bench.safe_output(target) + + def test_floating_or_wrong_baseline_is_rejected(self): + with self.assertRaises(ValueError): + bench.check_revisions("master", "1" * 40) + with self.assertRaises(ValueError): + bench.check_revisions("2" * 40, "HEAD") + bench.check_revisions(bench.UPSTREAM, "1" * 40) + + def test_primary_requires_five_pairs_and_resource_controls(self): + with self.assertRaises(ValueError): + bench.validate_profile("primary", 1, None) + with self.assertRaises(ValueError): + bench.validate_profile("primary", 5, None) + bench.validate_profile("smoke", 1, None) + + def test_epoch_shift_preserves_nanoseconds_and_nulls(self): + self.assertEqual(bench.shift_timestamp("2026-10-08T12:00:00.123456789Z", 3600), "2026-10-08T13:00:00.123456789Z") + self.assertEqual(bench.shift_timestamp("2026-10-08 12:00:00", -1), "2026-10-08 11:59:59") + self.assertIsNone(bench.shift_timestamp(None, 1)) + with self.assertRaises(ValueError): + bench.shift_timestamp("unexpected", 1) + + def test_public_bundle_excludes_databases_payloads_and_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / ".runtime").mkdir() + (root / ".runtime" / "events.jsonl").write_text("private packet payload") + (root / "manifest.json").write_text('{"path":"' + str(root).replace('\\', '\\\\') + '"}') + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + (run / "replay-config.json").write_text("private control") + (run / "requests.jsonl").write_text('{"class":"nodes"}\n') + bench.public_bundle(root) + files = {p.relative_to(root / "public").as_posix() for p in (root / "public").rglob("*") if p.is_file()} + self.assertEqual(files, {"manifest.json", "pair-00/sqlite/requests.jsonl"}) + + def test_visibility_is_bound_to_durable_sent_hashes(self): + sent = [{"hash":"a", "completed_ns":100, "new_transmission":True, "measured":True}, {"hash":"b", "completed_ns":200, "new_transmission":True, "measured":True}] + result = bench.visibility(sent,[{"hash":"a","received_ns":90},{"hash":"a","received_ns":110},{"hash":"unrelated","received_ns":300}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"],["b"]) + self.assertEqual(result["duplicate_hash_receipts"],1) + self.assertEqual(result["visible_before_ingest_return"],1) + self.assertIsNone(result["lag_ns"]["p99"]) + + def test_visibility_headline_excludes_warmup_but_keeps_earliest_negative_lags(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + received = [{"hash": "warm", "received_ns": 90}, {"hash": "warm", "received_ns": 95}, + {"hash": "warm", "received_ns": 99}, {"hash": "measured", "received_ns": 190}, + {"hash": "measured", "received_ns": 220}] + result = bench.visibility(sent, received) + self.assertTrue(result["verified"]) + self.assertEqual(result["samples_ns"], [-10]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertEqual(result["duplicate_hash_receipts"], 1) + self.assertEqual(result["duplicate_hash_receipts_all"], 3) + self.assertEqual(result["visible_before_ingest_return"], 1) + self.assertEqual(result["visible_before_ingest_return_all"], 2) + + def test_missing_warmup_sentinel_still_fails_verification(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + result = bench.visibility(sent, [{"hash": "measured", "received_ns": 230}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"], ["warm"]) + self.assertEqual(result["measured_missing"], 0) + self.assertEqual(result["samples_ns"], [30]) + + def test_warmup_does_not_make_sparse_visibility_p99_reportable(self): + sent = [{"hash": str(i), "completed_ns": 100, "new_transmission": True, "measured": i == 1000} for i in range(1001)] + result = bench.visibility(sent, [{"hash": str(i), "received_ns": 110} for i in range(1001)]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertIsNone(result["lag_ns"]["p99"]) + + def test_cgroup_launch_preserves_arguments_without_shell_interpolation(self): + budget = bench.Budget.__new__(bench.Budget) + budget.path = pathlib.Path("/sys/fs/cgroup/bench-test") + arguments = ["/tmp/application", "$(must-not-execute)", "a b", "a'b"] + launched = budget.arguments(arguments) + self.assertEqual(launched[-len(arguments):], arguments) + self.assertEqual(launched[-len(arguments)-1], str(budget.path)) + self.assertNotIn("must-not-execute", launched[2]) + budget.path = None + self.assertEqual(budget.arguments(arguments), arguments) + + def test_failure_diagnostic_keeps_errors_but_excludes_sensitive_log_data(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + log = root / "replay.log" + log.write_text('ordinary packet body: do not publish me\n' + ' zz_benchmark_ingestor_test.go:459: ERROR: missing relation "private_table" (SQLSTATE 42P01)\n' + ' zz_benchmark_ingestor_test.go:460: failed mqtt://writer:topsecret@private.example/db\n' + ' zz_benchmark_ingestor_test.go:461: failed at /home/private/state/config.json and C:\\Private\\state.json\n' + ' zz_benchmark_ingestor_test.go:462: payload={"text":"private body"} raw=00112233445566778899aabbccddeeff\n' + ' zz_benchmark_ingestor_test.go:463: host=private.example password=topsecret\n' + '--- FAIL: TestCoreScopeBenchmark (1.20s)\n') + diagnostic = bench.record_failure(log, 1) + artifact = root / "failure-replay.json" + text = artifact.read_text() + self.assertEqual(diagnostic["log"], "replay.log") + self.assertEqual(diagnostic["exit_code"], 1) + self.assertIn("SQLSTATE 42P01", text) + self.assertIn("TestCoreScopeBenchmark", text) + for private in ("do not publish me", "topsecret", "private.example", "/home/private", "Private", "private body", "00112233", "private_table"): + self.assertNotIn(private, text) + self.assertLess(len(text), 5000) + + def test_public_bundle_includes_only_sanitized_failure_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + logs = root / "logs" + logs.mkdir() + log = logs / "candidate-build.log" + log.write_text("private raw log\n./main.go:27: undefined: MissingSymbol\n") + bench.record_failure(log, 1) + bench.public_bundle(root) + self.assertTrue((root / "public/logs/failure-candidate-build.json").is_file()) + self.assertFalse((root / "public/logs/candidate-build.log").exists()) + + def test_failure_diagnostic_bounds_long_logs(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "worker.log" + log.write_text("ignored prefix\n" * 10000 + "panic: " + "x" * 20000 + "\n--- FAIL: TestCoreScopeBenchmark (1s)\n") + result = bench.record_failure(log, 2) + self.assertLessEqual(sum(len(x) for x in result["diagnostics"]), 2400) + self.assertIn("--- FAIL: TestCoreScopeBenchmark (1s)", result["diagnostics"]) + + + def test_finish_records_failed_child_before_raising(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "replay.log" + log.write_text(" zz_benchmark_ingestor_test.go:460: durable effects did not match\n") + child = types.SimpleNamespace(wait=lambda timeout: 1, poll=lambda: 1, bench_log=open(log, "a")) + with self.assertRaisesRegex(RuntimeError, "failure-replay.json"): + bench.finish(child) + self.assertTrue(child.bench_log.closed) + self.assertIn("durable effects did not match", (log.parent / "failure-replay.json").read_text()) + + def test_retained_load_requires_every_eligible_row(self): + with tempfile.TemporaryDirectory() as directory: + source = pathlib.Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE transmissions(id INTEGER,last_seen INTEGER);" + "CREATE TABLE observations(transmission_id INTEGER);" + "INSERT INTO transmissions VALUES(1,90),(2,100),(3,110);" + "INSERT INTO observations VALUES(1),(2),(2),(3);") + db.close() + expected = bench.retained_counts(source, 100) + self.assertEqual(expected, {"transmissions": 2, "observations": 3}) + bench.check_loaded_rows({"loadedTx": 2, "loadedObs": 3}, expected) + for incomplete in ({"loadedTx": 1, "loadedObs": 3}, {"loadedTx": 2, "loadedObs": 2}): + with self.assertRaisesRegex(RuntimeError, "retained corpus"): + bench.check_loaded_rows(incomplete, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/sqlite-benchmark/test_sqlite_controls.py b/scripts/sqlite-benchmark/test_sqlite_controls.py new file mode 100644 index 000000000..3b5b4390a --- /dev/null +++ b/scripts/sqlite-benchmark/test_sqlite_controls.py @@ -0,0 +1,267 @@ +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("sqlite_bench", HERE / "run.py") +bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench) + + +class SQLiteControls(unittest.TestCase): + def test_legacy_and_current_table_inventories_remain_complete(self): + additions = ["tx_rewrite_feed", "key_backfill_state", "key_backfill_done", "client_node_names"] + for extra_count in (0, 3, 4): + with self.subTest(extra_count=extra_count), tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + names = bench.TABLES + additions[:extra_count] + with bench.sqlite3.connect(source) as db: + for name in names: + db.execute(f'CREATE TABLE "{name}"(value TEXT)') + db.execute(f'INSERT INTO "{name}" VALUES(?)', ("synthetic",)) + db.close() + tables = bench.table_manifest(source) + self.assertEqual(len(tables), 18 + extra_count) + self.assertEqual(set(bench.validate(source, tables)), set(names)) + + def test_client_name_receipts_shift_without_changing_reported_clock(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE client_node_names(" + "id INTEGER PRIMARY KEY AUTOINCREMENT, target TEXT NOT NULL," + "rx_pubkey TEXT NOT NULL, observed_at TEXT NOT NULL," + "ingested_at TEXT NOT NULL, name TEXT NOT NULL, lat REAL," + "lon REAL, pos_acc_m REAL, repeater_clock INTEGER," + "UNIQUE(target, rx_pubkey, observed_at));") + db.execute("INSERT INTO client_node_names VALUES(?,?,?,?,?,?,?,?,?,?)", + (1, "synthetic-target", "synthetic-receiver", "2026-01-01T00:00:00.123Z", + "2026-01-01T00:00:01Z", "Synthetic node", None, None, None, 1767225590)) + db.close() + with mock.patch.object(bench, "TABLES", ["client_node_names"]): + tables = bench.table_manifest(source) + before = bench.validate(source, tables) + bench.freshen(source, 10) + after = bench.validate(source, tables) + self.assertEqual(tables, [{"name": "client_node_names", "columns": [ + "id", "target", "rx_pubkey", "observed_at", "ingested_at", "name", + "lat", "lon", "pos_acc_m", "repeater_clock"]}]) + self.assertNotEqual(before, after) + with bench.sqlite3.connect(source) as db: + row = db.execute("SELECT * FROM client_node_names").fetchone() + db.close() + self.assertEqual(row, (1, "synthetic-target", "synthetic-receiver", + "2026-01-01T00:00:10.123Z", "2026-01-01T00:00:11Z", + "Synthetic node", None, None, None, 1767225590)) + + def test_all_application_tables_are_freshened_and_digested(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE observers(id TEXT); INSERT INTO observers VALUES('synthetic');" + "CREATE TABLE key_backfill_state(started_at INTEGER, cursor INTEGER);" + "INSERT INTO key_backfill_state VALUES(100, 7);" + "CREATE TABLE key_backfill_done(done_at INTEGER);" + "INSERT INTO key_backfill_done VALUES(110);" + "CREATE TABLE tx_rewrite_feed(id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER);" + "INSERT INTO tx_rewrite_feed(created_at) VALUES(100);") + db.close() + with mock.patch.object(bench, "TABLES", ["observers"]): + tables = bench.table_manifest(source) + self.assertEqual({table["name"] for table in tables}, + {"observers", "key_backfill_state", "key_backfill_done", "tx_rewrite_feed"}) + before = bench.validate(source, tables) + bench.freshen(source, 10) + after = bench.validate(source, tables) + self.assertEqual(before["observers"], after["observers"]) + self.assertNotEqual(before["key_backfill_state"], after["key_backfill_state"]) + with bench.sqlite3.connect(source) as db: + state = db.execute("SELECT started_at, cursor FROM key_backfill_state").fetchone() + feed = db.execute("SELECT created_at FROM tx_rewrite_feed").fetchone() + done = db.execute("SELECT done_at FROM key_backfill_done").fetchone() + db.close() + self.assertEqual(state, (110, 7)) + self.assertEqual(feed, (110,)) + self.assertEqual(done, (120,)) + + def test_quoted_identifiers_are_freshened_and_digested(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript('CREATE TABLE observers(id TEXT);' + 'CREATE TABLE "extra""table"("value""column" TEXT, started_at INTEGER);' + 'INSERT INTO "extra""table" VALUES(\'synthetic\',100);') + db.close() + with mock.patch.object(bench, "TABLES", ["observers"]): + tables = bench.table_manifest(source) + before = bench.validate(source, tables) + self.assertIn('extra"table', before, "quoted application table was omitted from validation") + bench.freshen(source, 10) + after = bench.validate(source, tables) + self.assertNotEqual(before['extra"table'], after['extra"table']) + with bench.sqlite3.connect(source) as db: + values = db.execute('SELECT "value""column", started_at FROM "extra""table"').fetchone() + db.close() + self.assertEqual(values, ("synthetic", 110)) + + def test_validation_rejects_unaccounted_application_schema_changes(self): + for statement in ("CREATE TABLE extra(value TEXT)", "ALTER TABLE observers ADD COLUMN extra TEXT", + "DROP TABLE observers"): + with self.subTest(statement=statement), tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.execute("CREATE TABLE observers(id TEXT)") + db.close() + with mock.patch.object(bench, "TABLES", ["observers"]): + tables = bench.table_manifest(source) + with bench.sqlite3.connect(source) as db: + db.execute(statement) + db.close() + try: + bench.validate(source, tables) + except Exception as error: + self.assertIsInstance(error, RuntimeError) + self.assertRegex(str(error), "schema differs") + else: + self.fail("validation accepted unaccounted application schema changes") + + def test_manifest_still_requires_the_canonical_application_tables(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.execute("CREATE TABLE unrelated(value TEXT)") + db.close() + with self.assertRaisesRegex(ValueError, "corpus lacks canonical table"): + bench.table_manifest(source) + + def test_completed_pair_requires_unchanged_input_and_equal_cache_coverage(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + events = root / "events.jsonl" + events.write_text("synthetic input\n") + digest = bench.sha256(events) + for variant in ("baseline", "candidate"): + run = root / "pair-00" / variant + run.mkdir(parents=True) + for filename in ("startup.json", "startup-hot1.json"): + bench.write_json(run / filename, {"health": {"loadedTx": 10, "loadedObs": 30}}) + bench.verify_completed_pair(root, 0, events, digest) + changed = root / "pair-00/candidate/startup-hot1.json" + bench.write_json(changed, {"health": {"loadedTx": 10, "loadedObs": 29}}) + with self.assertRaisesRegex(RuntimeError, "coverage differs"): + bench.verify_completed_pair(root, 0, events, digest) + bench.write_json(changed, {"health": {"loadedTx": 10, "loadedObs": 30}}) + events.write_text("changed input\n") + with self.assertRaisesRegex(RuntimeError, "event stream changed"): + bench.verify_completed_pair(root, 0, events, digest) + + def test_server_and_workers_force_the_http_profiler_off(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + _,env=bench.server_settings("baseline",root/"data.sqlite",root,{"ENABLE_PPROF":"true"}) + self.assertEqual(env["ENABLE_PPROF"],"false") + with mock.patch.object(bench,"spawn") as spawn: + bench.worker(root/"server.test",dict(control_dir=str(root)),"server",{"ENABLE_PPROF":"true"},root/"log") + self.assertEqual(spawn.call_args.kwargs["env"]["ENABLE_PPROF"],"false") + self.assertIn("-test.run=^TestCoreScopeBenchmarkServer$",spawn.call_args.args[0]) + + def test_summary_keeps_queue_service_and_total_as_separate_distributions(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);run=root/"pair-00/baseline";run.mkdir(parents=True) + events=[dict(**{"class":"request"},measured=True,status=200,scheduled_ns=10,dispatched_ns=20,completed_ns=30), + dict(**{"class":"request"},measured=True,status=200,scheduled_ns=100,dispatched_ns=120,completed_ns=160)] + (run/"requests.jsonl").write_text("".join(json.dumps(row)+"\n" for row in events)) + summary=bench.summarize(root) + self.assertEqual({row.get("timing") for row in summary},{"queue","service","end_to_end"}) + medians={row["timing"]:row["p50_ns"] for row in summary} + self.assertEqual(medians,{"queue":10,"service":10,"end_to_end":20}) + + def test_harness_ref_is_verified_independently_and_dirty_content_fails(self): + with tempfile.TemporaryDirectory() as directory: + here = Path(directory) + (here / "run.py").write_text("print('reviewed')\n") + calls = [] + def git(args, **kwargs): + calls.append(args) + if args[1] == "rev-parse": return "4" * 40 + "\n" + if args[1] == "ls-tree": return "scripts/sqlite-benchmark/run.py\n" + return "print('reviewed')\n" + with mock.patch.object(bench, "command", side_effect=git): + digest = bench.verify_harness(here, "4" * 40, here) + self.assertEqual(len(digest), 64) + (here / "run.py").write_text("print('changed')\n") + with self.assertRaisesRegex(ValueError, "content differs"): + bench.verify_harness(here, "4" * 40, here) + (here / "extra.py").write_text("unexpected\n") + with self.assertRaisesRegex(ValueError, "files differ"): + bench.verify_harness(here, "4" * 40, here) + self.assertTrue(any("4" * 40 + ":scripts/sqlite-benchmark/run.py" in call for call in calls)) + + def test_bounded_cache_targets_cannot_relax_s_or_b(self): + for corpus in ("S", "B"): + self.assertIsNone(bench.declared_coverage(corpus, None, None)) + with self.assertRaises(ValueError): bench.declared_coverage(corpus, 1, 1) + for counts in ((None,None),(1,None),(0,1),(-1,1),(2_000_000,18_000_000)): + with self.assertRaises(ValueError): bench.declared_coverage("L", *counts) + self.assertEqual(bench.corpus("L")["observations"],18_000_000) + self.assertEqual(bench.declared_coverage("L",100000,1600000),dict(transmissions=100000,observations=1600000)) + + def test_capacity_guard_uses_actual_available_memory_and_disk(self): + from types import SimpleNamespace + with mock.patch.object(bench.shutil,"disk_usage",return_value=SimpleNamespace(free=129*1024**3)): + with mock.patch.object(bench.Path,"read_text",return_value="MemAvailable: 9000000 kB\n"): + self.assertGreater(bench.capacity_preflight("L",Path("."))["available_ram_bytes"],8*1024**3) + with mock.patch.object(bench.Path,"read_text",return_value="MemAvailable: 1000000 kB\n"): + with self.assertRaisesRegex(ValueError,"available RAM"):bench.capacity_preflight("L",Path(".")) + with mock.patch.object(bench.shutil,"disk_usage",return_value=SimpleNamespace(free=1024)): + with self.assertRaisesRegex(ValueError,"available disk"):bench.capacity_preflight("B",Path(".")) + + def test_diagnostic_is_separate_and_requires_a_common_budget(self): + with self.assertRaises(ValueError):bench.validate_profile("diagnostic",1,None) + with self.assertRaises(ValueError):bench.validate_profile("diagnostic",5,"owned") + bench.validate_profile("diagnostic",1,"owned") + + def test_private_raw_profiles_are_not_published(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);run=root/"pair-00/baseline" + private=run/".profiles/server";private.mkdir(parents=True) + (private/"cpu.pprof").write_bytes(b"private raw profile") + public=run/"profiles";public.mkdir() + (public/"server-cpu-top.txt").write_text("safe module.function\n") + bench.public_bundle(root) + self.assertFalse((root/"public/pair-00/baseline/.profiles").exists()) + self.assertEqual((root/"public/pair-00/baseline/profiles/server-cpu-top.txt").read_text(),"safe module.function\n") + + def test_two_native_variants_alternate(self): + self.assertEqual(bench.pair_order(2), [(0, "baseline"), (0, "candidate"), (1, "candidate"), (1, "baseline")]) + + def test_baseline_and_candidate_are_independent_exact_revisions(self): + bench.check_revisions("2" * 40, "3" * 40) + bench.check_revisions("2" * 40, "2" * 40) # A/A controller qualification. + for left, right in (("master", "3" * 40), ("2" * 40, "HEAD")): + with self.assertRaises(ValueError): + bench.check_revisions(left, right) + + def test_both_variants_build_the_native_adapter_with_trimmed_paths(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + binaries, logs = root / "bin", root / "logs" + binaries.mkdir(); logs.mkdir() + for variant in ("baseline", "candidate"): + with mock.patch.object(bench, "production_hash", return_value="stable"), mock.patch.object(bench, "command") as command: + bench.build(root, variant, binaries, {}, logs) + overlay = json.loads((binaries / (variant + "-overlay.json")).read_text()) + adapters = [p for p in overlay["Replace"].values() if p.endswith("_adapter_test.go")] + self.assertEqual(adapters, [str(HERE / "overlays/sqlite_adapter_test.go")]) + builds = [call.args[0] for call in command.call_args_list if call.args[0][0] == "go"] + self.assertEqual(len(builds), 3) + for args in builds: + self.assertIn("-trimpath", args) + self.assertFalse(any("migrate" in str(arg) for args in builds for arg in args)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test-all.sh b/test-all.sh index d84bca083..db13ff777 100755 --- a/test-all.sh +++ b/test-all.sh @@ -216,6 +216,7 @@ node tests/unit/test-rx-coverage-viewport.js node tests/unit/test-scope-audit-styles-linked.js node tests/unit/test-settings-sync.js node tests/unit/test-slideover-1056-rowsel-strict.js +node tests/unit/test-sqlite-benchmark-ci.js node tests/unit/test-top-routes-overlay.js node tests/unit/test-traces.js node tests/unit/test-url-state.js diff --git a/tests/unit/test-sqlite-benchmark-ci.js b/tests/unit/test-sqlite-benchmark-ci.js new file mode 100644 index 000000000..30527be5d --- /dev/null +++ b/tests/unit/test-sqlite-benchmark-ci.js @@ -0,0 +1,206 @@ +// Exercise the actual CI request parser and shell guard; no network or benchmarks. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const vm = require('node:vm'); +const { spawnSync } = require('node:child_process'); +const root = path.resolve(__dirname, '../..'); +const read = name => { + const file = path.join(root, '.github/workflows', name); + return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').replace(/\r/g, '') : ''; +}; +const deploy = read('deploy.yml'); +const workflow = read('sqlite-benchmark.yml'); +function block(source, key, indent) { + const lines = source.split('\n'); + const start = lines.findIndex(line => line.startsWith(' '.repeat(indent) + key + ':')); + if (start < 0) return ''; + let end = start + 1; + while (end < lines.length && (!lines[end].trim() || lines[end].search(/\S/) > indent)) end++; + return lines.slice(start, end).join('\n'); +} +function value(source, key, indent) { + const lines = block(source, key, indent).split('\n'); + if (!lines[0]) return ''; + const first = lines[0].slice(indent + key.length + 1).trim(); + return ['|', '>'].includes(first) ? lines.slice(1).map(line => line.slice(indent + 2)).join('\n') : first; +} +const step = (name, source = workflow) => source.split(/(?=^ - name:)/m).find(part => value(part, '- name', 6) === name) || ''; +const evaluate = (expression, context) => vm.runInNewContext(expression.replace(/\$\{\{|\}\}/g, '').trim(), { + ...context, startsWith: (text, prefix) => text.startsWith(prefix), cancelled: () => false, +}); +const needs = Object.fromEntries(['go-test', 'e2e-test', 'image-check', 'build-and-publish'].map(name => [name, { result: 'success' }])); +needs.changes = { outputs: { code: 'true', ingestor: 'true' }, result: 'success' }; +for (const ref of ['refs/heads/master', 'refs/tags/v9.8.7']) { + const context = { github: { event_name: 'workflow_dispatch', ref }, inputs: { sqlite_benchmark: true, images_published: false }, needs, vars: { ENABLE_STAGING_DEPLOY: 'true' } }; + for (const job of ['build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + assert.equal(Boolean(evaluate(value(block(deploy, job, 2), 'if', 4), context)), false, + `benchmark-only dispatch must never enter ${job} on ${ref}`); + } +} +console.log('PASS benchmark-only dispatch cannot publish or deploy even when normal gates are green'); +assert(workflow, 'SQLite benchmark workflow is missing'); +assert.equal(value(block(workflow, 'permissions', 0), 'contents', 2), 'read'); +assert(!/^\s*(secrets:|environment:|packages:|id-token:)/m.test(workflow), 'benchmark gains secrets or write permissions'); +assert(!block(deploy, 'pull_request', 2).includes('edited'), 'body edits must not overwrite ordinary required CI checks'); +const caller = block(deploy, 'sqlite-benchmark', 2); +assert.equal(value(caller, 'uses', 4), './.github/workflows/sqlite-benchmark.yml'); +assert.equal(value(block(caller, 'permissions', 4), 'contents', 6), 'read'); +assert(!caller.includes('secrets:'), 'dispatcher must not inherit secrets'); +assert(!block(deploy, 'permissions', 0).includes('write'), 'benchmark dispatcher inherits write permissions'); +assert(block(deploy, 'concurrency', 0).includes('inputs.sqlite_benchmark'), 'benchmark-only dispatch can block normal CI'); + +const parser = value(step('Parse benchmark request'), 'run', 8).split("python3 - <<'PY'\n")[1]?.split('\nPY')[0]; +assert(parser, 'missing executable Python request parser'); +assert(parser.includes('GITHUB_EVENT_PATH'), 'PR body must be parsed from the raw event, not inserted into shell source'); +assert(!parser.includes('${{'), 'request parser must not interpolate untrusted expressions'); +const A = 'a'.repeat(40), B = 'b'.repeat(40), H = 'c'.repeat(40); +const baseline = '7fa86fe1c1304b88e0f2a44052465dfb86abb8a6'; +const checked = '- [x] Run SQLite benchmark (primary)'; +const body = `${checked}\nsqlite-baseline-sha: ${A}\nsqlite-candidate-sha: ${B}\nsqlite-harness-sha: ${H}`; +function request(text = '', { action = 'synchronize', previous, changes, event = 'pull_request', inputs = {} } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-ci-request-')); + const eventFile = path.join(dir, 'event.json'), output = path.join(dir, 'output'); + fs.writeFileSync(eventFile, JSON.stringify({ action, pull_request: { body: text }, + changes: changes || (previous === undefined ? {} : { body: { from: previous } }) })); + fs.writeFileSync(output, ''); + try { + const result = spawnSync(process.platform === 'win32' ? 'python' : 'python3', ['-c', parser], { + encoding: 'utf8', env: { ...process.env, GITHUB_EVENT_PATH: eventFile, GITHUB_OUTPUT: output, + EVENT_NAME: event, EVENT_SHA: H, SOURCE_REPOSITORY: 'contributor/project', + INPUT_PROFILE: inputs.profile || '', INPUT_BASELINE: inputs.baseline || '', + INPUT_CANDIDATE: inputs.candidate || '', INPUT_HARNESS: inputs.harness || '' }, + }); + return { status: result.status, error: result.stderr, values: Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split(/\r?\n/).filter(Boolean).map(line => { + const at = line.indexOf('='); return [line.slice(0, at), line.slice(at + 1)]; + })) }; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} +const smoke = request(); +assert.equal(smoke.status, 0, smoke.error); +assert.deepEqual(smoke.values, { profile: 'smoke', baseline_sha: baseline, candidate_sha: H, harness_sha: H, repository: 'contributor/project' }); +const unchecked = request(body.replace('[x]', '[ ]')); +assert.equal(unchecked.status, 0, unchecked.error); +assert.deepEqual(unchecked.values, smoke.values, 'inactive historical comparison markers changed the smoke qualification'); +const dispatchedSmoke = request('', { event: 'workflow_dispatch' }); +assert.equal(dispatchedSmoke.status, 0, dispatchedSmoke.error); +assert.deepEqual(dispatchedSmoke.values, smoke.values, 'dispatch defaults differ from automatic smoke qualification'); +const primary = request(body); +assert.equal(primary.status, 0, primary.error); +assert.deepEqual(primary.values, { profile: 'primary', baseline_sha: A, candidate_sha: B, harness_sha: H, repository: 'contributor/project' }); +assert.equal(request(body.replace('(primary)', '(diagnostic)')).values.profile, 'diagnostic'); +for (const text of ['', '- [ ] Run SQLite benchmark (primary)', '```\n' + body + '\n```', '````\n```\n' + body + '\n````', '~~~text\n' + body + '\n~~~']) { + assert.equal(request(text, { action: 'edited', previous: body }).values.profile, 'off', 'unchecked/code-fenced body started a benchmark'); +} +assert.equal(request(body + '\nUnrelated prose.', { action: 'edited', previous: body }).values.profile, 'off'); +assert.equal(request(body, { action: 'edited', changes: { title: { from: 'old' } } }).values.profile, 'off'); +assert.equal(request(body, { action: 'edited', previous: '' }).values.profile, 'primary'); +assert.equal(request(body.replace(B, 'd'.repeat(40)), { action: 'edited', previous: body }).values.profile, 'primary'); +assert.equal(request(body, { action: 'edited', previous: body.replace(B, 'master') }).values.profile, 'primary'); +for (const bad of [body.replace(B, 'master'), body.replace(B, '$(' + 'echo unsafe)'), body + `\nsqlite-candidate-sha: ${A}`, + body + '\n- [x] Run SQLite benchmark (diagnostic)']) { + assert.notEqual(request(bad).status, 0, 'ambiguous or mutable request accepted'); +} +for (const profile of ['smoke', 'primary', 'diagnostic']) { + const result = request('', { event: 'workflow_dispatch', inputs: { profile, baseline: A, candidate: B, harness: H } }); + assert.equal(result.status, 0, result.error); + assert.equal(result.values.profile, profile); + assert.equal(result.values.candidate_sha, B); +} +assert.notEqual(request('', { event: 'workflow_dispatch', inputs: { profile: 'primary;echo unsafe' } }).status, 0); +console.log('PASS raw PR requests, edit gating, immutable independent refs, and explicit profiles'); + +const checkout = step('Checkout the exact harness'); +assert.equal(value(checkout, 'persist-credentials', 10), 'false'); +assert.equal(value(checkout, 'fetch-depth', 10), '0'); +assert(value(checkout, 'ref', 10).includes('harness_sha')); +const guard = value(step('Verify exact refs and workflow source'), 'run', 8); +assert(guard.includes('WORKFLOW_SHA') && guard.includes('EVENT_SHA') && guard.includes('git diff --quiet'), 'executed workflow is not pinned'); +let bash = process.env.BASH_PATH || 'bash'; +if (process.platform === 'win32' && !process.env.BASH_PATH) { + const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); + const found = path.resolve(git.stdout.trim(), '../../../bin/bash.exe'); + if (fs.existsSync(found)) bash = found; +} +function sourceGuard({ checkout = H, matches = true, candidate = B } = {}) { + const stub = `git(){ case "$1" in rev-parse) if [ "$2" = HEAD ]; then printf '%s\\n' "$ACTUAL_HEAD"; else printf '%s\\n' "\${2%\\^\\{commit\\}}"; fi;; fetch) return 0;; diff) [ "$MATCHES" = true ];; *) return 99;; esac; }\n`; + return spawnSync(bash, ['--noprofile', '--norc'], { input: stub + guard, encoding: 'utf8', env: { ...process.env, + BASELINE_SHA: A, CANDIDATE_SHA: candidate, HARNESS_SHA: H, EVENT_SHA: 'd'.repeat(40), WORKFLOW_SHA: 'e'.repeat(40), + ACTUAL_HEAD: checkout, MATCHES: String(matches), GITHUB_SERVER_URL: 'https://github.com', GITHUB_REPOSITORY: 'example/project' } }); +} +const goodGuard = sourceGuard(); +assert.equal(goodGuard.status, 0, goodGuard.stderr); +assert.notEqual(sourceGuard({ checkout: A }).status, 0, 'wrong harness checkout accepted'); +assert.notEqual(sourceGuard({ matches: false }).status, 0, 'mismatched workflow accepted'); +assert.notEqual(sourceGuard({ candidate: 'master' }).status, 0, 'mutable candidate accepted by execution guard'); +console.log('PASS actual shell guard rejects mismatched source and mutable refs'); + +const bashPath = file => process.platform === 'win32' ? file.replace(/\\/g, '/').replace(/^([A-Za-z]):/, (_, drive) => '/' + drive.toLowerCase()) : file; +const scopeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-ci-scope-')); +try { + const output = path.join(scopeDir, 'output'); + const scope = value(step('Decide whether anything but documentation changed', deploy), 'run', 8); + const result = spawnSync(bash, ['--noprofile', '--norc'], { + input: 'git(){ echo "unexpected checkout operation" >&2; return 99; }\n' + scope.replace(/\$\{\{.*?\}\}/g, 'unused'), + encoding: 'utf8', env: { ...process.env, BENCHMARK_ONLY: 'true', GITHUB_OUTPUT: bashPath(output) }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(fs.readFileSync(output, 'utf8').replace(/\r/g, ''), 'code=false\ningestor=false\n'); + const context = { github: { event_name: 'workflow_dispatch', ref: 'refs/heads/master' }, + inputs: { sqlite_benchmark: true }, needs: { changes: { outputs: { code: 'false', ingestor: 'false' } } } }; + for (const job of ['go-test', 'race-test', 'e2e-shard', 'e2e-test', 'image-check']) { + assert.equal(Boolean(evaluate(value(block(deploy, job, 2), 'if', 4), context)), false, `benchmark dispatch enters ${job}`); + } +} finally { fs.rmSync(scopeDir, { recursive: true, force: true }); } +console.log('PASS actual dispatch scope disables ordinary test and image jobs'); + +const run = value(step('Run bounded smoke and selected profile'), 'run', 8); +for (const flag of ['--baseline-sha "$BASELINE_SHA"', '--candidate-sha "$CANDIDATE_SHA"', '--harness-sha "$HARNESS_SHA"']) assert(run.includes(flag), 'missing independent ref ' + flag); +assert(run.includes('--corpus S --profile smoke --pairs 1'), 'mandatory S smoke missing'); +assert(run.includes('--corpus B --profile primary --pairs 5'), 'paired B comparison missing'); +assert(run.includes('--corpus B --profile diagnostic --pairs 1'), 'separate diagnostics missing'); +assert(run.includes('cpu memory io') && run.includes('cgroup.subtree_control'), 'required cgroup delegation missing'); +assert(workflow.includes('python3 -m unittest discover -s scripts/sqlite-benchmark'), 'cheap harness controls missing'); +assert(workflow.includes("go-version: '1.27.2'") && workflow.includes("CGO_ENABLED: '1'"), 'native compiler configuration changed'); +const artifacts = value(step('Upload only sanitized public evidence'), 'path', 10).trim().split('\n'); +assert(artifacts.length > 0 && artifacts.every(file => file.endsWith('/public/')), 'private database/profile files may be uploaded'); +assert(value(step('Remove only the owned cgroup'), 'run', 8).includes('realpath'), 'owned cleanup guard missing'); +for (const name of ['Validate runner capacity', 'Verify exact refs and workflow source', 'Run bounded smoke and selected profile', 'Remove only the owned cgroup']) { + const result = spawnSync(bash, ['-n'], { input: value(step(name), 'run', 8), encoding: 'utf8' }); + assert.equal(result.status, 0, `${name}: ${result.stderr}`); +} +for (const profile of ['smoke', 'primary', 'diagnostic']) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-ci-run-')); + try { + const log = path.join(dir, 'commands'); + fs.writeFileSync(log, ''); + // Replace privileged host setup and the workload with command capture only. + // Execute the actual workflow shell and assert exactly what it would run. + const stubs = ` + sudo(){ case "$1" in mkdir|chown) return 0;; tee) cat >/dev/null;; *) return 99;; esac; } + grep(){ [ "$1" = -qw ] && [[ "$2" =~ ^(cpu|memory|io)$ ]]; } + python3(){ node -e 'require("fs").appendFileSync(process.env.COMMAND_LOG, JSON.stringify(process.argv.slice(1))+"\\n")' "$@"; } + `; + const result = spawnSync(bash, ['--noprofile', '--norc'], { input: stubs + run, encoding: 'utf8', + env: { ...process.env, PROFILE: profile, BASELINE_SHA: A, CANDIDATE_SHA: B, HARNESS_SHA: H, + GITHUB_RUN_ID: 'unit-' + process.pid, GITHUB_RUN_ATTEMPT: '1', COMMAND_LOG: log, + SMOKE_OUTPUT: '/tmp/corescope-bench-smoke', BENCH_OUTPUT: '/tmp/corescope-bench-selected' } }); + assert.equal(result.status, 0, result.stderr); + const commands = fs.readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + assert.equal(commands.length, profile === 'smoke' ? 1 : 2); + for (const [index, args] of commands.entries()) { + const option = name => args[args.indexOf(name) + 1]; + assert.equal(args[0], 'scripts/sqlite-benchmark/run.py'); + assert.equal(option('--baseline-sha'), A); + assert.equal(option('--candidate-sha'), B); + assert.equal(option('--harness-sha'), H); + assert.equal(option('--corpus'), index === 0 ? 'S' : 'B'); + assert.equal(option('--profile'), index === 0 ? 'smoke' : profile); + assert.equal(option('--pairs'), index === 1 && profile === 'primary' ? '5' : '1'); + assert.equal(option('--ingest-rate'), '50'); + assert.equal(option('--http-rate'), '20'); + } + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} +console.log('PASS bounded native smoke, separate optional profiles, and public-only evidence');