From 22761bd9d7ddd0896f51050fd0428a8dec5c8a9e Mon Sep 17 00:00:00 2001 From: Daniel Worku Date: Tue, 13 Oct 2026 16:33:36 -0700 Subject: [PATCH] Run every benchmark through one reusable bench workflow The plan job, runner matrix, replicas, host info, bench-finish and summary job were duplicated across both workflows. They now live once in .github/workflows/bench.yml (workflow_call with suite and replicas inputs). Each suite's phases move into a composite action (.github/actions/), and the entry workflows shrink to their replicas input and a call to bench.yml. bench.yml puts bin/ on PATH so suites call mark directly. Failure paths: bench-finish only runs when the harness checkout succeeded, the summary skips artifacts without host.json instead of crashing and counts ok against the requested replicas (so a job that never reported is a failure, not absent), and the summary no longer runs when plan rejects the inputs. Docs: README gains a layout table and how to add a runner or a benchmark, and the matrix, mark, mise.toml and actionlint comments point at the new layout. --- .github/actionlint.yaml | 2 +- .../actions/rust-spotify-player/action.yml | 63 ++++++++ .github/actions/summarize/action.yml | 15 +- .github/actions/typescript-mastra/action.yml | 78 ++++++++++ .github/workflows/bench.yml | 103 +++++++++++++ .github/workflows/rust-spotify-player.yml | 116 +-------------- .github/workflows/typescript-mastra.yml | 140 +----------------- README.md | 47 ++++-- bin/mark | 5 +- mise.toml | 2 +- 10 files changed, 312 insertions(+), 259 deletions(-) create mode 100644 .github/actions/rust-spotify-player/action.yml create mode 100644 .github/actions/typescript-mastra/action.yml create mode 100644 .github/workflows/bench.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a517121..500dfed 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,7 +1,7 @@ # Runner labels outside actionlint's built-in set. The StarSling labels and the ubuntu-26.04-{2,8} # larger runners are registered at the org level; ubuntu-26.04 is a real GitHub-hosted label that # actionlint 1.7.12 predates (drop it once the pinned actionlint knows it). Keep this list in sync -# with the labels used across .github/workflows/. +# with the matrix labels in .github/workflows/bench.yml. self-hosted-runner: labels: - starsling-ubuntu-26.04-2 diff --git a/.github/actions/rust-spotify-player/action.yml b/.github/actions/rust-spotify-player/action.yml new file mode 100644 index 0000000..58c253f --- /dev/null +++ b/.github/actions/rust-spotify-player/action.yml @@ -0,0 +1,63 @@ +name: "Suite: Rust build (spotify-player)" +description: >- + aome510/spotify-player's CI, after runs-on-demo/benchmark-public's rust suite. Steps named + "phase: " are measured, each opened by `mark`. +runs: + using: composite + steps: + - name: Environment + shell: bash + run: | + { + echo "CARGO_TERM_COLOR=always" + echo "RUST_FEATURES=rodio-backend,media-control,image,notify,fzf" + } >> "$GITHUB_ENV" + - shell: bash + run: mark checkout + - name: "phase: checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: aome510/spotify-player + ref: 7dc9d17a7bf5671eb6129fdf9127729f659637fc + path: work + persist-credentials: false + - shell: bash + run: mark "system packages" + - name: "phase: system packages" + shell: bash + run: | + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + build-essential pkg-config libssl-dev libasound2-dev libdbus-1-dev \ + libxcb-shape0-dev libxcb-xfixes0-dev + - shell: bash + run: mark toolchain + - name: "phase: toolchain" + uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master, 2026-10-09 + with: + toolchain: 1.98.1 + components: clippy, rustfmt + - shell: bash + run: mark "cargo fmt" + - name: "phase: cargo fmt" + shell: bash + working-directory: work + run: cargo fmt --all -- --check + - shell: bash + run: mark "cargo test" + - name: "phase: cargo test" + shell: bash + working-directory: work + run: cargo test --no-default-features --features "$RUST_FEATURES" + - shell: bash + run: mark "cargo clippy" + - name: "phase: cargo clippy" + shell: bash + working-directory: work + run: cargo clippy --no-default-features --features "$RUST_FEATURES" -- -D warnings + - shell: bash + run: mark "cargo clippy (no features)" + - name: "phase: cargo clippy (no features)" + shell: bash + working-directory: work + run: cargo clippy --no-default-features -- -D warnings diff --git a/.github/actions/summarize/action.yml b/.github/actions/summarize/action.yml index cd6b540..90c25aa 100644 --- a/.github/actions/summarize/action.yml +++ b/.github/actions/summarize/action.yml @@ -1,5 +1,9 @@ name: Summarize description: Tabulate every bench job's phase timings into the run summary and a results artifact. +inputs: + replicas: + description: Jobs each runner was asked to run (the denominator of the ok column) + required: true runs: using: composite steps: @@ -11,12 +15,17 @@ runs: # lines, then bench-finish's `end <µs> `). A phase runs to the next boundary. The # table has one row per runner label, with p50 / p90 over that label's successful replicas. - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + REPLICAS: ${{ inputs.replicas }} with: script: | const fs = require("node:fs"); const dir = `${process.env.RUNNER_TEMP}/bench`; const read = (job, file) => fs.readFileSync(`${dir}/${job}/${file}`, "utf8"); - const jobs = (fs.existsSync(dir) ? fs.readdirSync(dir) : []).map((job) => { + // A job whose host-info step failed left no host.json; it only counts against its runner's ok + // column, through the replicas denominator. + const artifacts = fs.existsSync(dir) ? fs.readdirSync(dir).filter((job) => fs.existsSync(`${dir}/${job}/host.json`)) : []; + const jobs = artifacts.map((job) => { const rows = read(job, "phases.tsv").trim().split("\n").map((line) => line.split("\t")); const phases = rows.slice(0, -1).flatMap(([name, start], i) => (name === "-" ? [] : [{ name, seconds: (rows[i + 1][1] - start) / 1e6 }])); const totalSeconds = phases.reduce((sum, phase) => sum + phase.seconds, 0); @@ -41,14 +50,14 @@ runs: cells.sort((a, b) => a.advertisedVcpu - b.advertisedVcpu || a.p50 - b.p50); await core.summary .addHeading(context.workflow) - .addRaw("p50 / p90 seconds over each runner's successful replicas.", true) + .addRaw("p50 / p90 seconds over each runner's successful replicas; ok is successful / requested replicas.", true) .addTable([ ["vCPU", "runner", "RAM GiB", "ok", ...names, "total"].map((data) => ({ data, header: true })), ...cells.map((cell) => [ `${cell.vcpu}`, cell.label, `${cell.memoryGiB}`, - `${cell.ok.length}/${cell.runs.length}`, + `${cell.ok.length}/${process.env.REPLICAS}`, ...names.map((name) => stat(cell.ok.flatMap((job) => job.phases.filter((phase) => phase.name === name).map((phase) => phase.seconds)))), stat(cell.ok.map((job) => job.totalSeconds)), ]), diff --git a/.github/actions/typescript-mastra/action.yml b/.github/actions/typescript-mastra/action.yml new file mode 100644 index 0000000..12a88bc --- /dev/null +++ b/.github/actions/typescript-mastra/action.yml @@ -0,0 +1,78 @@ +name: "Suite: TypeScript build (mastra)" +description: >- + mastra-ai/mastra's checkout, cold install, lint and build. Mirrors the realworld-mastra profile in + hpc-sandbox-benchmarks (same pin, same commands, same cold-store rules). Steps named + "phase: " are measured, each opened by `mark`; the rest is setup and never timed. +runs: + using: composite + steps: + - name: Environment + shell: bash + # pnpm's store is what makes an install cold or warm. pnpm 11 resolves it from PNPM_HOME + # first, then /pnpm/store; images that set PNPM_HOME or carry a store in $HOME + # would otherwise turn the cold install into a warm one. Pin every knob into an empty + # directory under RUNNER_TEMP. A fresh checkout has no turbo cache, but TURBO_FORCE never lets + # a cache hit stand in for a build. + run: | + root="$RUNNER_TEMP/cold" + mkdir -p "$root/pnpm-home" "$root/xdg-data" "$root/xdg-cache" "$root/corepack" + { + echo "CI=true" + echo "COREPACK_ENABLE_DOWNLOAD_PROMPT=0" + echo "DO_NOT_TRACK=1" + echo "TURBO_TELEMETRY_DISABLED=1" + echo "TURBO_FORCE=true" + echo "PNPM_HOME=$root/pnpm-home" + echo "XDG_DATA_HOME=$root/xdg-data" + echo "XDG_CACHE_HOME=$root/xdg-cache" + echo "COREPACK_HOME=$root/corepack" + } >> "$GITHUB_ENV" + - shell: bash + run: mark checkout + - name: "phase: checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: mastra-ai/mastra + ref: b6eeb9d08ebeaf27bc6fd16b6b88087040aaf767 + path: work + persist-credentials: false + - shell: bash + run: mark - + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + - name: Setup pnpm + shell: bash + working-directory: work + # corepack fetches the pnpm that mastra's packageManager field pins (sha512-verified) here, so + # the download never lands inside the cold install. + run: | + corepack enable + corepack install + pnpm --version + # Refuse to time a warm install: the store must not exist yet. + store="$(pnpm store path)" + if [ -e "$store" ]; then + echo "::error::pnpm store already exists at $store; the install would not be cold" + exit 1 + fi + - shell: bash + run: mark "cold install" + - name: "phase: cold install" + shell: bash + working-directory: work + run: pnpm install --frozen-lockfile + - shell: bash + run: mark lint + - name: "phase: lint" + shell: bash + working-directory: work + run: pnpm lint:format + - shell: bash + run: mark build + - name: "phase: build" + shell: bash + working-directory: work + # Scoped to packages/core (and its workspace deps), as in the realworld-mastra profile. + run: pnpm build:core diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..7d55c86 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,103 @@ +name: Bench + +# The pipeline every benchmark shares: plan the replicas, run the suite once per runner and replica, +# then summarize. Each entry workflow (typescript-mastra.yml, rust-spotify-player.yml) calls this +# with its suite name; the suite's phases live in the composite action .github/actions/. + +on: + workflow_call: + inputs: + suite: + description: Suite to run (a composite action under .github/actions/) + type: string + required: true + replicas: + description: Jobs per runner (1-40) + type: number + required: true + +permissions: + contents: read + +jobs: + plan: + runs-on: starsling-ubuntu-26.04-2 + outputs: + replicas: ${{ steps.replicas.outputs.list }} + steps: + # Expressions have no range(): turn the count into the [1..N] list the matrix fans out over. + # 40 keeps 6 runners x N under GitHub's 256-job matrix limit. + - id: replicas + env: + SUITE: ${{ inputs.suite }} + REPLICAS: ${{ inputs.replicas }} + run: | + # Keep in sync with the suite steps of the bench job. + case "$SUITE" in + typescript-mastra | rust-spotify-player) ;; + *) echo "::error::unknown suite '$SUITE'"; exit 1 ;; + esac + if ! [[ "$REPLICAS" =~ ^[0-9]+$ ]] || (( REPLICAS < 1 || REPLICAS > 40 )); then + echo "::error::replicas must be 1-40, got '$REPLICAS'" + exit 1 + fi + echo "list=[$(seq 1 "$REPLICAS" | paste -sd, -)]" >> "$GITHUB_OUTPUT" + + bench: + needs: plan + name: ${{ matrix.vcpu }} vCPU · ${{ matrix.label }} · ${{ matrix.replica }} + strategy: + fail-fast: false + matrix: + vcpu: [2, 4, 8] + provider: [starsling, github] + replica: ${{ fromJSON(needs.plan.outputs.replicas) }} + include: + - { vcpu: 2, provider: starsling, label: starsling-ubuntu-26.04-2 } + - { vcpu: 2, provider: github, label: ubuntu-26.04-2 } + - { vcpu: 4, provider: starsling, label: starsling-ubuntu-26.04 } + # GitHub's standard runner, 4 vCPU in public repositories (2 and 8 are the org's larger + # runners). + - { vcpu: 4, provider: github, label: ubuntu-26.04 } + - { vcpu: 8, provider: starsling, label: starsling-ubuntu-26.04-8 } + - { vcpu: 8, provider: github, label: ubuntu-26.04-8 } + runs-on: ${{ matrix.label }} + timeout-minutes: 45 + steps: + - id: harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: .harness + persist-credentials: false + # Puts bin/mark on PATH for the suite's phase boundaries. + - run: echo "$GITHUB_WORKSPACE/.harness/bin" >> "$GITHUB_PATH" + - name: Host info + uses: ./.harness/.github/actions/host-info + with: + label: ${{ matrix.label }} + vcpu: ${{ matrix.vcpu }} + # `uses:` can't take an expression, so each suite is its own step. + - if: inputs.suite == 'typescript-mastra' + uses: ./.harness/.github/actions/typescript-mastra + - if: inputs.suite == 'rust-spotify-player' + uses: ./.harness/.github/actions/rust-spotify-player + # Needs the harness for the action itself; without it the job left no timings, and the summary + # counts it as a failed replica. + - name: Bench finish + if: always() && steps.harness.outcome == 'success' + uses: ./.harness/.github/actions/bench-finish + with: + key: ${{ matrix.label }}-${{ matrix.replica }} + + # Runs even when bench jobs fail or are cancelled, but not when plan rejected the inputs. + summary: + needs: [plan, bench] + if: always() && needs.plan.result == 'success' + runs-on: starsling-ubuntu-26.04-2 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/summarize + with: + replicas: ${{ inputs.replicas }} diff --git a/.github/workflows/rust-spotify-player.yml b/.github/workflows/rust-spotify-player.yml index 11af413..c6ed9c4 100644 --- a/.github/workflows/rust-spotify-player.yml +++ b/.github/workflows/rust-spotify-player.yml @@ -1,9 +1,7 @@ name: Rust build (spotify-player) -# aome510/spotify-player's CI, after runs-on-demo/benchmark-public's rust suite, on StarSling and -# GitHub-hosted runners. Steps named "phase: " are the measured ones, each opened by a -# `bin/mark` step (GitHub's API does not expose step timings); everything else is setup and never -# timed. +# aome510/spotify-player's CI (fmt, test, clippy) on every runner in bench.yml's matrix. The phases +# live in .github/actions/rust-spotify-player; the shared pipeline is .github/workflows/bench.yml. on: workflow_dispatch: @@ -21,109 +19,9 @@ concurrency: cancel-in-progress: false jobs: - plan: - runs-on: starsling-ubuntu-26.04-2 - outputs: - replicas: ${{ steps.replicas.outputs.list }} - steps: - # Expressions have no range(): turn the count into the [1..N] list the matrix fans out over. - # 40 keeps 6 runners x N under GitHub's 256-job matrix limit. - - id: replicas - env: - REPLICAS: ${{ inputs.replicas }} - run: | - if ! [[ "$REPLICAS" =~ ^[0-9]+$ ]] || (( REPLICAS < 1 || REPLICAS > 40 )); then - echo "::error::replicas must be 1-40, got '$REPLICAS'" - exit 1 - fi - echo "list=[$(seq 1 "$REPLICAS" | paste -sd, -)]" >> "$GITHUB_OUTPUT" - bench: - needs: plan - name: ${{ matrix.vcpu }} vCPU · ${{ matrix.label }} · ${{ matrix.replica }} - strategy: - fail-fast: false - matrix: - vcpu: [2, 4, 8] - provider: [starsling, github] - replica: ${{ fromJSON(needs.plan.outputs.replicas) }} - include: - - { vcpu: 2, provider: starsling, label: starsling-ubuntu-26.04-2 } - - { vcpu: 2, provider: github, label: ubuntu-26.04-2 } - - { vcpu: 4, provider: starsling, label: starsling-ubuntu-26.04 } - # GitHub's standard runner, 4 vCPU in public repositories (2 and 8 are the org's larger - # runners). - - { vcpu: 4, provider: github, label: ubuntu-26.04 } - - { vcpu: 8, provider: starsling, label: starsling-ubuntu-26.04-8 } - - { vcpu: 8, provider: github, label: ubuntu-26.04-8 } - runs-on: ${{ matrix.label }} - timeout-minutes: 45 - env: - SPOTIFY_PLAYER_REPOSITORY: aome510/spotify-player - SPOTIFY_PLAYER_COMMIT: 7dc9d17a7bf5671eb6129fdf9127729f659637fc - CARGO_TERM_COLOR: always - RUST_FEATURES: "rodio-backend,media-control,image,notify,fzf" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: .harness - persist-credentials: false - - name: Host info - uses: ./.harness/.github/actions/host-info - with: - label: ${{ matrix.label }} - vcpu: ${{ matrix.vcpu }} - - run: .harness/bin/mark "checkout" - - name: "phase: checkout" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SPOTIFY_PLAYER_REPOSITORY }} - ref: ${{ env.SPOTIFY_PLAYER_COMMIT }} - path: work - persist-credentials: false - - run: .harness/bin/mark "system packages" - - name: "phase: system packages" - run: | - sudo apt-get update -qq - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ - build-essential pkg-config libssl-dev libasound2-dev libdbus-1-dev \ - libxcb-shape0-dev libxcb-xfixes0-dev - - run: .harness/bin/mark "toolchain" - - name: "phase: toolchain" - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master, 2026-10-09 - with: - toolchain: 1.98.1 - components: clippy, rustfmt - - run: .harness/bin/mark "cargo fmt" - - name: "phase: cargo fmt" - working-directory: work - run: cargo fmt --all -- --check - - run: .harness/bin/mark "cargo test" - - name: "phase: cargo test" - working-directory: work - run: cargo test --no-default-features --features "$RUST_FEATURES" - - run: .harness/bin/mark "cargo clippy" - - name: "phase: cargo clippy" - working-directory: work - run: cargo clippy --no-default-features --features "$RUST_FEATURES" -- -D warnings - - run: .harness/bin/mark "cargo clippy (no features)" - - name: "phase: cargo clippy (no features)" - working-directory: work - run: cargo clippy --no-default-features -- -D warnings - - name: Bench finish - if: always() - uses: ./.harness/.github/actions/bench-finish - with: - key: ${{ matrix.label }}-${{ matrix.replica }} - - summary: - needs: bench - if: always() - runs-on: starsling-ubuntu-26.04-2 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: ./.github/actions/summarize + uses: ./.github/workflows/bench.yml + with: + suite: rust-spotify-player + # workflow_dispatch hands number inputs over as strings; bench.yml takes a number. + replicas: ${{ fromJSON(inputs.replicas) }} diff --git a/.github/workflows/typescript-mastra.yml b/.github/workflows/typescript-mastra.yml index 9d9d991..05c330b 100644 --- a/.github/workflows/typescript-mastra.yml +++ b/.github/workflows/typescript-mastra.yml @@ -1,10 +1,8 @@ name: TypeScript build (mastra) -# mastra-ai/mastra's CI phases, one step per phase, on StarSling and GitHub-hosted runners. Mirrors -# the realworld-mastra profile in hpc-sandbox-benchmarks (same pin, same commands, same cold-store -# rules), split into one Actions step per phase. -# Steps named "phase: " are the measured ones, each opened by a `bin/mark` step (GitHub's -# API does not expose step timings); everything else is setup and never timed. +# mastra-ai/mastra's checkout, cold install, lint and build on every runner in bench.yml's matrix. +# The phases live in .github/actions/typescript-mastra; the shared pipeline is +# .github/workflows/bench.yml. on: workflow_dispatch: @@ -22,131 +20,9 @@ concurrency: cancel-in-progress: false jobs: - plan: - runs-on: starsling-ubuntu-26.04-2 - outputs: - replicas: ${{ steps.replicas.outputs.list }} - steps: - # Expressions have no range(): turn the count into the [1..N] list the matrix fans out over. - # 40 keeps 6 runners x N under GitHub's 256-job matrix limit. - - id: replicas - env: - REPLICAS: ${{ inputs.replicas }} - run: | - if ! [[ "$REPLICAS" =~ ^[0-9]+$ ]] || (( REPLICAS < 1 || REPLICAS > 40 )); then - echo "::error::replicas must be 1-40, got '$REPLICAS'" - exit 1 - fi - echo "list=[$(seq 1 "$REPLICAS" | paste -sd, -)]" >> "$GITHUB_OUTPUT" - bench: - needs: plan - name: ${{ matrix.vcpu }} vCPU · ${{ matrix.label }} · ${{ matrix.replica }} - strategy: - fail-fast: false - matrix: - vcpu: [2, 4, 8] - provider: [starsling, github] - replica: ${{ fromJSON(needs.plan.outputs.replicas) }} - include: - - { vcpu: 2, provider: starsling, label: starsling-ubuntu-26.04-2 } - - { vcpu: 2, provider: github, label: ubuntu-26.04-2 } - - { vcpu: 4, provider: starsling, label: starsling-ubuntu-26.04 } - # GitHub's standard runner, 4 vCPU in public repositories (2 and 8 are the org's larger - # runners). - - { vcpu: 4, provider: github, label: ubuntu-26.04 } - - { vcpu: 8, provider: starsling, label: starsling-ubuntu-26.04-8 } - - { vcpu: 8, provider: github, label: ubuntu-26.04-8 } - runs-on: ${{ matrix.label }} - timeout-minutes: 45 - env: - MASTRA_REPOSITORY: mastra-ai/mastra - MASTRA_COMMIT: b6eeb9d08ebeaf27bc6fd16b6b88087040aaf767 - NODE_VERSION: "22" - CI: "true" - COREPACK_ENABLE_DOWNLOAD_PROMPT: "0" - DO_NOT_TRACK: "1" - TURBO_TELEMETRY_DISABLED: "1" - # A fresh checkout has no turbo cache, but never let a cache hit stand in for a build. - TURBO_FORCE: "true" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: .harness - persist-credentials: false - - name: Host info - uses: ./.harness/.github/actions/host-info - with: - label: ${{ matrix.label }} - vcpu: ${{ matrix.vcpu }} - - name: Isolate caches - # pnpm's store is what makes an install cold or warm. pnpm 11 resolves it from PNPM_HOME - # first, then /pnpm/store; images that set PNPM_HOME or carry a store in - # $HOME would otherwise turn the cold install into a warm one. Pin every knob into an empty - # directory under RUNNER_TEMP. - run: | - root="$RUNNER_TEMP/cold" - mkdir -p "$root/pnpm-home" "$root/xdg-data" "$root/xdg-cache" "$root/corepack" - { - echo "PNPM_HOME=$root/pnpm-home" - echo "XDG_DATA_HOME=$root/xdg-data" - echo "XDG_CACHE_HOME=$root/xdg-cache" - echo "COREPACK_HOME=$root/corepack" - } >> "$GITHUB_ENV" - - run: .harness/bin/mark "checkout" - - name: "phase: checkout" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.MASTRA_REPOSITORY }} - ref: ${{ env.MASTRA_COMMIT }} - path: work - persist-credentials: false - - run: .harness/bin/mark - - - name: Setup Node - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: ${{ env.NODE_VERSION }} - - name: Setup pnpm - working-directory: work - # corepack fetches the pnpm that mastra's packageManager field pins (sha512-verified) here, - # so the download never lands inside the cold install. - run: | - corepack enable - corepack install - pnpm --version - # Refuse to time a warm install: the store must not exist yet. - store="$(pnpm store path)" - if [ -e "$store" ]; then - echo "::error::pnpm store already exists at $store; the install would not be cold" - exit 1 - fi - - run: .harness/bin/mark "cold install" - - name: "phase: cold install" - working-directory: work - run: pnpm install --frozen-lockfile - - run: .harness/bin/mark "lint" - - name: "phase: lint" - working-directory: work - run: pnpm lint:format - - run: .harness/bin/mark "build" - - name: "phase: build" - working-directory: work - # Scoped to packages/core (and its workspace deps), as in the realworld-mastra profile. - run: pnpm build:core - - name: Bench finish - if: always() - uses: ./.harness/.github/actions/bench-finish - with: - key: ${{ matrix.label }}-${{ matrix.replica }} - - summary: - needs: bench - if: always() - runs-on: starsling-ubuntu-26.04-2 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: ./.github/actions/summarize + uses: ./.github/workflows/bench.yml + with: + suite: typescript-mastra + # workflow_dispatch hands number inputs over as strings; bench.yml takes a number. + replicas: ${{ fromJSON(inputs.replicas) }} diff --git a/README.md b/README.md index d2eb5e2..11416e7 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,10 @@ commands and cold caches on every runner. ## Benchmarks -| Workflow | Project | Measured phases | +| Workflow | Project | Measured phases ([suite action](.github/actions)) | |---|---|---| -| [`typescript-mastra.yml`](.github/workflows/typescript-mastra.yml) | [mastra-ai/mastra](https://github.com/mastra-ai/mastra) @ `b6eeb9d` | checkout, cold install (`pnpm install --frozen-lockfile`), lint (`pnpm lint:format`), build (`pnpm build:core`) | -| [`rust-spotify-player.yml`](.github/workflows/rust-spotify-player.yml) | [aome510/spotify-player](https://github.com/aome510/spotify-player) @ `7dc9d17` | checkout, system packages, toolchain, `cargo fmt`, `cargo test`, `cargo clippy`, `cargo clippy` (no features) | +| [`typescript-mastra.yml`](.github/workflows/typescript-mastra.yml) | [mastra-ai/mastra](https://github.com/mastra-ai/mastra) @ `b6eeb9d` | [checkout, cold install (`pnpm install --frozen-lockfile`), lint (`pnpm lint:format`), build (`pnpm build:core`)](.github/actions/typescript-mastra/action.yml) | +| [`rust-spotify-player.yml`](.github/workflows/rust-spotify-player.yml) | [aome510/spotify-player](https://github.com/aome510/spotify-player) @ `7dc9d17` | [checkout, system packages, toolchain, `cargo fmt`, `cargo test`, `cargo clippy`, `cargo clippy` (no features)](.github/actions/rust-spotify-player/action.yml) | The mastra phases follow the realworld mastra profile of [hpc-sandbox-benchmarks](https://github.com/starslingdev/hpc-sandbox-benchmarks); the Rust phases @@ -30,8 +30,9 @@ follow the rust suite of ## Runners -Each workflow runs a matrix of vCPU tier × provider × replica, with `replicas` jobs (default 5) per -cell: +Every benchmark runs the same matrix, defined once in +[`bench.yml`](.github/workflows/bench.yml): vCPU tier × provider × replica, with `replicas` jobs +(default 5) per cell: | vCPU | StarSling | GitHub | |---|---|---| @@ -46,18 +47,21 @@ build starts and warns if the vCPU count differs. Each run ends with a summary job that posts a table to the run's summary page: one row per runner, with the p50 / p90 of every phase and of the total over that runner's successful replicas, and how -many replicas succeeded. Rows are sorted by tier, then by p50 total. Every job's raw timings are +many of the requested replicas succeeded (a job that failed before reporting still counts against +it). Rows are sorted by tier, then by p50 total. Every job's raw timings are attached to the run as a `results.json` artifact. See the [Actions tab](../../actions) for runs. ## Methodology -- **Same work everywhere.** Both projects are pinned to a commit, the Rust toolchain to 1.98.1 and +- **Same work everywhere.** Every benchmark runs through the same pipeline on the same runners. Both + projects are pinned to a commit, the Rust toolchain to 1.98.1 and Node to 22, with pnpm pinned by mastra's `packageManager` field. - **Cold caches.** Nothing is restored from a cache. The mastra job points pnpm's store and corepack at empty directories and refuses to start if a pnpm store already exists, so the install is always cold. `TURBO_FORCE` rules out turbo cache hits. Cargo downloads every crate. - **Only the phases are timed.** Each measured phase is a `phase: ` step opened by - [`bin/mark`](bin/mark), which records a microsecond timestamp. Setup outside the phases (checking + [`bin/mark`](bin/mark), which records a microsecond timestamp, and the last one is closed when + the job finishes. Setup outside the phases (checking out this repository, recording host info, installing Node and pnpm for mastra) is not counted. - **Independent jobs.** `fail-fast` is off, so one runner failing never cancels the others. - **Replicas.** Every cell runs `replicas` independent jobs, each on a fresh runner, all dispatched @@ -76,6 +80,18 @@ gh workflow run typescript-mastra.yml gh workflow run rust-spotify-player.yml -f replicas=10 ``` +## Layout + +| Path | What it is | +|---|---| +| [`.github/workflows/.yml`](.github/workflows) | Entry point per benchmark: the `replicas` input, then a call to `bench.yml`. | +| [`.github/workflows/bench.yml`](.github/workflows/bench.yml) | The shared pipeline: a `plan` job validates the inputs, a `bench` job runs the suite once per runner and replica, and a `summary` job renders the table. | +| [`.github/actions/`](.github/actions) | The suite's phases, as a composite action. | +| [`.github/actions/host-info`](.github/actions/host-info/action.yml) | Records the host (vCPU, RAM, disk, OS image) before the suite starts. | +| [`.github/actions/bench-finish`](.github/actions/bench-finish/action.yml) | Closes the last phase with the job status and uploads host info and timings. | +| [`.github/actions/summarize`](.github/actions/summarize/action.yml) | Builds the p50 / p90 table with `core.summary` and uploads `results.json`. | +| [`bin/mark`](bin/mark) | Records a phase boundary. | + ## Development Tools are pinned in [`mise.toml`](mise.toml): @@ -87,9 +103,18 @@ mise exec -- zizmor . mise exec -- shellcheck bin/mark ``` -To add a runner, add an `include` entry (and its tier or provider if new) to both workflows' -matrices, and to [`.github/actionlint.yaml`](.github/actionlint.yaml) if actionlint doesn't know -the label. +To add a runner, add an `include` entry (and its tier or provider if new) to the matrix in +[`bench.yml`](.github/workflows/bench.yml), and the label to +[`.github/actionlint.yaml`](.github/actionlint.yaml) if actionlint doesn't know it. + +To add a benchmark: + +1. Write its phases as a composite action in `.github/actions//action.yml`: a `mark ` + step before each `phase: ` step, and `mark -` before untimed setup. +2. Add `` to the `plan` job's `case` and a matching `if: inputs.suite == ''` step to + the `bench` job in `bench.yml`. `uses:` can't take an expression, so each suite is its own step. +3. Add an entry workflow `.github/workflows/.yml` that calls `bench.yml` with + `suite: `, like the existing ones. ## License diff --git a/bin/mark b/bin/mark index f423e13..32f2cd1 100755 --- a/bin/mark +++ b/bin/mark @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Record a phase boundary for the summary: `mark ` starts a timed phase, `mark -` ends the # running one without starting another (untimed setup follows). bench-finish records the last -# boundary. Wall time in µs from bash's EPOCHREALTIME: `date +%s%3N` is not portable (Ubuntu -# 26.04's uutils date ignores the width and prints nanoseconds). +# boundary, and bench.yml puts this directory on PATH. Wall time in µs from bash's EPOCHREALTIME: +# `date +%s%3N` is not portable (Ubuntu 26.04's uutils date ignores the width and prints +# nanoseconds). # Usage: mark |- set -euo pipefail mkdir -p "$RUNNER_TEMP/bench" diff --git a/mise.toml b/mise.toml index cc45fa8..fb4474b 100644 --- a/mise.toml +++ b/mise.toml @@ -1,4 +1,4 @@ -# Non-Bun developer tools, version-pinned (aqua backend: checksum-verified official releases, no +# Developer tools, version-pinned (aqua backend: checksum-verified official releases, no # postinstall). `mise trust` once after cloning, then e.g. `mise exec -- actionlint`. [tools]