From 79bb026d7a21322e53faeddbb60c95ea935b75e7 Mon Sep 17 00:00:00 2001 From: Chris Laupama <42254688+chrislaupama@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:29:32 +0000 Subject: [PATCH 1/3] Build the engine tests and benchmarks with GCC 15 The hosted Linux runner moved to Ubuntu 26.04, whose GCC 15 reports -Wnull-dereference (an error in CI) in three files: - Copies of file-scope default vectors written through operator[]: GCC can't see that a dynamically initialised global is non-empty. Use .at() for those writes in test_mod_routes.cpp and bench_devices.cpp. - memcpy from an empty vector's data() in bench_devices.cpp: skip the copy when there is nothing to copy (memcpy from null is undefined even for zero bytes). - sorted.back() on a possibly empty timing vector in bench_render.cpp: guard it the way run_render already does. --- engine/bench/bench_devices.cpp | 8 ++++---- engine/bench/bench_render.cpp | 8 ++++---- engine/tests/devices/test_mod_routes.cpp | 13 ++++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/engine/bench/bench_devices.cpp b/engine/bench/bench_devices.cpp index 2ab20b5e..d456d738 100644 --- a/engine/bench/bench_devices.cpp +++ b/engine/bench/bench_devices.cpp @@ -56,7 +56,7 @@ std::vector blob_v2(const std::vector& v, const std::vector(v.size()), 0, 0, 0, static_cast(targets.size()), 0}; std::vector out(sizeof h + 4 * v.size() + sizeof(se_mod_route) * targets.size()); std::memcpy(out.data(), &h, sizeof h); - std::memcpy(out.data() + sizeof h, v.data(), 4 * v.size()); + if (!v.empty()) std::memcpy(out.data() + sizeof h, v.data(), 4 * v.size()); // memcpy from null is UB even for 0 bytes for (std::size_t i = 0; i < targets.size(); ++i) { se_mod_route r{targets[i], 0, SE_MOD_BUILTIN, 0, 0, 0, -0.25f, 0.25f, -60.f, 0.f, 1.f, 0.f}; std::memcpy(out.data() + sizeof h + 4 * v.size() + i * sizeof r, &r, sizeof r); @@ -145,7 +145,7 @@ std::vector blob(const std::vector& v, uint64_t sc = 0) { se_builtin_state h{SE_BUILTIN_STATE_MAGIC, static_cast(v.size()), sc}; std::vector out(sizeof h + 4 * v.size()); std::memcpy(out.data(), &h, sizeof h); - std::memcpy(out.data() + sizeof h, v.data(), 4 * v.size()); + if (!v.empty()) std::memcpy(out.data() + sizeof h, v.data(), 4 * v.size()); // memcpy from null is UB even for 0 bytes return out; } @@ -360,7 +360,7 @@ int main(int argc, char** argv) { const char* styles[] = {"Clean", "Opto", "Punch", "Smooth"}; for (int st = 0; st < 4; ++st) { std::vector v = kCompressor; - v[7] = static_cast(st); + v.at(7) = static_cast(st); char name[64]; std::snprintf(name, sizeof name, "compressor (%s)", styles[st]); row(name, bench_kernel("compressor", v, blocks, 0)); @@ -409,7 +409,7 @@ int main(int argc, char** argv) { const char* models[] = {"Clean", "Vari-Mu", "Bus VCA", "FET", "Opto", "RMS VCA"}; for (int m = 0; m < 6; ++m) { std::vector v = kCharacter; - v[0] = static_cast(m); + v.at(0) = static_cast(m); char name[64]; std::snprintf(name, sizeof name, "character-compressor (%s)", models[m]); row(name, bench_kernel("character-compressor", v, blocks, 0)); diff --git a/engine/bench/bench_render.cpp b/engine/bench/bench_render.cpp index 72bef290..a6e2dece 100644 --- a/engine/bench/bench_render.cpp +++ b/engine/bench/bench_render.cpp @@ -394,7 +394,7 @@ Result run_preview_notes(uint32_t tracks, uint32_t blocks, int mode) { Result r{}; r.us_per_block_avg = wall_us / blocks; r.us_per_block_p99 = sorted[static_cast(0.99 * (blocks - 1))]; - r.us_per_block_max = sorted.back(); + r.us_per_block_max = sorted.empty() ? 0.0 : sorted.back(); r.rt_ratio = static_cast(blocks) * kBlock / kRate * 1e6 / wall_us; return r; } @@ -459,7 +459,7 @@ Result run_midi(uint32_t tracks, uint32_t blocks, const se_synth_params* params) Result r{}; r.us_per_block_avg = wall_us / blocks; r.us_per_block_p99 = sorted[static_cast(0.99 * (blocks - 1))]; - r.us_per_block_max = sorted.back(); + r.us_per_block_max = sorted.empty() ? 0.0 : sorted.back(); r.rt_ratio = static_cast(blocks) * kBlock / kRate * 1e6 / wall_us; return r; } @@ -527,7 +527,7 @@ Result run_midi_input(uint32_t tracks, uint32_t blocks, int mode) { Result r{}; r.us_per_block_avg = wall_us / blocks; r.us_per_block_p99 = sorted[static_cast(0.99 * (blocks - 1))]; - r.us_per_block_max = sorted.back(); + r.us_per_block_max = sorted.empty() ? 0.0 : sorted.back(); r.rt_ratio = static_cast(blocks) * kBlock / kRate * 1e6 / wall_us; return r; } @@ -668,7 +668,7 @@ Result run_input(uint32_t tracks, uint32_t blocks, bool record) { Result r{}; r.us_per_block_avg = wall_us / blocks; r.us_per_block_p99 = sorted[static_cast(0.99 * (blocks - 1))]; - r.us_per_block_max = sorted.back(); + r.us_per_block_max = sorted.empty() ? 0.0 : sorted.back(); r.rt_ratio = static_cast(blocks) * kBlock / kRate * 1e6 / wall_us; return r; } diff --git a/engine/tests/devices/test_mod_routes.cpp b/engine/tests/devices/test_mod_routes.cpp index fd61a68e..b47fd732 100644 --- a/engine/tests/devices/test_mod_routes.cpp +++ b/engine/tests/devices/test_mod_routes.cpp @@ -73,8 +73,11 @@ TEST_CASE(lfo_shapes_follow_the_spec) { TEST_CASE(lfo_free_rate_and_synced_clock) { // Free, 1 Hz sine: the wave peaks at 0.25 s and bottoms out at 0.75 s. + // .at() on copies of the file-scope defaults: GCC 15 can't see that a + // dynamically initialised global vector is non-empty and flags [] writes + // as null dereferences (-Wnull-dereference, an error in CI). std::vector v = kLfo; - v[1] = 0.f; + v.at(1) = 0.f; DeviceRig rig("lfo", v); auto p = lfo_points(rig, 48000, false); CHECK_NEAR(p[0], 0.5, 1e-4); @@ -156,11 +159,11 @@ TEST_CASE(lfo_moves_a_builtin_parameter_in_the_engine) { EngineHandle e(48000, 256); const uint64_t a = load_asset(e, {sine(48000, 400.0, 48000.0, 0.1), sine(48000, 400.0, 48000.0, 0.1)}); std::vector comp = kComp; - comp[0] = 0.f; // threshold at 0 dB, ratio 1: no compression - comp[1] = 1.f; + comp.at(0) = 0.f; // threshold at 0 dB, ratio 1: no compression (.at(): see above) + comp.at(1) = 1.f; std::vector lfo = kLfo; - lfo[0] = 4.f; // square - lfo[1] = 0.f; // free, 1 Hz + lfo.at(0) = 4.f; // square + lfo.at(1) = 0.f; // free, 1 Hz uint64_t id = 10; // fresh instances per run: a free LFO starts its cycle when created const auto run = [&](bool lfo_on, float depth) { DeviceSession s; From d7895695ecfd4c35c3551baa886ab77a15fe316c Mon Sep 17 00:00:00 2001 From: Chris Laupama <42254688+chrislaupama@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:00:47 +0000 Subject: [PATCH 2/3] Watch the engine's pinned third-party sources - engine/CMakeLists.txt records the commit behind every pinned tag and the configure step fails if a fetched checkout is at a different commit. - tools/ci/engine_pins.py compares each pin with upstream (newer release tag, moved tag, or a change to a decoder file we compile). - The weekly Engine pins workflow keeps one issue open while something is behind and closes it when all pins are current. - docs/building.md describes how engine dependencies are updated. --- .github/dependabot.yml | 4 +- .github/workflows/ci.yml | 2 + .github/workflows/engine-pins.yml | 65 ++++++++++ docs/building.md | 33 ++++- engine/CMakeLists.txt | 36 ++++++ tools/ci/engine_pins.py | 204 ++++++++++++++++++++++++++++++ 6 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/engine-pins.yml create mode 100755 tools/ci/engine_pins.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73b0e1a2..ae3654c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,8 +7,8 @@ # they are updated by hand. For Cargo, security fixes still arrive as pull requests once # "Dependabot security updates" is on in the repository settings, and the Security workflow # checks Cargo.lock against the RustSec advisory database every week. Dependabot cannot see -# the C++ SDKs that engine/CMakeLists.txt fetches, so nothing watches those automatically: -# check their release notes and advisories by hand when updating them. +# the C++ SDKs that engine/CMakeLists.txt fetches: the weekly Engine pins workflow +# (tools/ci/engine_pins.py) reports those when they fall behind upstream. version: 2 updates: - package-ecosystem: github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08189ec..fb41d55a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,8 @@ jobs: run: tools/ci/test-changes.sh - name: Agent skills run: python3 tools/agents/check_skills.py + - name: Engine pin checker (offline self-test) + run: tools/ci/engine_pins.py --self-test tests: name: tests (${{ matrix.name }}) diff --git a/.github/workflows/engine-pins.yml b/.github/workflows/engine-pins.yml new file mode 100644 index 00000000..795804c1 --- /dev/null +++ b/.github/workflows/engine-pins.yml @@ -0,0 +1,65 @@ +name: Engine pins +# The C++ engine fetches its third-party code at pinned tags and commits +# (engine/CMakeLists.txt), which Dependabot and cargo audit cannot see. Once a +# week this compares every pin with its upstream (tools/ci/engine_pins.py) and +# keeps one issue, "Engine third-party pins need attention", up to date: it is +# opened or updated while a pin is behind or a pinned tag moved, and closed +# when everything is current again. +on: + schedule: + - cron: '41 5 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: engine-pins + cancel-in-progress: false + +jobs: + check: + name: check + # Scheduled runs only in the main repository, not in forks. + if: github.event_name != 'schedule' || github.repository == 'devicedaw/device' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Compare pins with upstream + id: pins + run: | + status=0 + tools/ci/engine_pins.py > "$RUNNER_TEMP/report.md" || status=$? + cat "$RUNNER_TEMP/report.md" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$status" -gt 1 ]; then echo "the check itself failed"; exit 1; fi + echo "status=$status" >> "$GITHUB_OUTPUT" + - name: Open, update or close the issue + env: + GH_TOKEN: ${{ github.token }} + STATUS: ${{ steps.pins.outputs.status }} + TITLE: Engine third-party pins need attention + run: | + number=$(gh api "repos/$GITHUB_REPOSITORY/issues?state=open&creator=github-actions%5Bbot%5D&per_page=100" \ + --jq "[.[] | select(.pull_request == null and .title == \"$TITLE\")][0].number // empty") + if [ "$STATUS" = 1 ]; then + { + echo "The weekly check of the engine's pinned third-party sources (\`engine/CMakeLists.txt\`) found something to do." + echo + cat "$RUNNER_TEMP/report.md" + echo + echo "_Updated by the Engine pins workflow; it closes this issue when every pin is current._" + } > "$RUNNER_TEMP/body.md" + if [ -n "$number" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -F "body=@$RUNNER_TEMP/body.md" --jq .html_url + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f "title=$TITLE" -F "body=@$RUNNER_TEMP/body.md" --jq .html_url + fi + elif [ -n "$number" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=closed -f state_reason=completed --jq .html_url + fi diff --git a/docs/building.md b/docs/building.md index afbe7f97..94573583 100644 --- a/docs/building.md +++ b/docs/building.md @@ -232,9 +232,36 @@ Every action in every workflow is pinned to a commit hash; Dependabot moves the hashes in one grouped pull request a week. Cargo crates and the engine's C++ SDKs are updated by hand. For Cargo crates, Dependabot opens security-fix pull requests once "Dependabot security updates" is on in the repository settings -(Settings › Code security). Dependabot does not see the SDKs that -`engine/CMakeLists.txt` fetches, so their advisories are checked by hand when -they are updated. +(Settings › Code security). + +### Engine dependencies + +The engine fetches its third-party code with CMake `FetchContent` at pins set +in `engine/CMakeLists.txt`: release tags for the CLAP and VST3 SDKs and the +Signalsmith libraries (each with the commit the tag pointed at, +`DSTUDIO__TAG_COMMIT`), and plain commits for dr_libs and stb, which +decode untrusted audio files. Dependabot cannot see these, so: + +- The configure step checks that every fetched checkout is at its pinned + commit and stops if a tag was moved upstream. +- The **Engine pins** workflow runs `tools/ci/engine_pins.py` every Monday. It + reports a newer release tag, a moved tag, and, for dr_libs and stb, a change + upstream to one of the files we compile. While there is something to do it + keeps the issue "Engine third-party pins need attention" open and current, + and closes it when every pin is current. Run the script by hand for the same + report. + +To update a pin: + +1. Read the upstream release notes or the compare link from the report, + looking for security fixes and API changes. +2. Change the tag or commit in `engine/CMakeLists.txt`. For a tag, also set + its `_TAG_COMMIT` (`git ls-remote 'refs/tags/^{}'`, or + without `^{}` for a lightweight tag). +3. Update the version, commit and date in `engine/third_party_licenses.md`, + and the licence text if upstream changed it. +4. Build and run the engine tests (`ctest`); for a decoder also run the + decode benchmark (`dstudio_decode_bench`) and compare with main. ### Performance checks diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 0127eb4e..441daa39 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -38,9 +38,17 @@ endif() # for the test plugins are used; nothing is added via add_subdirectory. # Set FETCHCONTENT_BASE_DIR to share one download between build trees, or # FETCHCONTENT_SOURCE_DIR_ to use a local checkout (offline builds). +# +# Each tag is recorded with the commit it pointed at when it was pinned +# (DSTUDIO__TAG_COMMIT). The configure step fails if a fetched tag is at +# a different commit, so a tag moved upstream cannot change what we build. +# tools/ci/engine_pins.py (weekly "Engine pins" workflow) reports pins that +# are behind upstream or whose tag moved; docs/building.md says how to update. include(FetchContent) set(DSTUDIO_CLAP_TAG 1.2.10) +set(DSTUDIO_CLAP_TAG_COMMIT 195b42a004144fab0b3cf95e9c067187d15365b7) set(DSTUDIO_VST3SDK_TAG v3.8.1_build_84) +set(DSTUDIO_VST3SDK_TAG_COMMIT 3cdf9ca5d1f5b1b21e0a86832aa4abe55607bd96) FetchContent_Declare(dstudio_clap GIT_REPOSITORY https://github.com/free-audio/clap.git GIT_TAG ${DSTUDIO_CLAP_TAG} @@ -55,7 +63,9 @@ FetchContent_Declare(dstudio_vst3sdk # Time stretching for warped audio clips (src/warp.cpp): signalsmith-stretch # and the FFT / STFT library it is built on. Header-only. set(DSTUDIO_STRETCH_TAG 1.4.0) +set(DSTUDIO_STRETCH_TAG_COMMIT a670068d9aeb64913331d5cc29337b19a457a7df) set(DSTUDIO_LINEAR_TAG 0.6.4) +set(DSTUDIO_LINEAR_TAG_COMMIT de55e6a50ffcf6f8f43f649692d94691c7025151) FetchContent_Declare(dstudio_signalsmith_stretch GIT_REPOSITORY https://github.com/Signalsmith-Audio/signalsmith-stretch.git GIT_TAG ${DSTUDIO_STRETCH_TAG} @@ -81,6 +91,32 @@ FetchContent_Declare(dstudio_stb GIT_TAG ${DSTUDIO_STB_COMMIT} SOURCE_SUBDIR _dstudio_headers_only) FetchContent_MakeAvailable(dstudio_clap dstudio_vst3sdk dstudio_signalsmith_stretch dstudio_signalsmith_linear dstudio_dr_libs dstudio_stb) +# Check every fetched checkout is at its pinned commit. Skipped for a local +# checkout given with FETCHCONTENT_SOURCE_DIR_ and for a source tree +# without git metadata (an unpacked archive). +function(dstudio_verify_pin name expected) + string(TOUPPER "${name}" upper) + if(FETCHCONTENT_SOURCE_DIR_${upper} OR NOT EXISTS "${${name}_SOURCE_DIR}/.git") + return() + endif() + find_package(Git QUIET) + if(NOT Git_FOUND) + return() + endif() + execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${${name}_SOURCE_DIR}" + OUTPUT_VARIABLE actual OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(actual AND NOT actual STREQUAL expected) + message(FATAL_ERROR "${name}: fetched commit ${actual} is not the pinned ${expected}. " + "The upstream tag may have moved; see docs/building.md (Engine dependencies).") + endif() +endfunction() +dstudio_verify_pin(dstudio_clap ${DSTUDIO_CLAP_TAG_COMMIT}) +dstudio_verify_pin(dstudio_vst3sdk ${DSTUDIO_VST3SDK_TAG_COMMIT}) +dstudio_verify_pin(dstudio_signalsmith_stretch ${DSTUDIO_STRETCH_TAG_COMMIT}) +dstudio_verify_pin(dstudio_signalsmith_linear ${DSTUDIO_LINEAR_TAG_COMMIT}) +dstudio_verify_pin(dstudio_dr_libs ${DSTUDIO_DR_LIBS_COMMIT}) +dstudio_verify_pin(dstudio_stb ${DSTUDIO_STB_COMMIT}) set(DSTUDIO_DECODER_INCLUDE "${dstudio_dr_libs_SOURCE_DIR}" "${dstudio_stb_SOURCE_DIR}") set(DSTUDIO_STRETCH_INCLUDE "${dstudio_signalsmith_stretch_SOURCE_DIR}" "${dstudio_signalsmith_linear_SOURCE_DIR}/include") set(DSTUDIO_CLAP_INCLUDE "${dstudio_clap_SOURCE_DIR}/include") diff --git a/tools/ci/engine_pins.py b/tools/ci/engine_pins.py new file mode 100755 index 00000000..103414e1 --- /dev/null +++ b/tools/ci/engine_pins.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Report engine third-party pins that are behind upstream or whose tag moved. + +Reads the pins from engine/CMakeLists.txt (the DSTUDIO_*_TAG, *_TAG_COMMIT and +*_COMMIT variables and the FetchContent_Declare blocks that use them) and asks +each upstream repository, with `git ls-remote`, for its tags and default +branch. Prints a Markdown report. Exit status: 0 everything current, 1 +something to do, 2 the check itself failed. + + tools/ci/engine_pins.py [--cmake engine/CMakeLists.txt] + +Needs git and network access to the upstream repositories, nothing else. +""" +import argparse +import re +import subprocess +import sys +import tempfile + +# Tags that count as releases, per tag pin. A commit pin (a repository +# without usable release tags) lists the files we compile instead: it is +# behind when the default branch changed one of them since the pinned commit. +RELEASE_TAGS = { + "DSTUDIO_CLAP_TAG": r"^\d+\.\d+\.\d+$", + "DSTUDIO_VST3SDK_TAG": r"^v\d+\.\d+\.\d+_build_\d+$", + "DSTUDIO_STRETCH_TAG": r"^\d+\.\d+\.\d+$", + "DSTUDIO_LINEAR_TAG": r"^\d+\.\d+\.\d+$", +} +COMMIT_PIN_FILES = { + "DSTUDIO_DR_LIBS_COMMIT": ["dr_mp3.h", "dr_flac.h", "dr_wav.h"], + "DSTUDIO_STB_COMMIT": ["stb_vorbis.c"], +} + + +def parse_pins(text): + """[(variable, repository URL, value, recorded tag commit or None)], in file order.""" + variables = dict(re.findall(r"^set\((DSTUDIO_\w+)\s+([^\s)]+)\)", text, re.M)) + pins = [] + for block in re.findall(r"FetchContent_Declare\((.*?)\)", text, re.S): + repo = re.search(r"GIT_REPOSITORY\s+(\S+)", block) + var = re.search(r"GIT_TAG\s+\$\{(\w+)\}", block) + if not repo or not var: + continue + name = var.group(1) + if name not in variables: + raise ValueError(f"{name} is used by FetchContent_Declare but never set") + pins.append((name, repo.group(1), variables[name], variables.get(name + "_COMMIT") if name.endswith("_TAG") else None)) + return pins + + +def version_key(tag): + return [int(n) for n in re.findall(r"\d+", tag)] + + +def newest(tags, pattern): + matching = [t for t in tags if re.match(pattern, t)] + return max(matching, key=version_key) if matching else None + + +def ls_remote(repo): + """({tag: commit}, default-branch commit). Annotated tags resolve to their commit.""" + out = subprocess.run(["git", "ls-remote", repo], check=True, capture_output=True, text=True, timeout=120).stdout + tags, head = {}, None + for line in out.splitlines(): + sha, ref = line.split("\t") + if ref == "HEAD": + head = sha + elif ref.startswith("refs/tags/"): + name = ref[len("refs/tags/"):] + if name.endswith("^{}"): + tags[name[:-3]] = sha # the commit an annotated tag points at + else: + tags.setdefault(name, sha) + return tags, head + + +def changed_files(repo, base, head): + """Files that differ between two commits of a repository, or None if unknown. + + Uses a blobless clone (commits and trees only), so it needs nothing but git. + """ + with tempfile.TemporaryDirectory() as tmp: + try: + subprocess.run(["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout", repo, tmp], check=True, capture_output=True, timeout=600) + out = subprocess.run(["git", "-C", tmp, "diff", "--name-only", base, head], check=True, capture_output=True, text=True, timeout=120) + except subprocess.SubprocessError: + return None + return out.stdout.split("\n") + + +def check(pins, remote=ls_remote, diff=changed_files): + """(findings, table rows). `remote` and `diff` are replaceable for the self-test.""" + findings, rows = [], [] + for name, repo, value, recorded in pins: + tags, head = remote(repo) + page = repo.removesuffix(".git") + if name in RELEASE_TAGS: + latest = newest(tags, RELEASE_TAGS[name]) + status = "current" + if value not in tags: + status = "pinned tag is gone upstream" + findings.append(f"`{name}`: tag `{value}` no longer exists in {page}.") + elif recorded is None: + status = "no recorded commit" + findings.append(f"`{name}`: add `{name}_COMMIT {tags[value]}` next to it.") + elif tags[value] != recorded: + status = "TAG MOVED" + findings.append(f"`{name}`: tag `{value}` now points at `{tags[value]}`, not the recorded `{recorded}`. Review the change before trusting it.") + if latest and latest != value and version_key(latest) > version_key(value): + status = f"behind: {latest}" if status == "current" else status + findings.append(f"`{name}`: `{value}` is behind `{latest}` ({page}/releases).") + rows.append((name, value, latest or "?", status)) + elif name in COMMIT_PIN_FILES: + status = "current" + if head is not None and head != value: + files = diff(repo, value, head) + compare = f"{page}/compare/{value}...{head}" + if files is None: + status = "default branch moved" + findings.append(f"`{name}`: the default branch moved and the changed files could not be listed; check {compare}.") + else: + ours = sorted(set(files) & set(COMMIT_PIN_FILES[name])) + if ours: + status = "behind: " + ", ".join(ours) + findings.append(f"`{name}`: {', '.join(f'`{f}`' for f in ours)} changed upstream since the pinned commit ({compare}).") + rows.append((name, value[:12], (head or "?")[:12], status)) + else: + findings.append(f"`{name}` is not known to tools/ci/engine_pins.py: add it to RELEASE_TAGS or COMMIT_PIN_FILES.") + rows.append((name, value, "?", "unknown pin")) + return findings, rows + + +def report(findings, rows): + lines = ["| Pin | Pinned | Upstream | Status |", "|---|---|---|---|"] + lines += [f"| `{n}` | `{v}` | `{u}` | {s} |" for n, v, u, s in rows] + lines.append("") + if findings: + lines += ["To do:", ""] + [f"- {f}" for f in findings] + lines += ["", "How to update a pin: docs/building.md, \"Engine dependencies\"."] + else: + lines.append("Every engine pin is current.") + return "\n".join(lines) + + +def self_test(): + cmake = """ +set(DSTUDIO_CLAP_TAG 1.2.10) +set(DSTUDIO_CLAP_TAG_COMMIT aaa) +set(DSTUDIO_LINEAR_TAG 0.6.4) +set(DSTUDIO_LINEAR_TAG_COMMIT bbb) +set(DSTUDIO_DR_LIBS_COMMIT ccc) +set(DSTUDIO_STB_COMMIT ddd) +FetchContent_Declare(a GIT_REPOSITORY https://example.org/clap.git GIT_TAG ${DSTUDIO_CLAP_TAG} GIT_SHALLOW TRUE) +FetchContent_Declare(b GIT_REPOSITORY https://example.org/linear.git GIT_TAG ${DSTUDIO_LINEAR_TAG}) +FetchContent_Declare(c GIT_REPOSITORY https://example.org/dr_libs.git GIT_TAG ${DSTUDIO_DR_LIBS_COMMIT}) +FetchContent_Declare(d GIT_REPOSITORY https://example.org/stb.git GIT_TAG ${DSTUDIO_STB_COMMIT}) +""" + pins = parse_pins(cmake) + assert [p[0] for p in pins] == ["DSTUDIO_CLAP_TAG", "DSTUDIO_LINEAR_TAG", "DSTUDIO_DR_LIBS_COMMIT", "DSTUDIO_STB_COMMIT"], pins + assert pins[0][3] == "aaa" and pins[2][3] is None + current = { + "https://example.org/clap.git": ({"1.2.9": "x", "1.2.10": "aaa", "1.2.10-rc1": "y"}, "h"), + "https://example.org/linear.git": ({"0.6.4": "bbb"}, "h"), + "https://example.org/dr_libs.git": ({"wav-0.14.6": "ccc"}, "ccc"), + "https://example.org/stb.git": ({}, "eee"), # moved, but not in a file we use + } + diffs = {"https://example.org/stb.git": ["stb_image.h"], "https://example.org/dr_libs.git": ["dr_mp3.h", "README.md"]} + findings, rows = check(pins, current.__getitem__, lambda repo, a, b: diffs[repo]) + assert not findings and all(r[3] == "current" for r in rows), (findings, rows) + stale = dict(current) + stale["https://example.org/clap.git"] = ({"1.2.10": "moved", "1.10.0": "z"}, "h") # moved tag, and 1.10 > 1.2.10 + stale["https://example.org/dr_libs.git"] = ({}, "new") + findings, rows = check(pins, stale.__getitem__, lambda repo, a, b: diffs[repo]) + text = report(findings, rows) + assert len(findings) == 3, findings + assert "TAG MOVED" in text and "`1.10.0`" in text and "`dr_mp3.h` changed upstream" in text, text + findings, _ = check(pins, current.__getitem__, lambda repo, a, b: None) + assert len(findings) == 1 and "could not be listed" in findings[0], findings + print("engine_pins self-test ok") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--cmake", default="engine/CMakeLists.txt") + ap.add_argument("--self-test", action="store_true", help="check the parser and the comparison offline") + args = ap.parse_args() + if args.self_test: + self_test() + return 0 + try: + with open(args.cmake, encoding="utf-8") as f: + pins = parse_pins(f.read()) + if not pins: + raise ValueError(f"no pins found in {args.cmake}") + findings, rows = check(pins) + except (OSError, ValueError, subprocess.SubprocessError) as e: + print(f"engine_pins: {e}", file=sys.stderr) + return 2 + print(report(findings, rows)) + return 1 if findings else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 3e8177b2f7490312901d1c4ead4ea6e8f278823c Mon Sep 17 00:00:00 2001 From: Chris Laupama <42254688+chrislaupama@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:40:51 +0000 Subject: [PATCH 3/3] Read the report without cat in the Engine pins workflow --- .github/workflows/engine-pins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/engine-pins.yml b/.github/workflows/engine-pins.yml index 795804c1..d6d86ece 100644 --- a/.github/workflows/engine-pins.yml +++ b/.github/workflows/engine-pins.yml @@ -36,7 +36,7 @@ jobs: run: | status=0 tools/ci/engine_pins.py > "$RUNNER_TEMP/report.md" || status=$? - cat "$RUNNER_TEMP/report.md" | tee -a "$GITHUB_STEP_SUMMARY" + tee -a "$GITHUB_STEP_SUMMARY" < "$RUNNER_TEMP/report.md" if [ "$status" -gt 1 ]; then echo "the check itself failed"; exit 1; fi echo "status=$status" >> "$GITHUB_OUTPUT" - name: Open, update or close the issue