diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73b0e1a2..ae3654c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,8 +7,8 @@ # they are updated by hand. For Cargo, security fixes still arrive as pull requests once # "Dependabot security updates" is on in the repository settings, and the Security workflow # checks Cargo.lock against the RustSec advisory database every week. Dependabot cannot see -# the C++ SDKs that engine/CMakeLists.txt fetches, so nothing watches those automatically: -# check their release notes and advisories by hand when updating them. +# the C++ SDKs that engine/CMakeLists.txt fetches: the weekly Engine pins workflow +# (tools/ci/engine_pins.py) reports those when they fall behind upstream. version: 2 updates: - package-ecosystem: github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08189ec..fb41d55a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,8 @@ jobs: run: tools/ci/test-changes.sh - name: Agent skills run: python3 tools/agents/check_skills.py + - name: Engine pin checker (offline self-test) + run: tools/ci/engine_pins.py --self-test tests: name: tests (${{ matrix.name }}) diff --git a/.github/workflows/engine-pins.yml b/.github/workflows/engine-pins.yml new file mode 100644 index 00000000..d6d86ece --- /dev/null +++ b/.github/workflows/engine-pins.yml @@ -0,0 +1,65 @@ +name: Engine pins +# The C++ engine fetches its third-party code at pinned tags and commits +# (engine/CMakeLists.txt), which Dependabot and cargo audit cannot see. Once a +# week this compares every pin with its upstream (tools/ci/engine_pins.py) and +# keeps one issue, "Engine third-party pins need attention", up to date: it is +# opened or updated while a pin is behind or a pinned tag moved, and closed +# when everything is current again. +on: + schedule: + - cron: '41 5 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: engine-pins + cancel-in-progress: false + +jobs: + check: + name: check + # Scheduled runs only in the main repository, not in forks. + if: github.event_name != 'schedule' || github.repository == 'devicedaw/device' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Compare pins with upstream + id: pins + run: | + status=0 + tools/ci/engine_pins.py > "$RUNNER_TEMP/report.md" || status=$? + tee -a "$GITHUB_STEP_SUMMARY" < "$RUNNER_TEMP/report.md" + if [ "$status" -gt 1 ]; then echo "the check itself failed"; exit 1; fi + echo "status=$status" >> "$GITHUB_OUTPUT" + - name: Open, update or close the issue + env: + GH_TOKEN: ${{ github.token }} + STATUS: ${{ steps.pins.outputs.status }} + TITLE: Engine third-party pins need attention + run: | + number=$(gh api "repos/$GITHUB_REPOSITORY/issues?state=open&creator=github-actions%5Bbot%5D&per_page=100" \ + --jq "[.[] | select(.pull_request == null and .title == \"$TITLE\")][0].number // empty") + if [ "$STATUS" = 1 ]; then + { + echo "The weekly check of the engine's pinned third-party sources (\`engine/CMakeLists.txt\`) found something to do." + echo + cat "$RUNNER_TEMP/report.md" + echo + echo "_Updated by the Engine pins workflow; it closes this issue when every pin is current._" + } > "$RUNNER_TEMP/body.md" + if [ -n "$number" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -F "body=@$RUNNER_TEMP/body.md" --jq .html_url + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f "title=$TITLE" -F "body=@$RUNNER_TEMP/body.md" --jq .html_url + fi + elif [ -n "$number" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=closed -f state_reason=completed --jq .html_url + fi diff --git a/docs/building.md b/docs/building.md index afbe7f97..94573583 100644 --- a/docs/building.md +++ b/docs/building.md @@ -232,9 +232,36 @@ Every action in every workflow is pinned to a commit hash; Dependabot moves the hashes in one grouped pull request a week. Cargo crates and the engine's C++ SDKs are updated by hand. For Cargo crates, Dependabot opens security-fix pull requests once "Dependabot security updates" is on in the repository settings -(Settings › Code security). Dependabot does not see the SDKs that -`engine/CMakeLists.txt` fetches, so their advisories are checked by hand when -they are updated. +(Settings › Code security). + +### Engine dependencies + +The engine fetches its third-party code with CMake `FetchContent` at pins set +in `engine/CMakeLists.txt`: release tags for the CLAP and VST3 SDKs and the +Signalsmith libraries (each with the commit the tag pointed at, +`DSTUDIO__TAG_COMMIT`), and plain commits for dr_libs and stb, which +decode untrusted audio files. Dependabot cannot see these, so: + +- The configure step checks that every fetched checkout is at its pinned + commit and stops if a tag was moved upstream. +- The **Engine pins** workflow runs `tools/ci/engine_pins.py` every Monday. It + reports a newer release tag, a moved tag, and, for dr_libs and stb, a change + upstream to one of the files we compile. While there is something to do it + keeps the issue "Engine third-party pins need attention" open and current, + and closes it when every pin is current. Run the script by hand for the same + report. + +To update a pin: + +1. Read the upstream release notes or the compare link from the report, + looking for security fixes and API changes. +2. Change the tag or commit in `engine/CMakeLists.txt`. For a tag, also set + its `_TAG_COMMIT` (`git ls-remote 'refs/tags/^{}'`, or + without `^{}` for a lightweight tag). +3. Update the version, commit and date in `engine/third_party_licenses.md`, + and the licence text if upstream changed it. +4. Build and run the engine tests (`ctest`); for a decoder also run the + decode benchmark (`dstudio_decode_bench`) and compare with main. ### Performance checks diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 0127eb4e..441daa39 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -38,9 +38,17 @@ endif() # for the test plugins are used; nothing is added via add_subdirectory. # Set FETCHCONTENT_BASE_DIR to share one download between build trees, or # FETCHCONTENT_SOURCE_DIR_ to use a local checkout (offline builds). +# +# Each tag is recorded with the commit it pointed at when it was pinned +# (DSTUDIO__TAG_COMMIT). The configure step fails if a fetched tag is at +# a different commit, so a tag moved upstream cannot change what we build. +# tools/ci/engine_pins.py (weekly "Engine pins" workflow) reports pins that +# are behind upstream or whose tag moved; docs/building.md says how to update. include(FetchContent) set(DSTUDIO_CLAP_TAG 1.2.10) +set(DSTUDIO_CLAP_TAG_COMMIT 195b42a004144fab0b3cf95e9c067187d15365b7) set(DSTUDIO_VST3SDK_TAG v3.8.1_build_84) +set(DSTUDIO_VST3SDK_TAG_COMMIT 3cdf9ca5d1f5b1b21e0a86832aa4abe55607bd96) FetchContent_Declare(dstudio_clap GIT_REPOSITORY https://github.com/free-audio/clap.git GIT_TAG ${DSTUDIO_CLAP_TAG} @@ -55,7 +63,9 @@ FetchContent_Declare(dstudio_vst3sdk # Time stretching for warped audio clips (src/warp.cpp): signalsmith-stretch # and the FFT / STFT library it is built on. Header-only. set(DSTUDIO_STRETCH_TAG 1.4.0) +set(DSTUDIO_STRETCH_TAG_COMMIT a670068d9aeb64913331d5cc29337b19a457a7df) set(DSTUDIO_LINEAR_TAG 0.6.4) +set(DSTUDIO_LINEAR_TAG_COMMIT de55e6a50ffcf6f8f43f649692d94691c7025151) FetchContent_Declare(dstudio_signalsmith_stretch GIT_REPOSITORY https://github.com/Signalsmith-Audio/signalsmith-stretch.git GIT_TAG ${DSTUDIO_STRETCH_TAG} @@ -81,6 +91,32 @@ FetchContent_Declare(dstudio_stb GIT_TAG ${DSTUDIO_STB_COMMIT} SOURCE_SUBDIR _dstudio_headers_only) FetchContent_MakeAvailable(dstudio_clap dstudio_vst3sdk dstudio_signalsmith_stretch dstudio_signalsmith_linear dstudio_dr_libs dstudio_stb) +# Check every fetched checkout is at its pinned commit. Skipped for a local +# checkout given with FETCHCONTENT_SOURCE_DIR_ and for a source tree +# without git metadata (an unpacked archive). +function(dstudio_verify_pin name expected) + string(TOUPPER "${name}" upper) + if(FETCHCONTENT_SOURCE_DIR_${upper} OR NOT EXISTS "${${name}_SOURCE_DIR}/.git") + return() + endif() + find_package(Git QUIET) + if(NOT Git_FOUND) + return() + endif() + execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${${name}_SOURCE_DIR}" + OUTPUT_VARIABLE actual OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(actual AND NOT actual STREQUAL expected) + message(FATAL_ERROR "${name}: fetched commit ${actual} is not the pinned ${expected}. " + "The upstream tag may have moved; see docs/building.md (Engine dependencies).") + endif() +endfunction() +dstudio_verify_pin(dstudio_clap ${DSTUDIO_CLAP_TAG_COMMIT}) +dstudio_verify_pin(dstudio_vst3sdk ${DSTUDIO_VST3SDK_TAG_COMMIT}) +dstudio_verify_pin(dstudio_signalsmith_stretch ${DSTUDIO_STRETCH_TAG_COMMIT}) +dstudio_verify_pin(dstudio_signalsmith_linear ${DSTUDIO_LINEAR_TAG_COMMIT}) +dstudio_verify_pin(dstudio_dr_libs ${DSTUDIO_DR_LIBS_COMMIT}) +dstudio_verify_pin(dstudio_stb ${DSTUDIO_STB_COMMIT}) set(DSTUDIO_DECODER_INCLUDE "${dstudio_dr_libs_SOURCE_DIR}" "${dstudio_stb_SOURCE_DIR}") set(DSTUDIO_STRETCH_INCLUDE "${dstudio_signalsmith_stretch_SOURCE_DIR}" "${dstudio_signalsmith_linear_SOURCE_DIR}/include") set(DSTUDIO_CLAP_INCLUDE "${dstudio_clap_SOURCE_DIR}/include") diff --git a/tools/ci/engine_pins.py b/tools/ci/engine_pins.py new file mode 100755 index 00000000..103414e1 --- /dev/null +++ b/tools/ci/engine_pins.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Report engine third-party pins that are behind upstream or whose tag moved. + +Reads the pins from engine/CMakeLists.txt (the DSTUDIO_*_TAG, *_TAG_COMMIT and +*_COMMIT variables and the FetchContent_Declare blocks that use them) and asks +each upstream repository, with `git ls-remote`, for its tags and default +branch. Prints a Markdown report. Exit status: 0 everything current, 1 +something to do, 2 the check itself failed. + + tools/ci/engine_pins.py [--cmake engine/CMakeLists.txt] + +Needs git and network access to the upstream repositories, nothing else. +""" +import argparse +import re +import subprocess +import sys +import tempfile + +# Tags that count as releases, per tag pin. A commit pin (a repository +# without usable release tags) lists the files we compile instead: it is +# behind when the default branch changed one of them since the pinned commit. +RELEASE_TAGS = { + "DSTUDIO_CLAP_TAG": r"^\d+\.\d+\.\d+$", + "DSTUDIO_VST3SDK_TAG": r"^v\d+\.\d+\.\d+_build_\d+$", + "DSTUDIO_STRETCH_TAG": r"^\d+\.\d+\.\d+$", + "DSTUDIO_LINEAR_TAG": r"^\d+\.\d+\.\d+$", +} +COMMIT_PIN_FILES = { + "DSTUDIO_DR_LIBS_COMMIT": ["dr_mp3.h", "dr_flac.h", "dr_wav.h"], + "DSTUDIO_STB_COMMIT": ["stb_vorbis.c"], +} + + +def parse_pins(text): + """[(variable, repository URL, value, recorded tag commit or None)], in file order.""" + variables = dict(re.findall(r"^set\((DSTUDIO_\w+)\s+([^\s)]+)\)", text, re.M)) + pins = [] + for block in re.findall(r"FetchContent_Declare\((.*?)\)", text, re.S): + repo = re.search(r"GIT_REPOSITORY\s+(\S+)", block) + var = re.search(r"GIT_TAG\s+\$\{(\w+)\}", block) + if not repo or not var: + continue + name = var.group(1) + if name not in variables: + raise ValueError(f"{name} is used by FetchContent_Declare but never set") + pins.append((name, repo.group(1), variables[name], variables.get(name + "_COMMIT") if name.endswith("_TAG") else None)) + return pins + + +def version_key(tag): + return [int(n) for n in re.findall(r"\d+", tag)] + + +def newest(tags, pattern): + matching = [t for t in tags if re.match(pattern, t)] + return max(matching, key=version_key) if matching else None + + +def ls_remote(repo): + """({tag: commit}, default-branch commit). Annotated tags resolve to their commit.""" + out = subprocess.run(["git", "ls-remote", repo], check=True, capture_output=True, text=True, timeout=120).stdout + tags, head = {}, None + for line in out.splitlines(): + sha, ref = line.split("\t") + if ref == "HEAD": + head = sha + elif ref.startswith("refs/tags/"): + name = ref[len("refs/tags/"):] + if name.endswith("^{}"): + tags[name[:-3]] = sha # the commit an annotated tag points at + else: + tags.setdefault(name, sha) + return tags, head + + +def changed_files(repo, base, head): + """Files that differ between two commits of a repository, or None if unknown. + + Uses a blobless clone (commits and trees only), so it needs nothing but git. + """ + with tempfile.TemporaryDirectory() as tmp: + try: + subprocess.run(["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout", repo, tmp], check=True, capture_output=True, timeout=600) + out = subprocess.run(["git", "-C", tmp, "diff", "--name-only", base, head], check=True, capture_output=True, text=True, timeout=120) + except subprocess.SubprocessError: + return None + return out.stdout.split("\n") + + +def check(pins, remote=ls_remote, diff=changed_files): + """(findings, table rows). `remote` and `diff` are replaceable for the self-test.""" + findings, rows = [], [] + for name, repo, value, recorded in pins: + tags, head = remote(repo) + page = repo.removesuffix(".git") + if name in RELEASE_TAGS: + latest = newest(tags, RELEASE_TAGS[name]) + status = "current" + if value not in tags: + status = "pinned tag is gone upstream" + findings.append(f"`{name}`: tag `{value}` no longer exists in {page}.") + elif recorded is None: + status = "no recorded commit" + findings.append(f"`{name}`: add `{name}_COMMIT {tags[value]}` next to it.") + elif tags[value] != recorded: + status = "TAG MOVED" + findings.append(f"`{name}`: tag `{value}` now points at `{tags[value]}`, not the recorded `{recorded}`. Review the change before trusting it.") + if latest and latest != value and version_key(latest) > version_key(value): + status = f"behind: {latest}" if status == "current" else status + findings.append(f"`{name}`: `{value}` is behind `{latest}` ({page}/releases).") + rows.append((name, value, latest or "?", status)) + elif name in COMMIT_PIN_FILES: + status = "current" + if head is not None and head != value: + files = diff(repo, value, head) + compare = f"{page}/compare/{value}...{head}" + if files is None: + status = "default branch moved" + findings.append(f"`{name}`: the default branch moved and the changed files could not be listed; check {compare}.") + else: + ours = sorted(set(files) & set(COMMIT_PIN_FILES[name])) + if ours: + status = "behind: " + ", ".join(ours) + findings.append(f"`{name}`: {', '.join(f'`{f}`' for f in ours)} changed upstream since the pinned commit ({compare}).") + rows.append((name, value[:12], (head or "?")[:12], status)) + else: + findings.append(f"`{name}` is not known to tools/ci/engine_pins.py: add it to RELEASE_TAGS or COMMIT_PIN_FILES.") + rows.append((name, value, "?", "unknown pin")) + return findings, rows + + +def report(findings, rows): + lines = ["| Pin | Pinned | Upstream | Status |", "|---|---|---|---|"] + lines += [f"| `{n}` | `{v}` | `{u}` | {s} |" for n, v, u, s in rows] + lines.append("") + if findings: + lines += ["To do:", ""] + [f"- {f}" for f in findings] + lines += ["", "How to update a pin: docs/building.md, \"Engine dependencies\"."] + else: + lines.append("Every engine pin is current.") + return "\n".join(lines) + + +def self_test(): + cmake = """ +set(DSTUDIO_CLAP_TAG 1.2.10) +set(DSTUDIO_CLAP_TAG_COMMIT aaa) +set(DSTUDIO_LINEAR_TAG 0.6.4) +set(DSTUDIO_LINEAR_TAG_COMMIT bbb) +set(DSTUDIO_DR_LIBS_COMMIT ccc) +set(DSTUDIO_STB_COMMIT ddd) +FetchContent_Declare(a GIT_REPOSITORY https://example.org/clap.git GIT_TAG ${DSTUDIO_CLAP_TAG} GIT_SHALLOW TRUE) +FetchContent_Declare(b GIT_REPOSITORY https://example.org/linear.git GIT_TAG ${DSTUDIO_LINEAR_TAG}) +FetchContent_Declare(c GIT_REPOSITORY https://example.org/dr_libs.git GIT_TAG ${DSTUDIO_DR_LIBS_COMMIT}) +FetchContent_Declare(d GIT_REPOSITORY https://example.org/stb.git GIT_TAG ${DSTUDIO_STB_COMMIT}) +""" + pins = parse_pins(cmake) + assert [p[0] for p in pins] == ["DSTUDIO_CLAP_TAG", "DSTUDIO_LINEAR_TAG", "DSTUDIO_DR_LIBS_COMMIT", "DSTUDIO_STB_COMMIT"], pins + assert pins[0][3] == "aaa" and pins[2][3] is None + current = { + "https://example.org/clap.git": ({"1.2.9": "x", "1.2.10": "aaa", "1.2.10-rc1": "y"}, "h"), + "https://example.org/linear.git": ({"0.6.4": "bbb"}, "h"), + "https://example.org/dr_libs.git": ({"wav-0.14.6": "ccc"}, "ccc"), + "https://example.org/stb.git": ({}, "eee"), # moved, but not in a file we use + } + diffs = {"https://example.org/stb.git": ["stb_image.h"], "https://example.org/dr_libs.git": ["dr_mp3.h", "README.md"]} + findings, rows = check(pins, current.__getitem__, lambda repo, a, b: diffs[repo]) + assert not findings and all(r[3] == "current" for r in rows), (findings, rows) + stale = dict(current) + stale["https://example.org/clap.git"] = ({"1.2.10": "moved", "1.10.0": "z"}, "h") # moved tag, and 1.10 > 1.2.10 + stale["https://example.org/dr_libs.git"] = ({}, "new") + findings, rows = check(pins, stale.__getitem__, lambda repo, a, b: diffs[repo]) + text = report(findings, rows) + assert len(findings) == 3, findings + assert "TAG MOVED" in text and "`1.10.0`" in text and "`dr_mp3.h` changed upstream" in text, text + findings, _ = check(pins, current.__getitem__, lambda repo, a, b: None) + assert len(findings) == 1 and "could not be listed" in findings[0], findings + print("engine_pins self-test ok") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--cmake", default="engine/CMakeLists.txt") + ap.add_argument("--self-test", action="store_true", help="check the parser and the comparison offline") + args = ap.parse_args() + if args.self_test: + self_test() + return 0 + try: + with open(args.cmake, encoding="utf-8") as f: + pins = parse_pins(f.read()) + if not pins: + raise ValueError(f"no pins found in {args.cmake}") + findings, rows = check(pins) + except (OSError, ValueError, subprocess.SubprocessError) as e: + print(f"engine_pins: {e}", file=sys.stderr) + return 2 + print(report(findings, rows)) + return 1 if findings else 0 + + +if __name__ == "__main__": + sys.exit(main())