From e93cfb3be49bc8f59a3c4574a0899f336a0ca9ce Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 11:22:24 -0700 Subject: [PATCH 01/25] help text fix --- tools/ci/src/ci_docs.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index f09114e6787..c63e2d5a82d 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -46,12 +46,16 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { .map(|l| format!("--{}", l)) .or_else(|| arg.get_short().map(|s| format!("-{}", s))) .unwrap_or_else(|| arg.get_id().to_string()); - let help = arg.get_long_help().unwrap_or_default(); + let help = arg + .get_long_help() + .or_else(|| arg.get_help()) + .map(|help| help.to_string()) + .unwrap_or_else(|| panic!("argument `{}` is missing help text", arg.get_id())); options.push_str(&format!( "- `{}`: {}\n{}", names, help, - if help.to_string().lines().count() > 1 { "\n" } else { "" } + if help.lines().count() > 1 { "\n" } else { "" } )); } From ac1dc96c389ecf0bff9fc91a153b55bea3523c85 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 11:25:27 -0700 Subject: [PATCH 02/25] readme --- tools/ci/README.md | 69 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index c01598359e7..d1548309139 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -44,7 +44,7 @@ Usage: test Lints the codebase -Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. +Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. **Usage:** ```bash @@ -102,8 +102,8 @@ Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. -- `--dotnet`: -- `args`: +- `--dotnet`: Whether to run smoketests that require the .NET toolchain +- `args`: Additional arguments to pass to the test runner - `--help`: Print help (see a summary with '-h') #### `prepare` @@ -130,7 +130,7 @@ Usage: check-mod-list **Options:** -- `--help`: +- `--help`: Print help #### `help` @@ -141,7 +141,7 @@ Usage: help [COMMAND]... **Options:** -- `subcommand`: +- `subcommand`: Print help for the subcommand(s) ### `update-flow` @@ -193,7 +193,62 @@ Usage: global-json-policy **Options:** -- `--help`: +- `--help`: Print help + +### `publish-checks` + +**Usage:** +```bash +Usage: publish-checks +``` + +**Options:** + +- `--help`: Print help + +### `typescript-test` + +**Usage:** +```bash +Usage: typescript-test +``` + +**Options:** + +- `--help`: Print help + +### `docs` + +**Usage:** +```bash +Usage: docs +``` + +**Options:** + +- `--help`: Print help + +### `csharp-tests` + +**Usage:** +```bash +Usage: csharp-tests +``` + +**Options:** + +- `--help`: Print help + +### `unity-tests` + +**Usage:** +```bash +Usage: unity-tests +``` + +**Options:** + +- `--help`: Print help ### `help` @@ -204,7 +259,7 @@ Usage: help [COMMAND]... **Options:** -- `subcommand`: +- `subcommand`: Print help for the subcommand(s) --- From 2c8803dae9dc3df1ba95fc1d0d2628a244548b52 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 13:56:08 -0700 Subject: [PATCH 03/25] comment --- tools/ci/src/smoketest.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/src/smoketest.rs b/tools/ci/src/smoketest.rs index 5799ba1a4ca..7e5d3cbab9f 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/src/smoketest.rs @@ -26,6 +26,7 @@ pub struct SmoketestsArgs { #[arg(long)] server: Option, + /// Whether to run smoketests that require the .NET toolchain. #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, From e9d387c35e3af629c19b8d28deb1fda381ed4b5b Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 14:17:45 -0700 Subject: [PATCH 04/25] WIP --- .github/workflows/ci.yml | 141 +------- .github/workflows/docs-publish.yaml | 9 +- .github/workflows/docs-test.yaml | 9 +- .github/workflows/typescript-test.yml | 50 +-- tools/ci/src/main.rs | 447 +++++++++++++++++++------- 5 files changed, 353 insertions(+), 303 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9c105f4c4c..5445171a134 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -336,24 +336,8 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - name: Set up Python env - run: | - test -d venv || python3 -m venv venv - venv/bin/pip3 install argparse toml - name: Run checks - run: | - set -ueo pipefail - FAILED=0 - ROOTS=(spacetimedb spacetimedb-sdk) - CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}") - for crate_dir in $CRATES; do - if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then - FAILED=$(( $FAILED + 1 )) - fi - done - if [ $FAILED -gt 0 ]; then - exit 1 - fi + run: cargo ci publish-checks update: name: Test spacetimedb-update flow (${{ matrix.target }}) @@ -383,28 +367,10 @@ jobs: shell: bash run: sudo apt install -y libssl-dev - - name: Build spacetimedb-update - run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update - if: runner.os == 'Windows' - - - name: Run self-install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - ROOT_DIR="$(mktemp -d)" - # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - # happens very frequently on the `macos-runner`, but we haven't seen it on any others). - cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes - "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help - if: runner.os == 'Windows' - - name: Test spacetimedb-update env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth - if: runner.os != 'Windows' unreal_engine_tests: name: Unreal Engine Tests @@ -647,31 +613,7 @@ jobs: with: global-json-file: global.json - - name: Override NuGet packages - run: | - dotnet pack crates/bindings-csharp/BSATN.Runtime - dotnet pack crates/bindings-csharp/Runtime - - # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository - # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if - # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. - # This means that (if version numbers match) we will test the local versions of the C# packages, even - # if they're not pushed to NuGet. - # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. - cd sdks/csharp - ./tools~/write-nuget-config.sh ../.. - - - name: Restore .NET solution - working-directory: sdks/csharp - run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln - # Now, setup the Unity tests. - - name: Patch spacetimedb dependency in Cargo.toml - working-directory: demo/Blackholio/server-rust - run: | - sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml - cat Cargo.toml - - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain @@ -709,19 +651,8 @@ jobs: # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - - name: Generate client bindings - working-directory: demo/Blackholio/server-rust - run: bash ./generate.sh -y - - - name: Check for changes - run: | - tools/check-diff.sh demo/Blackholio/client-unity/Assets/Scripts/autogen || { - echo 'Error: Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.' - exit 1 - } - - - name: Hydrate Unity SDK DLLs - run: cargo ci dlls + - name: Prepare Unity test workspace + run: cargo ci unity-tests - name: Check Unity meta files uses: DeNA/unity-meta-check@v3 @@ -731,23 +662,6 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - name: Start SpacetimeDB - run: | - spacetime start & - disown - - - name: Publish unity-tests module to SpacetimeDB - working-directory: demo/Blackholio/server-rust - run: | - spacetime logout && spacetime login --server-issued-login local - bash ./publish.sh - - - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json - working-directory: demo/Blackholio/client-unity/Packages - run: | - yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json - cat manifest.json - - uses: actions/cache@v3 with: path: demo/Blackholio/client-unity/Library @@ -792,32 +706,6 @@ jobs: with: global-json-file: global.json - - name: Override NuGet packages - run: | - dotnet pack crates/bindings-csharp/BSATN.Runtime - dotnet pack crates/bindings-csharp/Runtime - - # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository - # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if - # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. - # This means that (if version numbers match) we will test the local versions of the C# packages, even - # if they're not pushed to NuGet. - # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. - cd sdks/csharp - ./tools~/write-nuget-config.sh ../.. - - - name: Restore .NET solution - working-directory: sdks/csharp - run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln - - - name: Run .NET tests - working-directory: sdks/csharp - run: dotnet test -warnaserror --no-restore - - - name: Verify C# formatting - working-directory: sdks/csharp - run: dotnet format --no-restore --verify-no-changes SpacetimeDB.ClientSDK.sln - - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain @@ -861,15 +749,6 @@ jobs: # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - - name: Check quickstart-chat bindings are up to date - working-directory: sdks/csharp - run: | - bash tools~/gen-quickstart.sh - "${GITHUB_WORKSPACE}"/tools/check-diff.sh examples~/quickstart-chat || { - echo 'Error: quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.' - exit 1 - } - # TODO: Re-enable this once csharp is using the v2 ws api. # - name: Check client-api bindings are up to date # working-directory: sdks/csharp @@ -880,18 +759,8 @@ jobs: # exit 1 # } - - name: Start SpacetimeDB - run: | - spacetime start & - disown - - - name: Run regression tests - run: | - bash sdks/csharp/tools~/run-regression-tests.sh - tools/check-diff.sh sdks/csharp/examples~/regression-tests || { - echo 'Error: Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.' - exit 1 - } + - name: Run C# tests + run: cargo ci csharp-tests internal-tests: name: Internal Tests diff --git a/.github/workflows/docs-publish.yaml b/.github/workflows/docs-publish.yaml index 4b068d00bc3..41cb132e859 100644 --- a/.github/workflows/docs-publish.yaml +++ b/.github/workflows/docs-publish.yaml @@ -38,13 +38,12 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install dependencies - working-directory: docs - run: pnpm install + - uses: dsherret/rust-toolchain-file@v1 + - name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Docusaurus build - working-directory: docs - run: pnpm build + run: cargo ci docs - name: Publish docs to S3 uses: shallwefootball/s3-upload-action@master diff --git a/.github/workflows/docs-test.yaml b/.github/workflows/docs-test.yaml index b7233aad847..8550bcc4d74 100644 --- a/.github/workflows/docs-test.yaml +++ b/.github/workflows/docs-test.yaml @@ -35,10 +35,9 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install dependencies - working-directory: docs - run: pnpm install + - uses: dsherret/rust-toolchain-file@v1 + - name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Docusaurus build - working-directory: docs - run: pnpm build + run: cargo ci docs diff --git a/.github/workflows/typescript-test.yml b/.github/workflows/typescript-test.yml index edbbe89fc83..bc9bc960507 100644 --- a/.github/workflows/typescript-test.yml +++ b/.github/workflows/typescript-test.yml @@ -41,14 +41,6 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Build module library and SDK - working-directory: crates/bindings-typescript - run: pnpm build - - - name: Run module library and SDK tests - working-directory: crates/bindings-typescript - run: pnpm test - # - name: Extract SpacetimeDB branch name from file # id: extract-branch # run: | @@ -118,46 +110,8 @@ jobs: # Clear any existing information spacetime server clear -y - - name: Generate client bindings - working-directory: templates/chat-react-ts - run: | - pnpm generate - - - name: Check for changes - working-directory: templates/chat-react-ts - run: | - "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/module_bindings || { - echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." - exit 1 - } - - # - name: Start SpacetimeDB - # run: | - # spacetime start & - # disown - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logout && spacetime login --server-issued-login local - # spacetime publish -s local quickstart-chat -c -y - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logs quickstart-chat - - - name: Check that quickstart-chat builds - working-directory: templates/chat-react-ts - run: pnpm build - - - name: Check that templates build - working-directory: templates/ - run: pnpm -r --filter "./**" run build - - - name: Check that subdirectories build - working-directory: crates/bindings-typescript - run: pnpm -r --filter "./**" run build + - name: Run TypeScript tests + run: cargo ci typescript-test # - name: Run quickstart-chat tests # working-directory: examples/quickstart-chat diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index cef4b6fda34..23ea7bfb07b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,12 +1,14 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; +use regex::Regex; use std::ffi::OsStr; use std::ffi::OsString; use std::path::Path; use std::path::PathBuf; +use std::process::Command; use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; @@ -271,6 +273,16 @@ enum CiCmd { /// Verify that any non-root global.json files are symlinks to the root global.json. GlobalJsonPolicy, + /// Checks that publishable crates satisfy publish constraints. + PublishChecks, + /// Runs TypeScript workspace tests and template build checks. + TypescriptTest, + /// Builds the docs site. + Docs, + /// Runs the C# SDK test suite and binding checks. + CsharpTests, + /// Prepares the Unity test workspace and publishes the local test module. + UnityTests, } fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { @@ -299,6 +311,309 @@ fn tracked_rs_files_under(path: &str) -> Result> { .collect()) } +fn run_check_diff(path: &str, message: &str) -> Result<()> { + let status = Command::new("bash").args(["tools/check-diff.sh", path]).status()?; + if !status.success() { + bail!("{message}"); + } + Ok(()) +} + +fn prepare_csharp_sdk_solution() -> Result<()> { + cmd!( + "dotnet", + "pack", + "crates/bindings-csharp/BSATN.Runtime", + "-c", + "Release" + ) + .run()?; + cmd!("dotnet", "pack", "crates/bindings-csharp/Runtime", "-c", "Release").run()?; + cmd!("bash", "./tools~/write-nuget-config.sh", "../..") + .dir("sdks/csharp") + .run()?; + cmd!( + "dotnet", + "restore", + "--configfile", + "NuGet.Config", + "SpacetimeDB.ClientSDK.sln" + ) + .dir("sdks/csharp") + .run()?; + Ok(()) +} + +fn run_publish_checks() -> Result<()> { + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} + +fn run_typescript_tests() -> Result<()> { + cmd!("pnpm", "build").dir("crates/bindings-typescript").run()?; + cmd!("pnpm", "test").dir("crates/bindings-typescript").run()?; + cmd!("pnpm", "generate").dir("templates/chat-react-ts").run()?; + run_check_diff( + "templates/chat-react-ts/src/module_bindings", + "Bindings are dirty. Please generate bindings again and commit them to this branch.", + )?; + cmd!("pnpm", "build").dir("templates/chat-react-ts").run()?; + cmd!("pnpm", "-r", "--filter", "./**", "run", "build") + .dir("templates") + .run()?; + cmd!("pnpm", "-r", "--filter", "./**", "run", "build") + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} + +fn run_docs_build() -> Result<()> { + cmd!("pnpm", "install").dir("docs").run()?; + cmd!("pnpm", "build").dir("docs").run()?; + Ok(()) +} + +fn run_local_spacetime_script(script_name: &str, body: &str) -> Result<()> { + let script = format!( + r#"set -euo pipefail +spacetime start >"/tmp/{script_name}.log" 2>&1 & +STDB_PID=$! +trap 'kill "$STDB_PID" >/dev/null 2>&1 || true' EXIT +sleep 3 +{body} +"# + ); + cmd!("bash", "-lc", &script).run()?; + Ok(()) +} + +fn run_csharp_tests() -> Result<()> { + prepare_csharp_sdk_solution()?; + + cmd!("dotnet", "test", "-warnaserror", "--no-restore") + .dir("sdks/csharp") + .run()?; + cmd!( + "dotnet", + "format", + "--no-restore", + "--verify-no-changes", + "SpacetimeDB.ClientSDK.sln" + ) + .dir("sdks/csharp") + .run()?; + + cmd!("bash", "tools~/gen-quickstart.sh").dir("sdks/csharp").run()?; + run_check_diff( + "sdks/csharp/examples~/quickstart-chat", + "quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.", + )?; + + run_local_spacetime_script( + "spacetimedb-csharp-tests", + r#"bash sdks/csharp/tools~/run-regression-tests.sh"#, + )?; + run_check_diff( + "sdks/csharp/examples~/regression-tests", + "Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.", + )?; + + Ok(()) +} + +fn patch_blackholio_server_dependency() -> Result<()> { + let cargo_toml_path = Path::new("demo/Blackholio/server-rust/Cargo.toml"); + let existing = fs::read_to_string(cargo_toml_path)?; + let dependency_line = Regex::new(r#"(?m)^spacetimedb\s*=.*$"#)?; + let updated = dependency_line.replace(&existing, r#"spacetimedb = { path = "../../../crates/bindings" }"#); + + ensure!( + updated.as_ref() != existing, + "Failed to patch demo/Blackholio/server-rust/Cargo.toml with local spacetimedb dependency" + ); + + fs::write(cargo_toml_path, updated.as_ref())?; + Ok(()) +} + +fn run_unity_tests() -> Result<()> { + prepare_csharp_sdk_solution()?; + patch_blackholio_server_dependency()?; + + cmd!("bash", "./generate.sh", "-y") + .dir("demo/Blackholio/server-rust") + .run()?; + run_check_diff( + "demo/Blackholio/client-unity/Assets/Scripts/autogen", + "Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.", + )?; + + run_dlls()?; + + run_local_spacetime_script( + "spacetimedb-unity-tests", + r#"spacetime logout && spacetime login --server-issued-login local +cd demo/Blackholio/server-rust +bash ./publish.sh"#, + )?; + + cmd!( + "bash", + "-lc", + r#"cd demo/Blackholio/client-unity/Packages +yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json +cat manifest.json"# + ) + .run()?; + + Ok(()) +} + +fn run_dlls() -> Result<()> { + ensure_repo_root()?; + + cmd!( + "dotnet", + "pack", + "crates/bindings-csharp/BSATN.Runtime", + "-c", + "Release" + ) + .run()?; + cmd!("dotnet", "pack", "crates/bindings-csharp/Runtime", "-c", "Release").run()?; + + let repo_root = env::current_dir()?; + let bsatn_source = repo_root.join("crates/bindings-csharp/BSATN.Runtime/bin/Release"); + let runtime_source = repo_root.join("crates/bindings-csharp/Runtime/bin/Release"); + + let nuget_config_dir = tempfile::tempdir()?; + let nuget_config_path = nuget_config_dir.path().join("nuget.config"); + let nuget_config_contents = format!( + r#" + + + + + + + + + + + + + + + + + + + + "#, + bsatn_source.display(), + runtime_source.display(), + ); + fs::write(&nuget_config_path, nuget_config_contents)?; + + let nuget_config_path_str = nuget_config_path.to_string_lossy().to_string(); + + clear_restored_package_dirs("spacetimedb.bsatn.runtime")?; + clear_restored_package_dirs("spacetimedb.runtime")?; + + cmd!( + "dotnet", + "restore", + "SpacetimeDB.ClientSDK.csproj", + "--configfile", + &nuget_config_path_str, + ) + .dir("sdks/csharp") + .run()?; + + overlay_unity_meta_skeleton("spacetimedb.bsatn.runtime")?; + overlay_unity_meta_skeleton("spacetimedb.runtime")?; + + cmd!( + "dotnet", + "pack", + "SpacetimeDB.ClientSDK.csproj", + "-c", + "Release", + "--no-restore" + ) + .dir("sdks/csharp") + .run()?; + + Ok(()) +} + +fn run_update_flow(target: Option, github_token_auth: bool) -> Result<()> { + let mut common_args = vec![]; + if let Some(target) = target.as_ref() { + common_args.push("--target"); + common_args.push(target.as_str()); + log::info!("checking update flow for target: {target}"); + } else { + log::info!("checking update flow"); + } + if github_token_auth { + common_args.push("--features"); + common_args.push("github-token-auth"); + } + + cmd( + "cargo", + ["build", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.iter().copied()), + ) + .run()?; + + let root_dir = tempfile::tempdir()?; + let root_arg = format!("--root-dir={}", root_dir.path().display()); + cmd( + "cargo", + ["run", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.iter().copied()) + .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), + ) + .run()?; + + let mut spacetime_path = root_dir.path().join("spacetime"); + if !std::env::consts::EXE_EXTENSION.is_empty() { + spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); + } + cmd(spacetime_path, [&root_arg, "help"]).run()?; + + Ok(()) +} + fn main() -> Result<()> { env_logger::init(); @@ -459,79 +774,7 @@ fn main() -> Result<()> { } Some(CiCmd::Dlls) => { - ensure_repo_root()?; - - cmd!( - "dotnet", - "pack", - "crates/bindings-csharp/BSATN.Runtime", - "-c", - "Release" - ) - .run()?; - cmd!("dotnet", "pack", "crates/bindings-csharp/Runtime", "-c", "Release").run()?; - - let repo_root = env::current_dir()?; - let bsatn_source = repo_root.join("crates/bindings-csharp/BSATN.Runtime/bin/Release"); - let runtime_source = repo_root.join("crates/bindings-csharp/Runtime/bin/Release"); - - let nuget_config_dir = tempfile::tempdir()?; - let nuget_config_path = nuget_config_dir.path().join("nuget.config"); - let nuget_config_contents = format!( - r#" - - - - - - - - - - - - - - - - - - - - "#, - bsatn_source.display(), - runtime_source.display(), - ); - fs::write(&nuget_config_path, nuget_config_contents)?; - - let nuget_config_path_str = nuget_config_path.to_string_lossy().to_string(); - - clear_restored_package_dirs("spacetimedb.bsatn.runtime")?; - clear_restored_package_dirs("spacetimedb.runtime")?; - - cmd!( - "dotnet", - "restore", - "SpacetimeDB.ClientSDK.csproj", - "--configfile", - &nuget_config_path_str, - ) - .dir("sdks/csharp") - .run()?; - - overlay_unity_meta_skeleton("spacetimedb.bsatn.runtime")?; - overlay_unity_meta_skeleton("spacetimedb.runtime")?; - - cmd!( - "dotnet", - "pack", - "SpacetimeDB.ClientSDK.csproj", - "-c", - "Release", - "--no-restore" - ) - .dir("sdks/csharp") - .run()?; + run_dlls()?; } Some(CiCmd::Smoketests(args)) => { @@ -542,41 +785,7 @@ fn main() -> Result<()> { target, github_token_auth, }) => { - let mut common_args = vec![]; - if let Some(target) = target.as_ref() { - common_args.push("--target"); - common_args.push(target); - log::info!("checking update flow for target: {target}"); - } else { - log::info!("checking update flow"); - } - if github_token_auth { - common_args.push("--features"); - common_args.push("github-token-auth"); - } - - cmd( - "cargo", - ["build", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()), - ) - .run()?; - // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - // happens very frequently on the `macos-runner`, but we haven't seen it on any others). - let root_dir = tempfile::tempdir()?; - let root_dir_string = root_dir.path().to_string_lossy().to_string(); - let root_arg = format!("--root-dir={}", root_dir_string); - cmd( - "cargo", - ["run", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()) - .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), - ) - .run()?; - cmd!(format!("{}/spacetime", root_dir_string), &root_arg, "help",).run()?; + run_update_flow(target, github_token_auth)?; } Some(CiCmd::CliDocs { spacetime_path }) => { @@ -622,6 +831,26 @@ fn main() -> Result<()> { check_global_json_policy()?; } + Some(CiCmd::PublishChecks) => { + run_publish_checks()?; + } + + Some(CiCmd::TypescriptTest) => { + run_typescript_tests()?; + } + + Some(CiCmd::Docs) => { + run_docs_build()?; + } + + Some(CiCmd::CsharpTests) => { + run_csharp_tests()?; + } + + Some(CiCmd::UnityTests) => { + run_unity_tests()?; + } + None => run_all_clap_subcommands(&cli.skip)?, } From 358150c0312027557a65129ff4a516db622a2d69 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 14:42:00 -0700 Subject: [PATCH 05/25] move --- tools/ci/src/main.rs | 79 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 23ea7bfb07b..ea6a879c77a 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -573,45 +573,6 @@ fn run_dlls() -> Result<()> { } fn run_update_flow(target: Option, github_token_auth: bool) -> Result<()> { - let mut common_args = vec![]; - if let Some(target) = target.as_ref() { - common_args.push("--target"); - common_args.push(target.as_str()); - log::info!("checking update flow for target: {target}"); - } else { - log::info!("checking update flow"); - } - if github_token_auth { - common_args.push("--features"); - common_args.push("github-token-auth"); - } - - cmd( - "cargo", - ["build", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.iter().copied()), - ) - .run()?; - - let root_dir = tempfile::tempdir()?; - let root_arg = format!("--root-dir={}", root_dir.path().display()); - cmd( - "cargo", - ["run", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.iter().copied()) - .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), - ) - .run()?; - - let mut spacetime_path = root_dir.path().join("spacetime"); - if !std::env::consts::EXE_EXTENSION.is_empty() { - spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); - } - cmd(spacetime_path, [&root_arg, "help"]).run()?; - - Ok(()) } fn main() -> Result<()> { @@ -785,7 +746,45 @@ fn main() -> Result<()> { target, github_token_auth, }) => { - run_update_flow(target, github_token_auth)?; + let mut common_args = vec![]; + if let Some(target) = target.as_ref() { + common_args.push("--target"); + common_args.push(target.as_str()); + log::info!("checking update flow for target: {target}"); + } else { + log::info!("checking update flow"); + } + if github_token_auth { + common_args.push("--features"); + common_args.push("github-token-auth"); + } + + cmd( + "cargo", + ["build", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.iter().copied()), + ) + .run()?; + + let root_dir = tempfile::tempdir()?; + let root_arg = format!("--root-dir={}", root_dir.path().display()); + cmd( + "cargo", + ["run", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.iter().copied()) + .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), + ) + .run()?; + + let mut spacetime_path = root_dir.path().join("spacetime"); + if !std::env::consts::EXE_EXTENSION.is_empty() { + spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); + } + cmd(spacetime_path, [&root_arg, "help"]).run()?; + + Ok(()) } Some(CiCmd::CliDocs { spacetime_path }) => { From 675005c9b484026e6435ec67bf3ac38936fbe97a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 14:44:04 -0700 Subject: [PATCH 06/25] fix --- tools/ci/src/main.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index ea6a879c77a..085ba2cae4f 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -572,9 +572,6 @@ fn run_dlls() -> Result<()> { Ok(()) } -fn run_update_flow(target: Option, github_token_auth: bool) -> Result<()> { -} - fn main() -> Result<()> { env_logger::init(); @@ -783,8 +780,6 @@ fn main() -> Result<()> { spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); } cmd(spacetime_path, [&root_arg, "help"]).run()?; - - Ok(()) } Some(CiCmd::CliDocs { spacetime_path }) => { From f4ad5eb84796dcdf7f6827fb302a21450d1415c6 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 14:44:10 -0700 Subject: [PATCH 07/25] readme --- tools/ci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index d1548309139..309a16a9540 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -44,7 +44,7 @@ Usage: test Lints the codebase -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. +Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. **Usage:** ```bash From 34f1e792ea82bdb995c33657e3bd81adc99e99a4 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Tue, 21 Apr 2026 14:50:56 -0700 Subject: [PATCH 08/25] revert --- tools/ci/README.md | 22 +++++++++++----------- tools/ci/src/ci_docs.rs | 8 ++------ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index 309a16a9540..159ce20b50a 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -102,8 +102,8 @@ Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. -- `--dotnet`: Whether to run smoketests that require the .NET toolchain -- `args`: Additional arguments to pass to the test runner +- `--dotnet`: +- `args`: - `--help`: Print help (see a summary with '-h') #### `prepare` @@ -130,7 +130,7 @@ Usage: check-mod-list **Options:** -- `--help`: Print help +- `--help`: #### `help` @@ -141,7 +141,7 @@ Usage: help [COMMAND]... **Options:** -- `subcommand`: Print help for the subcommand(s) +- `subcommand`: ### `update-flow` @@ -193,7 +193,7 @@ Usage: global-json-policy **Options:** -- `--help`: Print help +- `--help`: ### `publish-checks` @@ -204,7 +204,7 @@ Usage: publish-checks **Options:** -- `--help`: Print help +- `--help`: ### `typescript-test` @@ -215,7 +215,7 @@ Usage: typescript-test **Options:** -- `--help`: Print help +- `--help`: ### `docs` @@ -226,7 +226,7 @@ Usage: docs **Options:** -- `--help`: Print help +- `--help`: ### `csharp-tests` @@ -237,7 +237,7 @@ Usage: csharp-tests **Options:** -- `--help`: Print help +- `--help`: ### `unity-tests` @@ -248,7 +248,7 @@ Usage: unity-tests **Options:** -- `--help`: Print help +- `--help`: ### `help` @@ -259,7 +259,7 @@ Usage: help [COMMAND]... **Options:** -- `subcommand`: Print help for the subcommand(s) +- `subcommand`: --- diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index c63e2d5a82d..f09114e6787 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -46,16 +46,12 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { .map(|l| format!("--{}", l)) .or_else(|| arg.get_short().map(|s| format!("-{}", s))) .unwrap_or_else(|| arg.get_id().to_string()); - let help = arg - .get_long_help() - .or_else(|| arg.get_help()) - .map(|help| help.to_string()) - .unwrap_or_else(|| panic!("argument `{}` is missing help text", arg.get_id())); + let help = arg.get_long_help().unwrap_or_default(); options.push_str(&format!( "- `{}`: {}\n{}", names, help, - if help.lines().count() > 1 { "\n" } else { "" } + if help.to_string().lines().count() > 1 { "\n" } else { "" } )); } From 107e1eda640a497fcd48271fce67d6cc79f758cc Mon Sep 17 00:00:00 2001 From: Zeke Foppa <196249+bfops@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:53:01 -0700 Subject: [PATCH 09/25] Apply suggestion from @bfops Signed-off-by: Zeke Foppa <196249+bfops@users.noreply.github.com> --- tools/ci/src/smoketest.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci/src/smoketest.rs b/tools/ci/src/smoketest.rs index 7e5d3cbab9f..5799ba1a4ca 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/src/smoketest.rs @@ -26,7 +26,6 @@ pub struct SmoketestsArgs { #[arg(long)] server: Option, - /// Whether to run smoketests that require the .NET toolchain. #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, From bf1d20ee44017393dfe88d40eb6bed31b4e726dc Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:18:34 -0700 Subject: [PATCH 10/25] refactor --- tools/ci/src/main.rs | 175 ++++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 85 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 1743b67a814..d4150dcdf28 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -299,6 +299,84 @@ fn tracked_rs_files_under(path: &str) -> Result> { .collect()) } +fn run_dlls() -> Result<()> { + ensure_repo_root()?; + + cmd!( + "dotnet", + "pack", + "crates/bindings-csharp/BSATN.Runtime", + "-c", + "Release" + ) + .run()?; + cmd!("dotnet", "pack", "crates/bindings-csharp/Runtime", "-c", "Release").run()?; + + let repo_root = env::current_dir()?; + let bsatn_source = repo_root.join("crates/bindings-csharp/BSATN.Runtime/bin/Release"); + let runtime_source = repo_root.join("crates/bindings-csharp/Runtime/bin/Release"); + + let nuget_config_dir = tempfile::tempdir()?; + let nuget_config_path = nuget_config_dir.path().join("nuget.config"); + let nuget_config_contents = format!( + r#" + + + + + + + + + + + + + + + + + + + + "#, + bsatn_source.display(), + runtime_source.display(), + ); + fs::write(&nuget_config_path, nuget_config_contents)?; + + let nuget_config_path_str = nuget_config_path.to_string_lossy().to_string(); + + clear_restored_package_dirs("spacetimedb.bsatn.runtime")?; + clear_restored_package_dirs("spacetimedb.runtime")?; + + cmd!( + "dotnet", + "restore", + "SpacetimeDB.ClientSDK.csproj", + "--configfile", + &nuget_config_path_str, + ) + .dir("sdks/csharp") + .run()?; + + overlay_unity_meta_skeleton("spacetimedb.bsatn.runtime")?; + overlay_unity_meta_skeleton("spacetimedb.runtime")?; + + cmd!( + "dotnet", + "pack", + "SpacetimeDB.ClientSDK.csproj", + "-c", + "Release", + "--no-restore" + ) + .dir("sdks/csharp") + .run()?; + + Ok(()) +} + fn main() -> Result<()> { env_logger::init(); @@ -306,8 +384,6 @@ fn main() -> Result<()> { match cli.cmd { Some(CiCmd::Test) => { - cmd!("pnpm", "build").dir("crates/bindings-typescript").run()?; - // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? // Exclude smoketests from `cargo test --all` since they require pre-built binaries. @@ -461,83 +537,10 @@ fn main() -> Result<()> { } Some(CiCmd::Dlls) => { - ensure_repo_root()?; - - cmd!( - "dotnet", - "pack", - "crates/bindings-csharp/BSATN.Runtime", - "-c", - "Release" - ) - .run()?; - cmd!("dotnet", "pack", "crates/bindings-csharp/Runtime", "-c", "Release").run()?; - - let repo_root = env::current_dir()?; - let bsatn_source = repo_root.join("crates/bindings-csharp/BSATN.Runtime/bin/Release"); - let runtime_source = repo_root.join("crates/bindings-csharp/Runtime/bin/Release"); - - let nuget_config_dir = tempfile::tempdir()?; - let nuget_config_path = nuget_config_dir.path().join("nuget.config"); - let nuget_config_contents = format!( - r#" - - - - - - - - - - - - - - - - - - - - "#, - bsatn_source.display(), - runtime_source.display(), - ); - fs::write(&nuget_config_path, nuget_config_contents)?; - - let nuget_config_path_str = nuget_config_path.to_string_lossy().to_string(); - - clear_restored_package_dirs("spacetimedb.bsatn.runtime")?; - clear_restored_package_dirs("spacetimedb.runtime")?; - - cmd!( - "dotnet", - "restore", - "SpacetimeDB.ClientSDK.csproj", - "--configfile", - &nuget_config_path_str, - ) - .dir("sdks/csharp") - .run()?; - - overlay_unity_meta_skeleton("spacetimedb.bsatn.runtime")?; - overlay_unity_meta_skeleton("spacetimedb.runtime")?; - - cmd!( - "dotnet", - "pack", - "SpacetimeDB.ClientSDK.csproj", - "-c", - "Release", - "--no-restore" - ) - .dir("sdks/csharp") - .run()?; + run_dlls()?; } Some(CiCmd::Smoketests(args)) => { - ensure_repo_root()?; smoketest::run(args)?; } @@ -548,7 +551,7 @@ fn main() -> Result<()> { let mut common_args = vec![]; if let Some(target) = target.as_ref() { common_args.push("--target"); - common_args.push(target); + common_args.push(target.as_str()); log::info!("checking update flow for target: {target}"); } else { log::info!("checking update flow"); @@ -562,24 +565,26 @@ fn main() -> Result<()> { "cargo", ["build", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.clone()), + .chain(common_args.iter().copied()), ) .run()?; - // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - // happens very frequently on the `macos-runner`, but we haven't seen it on any others). + let root_dir = tempfile::tempdir()?; - let root_dir_string = root_dir.path().to_string_lossy().to_string(); - let root_arg = format!("--root-dir={}", root_dir_string); + let root_arg = format!("--root-dir={}", root_dir.path().display()); cmd( "cargo", ["run", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.clone()) + .chain(common_args.iter().copied()) .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), ) .run()?; - cmd!(format!("{}/spacetime", root_dir_string), &root_arg, "help",).run()?; + + let mut spacetime_path = root_dir.path().join("spacetime"); + if !std::env::consts::EXE_EXTENSION.is_empty() { + spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); + } + cmd(spacetime_path, [&root_arg, "help"]).run()?; } Some(CiCmd::CliDocs { spacetime_path }) => { From c6813bdc0cd0081c3ebcb1e51cbc1f9e1894a33d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:21:05 -0700 Subject: [PATCH 11/25] revert --- tools/ci/src/main.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index d4150dcdf28..7143907c467 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -384,6 +384,8 @@ fn main() -> Result<()> { match cli.cmd { Some(CiCmd::Test) => { + cmd!("pnpm", "build").dir("crates/bindings-typescript").run()?; + // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? // Exclude smoketests from `cargo test --all` since they require pre-built binaries. @@ -541,6 +543,7 @@ fn main() -> Result<()> { } Some(CiCmd::Smoketests(args)) => { + ensure_repo_root()?; smoketest::run(args)?; } @@ -551,7 +554,7 @@ fn main() -> Result<()> { let mut common_args = vec![]; if let Some(target) = target.as_ref() { common_args.push("--target"); - common_args.push(target.as_str()); + common_args.push(target); log::info!("checking update flow for target: {target}"); } else { log::info!("checking update flow"); @@ -565,26 +568,24 @@ fn main() -> Result<()> { "cargo", ["build", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.iter().copied()), + .chain(common_args.clone()), ) .run()?; - + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). let root_dir = tempfile::tempdir()?; - let root_arg = format!("--root-dir={}", root_dir.path().display()); + let root_dir_string = root_dir.path().to_string_lossy().to_string(); + let root_arg = format!("--root-dir={}", root_dir_string); cmd( "cargo", ["run", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.iter().copied()) + .chain(common_args.clone()) .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), ) .run()?; - - let mut spacetime_path = root_dir.path().join("spacetime"); - if !std::env::consts::EXE_EXTENSION.is_empty() { - spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); - } - cmd(spacetime_path, [&root_arg, "help"]).run()?; + cmd!(format!("{}/spacetime", root_dir_string), &root_arg, "help",).run()?; } Some(CiCmd::CliDocs { spacetime_path }) => { From 322aa8202543a3f2c713172e6b9bfaafff29a387 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:33:56 -0700 Subject: [PATCH 12/25] review --- tools/ci/src/main.rs | 49 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index c426409d528..3c0ef03effa 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -8,7 +8,6 @@ use std::ffi::OsStr; use std::ffi::OsString; use std::path::Path; use std::path::PathBuf; -use std::process::Command; use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; @@ -311,14 +310,6 @@ fn tracked_rs_files_under(path: &str) -> Result> { .collect()) } -fn run_check_diff(path: &str, message: &str) -> Result<()> { - let status = Command::new("bash").args(["tools/check-diff.sh", path]).status()?; - if !status.success() { - bail!("{message}"); - } - Ok(()) -} - fn prepare_csharp_sdk_solution() -> Result<()> { cmd!( "dotnet", @@ -378,10 +369,15 @@ fn run_typescript_tests() -> Result<()> { cmd!("pnpm", "build").dir("crates/bindings-typescript").run()?; cmd!("pnpm", "test").dir("crates/bindings-typescript").run()?; cmd!("pnpm", "generate").dir("templates/chat-react-ts").run()?; - run_check_diff( - "templates/chat-react-ts/src/module_bindings", - "Bindings are dirty. Please generate bindings again and commit them to this branch.", - )?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } cmd!("pnpm", "build").dir("templates/chat-react-ts").run()?; cmd!("pnpm", "-r", "--filter", "./**", "run", "build") .dir("templates") @@ -429,19 +425,19 @@ fn run_csharp_tests() -> Result<()> { .run()?; cmd!("bash", "tools~/gen-quickstart.sh").dir("sdks/csharp").run()?; - run_check_diff( - "sdks/csharp/examples~/quickstart-chat", - "quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.", - )?; + let diff_status = cmd!("bash", "tools/check-diff.sh", "sdks/csharp/examples~/quickstart-chat").run()?; + if !diff_status.status.success() { + bail!("quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`."); + } run_local_spacetime_script( "spacetimedb-csharp-tests", r#"bash sdks/csharp/tools~/run-regression-tests.sh"#, )?; - run_check_diff( - "sdks/csharp/examples~/regression-tests", - "Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.", - )?; + let diff_status = cmd!("bash", "tools/check-diff.sh", "sdks/csharp/examples~/regression-tests").run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`."); + } Ok(()) } @@ -468,10 +464,15 @@ fn run_unity_tests() -> Result<()> { cmd!("bash", "./generate.sh", "-y") .dir("demo/Blackholio/server-rust") .run()?; - run_check_diff( + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", "demo/Blackholio/client-unity/Assets/Scripts/autogen", - "Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.", - )?; + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`."); + } run_dlls()?; From 7fa085929a82b6491fa0d1c687192a2e4330106a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:42:01 -0700 Subject: [PATCH 13/25] CI - Merge workflow files --- .github/workflows/ci.yml | 195 ++++++++++++++++++++++++++ .github/workflows/docs-test.yaml | 44 ------ .github/workflows/typescript-lint.yml | 40 ------ .github/workflows/typescript-test.yml | 169 ---------------------- 4 files changed, 195 insertions(+), 253 deletions(-) delete mode 100644 .github/workflows/docs-test.yaml delete mode 100644 .github/workflows/typescript-lint.yml delete mode 100644 .github/workflows/typescript-test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cf8dc70b47..7a1d8fbd4f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1134,3 +1134,198 @@ jobs: - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries run: | cargo ci smoketests check-mod-list + + docs-build: + name: Docs build + runs-on: spacetimedb-new-runner-2 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '22' + + - uses: pnpm/action-setup@v4 + with: + run_install: true + + - name: Get pnpm store directory + working-directory: sdks/typescript + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + name: Setup pnpm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + working-directory: docs + run: pnpm install + + - name: Docusaurus build + working-directory: docs + run: pnpm build + + typescript-test: + name: TypeScript - Tests + runs-on: spacetimedb-new-runner-2 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: pnpm/action-setup@v4 + with: + run_install: true + + - name: Get pnpm store directory + shell: bash + working-directory: crates/bindings-typescript + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + name: Setup pnpm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Build module library and SDK + working-directory: crates/bindings-typescript + run: pnpm build + + - name: Run module library and SDK tests + working-directory: crates/bindings-typescript + run: pnpm test + + # - name: Extract SpacetimeDB branch name from file + # id: extract-branch + # run: | + # # Define the path to the branch file + # BRANCH_FILE=".github/spacetimedb-branch.txt" + + # # Default to master if file doesn't exist + # if [ ! -f "$BRANCH_FILE" ]; then + # echo "::notice::No SpacetimeDB branch file found, using 'master'" + # echo "branch=master" >> $GITHUB_OUTPUT + # exit 0 + # fi + + # # Read and trim whitespace from the file + # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + + # # Fallback to master if empty + # if [ -z "$branch" ]; then + # echo "::warning::SpacetimeDB branch file is empty, using 'master'" + # branch="master" + # fi + + # echo "branch=$branch" >> $GITHUB_OUTPUT + # echo "Using SpacetimeDB branch from file: $branch" + + - name: Install Rust toolchain + uses: dsherret/rust-toolchain-file@v1 + - name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: ${{ github.workspace }} + shared-key: spacetimedb + # Let the main CI job save the cache since it builds the most things + save-if: false + prefix-key: v1 + + # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. + # ChatGPT suspects that this could be due to different build invocations using the same target dir, + # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with + # `cargo build --manifest-path` (which apparently build different dependency trees). + # However, we've been unable to fix it so... /shrug + - name: Check v8 outputs + run: | + find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true + if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then + echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + cargo clean -p v8 || true + cargo build -p v8 + fi + if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then + echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + cargo clean --release -p v8 || true + cargo build --release -p v8 + fi + + - name: Install SpacetimeDB CLI from the local checkout + run: | + export CARGO_HOME="$HOME/.cargo" + echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + cargo install --force --path crates/cli --locked --message-format=short + cargo install --force --path crates/standalone --locked --message-format=short + # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). + ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + # Clear any existing information + spacetime server clear -y + + - name: Generate client bindings + working-directory: templates/chat-react-ts + run: | + pnpm generate + + - name: Check for changes + working-directory: templates/chat-react-ts + run: | + "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/module_bindings || { + echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." + exit 1 + } + + # - name: Start SpacetimeDB + # run: | + # spacetime start & + # disown + + # - name: Publish module to SpacetimeDB + # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb + # run: | + # spacetime logout && spacetime login --server-issued-login local + # spacetime publish -s local quickstart-chat -c -y + + # - name: Publish module to SpacetimeDB + # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb + # run: | + # spacetime logs quickstart-chat + + - name: Check that quickstart-chat builds + working-directory: templates/chat-react-ts + run: pnpm build + + - name: Check that templates build + working-directory: templates/ + run: pnpm -r --filter "./**" run build + + - name: Check that subdirectories build + working-directory: crates/bindings-typescript + run: pnpm -r --filter "./**" run build + + # - name: Run quickstart-chat tests + # working-directory: examples/quickstart-chat + # run: pnpm test + # + # # Run this step always, even if the previous steps fail + # - name: Print rows in the user table + # if: always() + # run: spacetime sql quickstart-chat "SELECT * FROM user" diff --git a/.github/workflows/docs-test.yaml b/.github/workflows/docs-test.yaml deleted file mode 100644 index b7233aad847..00000000000 --- a/.github/workflows/docs-test.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Docs / Test -permissions: - contents: read - -on: - pull_request: - -jobs: - build: - runs-on: spacetimedb-new-runner-2 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '22' - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - working-directory: sdks/typescript - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install dependencies - working-directory: docs - run: pnpm install - - - name: Docusaurus build - working-directory: docs - run: pnpm build diff --git a/.github/workflows/typescript-lint.yml b/.github/workflows/typescript-lint.yml deleted file mode 100644 index d7f51fae5d5..00000000000 --- a/.github/workflows/typescript-lint.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: TypeScript - Lint - -on: - pull_request: - push: - branches: - - master - merge_group: - -jobs: - build: - runs-on: spacetimedb-new-runner-2 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Lint - run: pnpm lint diff --git a/.github/workflows/typescript-test.yml b/.github/workflows/typescript-test.yml deleted file mode 100644 index edbbe89fc83..00000000000 --- a/.github/workflows/typescript-test.yml +++ /dev/null @@ -1,169 +0,0 @@ -name: TypeScript - Tests - -on: - push: - branches: - - master - pull_request: - merge_group: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || format('sha-{0}', github.sha) }} - cancel-in-progress: true - -jobs: - build-and-test: - runs-on: spacetimedb-new-runner-2 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - shell: bash - working-directory: crates/bindings-typescript - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Build module library and SDK - working-directory: crates/bindings-typescript - run: pnpm build - - - name: Run module library and SDK tests - working-directory: crates/bindings-typescript - run: pnpm test - - # - name: Extract SpacetimeDB branch name from file - # id: extract-branch - # run: | - # # Define the path to the branch file - # BRANCH_FILE=".github/spacetimedb-branch.txt" - - # # Default to master if file doesn't exist - # if [ ! -f "$BRANCH_FILE" ]; then - # echo "::notice::No SpacetimeDB branch file found, using 'master'" - # echo "branch=master" >> $GITHUB_OUTPUT - # exit 0 - # fi - - # # Read and trim whitespace from the file - # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - # # Fallback to master if empty - # if [ -z "$branch" ]; then - # echo "::warning::SpacetimeDB branch file is empty, using 'master'" - # branch="master" - # fi - - # echo "branch=$branch" >> $GITHUB_OUTPUT - # echo "Using SpacetimeDB branch from file: $branch" - - - name: Install Rust toolchain - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: ${{ github.workspace }} - shared-key: spacetimedb - # Let the main CI job save the cache since it builds the most things - save-if: false - prefix-key: v1 - - # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. - # ChatGPT suspects that this could be due to different build invocations using the same target dir, - # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with - # `cargo build --manifest-path` (which apparently build different dependency trees). - # However, we've been unable to fix it so... /shrug - - name: Check v8 outputs - run: | - find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true - if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean -p v8 || true - cargo build -p v8 - fi - if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean --release -p v8 || true - cargo build --release -p v8 - fi - - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - # Clear any existing information - spacetime server clear -y - - - name: Generate client bindings - working-directory: templates/chat-react-ts - run: | - pnpm generate - - - name: Check for changes - working-directory: templates/chat-react-ts - run: | - "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/module_bindings || { - echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." - exit 1 - } - - # - name: Start SpacetimeDB - # run: | - # spacetime start & - # disown - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logout && spacetime login --server-issued-login local - # spacetime publish -s local quickstart-chat -c -y - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logs quickstart-chat - - - name: Check that quickstart-chat builds - working-directory: templates/chat-react-ts - run: pnpm build - - - name: Check that templates build - working-directory: templates/ - run: pnpm -r --filter "./**" run build - - - name: Check that subdirectories build - working-directory: crates/bindings-typescript - run: pnpm -r --filter "./**" run build - - # - name: Run quickstart-chat tests - # working-directory: examples/quickstart-chat - # run: pnpm test - # - # # Run this step always, even if the previous steps fail - # - name: Print rows in the user table - # if: always() - # run: spacetime sql quickstart-chat "SELECT * FROM user" From 428d0f553ba21b7a0c4fa62de4cdde591b533677 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:42:44 -0700 Subject: [PATCH 14/25] revert --- .github/workflows/typescript-lint.yml | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/typescript-lint.yml diff --git a/.github/workflows/typescript-lint.yml b/.github/workflows/typescript-lint.yml new file mode 100644 index 00000000000..d7f51fae5d5 --- /dev/null +++ b/.github/workflows/typescript-lint.yml @@ -0,0 +1,40 @@ +name: TypeScript - Lint + +on: + pull_request: + push: + branches: + - master + merge_group: + +jobs: + build: + runs-on: spacetimedb-new-runner-2 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: pnpm/action-setup@v4 + with: + run_install: true + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + name: Setup pnpm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Lint + run: pnpm lint From 6a8990267223f511541710f1d704f3363a9cc997 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:44:06 -0700 Subject: [PATCH 15/25] fix warnings --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a1d8fbd4f9..c587fc6ba7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }} cancel-in-progress: true +permissions: + contents: read + jobs: smoketests: needs: [lints] From 85c08a5663c2f3a5a3569d1935b01c4e90b6935e Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 10:48:58 -0700 Subject: [PATCH 16/25] fix permissions --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c587fc6ba7d..9173205f1a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ concurrency: permissions: contents: read + pull-requests: read jobs: smoketests: From 8ae4217b8eca25ba74db4e5f153cce0a25bb6316 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:10:52 -0700 Subject: [PATCH 17/25] revert --- .github/workflows/docs-publish.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-publish.yaml b/.github/workflows/docs-publish.yaml index 41cb132e859..4b068d00bc3 100644 --- a/.github/workflows/docs-publish.yaml +++ b/.github/workflows/docs-publish.yaml @@ -38,12 +38,13 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + - name: Install dependencies + working-directory: docs + run: pnpm install - name: Docusaurus build - run: cargo ci docs + working-directory: docs + run: pnpm build - name: Publish docs to S3 uses: shallwefootball/s3-upload-action@master From 545a53f202a028985f8ce44ece342fc542cb366e Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:38:30 -0700 Subject: [PATCH 18/25] CI - Move simple jobs into `cargo ci` --- .github/workflows/ci.yml | 77 ++++---------------------------------- tools/ci/src/main.rs | 80 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f7319726f1..b66b30d7fec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -342,24 +342,8 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - name: Set up Python env - run: | - test -d venv || python3 -m venv venv - venv/bin/pip3 install argparse toml - name: Run checks - run: | - set -ueo pipefail - FAILED=0 - ROOTS=(spacetimedb spacetimedb-sdk) - CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}") - for crate_dir in $CRATES; do - if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then - FAILED=$(( $FAILED + 1 )) - fi - done - if [ $FAILED -gt 0 ]; then - exit 1 - fi + run: cargo ci publish-checks update: name: Test spacetimedb-update flow (${{ matrix.target }}) @@ -1178,13 +1162,12 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install dependencies - working-directory: docs - run: pnpm install + - uses: dsherret/rust-toolchain-file@v1 + - name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Docusaurus build - working-directory: docs - run: pnpm build + run: cargo ci docs typescript-test: name: TypeScript - Tests @@ -1216,14 +1199,6 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Build module library and SDK - working-directory: crates/bindings-typescript - run: pnpm build - - - name: Run module library and SDK tests - working-directory: crates/bindings-typescript - run: pnpm test - # - name: Extract SpacetimeDB branch name from file # id: extract-branch # run: | @@ -1293,46 +1268,8 @@ jobs: # Clear any existing information spacetime server clear -y - - name: Generate client bindings - working-directory: templates/chat-react-ts - run: | - pnpm generate - - - name: Check for changes - working-directory: templates/chat-react-ts - run: | - "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/module_bindings || { - echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." - exit 1 - } - - # - name: Start SpacetimeDB - # run: | - # spacetime start & - # disown - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logout && spacetime login --server-issued-login local - # spacetime publish -s local quickstart-chat -c -y - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/templates/quickstart-chat-typescript/spacetimedb - # run: | - # spacetime logs quickstart-chat - - - name: Check that quickstart-chat builds - working-directory: templates/chat-react-ts - run: pnpm build - - - name: Check that templates build - working-directory: templates/ - run: pnpm -r --filter "./**" run build - - - name: Check that subdirectories build - working-directory: crates/bindings-typescript - run: pnpm -r --filter "./**" run build + - name: Run TypeScript tests + run: cargo ci typescript-test # - name: Run quickstart-chat tests # working-directory: examples/quickstart-chat diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index fd6c9ec4238..606291c16e2 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,8 +1,9 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; +use regex::Regex; use std::ffi::OsStr; use std::ffi::OsString; use std::path::Path; @@ -272,6 +273,12 @@ enum CiCmd { /// Verify that any non-root global.json files are symlinks to the root global.json. GlobalJsonPolicy, + /// Checks that publishable crates satisfy publish constraints. + PublishChecks, + /// Runs TypeScript workspace tests and template build checks. + TypescriptTest, + /// Builds the docs site. + Docs, } fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { @@ -300,6 +307,65 @@ fn tracked_rs_files_under(path: &str) -> Result> { .collect()) } +fn run_publish_checks() -> Result<()> { + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} + +fn run_typescript_tests() -> Result<()> { + cmd!("pnpm", "build").dir("crates/bindings-typescript").run()?; + cmd!("pnpm", "test").dir("crates/bindings-typescript").run()?; + cmd!("pnpm", "generate").dir("templates/chat-react-ts").run()?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } + cmd!("pnpm", "build").dir("templates/chat-react-ts").run()?; + cmd!("pnpm", "-r", "--filter", "./**", "run", "build") + .dir("templates") + .run()?; + cmd!("pnpm", "-r", "--filter", "./**", "run", "build") + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} + +fn run_docs_build() -> Result<()> { + cmd!("pnpm", "install").dir("docs").run()?; + cmd!("pnpm", "build").dir("docs").run()?; + Ok(()) +} + fn main() -> Result<()> { env_logger::init(); @@ -627,6 +693,18 @@ fn main() -> Result<()> { check_global_json_policy()?; } + Some(CiCmd::PublishChecks) => { + run_publish_checks()?; + } + + Some(CiCmd::TypescriptTest) => { + run_typescript_tests()?; + } + + Some(CiCmd::Docs) => { + run_docs_build()?; + } + None => run_all_clap_subcommands(&cli.skip)?, } From 784c13dfc534311fefcec2a0af445711e037d35d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:42:18 -0700 Subject: [PATCH 19/25] remove --- tools/ci/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 606291c16e2..507bf3f07df 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,9 +1,8 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, ensure, Result}; +use anyhow::{bail, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; -use regex::Regex; use std::ffi::OsStr; use std::ffi::OsString; use std::path::Path; From ec7dcc14486a72381cc8375996e74c4a2a07e3e6 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:49:16 -0700 Subject: [PATCH 20/25] remove unnecessary --- .github/workflows/ci.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b66b30d7fec..a93d1da3e27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1238,36 +1238,6 @@ jobs: save-if: false prefix-key: v1 - # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. - # ChatGPT suspects that this could be due to different build invocations using the same target dir, - # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with - # `cargo build --manifest-path` (which apparently build different dependency trees). - # However, we've been unable to fix it so... /shrug - - name: Check v8 outputs - run: | - find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true - if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean -p v8 || true - cargo build -p v8 - fi - if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean --release -p v8 || true - cargo build --release -p v8 - fi - - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - # Clear any existing information - spacetime server clear -y - - name: Run TypeScript tests run: cargo ci typescript-test From cdbe9bf28feae2a3554c410b4374bf42e4dcd87e Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:50:35 -0700 Subject: [PATCH 21/25] comment out unused --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a93d1da3e27..9236eaa213f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1238,6 +1238,36 @@ jobs: save-if: false prefix-key: v1 + # # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. + # # ChatGPT suspects that this could be due to different build invocations using the same target dir, + # # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with + # # `cargo build --manifest-path` (which apparently build different dependency trees). + # # However, we've been unable to fix it so... /shrug + # - name: Check v8 outputs + # run: | + # find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true + # if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then + # echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + # cargo clean -p v8 || true + # cargo build -p v8 + # fi + # if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then + # echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + # cargo clean --release -p v8 || true + # cargo build --release -p v8 + # fi + + # - name: Install SpacetimeDB CLI from the local checkout + # run: | + # export CARGO_HOME="$HOME/.cargo" + # echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + # cargo install --force --path crates/cli --locked --message-format=short + # cargo install --force --path crates/standalone --locked --message-format=short + # # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). + # ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + # # Clear any existing information + # spacetime server clear -y + - name: Run TypeScript tests run: cargo ci typescript-test From 13f930f86c2fbd9a0ce889293dda7b212b629b75 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:57:54 -0700 Subject: [PATCH 22/25] revert --- tools/ci/src/main.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 440d2ff6a51..e4fd7c971c3 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -752,7 +752,7 @@ fn main() -> Result<()> { let mut common_args = vec![]; if let Some(target) = target.as_ref() { common_args.push("--target"); - common_args.push(target.as_str()); + common_args.push(target); log::info!("checking update flow for target: {target}"); } else { log::info!("checking update flow"); @@ -766,17 +766,20 @@ fn main() -> Result<()> { "cargo", ["build", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.iter().copied()), + .chain(common_args.clone()), ) .run()?; - + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). let root_dir = tempfile::tempdir()?; - let root_arg = format!("--root-dir={}", root_dir.path().display()); + let root_dir_string = root_dir.path().to_string_lossy().to_string(); + let root_arg = format!("--root-dir={}", root_dir_string); cmd( "cargo", ["run", "-p", "spacetimedb-update"] .into_iter() - .chain(common_args.iter().copied()) + .chain(common_args.clone()) .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), ) .run()?; From 5fd2dabc46a1b770258ddd8b62fc3f361277b8b3 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 11:58:12 -0700 Subject: [PATCH 23/25] update README --- tools/ci/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tools/ci/README.md b/tools/ci/README.md index 91bca8784f4..b87db93ce63 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -195,6 +195,39 @@ Usage: global-json-policy - `--help`: +### `publish-checks` + +**Usage:** +```bash +Usage: publish-checks +``` + +**Options:** + +- `--help`: + +### `typescript-test` + +**Usage:** +```bash +Usage: typescript-test +``` + +**Options:** + +- `--help`: + +### `docs` + +**Usage:** +```bash +Usage: docs +``` + +**Options:** + +- `--help`: + ### `help` **Usage:** From 71c6c084a1ae71dbd6fed8bc636855f278ab09b8 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 12:53:15 -0700 Subject: [PATCH 24/25] Revert "CI - Upgrade job runs on windows" This reverts commit e4941a187c802f90878ff7b1529d3060c3fd2e69. --- .github/workflows/ci.yml | 18 ++++++++++++++++++ tools/ci/src/main.rs | 7 +------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 735ab8f62ee..f9b7e6297ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -373,10 +373,28 @@ jobs: shell: bash run: sudo apt install -y libssl-dev + - name: Build spacetimedb-update + run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update + if: runner.os == 'Windows' + + - name: Run self-install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + ROOT_DIR="$(mktemp -d)" + # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + # happens very frequently on the `macos-runner`, but we haven't seen it on any others). + cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes + "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help + if: runner.os == 'Windows' + - name: Test spacetimedb-update env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth + if: runner.os != 'Windows' unreal_engine_tests: name: Unreal Engine Tests diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index e4fd7c971c3..bec9ab327d0 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -783,12 +783,7 @@ fn main() -> Result<()> { .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), ) .run()?; - - let mut spacetime_path = root_dir.path().join("spacetime"); - if !std::env::consts::EXE_EXTENSION.is_empty() { - spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); - } - cmd(spacetime_path, [&root_arg, "help"]).run()?; + cmd!(format!("{}/spacetime", root_dir_string), &root_arg, "help",).run()?; } Some(CiCmd::CliDocs { spacetime_path }) => { From f6e71be211db8f6c38f22270a074dd938891061c Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Apr 2026 16:59:16 -0700 Subject: [PATCH 25/25] readme --- tools/ci/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tools/ci/README.md b/tools/ci/README.md index c0bafdeb5bb..dbe452243f0 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -195,6 +195,39 @@ Usage: global-json-policy - `--help`: Print help +### `publish-checks` + +**Usage:** +```bash +Usage: publish-checks +``` + +**Options:** + +- `--help`: Print help + +### `typescript-test` + +**Usage:** +```bash +Usage: typescript-test +``` + +**Options:** + +- `--help`: Print help + +### `docs` + +**Usage:** +```bash +Usage: docs +``` + +**Options:** + +- `--help`: Print help + ### `help` **Usage:**