scaffold-e2e.yml

# Scaffold E2E — the first-run experience as a regression gate (#2908).
#
# Two failure classes broke `npm create objectstack` in the past and neither
# was visible to unit tests:
#   1. The bundled template drifted from the published framework (pinned
#      ^6.0.0 while the registry was at 14.x; used APIs removed in 14.x).
#   2. Nothing ever exercised the published package end-to-end, so registry
#      breakage would only be found by a real new user.
#
# Job 1 (PRs touching the scaffolder / docker example) scaffolds with the
# repo-built scaffolder and runs the generated project against the registry:
# install → validate → build → boot → health probes → docker build/run.
# Job 2 (nightly) does the same via the *published* `create-objectstack@latest`
# across every template in the registry — the new-user canary.

name: Scaffold E2E

on:
  pull_request:
    paths:
      - 'packages/create-objectstack/**'
      - 'docker/**'
      - '.github/workflows/scaffold-e2e.yml'
  schedule:
    - cron: '23 3 * * *'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  scaffold-local:
    name: Scaffold with repo dist
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Get pnpm store directory
        shell: bash
        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build the scaffolder
        run: pnpm --filter create-objectstack build

      - name: Scaffold a project
        run: |
          cd "$RUNNER_TEMP"
          node "$GITHUB_WORKSPACE/packages/create-objectstack/bin/create-objectstack.js" e2e-app --skip-install --skip-skills

      - name: Install generated project (registry deps)
        run: |
          cd "$RUNNER_TEMP/e2e-app"
          # The scaffolder pins ^<repo version>. Right after a version bump
          # lands but before the release publishes, that version is not on the
          # registry yet — fall back to the latest published release so the
          # template is still exercised against the real registry.
          if ! npm install --no-fund --no-audit; then
            echo "::warning::repo version not published yet — falling back to latest"
            node -e '
              const fs = require("fs");
              const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
              for (const deps of [pkg.dependencies, pkg.devDependencies]) {
                if (!deps) continue;
                for (const d of Object.keys(deps)) {
                  if (d.startsWith("@objectstack/")) deps[d] = "latest";
                }
              }
              fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
            '
            npm install --no-fund --no-audit
            # The fallback just swapped the repo's unpublished version for
            # whatever `latest` points at — which during an RC window is the
            # PREVIOUS major (17.0.0-rc.N unpublished → latest is 16.x). The
            # template's manifest still stamps the repo's protocol major
            # (`engines: { protocol: '^17' }`, written by
            # scripts/sync-template-versions.mjs), so the ADR-0087 D1 handshake
            # in `os start` correctly refuses to boot the artifact:
            #   ✗ package 'e2e-app' targets protocol ^17 (engines.protocol)
            #     but this runtime is protocol 16.0.0
            # That is the gate working, on a skew this step introduced (#4894).
            # Re-stamp the manifest to the protocol major actually installed so
            # the rest of the job exercises the template against a coherent
            # runtime. Same alignment the Docker step below already does by
            # reading the resolved CLI version; done here BEFORE `npm run
            # build`, so the artifact carries the corrected range too.
            #
            # The major is read off the installed @objectstack/spec package:
            # PROTOCOL_VERSION is kept in lockstep with that package's own major
            # (packages/spec/src/kernel/protocol-version.ts, asserted by
            # protocol-version.test.ts), so the two cannot drift.
            #
            # Deliberately confined to the fallback branch: on the normal path
            # the project installs the repo's own version and the majors agree
            # by construction — a template that stamped the wrong major would
            # still fail, which is what template-consistency.test.ts is for.
            node -e '
              const fs = require("fs");
              const major = JSON.parse(
                fs.readFileSync("node_modules/@objectstack/spec/package.json", "utf8"),
              ).version.split(".")[0];
              const path = "objectstack.config.ts";
              const src = fs.readFileSync(path, "utf8");
              const stamp = /engines:\s*\{\s*protocol:\s*[\x27"][^\x27"]*[\x27"]\s*\}/;
              if (!stamp.test(src)) {
                console.log("::error::fallback cannot re-stamp engines.protocol — no stamp found in " + path);
                process.exit(1);
              }
              const out = src.replace(stamp, "engines: { protocol: \x27^" + major + "\x27 }");
              fs.writeFileSync(path, out);
              console.log("::notice::fallback re-stamped engines.protocol to ^" + major + " (installed @objectstack/spec major) — this run exercises the template against protocol " + major + ", not the repo\x27s");
            '
          fi

      - name: Validate and build the generated project
        run: |
          cd "$RUNNER_TEMP/e2e-app"
          npm run validate
          npm run build

      - name: Boot from the artifact and probe health
        run: |
          cd "$RUNNER_TEMP/e2e-app"
          npx os start --artifact ./dist/objectstack.json --port 8080 > server.log 2>&1 &
          SERVER_PID=$!
          ok=""
          for i in $(seq 1 30); do
            if curl -fsS http://localhost:8080/api/v1/health > /dev/null 2>&1; then ok=1; break; fi
            sleep 2
          done
          if [ -z "$ok" ]; then
            echo "::error::server never became healthy"; cat server.log; exit 1
          fi
          curl -fsS http://localhost:8080/api/v1/ready
          kill "$SERVER_PID"

      - name: 'Skills boundary: only the curated skills/ catalog installs'
        # 15.1 third-party eval: the repo-internal dogfood-verification skill
        # (.claude/skills/) leaked into scaffolded projects. Two leak paths,
        # two assertions against THIS checkout:
        #   1. The scaffolder/docs command is `skills add …/objectstack/skills
        #      --all` — the /skills subpath is the hard boundary, because the
        #      skills CLI's --all implies --skill '*' which INCLUDES
        #      metadata.internal skills. Install from the subpath and assert
        #      set-equality with the curated catalog.
        #   2. Interactive `skills add …/framework` (no --all) relies on
        #      metadata.internal hiding — assert repo-root discovery surfaces
        #      exactly the curated set and nothing internal.
        run: |
          cd "$RUNNER_TEMP"
          node "$GITHUB_WORKSPACE/packages/create-objectstack/bin/create-objectstack.js" skills-probe --skip-install --skip-skills
          cd skills-probe
          expected=$(cd "$GITHUB_WORKSPACE/skills" && ls -d objectstack-*/ | tr -d '/' | sort)
          expected_count=$(echo "$expected" | wc -l | tr -d ' ')

          npx -y skills add "$GITHUB_WORKSPACE/skills" --all --copy
          installed=$(find . -name SKILL.md -not -path '*/node_modules/*' -exec dirname {} \; | xargs -rn1 basename | sort -u)
          echo "installed skills:"; echo "$installed"
          if [ "$installed" != "$expected" ]; then
            echo "::error::scaffolded skill set != curated skills/ catalog (internal-skill leak or missing skill)"
            printf 'expected:\n%s\n' "$expected"
            exit 1
          fi

          # Strip ANSI color codes — the skills CLI colors the count even
          # when piped ("Found \e[32m9\e[39m skills"), which broke the greps.
          listing=$(npx -y skills add "$GITHUB_WORKSPACE" --list 2>&1 | sed 's/\x1b\[[0-9;]*m//g')
          echo "$listing"
          if echo "$listing" | grep -q 'dogfood-verification'; then
            echo "::error::repo-root discovery surfaced an internal skill (metadata.internal marker missing?)"
            exit 1
          fi
          if ! echo "$listing" | grep -q "Found $expected_count skill"; then
            echo "::error::repo-root discovery does not match the curated catalog (expected $expected_count skills) — a SKILL.md outside skills/ is missing metadata.internal"
            exit 1
          fi

      - name: Build official runtime image from this checkout
        # The scaffolded app's Dockerfile builds FROM
        # ghcr.io/objectstack-ai/objectstack. Build that base HERE from
        # docker/Dockerfile instead of pulling, so
        # (a) PRs exercise the official runtime Dockerfile itself, and
        # (b) the e2e stays hermetic — no dependency on a prior release
        # having published the tag (chicken-and-egg on the very first one).
        #
        # Pin the runtime's CLI to the SAME version the generated project
        # actually resolved to — NOT a hardcoded `latest`. The scaffolded
        # artifact is built by that project's `@objectstack/cli` and carries
        # its `engines.protocol` major; `os start` in the runtime image rejects
        # any artifact from a different protocol major. During an RC window the
        # project pins `^<repo rc>` (e.g. 16.0.0-rc.1, protocol 16) while the
        # `latest` dist-tag still points at the last STABLE cli (protocol 15),
        # so a hardcoded `latest` skews the two and boot fails. Reading the
        # resolved version keeps the image in lockstep whether the project got
        # the repo RC, the `latest` fallback (see the install step), or a
        # published stable.
        run: |
          CLI_VERSION=$(node -p "require('$RUNNER_TEMP/e2e-app/node_modules/@objectstack/cli/package.json').version")
          echo "Runtime image will bundle @objectstack/cli@$CLI_VERSION (matches the scaffolded artifact's protocol)"
          docker build -t ghcr.io/objectstack-ai/objectstack:latest \
            --build-arg OS_CLI_VERSION="$CLI_VERSION" \
            "$GITHUB_WORKSPACE/docker"

      - name: Docker build and run (scaffolded Dockerfile)
        # The blank template ships its own Dockerfile / docker-compose.yml /
        # .dockerignore, so build the generated project as-is — this exercises
        # the exact Docker path a real `npm create objectstack` user ships (the
        # standalone examples/docker copy was removed in 6c28bac).
        run: |
          cd "$RUNNER_TEMP/e2e-app"
          docker build --pull=false -t e2e-app .
          docker run -d --name e2e -p 18080:8080 \
            -e OS_SECRET_KEY="$(openssl rand -hex 32)" \
            -e OS_AUTH_SECRET="$(openssl rand -hex 32)" \
            e2e-app
          ok=""
          for i in $(seq 1 30); do
            if curl -fsS http://localhost:18080/api/v1/health > /dev/null 2>&1; then ok=1; break; fi
            sleep 2
          done
          if [ -z "$ok" ]; then
            echo "::error::container never became healthy"; docker logs e2e; exit 1
          fi
          docker rm -f e2e

  registry-canary:
    name: 'Registry canary: ${{ matrix.template }}'
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        template: [blank, todo, compliance, content, contracts, procurement]
    steps:
      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Scaffold with published create-objectstack@latest
        run: |
          cd "$RUNNER_TEMP"
          npx -y create-objectstack@latest canary-app -t ${{ matrix.template }} --skip-skills

      # Remote templates ship `build` (same gates) but not always `validate`.
      - name: Gate the generated project
        run: |
          cd "$RUNNER_TEMP/canary-app"
          if node -e "process.exit(JSON.parse(require('fs').readFileSync('package.json','utf8')).scripts?.validate ? 0 : 1)"; then
            npm run validate
          fi
          npm run build

      - name: Boot and probe health (blank only)
        if: matrix.template == 'blank'
        run: |
          cd "$RUNNER_TEMP/canary-app"
          npx os start --artifact ./dist/objectstack.json --port 8080 > server.log 2>&1 &
          SERVER_PID=$!
          ok=""
          for i in $(seq 1 30); do
            if curl -fsS http://localhost:8080/api/v1/health > /dev/null 2>&1; then ok=1; break; fi
            sleep 2
          done
          if [ -z "$ok" ]; then
            echo "::error::server never became healthy"; cat server.log; exit 1
          fi
          curl -fsS http://localhost:8080/api/v1/ready
          kill "$SERVER_PID"