Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/remote-docs-preview.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ jobs:
DOCS_REMOTE_REF: ${{ steps.pull-request.outputs.head_sha }}
DOCS_REMOTE_REPOSITORY: ${{ inputs.source_repository }}
DOCS_REMOTE_SOURCE_REPOSITORY: ${{ steps.pull-request.outputs.head_repository }}
DOCS_REMOTE_PULL_REQUEST_REF: refs/pull/${{ inputs.pull_request_number }}/head
PULL_REQUEST_NUMBER: ${{ inputs.pull_request_number }}
SITE_SHA: ${{ steps.site.outputs.sha }}
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
Expand Down Expand Up @@ -310,6 +311,7 @@ jobs:
--arg remote_name "$DOCS_REMOTE_NAME" \
--arg remote_repository "$DOCS_REMOTE_REPOSITORY" \
--arg source_repository "$DOCS_REMOTE_SOURCE_REPOSITORY" \
--arg pull_request_ref "$DOCS_REMOTE_PULL_REQUEST_REF" \
--arg site_sha "$SITE_SHA" \
'{
name: $project_name,
Expand All @@ -330,6 +332,7 @@ jobs:
DOCS_REMOTE_NAME: $remote_name,
DOCS_REMOTE_REPOSITORY: $remote_repository,
DOCS_REMOTE_SOURCE_REPOSITORY: $source_repository,
DOCS_REMOTE_PULL_REQUEST_REF: $pull_request_ref,
DOCS_REMOTE_REF: $approved_sha
}},
meta: {
Expand Down
65 changes: 47 additions & 18 deletions bin/fetch-remotes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
// 3. a shallow, blob-filtered Git SSH checkout for local private remotes.
// Production always reads each repository's `main` branch. A source-repository
// preview replaces exactly one source with an immutable commit SHA and omits
// the other remote sources. Base-repository previews omit every remote. Only
// the other remote sources. Fork previews fetch the PR ref from the registered
// base repository, then verify it resolves to that immutable SHA.
// Base-repository previews omit every remote. Only
// production and the `connect-preview` custom environment may use Vercel
// Connect. Remote content is copied but never parsed or imported here.
// Usage: node bin/fetch-remotes.ts
Expand Down Expand Up @@ -91,13 +93,19 @@ function gitEnvironment(token?: string): NodeJS.ProcessEnv {
};
}

interface CheckedOutSource {
source: string;
commit: string;
}

function checkoutSparseGit(
repositoryUrl: string,
ref: string,
sourcePath: string,
destination: string,
token?: string,
): string {
expectedCommit?: string,
): CheckedOutSource {
const environment = gitEnvironment(token);
const checkout = path.join(destination, "checkout");
const git = (args: string[]) => execFileSync("git", args, { env: environment, stdio: "pipe" });
Expand All @@ -108,12 +116,16 @@ function checkoutSparseGit(
git(["-C", checkout, "sparse-checkout", "init", "--cone"]);
git(["-C", checkout, "sparse-checkout", "set", "--cone", sourcePath]);
}
// Fetching the requested ref directly supports immutable pull-request SHAs
// as well as the normal main branch. With sparse checkout enabled, Git only
// downloads blobs required by sourcePath during checkout.
// Fetching the requested ref directly supports base-repository PR refs,
// immutable commit SHAs, and the normal main branch. With sparse checkout
// enabled, Git only downloads blobs required by sourcePath during checkout.
git(["-C", checkout, "fetch", "--depth", "1", "--filter=blob:none", "origin", ref]);
git(["-C", checkout, "checkout", "--detach", "FETCH_HEAD"]);
return path.join(checkout, sourcePath);
const commit = git(["-C", checkout, "rev-parse", "HEAD"]).toString().trim().toLowerCase();
if (expectedCommit && commit !== expectedCommit) {
throw new Error(`fetch-remotes: ${repositoryUrl}@${ref} resolved to ${commit}, expected ${expectedCommit}`);
}
return { source: path.join(checkout, sourcePath), commit };
}

async function downloadGitHubArchive(
Expand Down Expand Up @@ -221,6 +233,9 @@ for (const r of manifest.remotes) {
);
}
const ref = selectedPreview?.ref ?? "main";
const fetchRef = selectedPreview?.pullRequestRef ?? ref;
const fetchRepository = selectedPreview?.repository ?? r.repo;
const expectedCommit = selectedPreview?.ref;
const sourceRepository = selectedPreview?.sourceRepository ?? r.repo;
if (usePrefetchedRemotes) {
const stateFile = path.join(stateDir, `${r.name}.json`);
Expand Down Expand Up @@ -253,47 +268,61 @@ for (const r of manifest.remotes) {
let commit = "unknown";

try {
const credentials = await githubAuthentication(r, sourceRepository);
const credentials = await githubAuthentication(r, fetchRepository);
if (!r.private || credentials.token) {
const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`));
temporaryDirectory = tmp;
try {
source = checkoutSparseGit(
`https://github.com/${sourceRepository}.git`,
ref,
const checkedOut = checkoutSparseGit(
`https://github.com/${fetchRepository}.git`,
fetchRef,
r.path,
tmp,
credentials.token,
expectedCommit,
);
source = checkedOut.source;
sourceKind = "github-sparse";
authentication = credentials.authentication;
commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
} catch {
commit = checkedOut.commit;
} catch (error) {
if (selectedPreview?.pullRequestRef) throw error;
// GitHub archives remain a portable fallback for hosts without partial
// clone support. Do not surface child-process output because it may
// include transport details from an authenticated request.
fs.rmSync(path.join(tmp, "checkout"), { recursive: true, force: true });
const tar = path.join(tmp, "src.tgz");
authentication = await downloadGitHubArchive(r, sourceRepository, ref, tar, credentials);
authentication = await downloadGitHubArchive(r, fetchRepository, fetchRef, tar, credentials);
execFileSync("tar", ["-xzf", tar, "-C", tmp]);
const extracted = fs.readdirSync(tmp).find((directory) => directory !== "src.tgz");
if (!extracted) throw new Error(`fetch-remotes: archive for ${r.name} contained no root directory`);
source = path.join(tmp, extracted, r.path);
sourceKind = "github-api";
commit = extracted.split("-").pop() ?? "unknown";
commit = (extracted.split("-").pop() ?? "unknown").toLowerCase();
if (expectedCommit && commit !== expectedCommit) {
throw new Error(`fetch-remotes: archive for ${r.name} resolved to ${commit}, expected ${expectedCommit}`);
}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Archive SHA check always fails

Low Severity

The new archive fallback treats the tarball root directory suffix as the fetched commit and compares it to expectedCommit, a 40-character SHA. GitHub's tarball API names that directory with a short SHA, so the check rejects a valid archive whenever a SHA-only preview falls back after sparse clone fails.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 59b2682. Configure here.

}
} else if (!process.env.CI) {
const tmp = fs.mkdtempSync(path.join(stateDir, `${r.name}-`));
temporaryDirectory = tmp;
source = checkoutSparseGit(`git@github.com:${sourceRepository}.git`, ref, r.path, tmp);
const checkedOut = checkoutSparseGit(
`git@github.com:${fetchRepository}.git`,
fetchRef,
r.path,
tmp,
undefined,
expectedCommit,
);
source = checkedOut.source;
sourceKind = "github-ssh-sparse";
authentication = "ssh";
commit = execFileSync("git", ["-C", path.join(tmp, "checkout"), "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
commit = checkedOut.commit;
}

if (!source) {
throw new Error(
`fetch-remotes: private remote ${r.name} (${sourceRepository}@${ref}) requires Vercel Connect, GH_TOKEN/GITHUB_TOKEN, or local SSH access`,
`fetch-remotes: private remote ${r.name} (${fetchRepository}@${fetchRef}) requires Vercel Connect, GH_TOKEN/GITHUB_TOKEN, or local SSH access`,
);
}
if (!fs.existsSync(source)) throw new Error(`fetch-remotes: source path does not exist for ${r.name}: ${source}`);
Expand All @@ -304,7 +333,7 @@ for (const r of manifest.remotes) {
path.join(stateDir, `${r.name}.json`),
JSON.stringify({ name: r.name, repo: r.repo, sourceRepository, ref, source: sourceKind, authentication, commit, fetchedAt: new Date().toISOString(), files }, null, 2),
);
console.log(`fetch-remotes: ${r.name} <- ${sourceKind}/${authentication} ${sourceRepository}@${ref} (${commit.slice(0, 12)}): ${files} files into ${r.mount}`);
console.log(`fetch-remotes: ${r.name} <- ${sourceKind}/${authentication} ${fetchRepository}@${fetchRef} (${commit.slice(0, 12)}): ${files} files into ${r.mount}`);
} finally {
if (temporaryDirectory) fs.rmSync(temporaryDirectory, { recursive: true, force: true });
}
Expand Down
5 changes: 5 additions & 0 deletions bin/write-remote-preview-scope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const root = process.cwd();
const name = (process.env.DOCS_REMOTE_NAME ?? "").trim();
const repo = (process.env.DOCS_REMOTE_REPOSITORY ?? "").trim();
const sourceRepo = (process.env.DOCS_REMOTE_SOURCE_REPOSITORY ?? "").trim() || repo;
const pullRequestRef = (process.env.DOCS_REMOTE_PULL_REQUEST_REF ?? "").trim();
const ref = (process.env.DOCS_REMOTE_REF ?? "").trim().toLowerCase();
if (!name || !repo || !ref) {
throw new Error("write-remote-preview-scope: DOCS_REMOTE_NAME, DOCS_REMOTE_REPOSITORY and DOCS_REMOTE_REF are required");
Expand All @@ -18,6 +19,9 @@ if (!/^[0-9a-f]{40}$/.test(ref)) {
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(sourceRepo)) {
throw new Error("write-remote-preview-scope: DOCS_REMOTE_SOURCE_REPOSITORY must use the owner/name form");
}
if (pullRequestRef && !/^refs\/pull\/[1-9][0-9]*\/head$/.test(pullRequestRef)) {
throw new Error("write-remote-preview-scope: DOCS_REMOTE_PULL_REQUEST_REF must use the refs/pull/<number>/head form");
}

const manifest = JSON.parse(fs.readFileSync(path.join(root, "remotes.json"), "utf8")) as { remotes: Remote[] };
const remote = manifest.remotes.find((candidate) => candidate.name === name);
Expand All @@ -33,6 +37,7 @@ const scope = {
name: remote.name,
repository: remote.repo,
sourceRepository: sourceRepo,
pullRequestRef: pullRequestRef || undefined,
ref,
},
};
Expand Down
17 changes: 10 additions & 7 deletions src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ Mintlify-flavoured MDX build (see `src/plugins/vite-mintlify-snippets.ts` and
| `DOCS_LOCALES` | Translations included in a `translations` or legacy `combined` artifact: `none`, `all`, or a comma-separated list such as `es,fr`. The translations project uses `all`. |
| `DOCS_REMOTES` | Registered remote-source scope: `none` for an ordinary base-repository preview and `all` for source previews, production, and a base preview carrying `docs-preview-show-remotes`. |
| `DOCS_REMOTE_NAME`, `DOCS_REMOTE_REPOSITORY`, `DOCS_REMOTE_REF` | CI-only tuple selecting one registered remote at an immutable commit for an English pull-request preview. |
| `DOCS_REMOTE_SOURCE_REPOSITORY` | CI-derived repository that owns the preview SHA. It defaults to the registered repository and differs only for a fork PR. |
| `DOCS_REMOTE_SOURCE_REPOSITORY` | CI-derived PR-head repository, retained only for preview attribution and edit links. It defaults to the registered repository. |
| `DOCS_REMOTE_PULL_REQUEST_REF` | CI-only base-repository PR ref (`refs/pull/<number>/head`) to fetch and verify against `DOCS_REMOTE_REF`. |
| `DOCS_REMOTES_PREFETCHED=1` | Requires the remote mounts and fetch-state files supplied by the credentialed CI fetch job. |
| `DOCS_PREVIEW_ALIAS` | Lowercase Cloudflare alias used by `pnpm run deploy:preview`. |
| `DOCS_GITHUB_CONNECTOR` | Vercel Connect GitHub connector UID, for example `github/clickhouse-docs`. Configure it only for `production` and the `connect-preview` Custom Environment. |
Expand Down Expand Up @@ -59,8 +60,10 @@ GitHub App token only to resolve the immutable head SHA and the branch or fork
repository that owns it, then asks Vercel to build trusted Nimbus `main` in the
`connect-preview` environment. The Vercel build uses
its OIDC identity to request a short-lived, `contents:read` token from Vercel
Connect for the branch or fork repository. Public repositories are fetched
anonymously. `bin/fetch-remotes.ts` exits before
Connect for the registered source repository. For a fork, it fetches the
base repository's `refs/pull/<number>/head` ref and verifies it resolves to the
approved SHA; the fork repository is metadata only. Public repositories are
fetched anonymously. `bin/fetch-remotes.ts` exits before
`bin/vercel-build.ts` removes `VERCEL_OIDC_TOKEN` and starts any Markdown or MDX
processing. The Actions token never enters Vercel. Maintainers can also run the
workflow directly from the `mintlify-docs-dev` Actions page by providing the
Expand All @@ -87,10 +90,10 @@ registered remote source.
Source-repository pull requests use
`.github/workflows/remote-docs-preview.yml`. Vercel builds trusted Nimbus
`main`, selects exactly one registered source with `DOCS_REMOTE_*`, and fetches
the pull request's immutable head revision. Trusted branches and forks have the
same source-only build scope. They use `connect-preview` so private registered
sources can obtain a short-lived token; public sources remain anonymously
fetchable.
the pull request's immutable head revision from the registered repository's PR
ref. Trusted branches and forks have the same source-only build scope. They use
`connect-preview` so private registered sources can obtain a short-lived token;
public sources remain anonymously fetchable.

After a source-repository change reaches its trusted production branch, that
repository calls `.github/workflows/site-production.yml`. The reusable workflow
Expand Down
21 changes: 19 additions & 2 deletions src/lib/scope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
* "name": "clickhouse-private",
* "repository": "ClickHouse/airgap-docs",
* "sourceRepository": "contributor/airgap-docs",
* "ref": "<40-character commit SHA>"
* "ref": "<40-character approved commit SHA>",
* "pullRequestRef": "refs/pull/123/head"
* }
* }
*/
Expand All @@ -32,6 +33,9 @@ export interface RemotePreview {
repository: string;
/** Repository that owns the immutable preview SHA; differs for fork PRs. */
sourceRepository: string;
/** Base-repository PR ref to fetch before checking it matches ref. */
pullRequestRef?: string;
/** Immutable commit SHA approved by the triggering GitHub event. */
ref: string;
}

Expand Down Expand Up @@ -111,6 +115,9 @@ function parseRemotePreview(value: unknown, source: string): RemotePreview | und
const sourceRepository = typeof candidate.sourceRepository === "string"
? candidate.sourceRepository.trim()
: repository;
const pullRequestRef = typeof candidate.pullRequestRef === "string"
? candidate.pullRequestRef.trim()
: "";
const ref = typeof candidate.ref === "string" ? candidate.ref.trim().toLowerCase() : "";
if (!name || !repository || !sourceRepository || !ref) {
throw new Error(`${source} must contain non-empty name, repository and ref values`);
Expand All @@ -122,7 +129,10 @@ function parseRemotePreview(value: unknown, source: string): RemotePreview | und
if (!/^[0-9a-f]{40}$/.test(ref)) {
throw new Error(`${source}.ref must be an immutable 40-character commit SHA`);
}
return { name, repository, sourceRepository, ref };
if (pullRequestRef && !/^refs\/pull\/[1-9][0-9]*\/head$/.test(pullRequestRef)) {
throw new Error(`${source}.pullRequestRef must use the refs/pull/<number>/head form`);
}
return { name, repository, sourceRepository, pullRequestRef: pullRequestRef || undefined, ref };
}

function parseReference(value: unknown, source: string): boolean {
Expand Down Expand Up @@ -184,6 +194,7 @@ export function readScope(root = process.cwd()): BuildScope {
const envRemoteName = (process.env.DOCS_REMOTE_NAME ?? "").trim();
const envRemoteRepository = (process.env.DOCS_REMOTE_REPOSITORY ?? "").trim();
const envRemoteSourceRepository = (process.env.DOCS_REMOTE_SOURCE_REPOSITORY ?? "").trim();
const envRemotePullRequestRef = (process.env.DOCS_REMOTE_PULL_REQUEST_REF ?? "").trim();
const envRemoteRef = (process.env.DOCS_REMOTE_REF ?? "").trim();
const file = path.join(root, ".preview-scope.json");
const fileScope = fs.existsSync(file)
Expand All @@ -206,6 +217,11 @@ export function readScope(root = process.cwd()): BuildScope {
"DOCS_REMOTE_SOURCE_REPOSITORY requires DOCS_REMOTE_NAME, DOCS_REMOTE_REPOSITORY and DOCS_REMOTE_REF",
);
}
if (envRemotePullRequestRef && !hasRemoteEnvironment) {
throw new Error(
"DOCS_REMOTE_PULL_REQUEST_REF requires DOCS_REMOTE_NAME, DOCS_REMOTE_REPOSITORY and DOCS_REMOTE_REF",
);
}

let source: BuildScope["source"] = fileScope ? "file" : "default";
const vercelTarget = (process.env.VERCEL_TARGET_ENV ?? process.env.VERCEL_ENV ?? "")
Expand Down Expand Up @@ -276,6 +292,7 @@ export function readScope(root = process.cwd()): BuildScope {
name: envRemoteName,
repository: envRemoteRepository,
sourceRepository: envRemoteSourceRepository || envRemoteRepository,
pullRequestRef: envRemotePullRequestRef || undefined,
ref: envRemoteRef,
},
"remote preview environment",
Expand Down
Loading