diff --git a/AGENTS.md b/AGENTS.md index 78dfea438c2..4bf6032d8c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,34 +16,35 @@ This is the source for [developers.cloudflare.com](https://developers.cloudflare cloudflare-docs/ ├── src/ │ ├── content/ -│ │ ├── docs/ # 5,400+ MDX pages — the user-facing documentation -│ │ ├── partials/ # 1,200+ reusable MDX snippets (by product) -│ │ ├── changelog/ # Product changelogs (by product subdirectory) -│ │ ├── glossary/ # Glossary term definitions (YAML) -│ │ ├── products/ # Product metadata (YAML, 135 files) -│ │ └── ... # Other data collections (plans, fields, models, etc.) -│ ├── components/ # Astro + React components (barrel: components.ts) -│ ├── components.ts # MDX component barrel — all MDX imports come from here -│ ├── layouts/ # Page layout components -│ ├── pages/ # Dynamic route pages (changelog, llms.txt, RSS, etc.) -│ ├── schemas/ # Zod schemas for all content collections -│ ├── plugins/ # Satteri hast pipeline plugins -│ ├── scripts/ # Client-side scripts (analytics, mermaid, webmcp, etc.) -│ ├── styles/ # CSS (Tailwind 4) -│ ├── icons/ # Product SVG icons (~110) -│ ├── assets/ # Processed images (optimized by Astro) -│ ├── util/ # App utility functions -│ └── content.config.ts # Content collection definitions -├── public/ # Static files served as-is (images, redirects, robots.txt) -├── worker/ # Cloudflare Worker for serving the site -├── bin/ # Build scripts and CI helpers -│ └── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/ -├── skills/ # Agent Skills served at /.well-known/skills/ — GENERATED, do not edit -│ # Fetched from https://middlecache.ced.cloudflare.com/v1/cloudflare-skills/skills.tar.gz -│ # by bin/fetch-skills.ts, which runs automatically via prebuild/predev hooks. -│ # skills/ is in .gitignore and is NOT committed to the repository. -├── .flue/ # Flue cloudflare-docs-bot — see .flue/AGENTS.md -├── astro.config.ts # Astro + Nimbus configuration +│ │ ├── docs/ # 5,400+ MDX pages — the user-facing documentation +│ │ ├── partials/ # 1,200+ reusable MDX snippets (by product) +│ │ ├── changelog/ # Product changelogs (by product subdirectory) +│ │ ├── glossary/ # Glossary term definitions (YAML) +│ │ ├── products/ # Product metadata (YAML, 135 files) +│ │ └── ... # Other data collections (plans, fields, models, etc.) +│ ├── components/ # Astro + React components (barrel: components.ts) +│ ├── components.ts # MDX component barrel — all MDX imports come from here +│ ├── layouts/ # Page layout components +│ ├── pages/ # Dynamic route pages (changelog, llms.txt, RSS, etc.) +│ ├── schemas/ # Zod schemas for all content collections +│ ├── plugins/ # Satteri hast pipeline plugins +│ ├── scripts/ # Client-side scripts (analytics, mermaid, webmcp, etc.) +│ ├── styles/ # CSS (Tailwind 4) +│ ├── icons/ # Product SVG icons (~110) +│ ├── assets/ # Processed images (optimized by Astro) +│ ├── util/ # App utility functions +│ └── content.config.ts # Content collection definitions +├── public/ # Static files served as-is (images, redirects, robots.txt) +├── worker/ # Cloudflare Worker for serving the site +├── bin/ # Build scripts and CI helpers +│ ├── fetch-skills.ts # Downloads skills.tar.gz from middlecache, extracts to skills/ +│ └── fetch-logpush-datasets.ts # Downloads generated Logpush dataset pages; fetch failures retain checked-in pages +├── skills/ # Agent Skills served at /.well-known/skills/ — GENERATED, do not edit +│ # Fetched from https://middlecache.ced.cloudflare.com/v1/cloudflare-skills/skills.tar.gz +│ # by bin/fetch-skills.ts, which runs automatically via prebuild/predev hooks. +│ # skills/ is in .gitignore and is NOT committed to the repository. +├── .flue/ # Flue cloudflare-docs-bot — see .flue/AGENTS.md +├── astro.config.ts # Astro + Nimbus configuration ├── package.json └── tsconfig.json ``` diff --git a/bin/fetch-logpush-datasets.ts b/bin/fetch-logpush-datasets.ts new file mode 100644 index 00000000000..717c5824fa7 --- /dev/null +++ b/bin/fetch-logpush-datasets.ts @@ -0,0 +1,139 @@ +#!/usr/bin/env tsx + +import fs from "fs"; +import { randomUUID } from "crypto"; +import { join } from "path"; + +import { + downloadToDotTempIfNotPresent, + extractTarGz, +} from "../src/util/custom-loaders"; + +const MIDDLECACHE_BASE_URL = + process.env.MIDDLECACHE_BASE_URL ?? "https://middlecache.ced.cloudflare.com/"; +const ARCHIVE_MIDDLECACHE_PATH = "v1/logpush-datasets/datasets.tar.gz"; +const ARCHIVE_DOT_TMP_PATH = `middlecache/${ARCHIVE_MIDDLECACHE_PATH}`; +const DATASETS_DIR = + process.env.LOGPUSH_DATASETS_DIR ?? + "src/content/docs/logs/logpush/logpush-job/datasets"; +const EXTRACTED_DIR = join(".tmp", "logpush-datasets"); + +// Builds soft-fail so middlecache availability cannot block unrelated docs changes. +const soft = process.argv.includes("--soft"); +const force = process.argv.includes("--force"); + +const fail = (message: string): never => { + if (soft) { + console.warn( + `Warning: ${message} - continuing with checked-in Logpush dataset pages`, + ); + process.exit(0); + } + console.error(`Error: ${message}`); + process.exit(1); +}; + +const archivePath = join(".tmp", ...ARCHIVE_DOT_TMP_PATH.split("/")); +if (!fs.existsSync(DATASETS_DIR) || !fs.statSync(DATASETS_DIR).isDirectory()) { + fail(`Logpush dataset directory does not exist: ${DATASETS_DIR}`); +} +if (force) { + fs.rmSync(archivePath, { force: true }); +} + +console.log("Fetching Logpush dataset pages from middlecache"); + +let mutationStarted = false; +try { + await downloadToDotTempIfNotPresent( + `${MIDDLECACHE_BASE_URL}${ARCHIVE_MIDDLECACHE_PATH}`, + ARCHIVE_DOT_TMP_PATH, + ); + + fs.rmSync(EXTRACTED_DIR, { recursive: true, force: true }); + await extractTarGz(archivePath, EXTRACTED_DIR); + + const preparedScopes: Array<{ + destination: string; + pageNames: Set; + stagedPages: Array<{ source: string; destination: string }>; + }> = []; + const stagedPaths: string[] = []; + try { + for (const scope of fs.readdirSync(EXTRACTED_DIR, { + withFileTypes: true, + })) { + if (!scope.isDirectory()) continue; + + const destination = join(DATASETS_DIR, scope.name); + if ( + !fs.existsSync(destination) || + !fs.statSync(destination).isDirectory() + ) { + console.log(`Skipping ${scope.name} scope: no destination directory`); + continue; + } + + const source = join(EXTRACTED_DIR, scope.name); + for (const page of fs.readdirSync(destination)) { + if (page.startsWith(".logpush-") && page.endsWith(".tmp")) { + fs.rmSync(join(destination, page), { force: true }); + } + } + const pages = fs + .readdirSync(source, { withFileTypes: true }) + .filter((page) => page.isFile() && page.name.endsWith(".md")); + if (pages.length === 0) { + throw new Error(`Logpush dataset scope is empty: ${scope.name}`); + } + + const stagedPages = pages.map((page) => { + const stagedPath = join(destination, `.logpush-${randomUUID()}.tmp`); + stagedPaths.push(stagedPath); + fs.copyFileSync(join(source, page.name), stagedPath); + return { + source: stagedPath, + destination: join(destination, page.name), + }; + }); + preparedScopes.push({ + destination, + pageNames: new Set(pages.map((page) => page.name)), + stagedPages, + }); + } + + if (preparedScopes.length === 0) { + throw new Error( + "No Logpush dataset scopes matched destination directories", + ); + } + + mutationStarted = true; + for (const scope of preparedScopes) { + for (const page of scope.stagedPages) { + fs.renameSync(page.source, page.destination); + } + } + for (const scope of preparedScopes) { + for (const page of fs.readdirSync(scope.destination)) { + if (page.endsWith(".md") && !scope.pageNames.has(page)) { + fs.rmSync(join(scope.destination, page)); + } + } + } + } finally { + for (const stagedPath of stagedPaths) { + fs.rmSync(stagedPath, { force: true }); + } + } + console.log("Logpush dataset pages ready"); +} catch (err) { + if (mutationStarted) { + console.error( + `Error: Logpush dataset replacement failed: ${(err as Error).message}`, + ); + process.exit(1); + } + fail(`Logpush dataset fetch failed: ${(err as Error).message}`); +} diff --git a/package.json b/package.json index 75a6d7102ce..fc0bc5a2cc8 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "check": "pnpm run check:astro && pnpm run check:worker", "check:astro": "astro check --minimumFailingSeverity=hint", "check:worker": "tsc --noEmit -p ./worker/tsconfig.json", - "predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft", + "predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft && tsx bin/fetch-logpush-datasets.ts --soft --force", "dev": "astro dev", "format": "pnpm run format:core:fix && pnpm run format:data:fix && pnpm run format:content:fix", "format:check": "pnpm run format:core:check && pnpm run format:data:check && pnpm run format:content:check", @@ -40,7 +40,7 @@ "lint": "eslint", "prepare": "husky", "prebuild:incremental": "pnpm run fetch:assets", - "fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts" + "fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts && tsx bin/fetch-logpush-datasets.ts --soft --force" }, "devDependencies": { "@actions/core": "3.0.1",