diff --git a/.github/workflows/deploy-web.yml b/.github/workflows/deploy-web.yml index 3f73b24..b3cbb46 100644 --- a/.github/workflows/deploy-web.yml +++ b/.github/workflows/deploy-web.yml @@ -14,7 +14,7 @@ on: branches: [main] paths: - 'web/**' - - 'src/sitemap.js' + - 'src/**' - '.github/workflows/deploy-web.yml' workflow_dispatch: {} diff --git a/CHANGELOG.md b/CHANGELOG.md index a67b0e0..11e378b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ All notable changes to `@geosuite/sitemap-builder` are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Same results as the GeoSuite platform's Sitemap Builder, checked by golden + cases both repos run (`test/fixtures/golden-sitemap.json`, `test/golden.test.js`). +- The crawler uses `fetch` instead of `node:http`, so the hosted page runs the + package's own `crawlSite()` instead of a separate copy. +- Redirects are followed and pages are listed at their final URL; a start URL + redirecting to another host (apex to `www`) moves the crawl there. +- Only 2xx HTML pages go in the sitemap: 404s, 500s and non-HTML responses no + longer do. +- Links are normalized (lowercase host, no default port, `&` decoded) and + followed in sorted order; pages are sorted by depth, then by URL in code-point + order (it was locale order). +- `` accepts only HTTP dates and is the UTC date. +- `crawlSite()` accepts a bare domain and returns the `origin` it crawled; + `--respect-robots` follows `Allow` rules and groups of several `User-agent` + lines (longest match wins, `Allow` on a tie). +- Hosted page: depth 3 (was 2), the platform's timeouts and budget, and + `` in the output; still at most 40 pages. + ## [0.3.2] - 2026-05-23 ### Changed diff --git a/README.md b/README.md index 571aab8..94f0621 100644 --- a/README.md +++ b/README.md @@ -80,16 +80,30 @@ geosuite-sitemap-builder https://example.com --json ## What it actually does -1. Starts at the URL you pass. -2. BFS-crawls **same-origin** `` links only (never wanders off the host). -3. Drops obvious non-HTML extensions (`.png`, `.css`, `.pdf`, …) so the +1. Starts at the URL you pass (a bare `example.com` is crawled over https). +2. BFS-crawls **same-origin** `` links only (never wanders off the host), + in sorted order. Links are normalized the way a browser does: lowercase + host, no default port, no `#fragment`, `.`/`..` resolved, `&` decoded, + path and query percent-encoded. +3. Follows redirects and lists each page at its **final URL**. If the start URL + redirects to another host (`example.com` to `www.example.com`), the crawl + continues there; any other page that redirects off the site is dropped. +4. Lists only pages that answer **2xx with HTML**: a 404, a 500 or a PDF + served without an extension stays out. +5. Drops obvious non-HTML extensions (`.png`, `.css`, `.pdf`, …) so the sitemap doesn't get polluted with assets. -4. Skips fragment-only links (`#section`), `mailto:`, `tel:`, and `javascript:`. -5. Stops at any of three caps (whichever fires first): +6. Skips fragment-only links (`#section`), `mailto:`, `tel:`, and `javascript:`. +7. Stops at any of three caps (whichever fires first): - page count (`--max-pages`) - BFS depth (`--max-depth`) - wall-clock budget (`--budget-s`) -6. Renders the discovered URLs as a sitemaps.org-compliant ``. +8. Renders the discovered URLs as a sitemaps.org-compliant ``, by + depth and then by URL, with `` from the `Last-Modified` header + when the site sends one. + +The same rules run in the [GeoSuite](https://trygeosuite.it) platform's +Sitemap Builder: both run the cases in `test/fixtures/golden-sitemap.json`, so +for the same site and the same caps the two give the same sitemap. The output is intentionally minimal: `` plus an optional ``. We skip `` and `` — the spec deprecates them and the major @@ -102,12 +116,12 @@ search engines have ignored them for years. - **JavaScript rendering**. The crawler is HTTP + regex. Single-page apps whose links only appear after client-side hydration won't be discovered. Build-time pre-rendering or an SSR layer is the right fix. -- **Robots.txt awareness**. By default the tool runs against the site - owner's own domain and honoring `robots.txt` would silently strip the - pages they want to publish. (Adding an opt-in `--respect-robots` flag is - on the roadmap.) -- **`` accuracy**. Today we don't fill `` from `Last-Modified` - response headers. Coming in 0.2. +- **Robots.txt by default**. The tool runs against the site owner's own + domain, and honoring `robots.txt` would silently strip the pages they want + to publish. Pass `--respect-robots` to skip the paths disallowed for + `User-agent: *` (longest match wins, `Allow` on a tie). +- **`` beyond the header**. `` comes only from the + `Last-Modified` response header; pages that don't send it get none. - **LLM-powered grouping or summaries**. The deterministic 0.1 ships without a network dependency on any model. An opt-in `--ai` mode is on the roadmap (provide `OPENAI_API_KEY` or `ANTHROPIC_API_KEY` to enable). diff --git a/package.json b/package.json index bd677d8..68615c8 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "CHANGELOG.md" ], "scripts": { - "lint": "node --check src/index.js && node --check src/crawler.js && node --check src/sitemap.js && node --check src/ai.js && node --check bin/cli.js && node --check test/test-sitemap.js && node --check test/test-crawler.js && node --check test/test-ai.js", + "lint": "node --check src/index.js && node --check src/crawler.js && node --check src/robots.js && node --check src/sitemap.js && node --check src/ai.js && node --check bin/cli.js && node --check test/test-sitemap.js && node --check test/test-crawler.js && node --check test/test-ai.js && node --check test/golden.test.js && node --check test/golden-run.js && node --check scripts/update-golden.mjs", "test": "node --test test/*.js", "coverage": "node --experimental-test-coverage --test test/*.js" }, diff --git a/scripts/update-golden.mjs b/scripts/update-golden.mjs new file mode 100644 index 0000000..324aa96 --- /dev/null +++ b/scripts/update-golden.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node +// Recompute the `expected` blocks of test/fixtures/golden-sitemap.json from +// the current code. Run it only after an intentional behavior change, read the +// diff, then copy the file to the platform repo +// (geo: backend/tests/fixtures/golden-sitemap.json) and make its Python +// service give the same answers. +// +// node scripts/update-golden.mjs + +import { readFile, writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { SECTIONS, computeGolden } from '../test/golden-run.js'; + +const file = join(dirname(fileURLToPath(import.meta.url)), '..', 'test', 'fixtures', 'golden-sitemap.json'); +const golden = JSON.parse(await readFile(file, 'utf8')); +for (const section of SECTIONS) { + for (const c of golden[section]) c.expected = await computeGolden(section, c); +} +await writeFile(file, JSON.stringify(golden, null, 2) + '\n'); +process.stdout.write(`updated ${file}\n`); diff --git a/src/crawler.js b/src/crawler.js index d5fc80f..4ef69e7 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -6,16 +6,27 @@ // - same-origin only (never wanders off the host) // - User-Agent identifies the tool // - small concurrency cap (caller decides) -// - per-page timeout via AbortController +// - per-page timeout via AbortSignal // - hard time budget (caller-enforced via Date-based deadline) // - skips obvious non-html / asset extensions // // `respectRobots` is opt-in: by default the tool runs against the site // owner's own domain so honoring robots.txt would silently strip the very // pages they want to publish. Public-facing crawls should flip it on. +// +// It uses the platform `fetch`, so the same code runs in the CLI (Node 20+) +// and in the hosted Worker (web/worker.js). The GeoSuite platform crawls with +// the same rules in Python (backend/services/tools/sitemap_builder_service.py); +// both run the cases in test/fixtures/golden-sitemap.json: +// - redirects are followed and the page is listed at its final URL; when the +// start URL redirects to another host (example.com -> www.example.com) the +// crawl continues on that host; +// - only 2xx HTML pages go in the sitemap (a 404 or a PDF does not); +// - links are read from only, `&` decoded, normalized (lowercase +// host, no default port, no fragment, percent-encoded) and followed in +// sorted order; +// - pages are listed by depth, then by URL (code-point order). -import http from 'node:http'; -import https from 'node:https'; import { loadRobotsChecker } from './robots.js'; const DEFAULT_USER_AGENT = @@ -25,6 +36,8 @@ const DEFAULT_TIMEOUT_MS = 8_000; const DEFAULT_MAX_PAGES = 200; const DEFAULT_MAX_DEPTH = 3; const DEFAULT_CONCURRENCY = 6; +// Only the first 512 KB of a page are read for links and title. +const MAX_BODY_CHARS = 512 * 1024; const ASSET_EXT = new Set([ '.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.ico', '.bmp', @@ -39,7 +52,7 @@ const HREF_RE = /]*\bhref\s*=\s*["']([^"']+)["']/gi; /** * @typedef {Object} CrawledPage - * @property {string} url Final URL (after dropping hash). + * @property {string} url Final URL (after redirects, without the hash). * @property {number} depth BFS depth (0 = start URL). * @property {number} status HTTP status code that returned this page. * @property {string|null} title HTML if discovered. @@ -57,36 +70,40 @@ const HREF_RE = /<a\b[^>]*\bhref\s*=\s*["']([^"']+)["']/gi; * @property {string} [userAgent] * @property {boolean} [respectRobots] When true, fetch /robots.txt and skip * paths disallowed for User-Agent *. + * @property {typeof fetch} [fetch] Fetch implementation (tests pass a fake). */ /** * Crawl an origin starting at `startUrl` and return discovered pages. + * A bare domain (`example.com`) is crawled over https. * * @param {string} startUrl * @param {CrawlOptions} [opts] - * @returns {Promise<{ pages: CrawledPage[], hitCap: boolean, hitDeadline: boolean }>} + * @returns {Promise<{ origin: string, pages: CrawledPage[], hitCap: boolean, hitDeadline: boolean }>} */ export async function crawlSite(startUrl, opts = {}) { - const start = normalizeUrl(startUrl); + const start = normalizeStartUrl(startUrl); if (!start) throw new Error(`Invalid start URL: ${startUrl}`); - const origin = `${start.protocol}//${start.host}`; + let origin = start.origin; const maxPages = clamp(opts.maxPages ?? DEFAULT_MAX_PAGES, 1, 1000); const maxDepth = clamp(opts.maxDepth ?? DEFAULT_MAX_DEPTH, 0, 6); const concurrency = clamp(opts.concurrency ?? DEFAULT_CONCURRENCY, 1, 16); const timeoutMs = opts.perPageTimeoutMs ?? DEFAULT_TIMEOUT_MS; const deadline = opts.deadlineMs ?? Number.POSITIVE_INFINITY; const userAgent = opts.userAgent ?? DEFAULT_USER_AGENT; + const doFetch = opts.fetch ?? fetch; const isAllowed = opts.respectRobots - ? await loadRobotsChecker(origin, { timeoutMs, userAgent }) + ? await loadRobotsChecker(origin, { timeoutMs, userAgent, fetch: doFetch }) : () => true; - /** @type {Map<string, CrawledPage>} */ - const visited = new Map(); + /** @type {Map<string, CrawledPage>} final URL -> page */ + const pages = new Map(); + /** URLs already fetched or queued for fetching (requested and final). */ + const seen = new Set(); /** @type {{ url: string, depth: number }[]} */ - const startPath = start.pathname || '/'; - const queue = isAllowed(startPath) ? [{ url: start.href, depth: 0 }] : []; + const queue = isAllowed(new URL(start.href).pathname) ? [{ url: start.href, depth: 0 }] : []; let hitCap = false; let hitDeadline = false; @@ -98,7 +115,7 @@ export async function crawlSite(startUrl, opts = {}) { for (let i = 0; i < concurrency; i++) { workers.push((async () => { while (true) { - if (visited.size >= maxPages) { hitCap = true; return; } + if (pages.size >= maxPages) { hitCap = true; return; } if (Date.now() >= deadline) { hitDeadline = true; return; } const job = queue.shift(); if (!job) { @@ -106,29 +123,42 @@ export async function crawlSite(startUrl, opts = {}) { await sleep(20); continue; } - if (visited.has(job.url)) continue; - if (visited.size >= maxPages) { hitCap = true; return; } + if (seen.has(job.url)) continue; + seen.add(job.url); inFlight++; try { - const result = await fetchHtml(job.url, { timeoutMs, userAgent }); + const result = await fetchHtml(job.url, { timeoutMs, userAgent, doFetch }); if (!result) continue; - visited.set(job.url, { - url: job.url, + const finalUrl = normalizeUrl(result.url)?.href ?? job.url; + const finalOrigin = new URL(finalUrl).origin; + if (finalOrigin !== origin) { + // The start page may move the crawl to its canonical host + // (example.com -> www.example.com); any other page that leaves + // the site is dropped. + if (job.depth !== 0 || pages.size) continue; + origin = finalOrigin; + } + if (finalUrl !== job.url) { + if (seen.has(finalUrl)) continue; + seen.add(finalUrl); + } + if (!result.isHtml || pages.size >= maxPages) continue; + pages.set(finalUrl, { + url: finalUrl, depth: job.depth, status: result.status, - title: result.title, - lastModified: result.lastModified ?? null, + title: extractTitle(result.body), + lastModified: result.lastModified, }); if (job.depth >= maxDepth) continue; - for (const link of extractInternalLinks(result.body, job.url, origin)) { - if (visited.has(link)) continue; - const linkPath = new URL(link).pathname; - if (!isAllowed(linkPath)) continue; + for (const link of extractInternalLinks(result.body, finalUrl, origin)) { + if (seen.has(link)) continue; + if (!isAllowed(new URL(link).pathname)) continue; queue.push({ url: link, depth: job.depth + 1 }); } } catch { - // Per-page errors are swallowed — the crawler is best-effort. + // Per-page errors are swallowed: the crawler is best-effort. } finally { inFlight--; } @@ -138,14 +168,14 @@ export async function crawlSite(startUrl, opts = {}) { await Promise.all(workers); - // Stable sort: depth asc, then path lexicographic. Mirrors the typical + // Stable sort: depth asc, then URL by code point. Mirrors the typical // "homepage first, then sections" ordering search engines prefer. - const pages = [...visited.values()].sort((a, b) => { + const sorted = [...pages.values()].sort((a, b) => { if (a.depth !== b.depth) return a.depth - b.depth; - return a.url.localeCompare(b.url); + return a.url < b.url ? -1 : a.url > b.url ? 1 : 0; }); - return { pages, hitCap, hitDeadline }; + return { origin, pages: sorted, hitCap, hitDeadline }; } // --- helpers --------------------------------------------------------------- @@ -159,8 +189,8 @@ function sleep(ms) { } /** - * Strip the fragment, lowercase the host, drop trailing single slashes that - * would otherwise dedupe weirdly. Keeps everything else intact. + * Strip the fragment, lowercase the host, drop a default port. The WHATWG URL + * parser also resolves dot segments and percent-encodes the path and query. * * @returns {URL | null} */ @@ -180,6 +210,22 @@ export function normalizeUrl(input) { } } +/** + * The start URL as typed by a person: trimmed, `https://` added to a bare + * domain, and a host that is `localhost` or has a dot. Same rule as the + * platform's `_normalize_origin`. + * + * @returns {URL | null} + */ +export function normalizeStartUrl(input) { + const cleaned = String(input ?? '').trim(); + if (!cleaned || /\s/.test(cleaned)) return null; + const u = normalizeUrl(cleaned.includes('://') ? cleaned : `https://${cleaned}`); + if (!u) return null; + if (!u.host.includes('.') && u.hostname !== 'localhost') return null; + return u; +} + function isAssetPath(pathname) { const lastDot = pathname.lastIndexOf('.'); if (lastDot < 0) return false; @@ -190,75 +236,50 @@ function isAssetPath(pathname) { function extractInternalLinks(html, currentUrl, origin) { const found = new Set(); for (const match of html.matchAll(HREF_RE)) { - const raw = match[1]; + // An href inside HTML escapes its ampersands: /p?a=1&b=2 is /p?a=1&b=2. + const raw = match[1].trim().replace(/&/g, '&'); if (!raw || raw.startsWith('#')) continue; if (raw.startsWith('mailto:') || raw.startsWith('tel:') || raw.startsWith('javascript:')) continue; const resolved = (() => { - try { return new URL(raw, currentUrl); } catch { return null; } + try { return normalizeUrl(new URL(raw, currentUrl)); } catch { return null; } })(); if (!resolved) continue; - if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') continue; - if (`${resolved.protocol}//${resolved.host}` !== origin) continue; + if (resolved.origin !== origin) continue; if (isAssetPath(resolved.pathname)) continue; - resolved.hash = ''; found.add(resolved.href); } - return [...found]; + return [...found].sort(); } /** - * @returns {Promise<{ status: number, body: string, title: string | null } | null>} + * GET a page, following redirects. Returns null on a transport error. + * + * @returns {Promise<{ url: string, status: number, isHtml: boolean, body: string, lastModified: string | null } | null>} */ -function fetchHtml(url, { timeoutMs, userAgent }) { - const parsed = (() => { try { return new URL(url); } catch { return null; } })(); - if (!parsed) return Promise.resolve(null); - const lib = parsed.protocol === 'https:' ? https : http; - return new Promise((resolve) => { - const req = lib.request( - { - method: 'GET', - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80), - path: `${parsed.pathname || '/'}${parsed.search || ''}`, - headers: { - 'User-Agent': userAgent, - Accept: 'text/html, */*', - 'Accept-Encoding': 'identity', - }, - }, - (res) => { - const status = res.statusCode ?? 0; - const ctype = String(res.headers['content-type'] || '').toLowerCase(); - const lastModified = parseLastModified(res.headers['last-modified']); - if (status < 200 || status >= 300 || !ctype.includes('html')) { - res.resume(); - resolve({ status, body: '', title: null, lastModified }); - return; - } - let buffered = ''; - res.setEncoding('utf8'); - res.on('data', (chunk) => { - buffered += chunk; - // Cap the buffer at ~512 KB — sitemap building doesn't need more. - if (buffered.length > 512 * 1024) { - res.destroy(); - } - }); - res.on('end', () => { - resolve({ - status, - body: buffered, - title: extractTitle(buffered), - lastModified, - }); - }); - }, - ); - req.setTimeout(timeoutMs, () => req.destroy()); - req.on('error', () => resolve(null)); - req.end(); - }); +async function fetchHtml(url, { timeoutMs, userAgent, doFetch }) { + let res; + try { + res = await doFetch(url, { + headers: { 'user-agent': userAgent, accept: 'text/html, */*' }, + redirect: 'follow', + signal: AbortSignal.timeout(timeoutMs), + }); + } catch { + return null; + } + const status = res.status; + const ctype = String(res.headers.get('content-type') || '').toLowerCase(); + const lastModified = parseLastModified(res.headers.get('last-modified')); + const isHtml = status >= 200 && status < 300 && ctype.includes('html'); + let body = ''; + if (isHtml) { + try { + body = (await res.text()).slice(0, MAX_BODY_CHARS); + } catch { + return null; + } + } + return { url: res.url || url, status, isHtml, body, lastModified }; } function extractTitle(body) { @@ -267,21 +288,24 @@ function extractTitle(body) { return match[1].replace(/\s+/g, ' ').trim() || null; } +// HTTP dates (RFC 1123, RFC 850, asctime) all start with a day name. +const HTTP_DATE_RE = /^[A-Za-z]{3}/; + /** * Parse a `Last-Modified` HTTP header into the ISO-8601 date string * sitemaps.org expects in `<lastmod>`. Sites serve this header in RFC - * 1123 format (e.g. `Wed, 21 Oct 2026 07:28:00 GMT`); we accept anything - * `Date` can parse and emit `YYYY-MM-DD`. Returns null on failure. + * 1123 format (e.g. `Wed, 21 Oct 2026 07:28:00 GMT`); anything that is not + * an HTTP date returns null. Emits the UTC `YYYY-MM-DD`. * - * @param {string | string[] | undefined} raw + * @param {string | string[] | null | undefined} raw * @returns {string | null} */ function parseLastModified(raw) { const value = Array.isArray(raw) ? raw[0] : raw; - if (!value || typeof value !== 'string') return null; + if (!value || typeof value !== 'string' || !HTTP_DATE_RE.test(value.trim())) return null; const parsed = new Date(value); if (Number.isNaN(parsed.getTime())) return null; - // Use the date part only — sitemaps.org accepts both YYYY-MM-DD and + // Use the date part only: sitemaps.org accepts both YYYY-MM-DD and // full ISO timestamps; the date form is concise and avoids advertising // false precision. return parsed.toISOString().slice(0, 10); @@ -290,6 +314,7 @@ function parseLastModified(raw) { export const _internals = { extractInternalLinks, normalizeUrl, + normalizeStartUrl, isAssetPath, extractTitle, parseLastModified, diff --git a/src/robots.js b/src/robots.js index 86dcf7d..2a8d351 100644 --- a/src/robots.js +++ b/src/robots.js @@ -1,64 +1,135 @@ -// Minimal robots.txt checker for sitemap-builder. -// Fetches /robots.txt from the crawl origin and returns an isAllowed(path) -// predicate that applies longest-match semantics for User-Agent: *. -// Zero runtime dependencies; fails permissively on any error. +// Minimal robots.txt checker. +// Fetches /robots.txt and returns an isAllowed(path) predicate for a user +// agent (default `*`). Zero runtime dependencies; fails permissively on any +// error. +// +// The rules are the GeoSuite platform's (backend/services/tools/ +// robots_check_service.py, `is_url_allowed`), checked by the golden cases: +// - consecutive User-agent lines share one group; a rule line closes it; +// - groups naming the agent win over `*` groups; +// - inside a group the longest matching pattern wins, Allow on a tie; +// - the first group with a matching rule decides; no match means allowed; +// - `*` matches anything, a final `$` anchors the end, and both the +// pattern and the path are percent-decoded before matching. +// +// The same file ships in llms-txt-generator and sitemap-builder: keep them +// identical. -import http from 'node:http'; -import https from 'node:https'; +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_USER_AGENT = 'geosuite-robots-check'; /** - * Fetch /robots.txt from `origin` and build a path-checker for the wildcard - * User-Agent group. Returns `() => true` (allow all) on any fetch or parse - * error so callers never need to handle failures. + * Fetch /robots.txt from the origin of `siteUrl` and return a path-checker. + * Returns `() => true` (allow all) when robots.txt can't be fetched. * - * @param {string} origin e.g. "https://example.com" - * @param {{ timeoutMs?: number, userAgent?: string }} [opts] + * @param {string} siteUrl Any URL on the target site. + * @param {{ timeoutMs?: number, userAgent?: string, fetch?: typeof fetch }} [opts] * @returns {Promise<(path: string) => boolean>} */ -export async function loadRobotsChecker(origin, opts = {}) { - const timeoutMs = opts.timeoutMs ?? 10_000; - const ua = opts.userAgent ?? 'geosuite-sitemap-builder'; - const body = await fetchText(`${origin}/robots.txt`, timeoutMs, ua); - if (!body) return () => true; - const disallowed = parseWildcardDisallowed(body); - if (!disallowed.length) return () => true; - return (path) => !matchesAny(path, disallowed); -} +export async function loadRobotsChecker(siteUrl, opts = {}) { + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const userAgent = opts.userAgent ?? DEFAULT_USER_AGENT; + const doFetch = opts.fetch ?? fetch; + let origin; + try { + const u = new URL(siteUrl); + origin = `${u.protocol}//${u.host}`; + } catch { + return () => true; + } + + let body; + try { + const res = await doFetch(`${origin}/robots.txt`, { + headers: { 'user-agent': userAgent, accept: 'text/plain, */*' }, + redirect: 'follow', + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) return () => true; + body = await res.text(); + } catch { + return () => true; + } -// ---- parser ------------------------------------------------------------------ + const groups = parseRobots(body); + if (!groups.length) return () => true; + return (path) => isPathAllowed(groups, path); +} -function parseWildcardDisallowed(raw) { - const out = []; - let inWildcard = false; - for (const rawLine of raw.split(/\r?\n/)) { +/** + * Parse robots.txt into groups of user agents and their Allow/Disallow rules. + * + * @param {string} raw + * @returns {Array<{ agents: string[], rules: Array<{ allow: boolean, pattern: string }> }>} + */ +export function parseRobots(raw) { + const groups = []; + let current = null; + let lastWasRule = false; + for (const rawLine of String(raw || '').split(/\r\n|\r|\n/)) { const line = rawLine.split('#')[0].trim(); if (!line || !line.includes(':')) continue; const colon = line.indexOf(':'); const key = line.slice(0, colon).trim().toLowerCase(); - const val = line.slice(colon + 1).trim(); + const value = line.slice(colon + 1).trim(); if (key === 'user-agent') { - inWildcard = val === '*'; - } else if (key === 'disallow' && inWildcard && val) { - out.push(val); + if (!value) continue; + if (!current || lastWasRule) { + current = { agents: [], rules: [] }; + groups.push(current); + } + current.agents.push(value); + lastWasRule = false; + } else if (key === 'allow' || key === 'disallow') { + if (!current) continue; + current.rules.push({ allow: key === 'allow', pattern: value }); + lastWasRule = true; } + // Sitemap, Crawl-delay, Host, Content-Signal... neither apply nor split groups. } - return out; + return groups; } -// Longest-matching pattern wins; returns true if the path is blocked. -function matchesAny(path, patterns) { - let winner = null; - for (const p of patterns) { - if (pathMatches(p, path) && (!winner || p.length > winner.length)) winner = p; +/** + * Is `path` allowed for `agent` under the parsed groups? + * + * @param {ReturnType<typeof parseRobots>} groups + * @param {string} path URL path (no query). + * @param {string} [agent] + */ +export function isPathAllowed(groups, path, agent = '*') { + if (!groups || !groups.length) return true; + const token = agent.toLowerCase(); + const specific = []; + const wildcard = []; + for (const g of groups) { + const agents = g.agents.map((a) => a.toLowerCase()); + if (agents.includes(token)) specific.push(g); + else if (agents.includes('*')) wildcard.push(g); } - return winner !== null; + const pool = specific.length ? specific : wildcard; + const target = path || '/'; + for (const g of pool) { + let best = null; + for (const rule of g.rules) { + if (!pathMatches(rule.pattern, target)) continue; + const len = rule.pattern.length; + if (!best || len > best.len || (len === best.len && rule.allow && !best.allow)) { + best = { len, allow: rule.allow }; + } + } + if (best) return best.allow; + } + return true; } -// robots.txt wildcard matching: * = any chars, $ = must end here. +// robots.txt wildcard matching: * = any chars, a final $ = must end here. +// An empty pattern matches nothing. function pathMatches(pattern, path) { if (!pattern) return false; - const endAnchor = pattern.endsWith('$'); - const body = endAnchor ? pattern.slice(0, -1) : pattern; + const p = percentDecode(pattern); + const endAnchor = p.endsWith('$'); + const body = endAnchor ? p.slice(0, -1) : p; const re = new RegExp( '^' + body @@ -67,52 +138,13 @@ function pathMatches(pattern, path) { .join('.*') + (endAnchor ? '$' : ''), ); + return re.test(percentDecode(path)); +} + +function percentDecode(s) { try { - return re.test(decodeURIComponent(path)); + return decodeURIComponent(s); } catch { - return re.test(path); + return s; } } - -// ---- fetcher ----------------------------------------------------------------- - -function fetchText(url, timeoutMs, userAgent) { - return new Promise((resolve) => { - let parsed; - try { - parsed = new URL(url); - } catch { - resolve(null); - return; - } - const lib = parsed.protocol === 'https:' ? https : http; - const req = lib.request( - { - method: 'GET', - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80), - path: parsed.pathname + parsed.search, - headers: { - 'User-Agent': userAgent, - Accept: 'text/plain, */*', - 'Accept-Encoding': 'identity', - }, - }, - (res) => { - if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) { - res.resume(); - resolve(null); - return; - } - const chunks = []; - res.on('data', (c) => chunks.push(c)); - res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); - res.on('error', () => resolve(null)); - }, - ); - req.setTimeout(timeoutMs, () => req.destroy()); - req.on('error', () => resolve(null)); - req.end(); - }); -} diff --git a/test/fixtures/golden-sitemap.json b/test/fixtures/golden-sitemap.json new file mode 100644 index 0000000..a2bfd80 --- /dev/null +++ b/test/fixtures/golden-sitemap.json @@ -0,0 +1,675 @@ +{ + "about": "Golden cases for the sitemap crawl, shared by TryGeoSuite/sitemap-builder (test/fixtures/golden-sitemap.json, run by test/golden.test.js) and the GeoSuite platform (geo: backend/tests/fixtures/golden-sitemap.json, run by backend/tests/tools/test_sitemap_builder_golden.py). The two files must stay identical. crawl cases run with concurrency 1 against the `site` map: a URL not in it answers 404, `location` redirects, everything else is served with its status, content_type (default text/html), last_modified and body. After an intended change: node scripts/update-golden.mjs in the package, review the diff, copy the file to geo, make the Python side pass.", + "start_url": [ + { + "name": "bare domain gets https", + "input": "example.com", + "expected": "https://example.com/" + }, + { + "name": "spaces trimmed, host lowercased, default port and fragment dropped", + "input": " https://Example.COM:443/Path?q=1#top ", + "expected": "https://example.com/Path?q=1" + }, + { + "name": "other port kept", + "input": "http://example.com:8080/x", + "expected": "http://example.com:8080/x" + }, + { + "name": "localhost with a port", + "input": "localhost:3000", + "expected": "https://localhost:3000/" + }, + { + "name": "host without a dot", + "input": "intranet", + "expected": null + }, + { + "name": "not http", + "input": "ftp://example.com/", + "expected": null + }, + { + "name": "spaces inside", + "input": "not a url", + "expected": null + }, + { + "name": "empty", + "input": "", + "expected": null + } + ], + "links": [ + { + "name": "same origin only, normalized and sorted", + "current_url": "https://example.com/blog/post", + "origin": "https://example.com", + "html": "<a href=\"/zeta\">x</a><a href=\"../sibling\">x</a><a href=\"?q=1\">x</a><a href=\"//example.com/proto-relative\">x</a><a href=\" /spaced \">x</a><a href=\"/a b\">x</a><a href=\"/x#frag\">x</a><a href=\"#only\">x</a><a href=\"javascript:void(0)\">x</a><a href=\"/style.CSS\">x</a><a href=\"/path-with.dot/in-segment\">x</a><a href=\"http://example.com/other-scheme\">x</a><a href=\"/p?q=a b&c=d\">x</a><a href=\"/über\">x</a><a href=\"https://example.com:443/port\">x</a><a href=\"/%7Euser\">x</a><a href=\"/a|b\">x</a><a href=\"./\">x</a><a href=\"https://other.com/\">x</a><a href=\"tel:+390000\">x</a><a href=\"/About\">x</a><link href=\"/not-an-anchor\"><A HREF='/upper-tag'>u</A>", + "expected": [ + "https://example.com/%7Euser", + "https://example.com/%C3%BCber", + "https://example.com/About", + "https://example.com/a%20b", + "https://example.com/a|b", + "https://example.com/blog/", + "https://example.com/blog/post?q=1", + "https://example.com/p?q=a%20b&c=d", + "https://example.com/path-with.dot/in-segment", + "https://example.com/port", + "https://example.com/proto-relative", + "https://example.com/sibling", + "https://example.com/spaced", + "https://example.com/upper-tag", + "https://example.com/x", + "https://example.com/zeta" + ] + }, + { + "name": "nothing to follow", + "current_url": "https://example.com/", + "origin": "https://example.com", + "html": "<p>no links</p>", + "expected": [] + } + ], + "last_modified": [ + { + "name": "RFC 1123", + "header": "Wed, 21 Oct 2026 07:28:00 GMT", + "expected": "2026-10-21" + }, + { + "name": "offset converted to UTC", + "header": "Thu, 22 Oct 2026 01:30:00 +0200", + "expected": "2026-10-21" + }, + { + "name": "ISO date is not an HTTP date", + "header": "2026-10-21", + "expected": null + }, + { + "name": "garbage", + "header": "garbage", + "expected": null + }, + { + "name": "missing", + "header": null, + "expected": null + } + ], + "render": [ + { + "name": "loc escaped, lastmod only when present", + "pages": [ + { + "url": "https://example.com/" + }, + { + "url": "https://example.com/p?a=1&b=<2>", + "lastmod": "2026-10-21" + }, + { + "url": "" + } + ], + "expected": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n <url>\n <loc>https://example.com/</loc>\n </url>\n <url>\n <loc>https://example.com/p?a=1&b=<2></loc>\n <lastmod>2026-10-21</lastmod>\n </url>\n</urlset>\n" + }, + { + "name": "no pages", + "pages": [], + "expected": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n</urlset>\n" + } + ], + "robots": [ + { + "name": "consecutive user-agent lines share a group", + "agent": "*", + "robots_txt": "User-agent: *\nUser-agent: Googlebot\nDisallow: /private\n", + "paths": [ + "/private/x", + "/public" + ], + "expected": { + "/private/x": false, + "/public": true + } + }, + { + "name": "longest match wins, Allow on a tie", + "agent": "*", + "robots_txt": "User-agent: *\nDisallow: /shop\nAllow: /shop/public\nDisallow: /page\nAllow: /page\n", + "paths": [ + "/shop/x", + "/shop/public/a", + "/page", + "/pages" + ], + "expected": { + "/shop/x": false, + "/shop/public/a": true, + "/page": true, + "/pages": true + } + }, + { + "name": "wildcards and end anchor", + "agent": "*", + "robots_txt": "User-agent: *\nDisallow: /*.pdf$\nDisallow: /*?\nDisallow: /tmp*/cache\n", + "paths": [ + "/a/b.pdf", + "/a/b.pdfx", + "/tmp1/cache/x", + "/tmp/x" + ], + "expected": { + "/a/b.pdf": false, + "/a/b.pdfx": true, + "/tmp1/cache/x": false, + "/tmp/x": true + } + }, + { + "name": "empty Disallow allows everything", + "agent": "*", + "robots_txt": "User-agent: *\nDisallow:\n", + "paths": [ + "/", + "/anything" + ], + "expected": { + "/": true, + "/anything": true + } + }, + { + "name": "a group for another bot does not apply to *", + "agent": "*", + "robots_txt": "User-agent: GPTBot\nDisallow: /\n", + "paths": [ + "/", + "/x" + ], + "expected": { + "/": true, + "/x": true + } + }, + { + "name": "named agent group wins over *", + "agent": "GPTBot", + "robots_txt": "User-agent: *\nDisallow: /\n\nUser-agent: gptbot\nAllow: /\nDisallow: /secret\n", + "paths": [ + "/", + "/secret/a" + ], + "expected": { + "/": true, + "/secret/a": false + } + }, + { + "name": "percent-encoding is decoded on both sides", + "agent": "*", + "robots_txt": "User-agent: *\nDisallow: /caf%C3%A9\n", + "paths": [ + "/café/menu", + "/caf%C3%A9/menu", + "/cafe" + ], + "expected": { + "/café/menu": false, + "/caf%C3%A9/menu": false, + "/cafe": true + } + }, + { + "name": "rules before any user-agent are ignored, comments stripped", + "agent": "*", + "robots_txt": "Disallow: /early\n# comment\nUser-agent: * # everyone\nDisallow: /late # trailing\n", + "paths": [ + "/early", + "/late" + ], + "expected": { + "/early": true, + "/late": false + } + }, + { + "name": "the first * group with a matching rule decides", + "agent": "*", + "robots_txt": "User-agent: *\nDisallow: /a\n\nUser-agent: *\nAllow: /a/b\n", + "paths": [ + "/a/b", + "/a/c" + ], + "expected": { + "/a/b": false, + "/a/c": false + } + }, + { + "name": "non-rule lines do not split a group", + "agent": "*", + "robots_txt": "User-agent: bingbot\nCrawl-delay: 5\nUser-agent: *\nDisallow: /x\nSitemap: https://example.com/sitemap.xml\n", + "paths": [ + "/x", + "/y" + ], + "expected": { + "/x": false, + "/y": true + } + } + ], + "crawl": [ + { + "name": "redirect to www, final URLs, only 2xx HTML, depth cap", + "start_url": "example.com", + "max_pages": 100, + "max_depth": 3, + "respect_robots": false, + "site": { + "https://example.com/": { + "status": 301, + "location": "https://www.example.com/" + }, + "https://www.example.com/": { + "body": "<html><head><title> Home \n page xxxxxxxxxxxxx", + "last_modified": "Wed, 21 Oct 2026 07:28:00 GMT" + }, + "https://www.example.com/about": { + "body": "Abouthomedeep" + }, + "https://www.example.com/old-page": { + "status": 301, + "location": "/new-page" + }, + "https://www.example.com/new-page": { + "body": "New" + }, + "https://www.example.com/files/report": { + "status": 200, + "content_type": "application/pdf", + "body": "%PDF" + }, + "https://www.example.com/p?a=1&b=2": { + "body": "Query" + }, + "https://www.example.com/Blog/": { + "body": "Wrong case" + }, + "https://www.example.com/blog/": { + "body": "Blogp" + }, + "https://www.example.com/blog/post-1": { + "body": "Post 1" + }, + "https://www.example.com/contact": { + "body": "Contactapex" + }, + "https://www.example.com/caf%C3%A9": { + "body": "Café" + }, + "https://www.example.com/deep/one": { + "body": "2" + }, + "https://www.example.com/deep/two": { + "body": "3" + }, + "https://www.example.com/deep/three": { + "body": "Too deep" + } + }, + "expected": { + "origin": "https://www.example.com", + "pages": [ + { + "url": "https://www.example.com/", + "depth": 0, + "status": 200, + "title": "Home page", + "last_modified": "2026-10-21" + }, + { + "url": "https://www.example.com/about", + "depth": 1, + "status": 200, + "title": "About", + "last_modified": null + }, + { + "url": "https://www.example.com/blog/", + "depth": 1, + "status": 200, + "title": "Blog", + "last_modified": null + }, + { + "url": "https://www.example.com/caf%C3%A9", + "depth": 1, + "status": 200, + "title": "Café", + "last_modified": null + }, + { + "url": "https://www.example.com/contact", + "depth": 1, + "status": 200, + "title": "Contact", + "last_modified": null + }, + { + "url": "https://www.example.com/new-page", + "depth": 1, + "status": 200, + "title": "New", + "last_modified": null + }, + { + "url": "https://www.example.com/p?a=1&b=2", + "depth": 1, + "status": 200, + "title": "Query", + "last_modified": null + }, + { + "url": "https://www.example.com/blog/post-1", + "depth": 2, + "status": 200, + "title": "Post 1", + "last_modified": null + }, + { + "url": "https://www.example.com/deep/one", + "depth": 2, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://www.example.com/deep/two", + "depth": 3, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": false, + "xml": "\n\n \n https://www.example.com/\n 2026-10-21\n \n \n https://www.example.com/about\n \n \n https://www.example.com/blog/\n \n \n https://www.example.com/caf%C3%A9\n \n \n https://www.example.com/contact\n \n \n https://www.example.com/new-page\n \n \n https://www.example.com/p?a=1&b=2\n \n \n https://www.example.com/blog/post-1\n \n \n https://www.example.com/deep/one\n \n \n https://www.example.com/deep/two\n \n\n" + } + }, + { + "name": "page cap", + "start_url": "https://shop.it", + "max_pages": 3, + "max_depth": 3, + "respect_robots": false, + "site": { + "https://shop.it/": { + "body": "abcd" + }, + "https://shop.it/a": { + "body": "abcd" + }, + "https://shop.it/b": { + "body": "abcd" + }, + "https://shop.it/c": { + "body": "abcd" + }, + "https://shop.it/d": { + "body": "abcd" + } + }, + "expected": { + "origin": "https://shop.it", + "pages": [ + { + "url": "https://shop.it/", + "depth": 0, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://shop.it/a", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://shop.it/b", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": true, + "xml": "\n\n \n https://shop.it/\n \n \n https://shop.it/a\n \n \n https://shop.it/b\n \n\n" + } + }, + { + "name": "robots respected when asked", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 3, + "respect_robots": true, + "site": { + "https://site.it/robots.txt": { + "status": 200, + "content_type": "text/plain", + "body": "User-agent: *\nDisallow: /private\nAllow: /private/ok\n" + }, + "https://site.it/": { + "body": "aokp" + }, + "https://site.it/private/a": { + "body": "secret" + }, + "https://site.it/private/ok": { + "body": "fine" + }, + "https://site.it/public": { + "body": "public" + } + }, + "expected": { + "origin": "https://site.it", + "pages": [ + { + "url": "https://site.it/", + "depth": 0, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://site.it/private/ok", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://site.it/public", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": false, + "xml": "\n\n \n https://site.it/\n \n \n https://site.it/private/ok\n \n \n https://site.it/public\n \n\n" + } + }, + { + "name": "robots ignored when not asked", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 3, + "respect_robots": false, + "site": { + "https://site.it/robots.txt": { + "status": 200, + "content_type": "text/plain", + "body": "User-agent: *\nDisallow: /private\nAllow: /private/ok\n" + }, + "https://site.it/": { + "body": "aokp" + }, + "https://site.it/private/a": { + "body": "secret" + }, + "https://site.it/private/ok": { + "body": "fine" + }, + "https://site.it/public": { + "body": "public" + } + }, + "expected": { + "origin": "https://site.it", + "pages": [ + { + "url": "https://site.it/", + "depth": 0, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://site.it/private/a", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://site.it/private/ok", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + }, + { + "url": "https://site.it/public", + "depth": 1, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": false, + "xml": "\n\n \n https://site.it/\n \n \n https://site.it/private/a\n \n \n https://site.it/private/ok\n \n \n https://site.it/public\n \n\n" + } + }, + { + "name": "start disallowed by robots", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 3, + "respect_robots": true, + "site": { + "https://site.it/robots.txt": { + "status": 200, + "content_type": "text/plain", + "body": "User-agent: *\nDisallow: /\n" + }, + "https://site.it/": { + "body": "a" + } + }, + "expected": { + "origin": "https://site.it", + "pages": [], + "hit_page_cap": false, + "xml": "\n\n\n" + } + }, + { + "name": "start page missing", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 3, + "respect_robots": false, + "site": {}, + "expected": { + "origin": "https://site.it", + "pages": [], + "hit_page_cap": false, + "xml": "\n\n\n" + } + }, + { + "name": "depth 0 lists the start page only", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 0, + "respect_robots": false, + "site": { + "https://site.it/": { + "body": "a" + }, + "https://site.it/a": { + "body": "b" + } + }, + "expected": { + "origin": "https://site.it", + "pages": [ + { + "url": "https://site.it/", + "depth": 0, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": false, + "xml": "\n\n \n https://site.it/\n \n\n" + } + }, + { + "name": "a page redirecting off the site and a 500 are dropped", + "start_url": "https://site.it/", + "max_pages": 100, + "max_depth": 3, + "respect_robots": false, + "site": { + "https://site.it/": { + "body": "xy" + }, + "https://site.it/x": { + "status": 302, + "location": "https://elsewhere.com/x" + }, + "https://site.it/y": { + "status": 500, + "body": "oops" + } + }, + "expected": { + "origin": "https://site.it", + "pages": [ + { + "url": "https://site.it/", + "depth": 0, + "status": 200, + "title": null, + "last_modified": null + } + ], + "hit_page_cap": false, + "xml": "\n\n \n https://site.it/\n \n\n" + } + } + ] +} diff --git a/test/golden-run.js b/test/golden-run.js new file mode 100644 index 0000000..408874e --- /dev/null +++ b/test/golden-run.js @@ -0,0 +1,65 @@ +// Runs one golden case through the package code. Shared by the golden test and +// scripts/update-golden.mjs, so both read the cases the same way. + +import { crawlSite, _internals } from '../src/crawler.js'; +import { renderSitemapXml } from '../src/sitemap.js'; +import { parseRobots, isPathAllowed } from '../src/robots.js'; + +const { extractInternalLinks, normalizeStartUrl, parseLastModified } = _internals; + +export const SECTIONS = ['start_url', 'links', 'last_modified', 'render', 'robots', 'crawl']; + +// A fetch that answers from the case's `site` map: a missing URL is a 404, an +// entry with `location` redirects (followed, like fetch does), anything else +// is served with its status, content type, Last-Modified and body. +export function fakeFetch(site) { + return async (url) => { + let current = url; + for (let hop = 0; hop < 10; hop++) { + const page = site[current]; + if (page && page.location) { + current = new URL(page.location, current).href; + continue; + } + const headers = new Headers({ 'content-type': page?.content_type ?? 'text/html; charset=utf-8' }); + if (page?.last_modified) headers.set('last-modified', page.last_modified); + const status = page ? page.status ?? 200 : 404; + const body = page?.body ?? ''; + return { url: current, status, ok: status >= 200 && status < 300, headers, text: async () => body }; + } + throw new Error('too many redirects'); + }; +} + +export async function computeGolden(section, c) { + if (section === 'start_url') return normalizeStartUrl(c.input)?.href ?? null; + if (section === 'links') return extractInternalLinks(c.html, c.current_url, c.origin); + if (section === 'last_modified') return parseLastModified(c.header); + if (section === 'render') return renderSitemapXml(c.pages); + if (section === 'robots') { + const groups = parseRobots(c.robots_txt); + return Object.fromEntries(c.paths.map((p) => [p, isPathAllowed(groups, p, c.agent)])); + } + if (section === 'crawl') { + const { origin, pages, hitCap } = await crawlSite(c.start_url, { + maxPages: c.max_pages, + maxDepth: c.max_depth, + respectRobots: c.respect_robots, + concurrency: 1, + fetch: fakeFetch(c.site), + }); + return { + origin, + pages: pages.map((p) => ({ + url: p.url, + depth: p.depth, + status: p.status, + title: p.title, + last_modified: p.lastModified ?? null, + })), + hit_page_cap: hitCap, + xml: renderSitemapXml(pages.map((p) => ({ url: p.url, lastmod: p.lastModified || undefined }))), + }; + } + throw new Error(`unknown golden section ${section}`); +} diff --git a/test/golden.test.js b/test/golden.test.js new file mode 100644 index 0000000..262e2de --- /dev/null +++ b/test/golden.test.js @@ -0,0 +1,23 @@ +// Golden cases shared with the GeoSuite platform. The same JSON file lives in +// the platform repo (geo: backend/tests/fixtures/golden-sitemap.json) and its +// Python Sitemap Builder must give the same answers: if this test fails, the +// package changed behavior and the platform no longer matches. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { SECTIONS, computeGolden } from './golden-run.js'; + +const golden = JSON.parse( + readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'golden-sitemap.json'), 'utf8'), +); + +for (const section of SECTIONS) { + for (const c of golden[section]) { + test(`golden ${section}: ${c.name}`, async () => { + assert.deepEqual(await computeGolden(section, c), c.expected); + }); + } +} diff --git a/web/README.md b/web/README.md index 7ad1548..ceabf53 100644 --- a/web/README.md +++ b/web/README.md @@ -7,13 +7,14 @@ a sitemaps.org-compliant `sitemap.xml` to copy or download. - `GET /` — the page (`page.js`), bilingual **en/it** (auto-detected from `Accept-Language`; `/en` · `/it` force a locale) - `GET /og.png` · `GET /favicon.png` (also `/favicon.ico`, `/favicon.svg`, `/apple-touch-icon.png`) — Open Graph share image (1200×630) + GeoSuite's favicon - `GET /logos/{chatgpt,gemini,perplexity}.svg` — engine logos for the call to action -- `GET /api/crawl?url=https://example.com` — `{ site, count, capped, xml }` - -> ⚠️ **Capped edge version.** The crawl is a small BFS (≤40 pages, depth ≤2) -> with the platform `fetch`, to stay under the Workers subrequest/time limits. -> The package's full crawler (`src/crawler.js`) uses `node:http` and isn't -> Worker-compatible, so the crawl is reimplemented here and only the pure -> `renderSitemapXml()` is reused. **For a complete crawl, use the CLI:** +- `GET /api/crawl?url=https://example.com`: `{ site, count, capped, hitDeadline, xml }` + +> **Same crawler as the CLI and the platform.** The Worker runs the package's +> `crawlSite()` (`src/crawler.js`, platform `fetch`) and `renderSitemapXml()` +> with the GeoSuite platform's settings: depth 3, 6 parallel fetches, 6 s per +> page, 25 s budget. The one difference is the page cap, 40, to stay under the +> Workers subrequest limit: the platform's Sitemap Builder with "Max pages" 40 +> returns the same file. **For a bigger crawl, use the CLI:** > `npx @geosuite/sitemap-builder `. ## Run locally @@ -38,7 +39,7 @@ npx wrangler deploy ## Auto-deploy (CI) [`.github/workflows/deploy-web.yml`](../.github/workflows/deploy-web.yml) -redeploys on every push to `main` touching `web/` or `src/sitemap.js`. Add two +redeploys on every push to `main` touching `web/` or `src/`. Add two repo secrets (Settings → Secrets and variables → Actions): - `CLOUDFLARE_API_TOKEN` — token scoped **Edit Cloudflare Workers** from the @@ -47,7 +48,7 @@ repo secrets (Settings → Secrets and variables → Actions): ## Notes -- Zero deps, no `nodejs_compat`. The crawl uses the platform `fetch`; only - `renderSitemapXml()` is imported from the package. +- Zero deps, no `nodejs_compat`: the crawler and the robots checker use the + platform `fetch` only. - This directory is **not** part of the npm package, so it never ships to registry consumers. diff --git a/web/page.js b/web/page.js index fd6ecad..beac1a6 100644 --- a/web/page.js +++ b/web/page.js @@ -44,11 +44,11 @@ const S = { howLabel: 'How it works', how: [ 'We start from the page you paste and follow the links that stay on your site.', - 'Up to ~40 pages, two clicks deep, so the answer comes back in seconds.', + 'Up to 40 pages, three clicks deep, so the answer comes back in seconds. Redirects are followed; only pages that load go in.', 'You get a valid sitemap.xml to copy or download, ready for your site root.', ], cliLabel: 'From the terminal', - hintPre: 'Quick crawl, capped at ~40 pages (depth 2) to stay fast. For a full crawl of a large site, use the CLI:', + hintPre: 'Quick crawl, capped at 40 pages (depth 3) to stay fast. For a full crawl of a large site, use the CLI:', copyCmd: 'Copy', copiedCmd: 'Copied', copyLabel: 'Copy the command', @@ -87,11 +87,11 @@ const S = { howLabel: 'Come funziona', how: [ 'Partiamo dalla pagina che incolli e seguiamo i link che restano sul tuo sito.', - 'Fino a ~40 pagine, a due clic di profondità, così la risposta arriva in pochi secondi.', + 'Fino a 40 pagine, a tre clic di profondità, così la risposta arriva in pochi secondi. Seguiamo i redirect; entrano solo le pagine che si aprono.', 'Ottieni un sitemap.xml valido da copiare o scaricare, pronto per la radice del sito.', ], cliLabel: 'Dal terminale', - hintPre: 'Crawl rapido, limitato a ~40 pagine (profondità 2) per restare veloce. Per un crawl completo di un sito grande, usa la CLI:', + hintPre: 'Crawl rapido, limitato a 40 pagine (profondità 3) per restare veloce. Per un crawl completo di un sito grande, usa la CLI:', copyCmd: 'Copia', copiedCmd: 'Copiato', copyLabel: 'Copia il comando', diff --git a/web/worker.js b/web/worker.js index f0be9d2..dcb1f41 100644 --- a/web/worker.js +++ b/web/worker.js @@ -1,22 +1,24 @@ // "Sitemap Builder" — hosted free tool (Cloudflare Worker). // -// ⚠️ This is the CAPPED edge version: a small BFS crawl (≤40 pages, depth ≤2) -// using the platform `fetch`, to stay under the Workers subrequest/time limits. -// The package's full crawler (src/crawler.js) uses node:http and isn't -// Worker-compatible, so the crawl is reimplemented here with `fetch`; only the -// pure renderSitemapXml() is reused. For a complete crawl of a large site, use -// the CLI: `npx @geosuite/sitemap-builder `. +// It runs the package's own crawler (src/crawler.js, platform `fetch`) and +// renderSitemapXml(), with the GeoSuite platform's crawl settings (depth 3, +// 6 parallel fetches, 6 s per page, 25 s budget) and one difference: at most +// 40 pages, to stay under the Workers subrequest limit. For the same site the +// page gives the same sitemap.xml as the platform's Sitemap Builder run with +// "Max pages" 40. For a bigger crawl, use the CLI: +// `npx @geosuite/sitemap-builder `. // // Routes: // GET / → the page, locale from Accept-Language (it → Italian) // GET /en /it → the page in that locale -// GET /api/crawl?url=... → { site, count, capped, xml, error } +// GET /api/crawl?url=... → { site, count, capped, hitDeadline, xml, error } // GET /og.png → Open Graph share image // GET /favicon.png /favicon.ico /favicon.svg /apple-touch-icon.png // → GeoSuite's favicon (PNG; /favicon.svg is the old // path, kept so cached links still get an icon) // GET /logos/{chatgpt,gemini,perplexity}.svg → engine logos for the CTA +import { crawlSite } from '../src/crawler.js'; import { renderSitemapXml } from '../src/sitemap.js'; import { renderPage } from './page.js'; import OG_PNG from './og.png'; // bundled as ArrayBuffer via the wrangler "Data" rule @@ -40,79 +42,31 @@ function pickLang(request, path) { return first.startsWith('it') ? 'it' : 'en'; } +// The platform's defaults (sitemap_builder_service.py), except the page cap. const MAX_PAGES = 40; -const MAX_DEPTH = 2; +const MAX_DEPTH = 3; +const CONCURRENCY = 6; const TIMEOUT_MS = 6000; +const BUDGET_MS = 25_000; const UA = 'sitemap-builder-web/1.0 (+https://github.com/TryGeoSuite/sitemap-builder)'; -const SKIP_EXT = /\.(?:png|jpe?g|gif|svg|webp|ico|css|js|mjs|json|xml|txt|pdf|zip|gz|mp4|webm|mp3|woff2?|ttf|eot)(?:$|\?)/i; - -function extractLinks(html, base, origin) { - const out = new Set(); - const re = /href\s*=\s*["']([^"']+)["']/gi; - let m; - while ((m = re.exec(html))) { - const href = m[1].trim(); - if (!href || /^(mailto:|tel:|javascript:|#|data:)/i.test(href)) continue; - let abs; - try { - abs = new URL(href, base); - } catch { - continue; - } - if (abs.protocol !== 'http:' && abs.protocol !== 'https:') continue; - if (abs.origin !== origin) continue; - if (SKIP_EXT.test(abs.pathname)) continue; - abs.hash = ''; - out.add(abs.toString()); - } - return out; -} async function crawl(input) { - let start; + let result; try { - start = new URL(/^https?:\/\//i.test(input) ? input : 'https://' + input); + result = await crawlSite(input, { + maxPages: MAX_PAGES, + maxDepth: MAX_DEPTH, + concurrency: CONCURRENCY, + perPageTimeoutMs: TIMEOUT_MS, + deadlineMs: Date.now() + BUDGET_MS, + userAgent: UA, + }); } catch { return { error: 'Please enter a valid URL.' }; } - if (start.protocol !== 'http:' && start.protocol !== 'https:') { - return { error: 'Please enter an http(s) URL.' }; - } - const origin = start.origin; - start.hash = ''; - - const seen = new Set([start.toString()]); - const queue = [{ url: start.toString(), depth: 0 }]; - const pages = []; - - while (queue.length && pages.length < MAX_PAGES) { - const { url, depth } = queue.shift(); - let html = ''; - try { - const res = await fetch(url, { - headers: { 'user-agent': UA, accept: 'text/html,*/*' }, - redirect: 'follow', - signal: AbortSignal.timeout(TIMEOUT_MS), - }); - if (!res.ok) continue; - if (!(res.headers.get('content-type') || '').toLowerCase().includes('html')) continue; - pages.push({ url }); - if (depth < MAX_DEPTH) html = await res.text(); - } catch { - continue; - } - if (!html) continue; - for (const link of extractLinks(html, url, origin)) { - if (!seen.has(link) && seen.size < MAX_PAGES * 5) { - seen.add(link); - queue.push({ url: link, depth: depth + 1 }); - } - } - } - - const capped = queue.length > 0 || pages.length >= MAX_PAGES; - const xml = renderSitemapXml(pages.map((p) => ({ url: p.url }))); - return { site: origin, count: pages.length, capped, xml, error: null }; + const { origin, pages, hitCap, hitDeadline } = result; + const xml = renderSitemapXml(pages.map((p) => ({ url: p.url, lastmod: p.lastModified || undefined }))); + return { site: origin, count: pages.length, capped: hitCap || hitDeadline, hitDeadline, xml, error: null }; } const JSON_HEADERS = { diff --git a/web/wrangler.toml b/web/wrangler.toml index 7de17b0..f3d56a1 100644 --- a/web/wrangler.toml +++ b/web/wrangler.toml @@ -10,8 +10,8 @@ rules = [ { type = "Text", globs = ["**/*.svg"], fallthrough = false } ] -# No nodejs_compat / npm install needed: the Worker reuses only the pure -# renderSitemapXml() from ../src/sitemap.js and crawls with the platform fetch. +# No nodejs_compat / npm install needed: the Worker reuses ../src/crawler.js, +# ../src/robots.js and ../src/sitemap.js, which use only the platform fetch. # # ⚠️ Deploy to your PERSONAL / GeoSuite Cloudflare account — NOT the work/Cato # one (`wrangler whoami` to check). Pin it to be safe: