diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index c28880073..c4a0c7189 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -6,10 +6,20 @@ on: deployment_status: # Allow manual trigger workflow_dispatch: + inputs: + full: + description: 'Submit every sitemap URL instead of only what changed' + type: boolean + default: false permissions: contents: read +# Deploys overlap; serialise so runs can't race on the submission-state cache. +concurrency: + group: indexnow + cancel-in-progress: false + jobs: ping: runs-on: ubuntu-latest @@ -24,16 +34,95 @@ jobs: with: submodules: true token: ${{ secrets.SUBMODULE_TOKEN }} + # Deep enough to diff against the content commit we last submitted for. + fetch-depth: 100 + + - uses: actions/cache/restore@v4 + with: + path: .indexnow-state + key: indexnow-state-${{ github.sha }} + restore-keys: indexnow-state- + + - name: Resolve what changed since the last submission + id: changes + run: | + state=.indexnow-state/urls.json + content_sha=$(git -C src/content rev-parse HEAD) + echo "content_sha=$content_sha" >> "$GITHUB_OUTPUT" + + if [ ! -f "$state" ]; then + echo "No submission state cached — the script will submit the full sitemap." + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # checkout clones submodules at the superproject's depth; the commit we last + # submitted for can sit outside it, so take the whole content history. + if [ "$(git -C src/content rev-parse --is-shallow-repository)" = 'true' ]; then + git -C src/content fetch --unshallow --quiet || true + fi + + prev_content_sha=$(jq -r '.contentSha // empty' "$state") + if [ -n "$prev_content_sha" ] && git -C src/content cat-file -e "$prev_content_sha^{commit}" 2>/dev/null; then + changed=$(git -C src/content diff --name-only "$prev_content_sha" "$content_sha") + force_full=false + else + # Without the diff we cannot tell which existing pages were edited, and the + # sitemap diff only surfaces new URLs. Resubmit everything rather than skip. + echo "Previous content commit unavailable — falling back to a full submission." + changed="" + force_full=true + fi + echo "force_full=$force_full" >> "$GITHUB_OUTPUT" + + { + echo 'changed_files<> "$GITHUB_OUTPUT" + + # Everything outside the submodule that can add or remove a sitemap URL. + prev_sha=$(jq -r '.sha // empty' "$state") + if [ -n "$prev_sha" ] && git cat-file -e "$prev_sha^{commit}" 2>/dev/null; then + sources_changed=$(git diff --name-only "$prev_sha" "$GITHUB_SHA" -- \ + src/app/sitemap.ts src/data/seo src/i18n src/lib/content.ts src/lib/blog.ts) + else + echo "Previous commit outside fetch depth — running rather than risk missing new URLs." + sources_changed="unknown" + fi + + if [ -z "$changed" ] && [ -z "$sources_changed" ] && [ "$prev_content_sha" = "$content_sha" ] \ + && [ "$force_full" != 'true' ] && [ "${{ inputs.full }}" != 'true' ]; then + echo 'Nothing that affects the sitemap changed since the last submission — skipping.' + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi - uses: pnpm/action-setup@7088e561eb65bb68695d245aa206f005ef30921d # v4.1.0 + if: steps.changes.outputs.run == 'true' + - uses: actions/setup-node@v4 + if: steps.changes.outputs.run == 'true' with: - node-version-file: '.node-version' + node-version: '20' cache: 'pnpm' - run: pnpm install --frozen-lockfile + if: steps.changes.outputs.run == 'true' - name: Ping IndexNow + if: steps.changes.outputs.run == 'true' env: + # Not a secret by design — IndexNow requires it to be served at public/.txt. INDEXNOW_KEY: '054e10e6239a45cb2d06e92d669f5b6f' + INDEXNOW_FULL: ${{ inputs.full == true || steps.changes.outputs.force_full == 'true' }} + INDEXNOW_CHANGED_FILES: ${{ steps.changes.outputs.changed_files }} + INDEXNOW_CONTENT_SHA: ${{ steps.changes.outputs.content_sha }} run: pnpm tsx scripts/ping-indexnow.ts + + - uses: actions/cache/save@v4 + if: steps.changes.outputs.run == 'true' + with: + path: .indexnow-state + key: indexnow-state-${{ github.sha }} diff --git a/.gitignore b/.gitignore index 70d2ea2ed..09967530f 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ coverage/ # See .github/workflows/tests.yml — `Fetch BE render-snapshot baseline` step. # Never commit; the canonical 49-entry baseline lives in the BE repo. src/components/TransactionDetails/__tests__/fixtures/be-entries.json + +# IndexNow submission state, restored from the Actions cache (see .github/workflows/indexnow.yml). +.indexnow-state/ diff --git a/scripts/ping-indexnow.ts b/scripts/ping-indexnow.ts index f0d0e0c1b..9b737c85c 100644 --- a/scripts/ping-indexnow.ts +++ b/scripts/ping-indexnow.ts @@ -1,98 +1,156 @@ /** - * Pings IndexNow (Bing, Yandex, etc.) with all sitemap URLs. + * Submits URLs to IndexNow (Bing, Yandex, Seznam, Naver). + * + * IndexNow is for URLs that were added, updated or deleted — resubmitting the whole + * site on every deploy burns the daily quota and gets the host deprioritised. So the + * default mode is a delta: the sitemap's URL set is diffed against the set submitted + * last time, and content files changed since then are mapped back to the pages they + * render. * * Usage: - * INDEXNOW_KEY=your-key-here tsx scripts/ping-indexnow.ts + * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts # delta vs. previous run + * INDEXNOW_KEY=xxx INDEXNOW_FULL=true tsx … # every sitemap URL + * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts /en/brazil # explicit paths * - * Or pass specific paths: - * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts /en/argentina /en/brazil + * Env: + * INDEXNOW_KEY required; must match public/.txt + * INDEXNOW_FULL 'true' to submit the full sitemap + * INDEXNOW_CHANGED_FILES newline-separated paths, relative to the content submodule + * INDEXNOW_CONTENT_SHA content submodule commit, recorded for the next run's diff + * GITHUB_SHA superproject commit, recorded for the next run's diff + * INDEXNOW_STATE_FILE defaults to .indexnow-state/urls.json */ -const BASE_URL = 'https://peanut.me' +import fs from 'fs' +import path from 'path' +import generateSitemap from '../src/app/sitemap' +import { BASE_URL } from '../src/constants/general.consts' + +const PRODUCTION_ORIGIN = 'https://peanut.me' const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/IndexNow' +const MAX_URLS_PER_REQUEST = 10_000 + const KEY = process.env.INDEXNOW_KEY +const STATE_FILE = process.env.INDEXNOW_STATE_FILE || path.join(process.cwd(), '.indexnow-state/urls.json') if (!KEY) { console.error('INDEXNOW_KEY environment variable is required') process.exit(1) } -// If CLI args provided, use those. Otherwise build full URL list. -const cliPaths = process.argv.slice(2) - -async function getAllPaths(): Promise { - // Dynamic import to reuse the same data sources as sitemap.ts - const { COUNTRIES_SEO, CORRIDORS, COMPETITORS, EXCHANGES, PAYMENT_METHOD_SLUGS } = - await import('../src/data/seo/index') - const { SUPPORTED_LOCALES } = await import('../src/i18n/types') - const { listContentSlugs } = await import('../src/lib/content') +interface State { + /** Commits the last submission accounted for, so the next run knows what to diff against. */ + sha?: string + contentSha?: string + submittedAt?: string + urls: string[] +} - const paths: string[] = ['/', '/lp/card', '/careers', '/exchange', '/privacy', '/terms'] +function readState(): State | null { + try { + const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')) as State + return Array.isArray(parsed.urls) ? parsed : null + } catch { + return null + } +} - for (const locale of SUPPORTED_LOCALES) { - for (const country of Object.keys(COUNTRIES_SEO)) { - paths.push(`/${locale}/${country}`) - paths.push(`/${locale}/send-money-to/${country}`) - } - for (const corridor of CORRIDORS) { - paths.push(`/${locale}/send-money-from/${corridor.from}/to/${corridor.to}`) - } - const receiveSources = [...new Set(CORRIDORS.map((c: { from: string }) => c.from))] - for (const source of receiveSources) { - paths.push(`/${locale}/receive-money-from/${source}`) - } - for (const slug of Object.keys(COMPETITORS)) { - paths.push(`/${locale}/compare/peanut-vs-${slug}`) - } - for (const slug of Object.keys(EXCHANGES)) { - paths.push(`/${locale}/deposit/from-${slug}`) - } - for (const method of PAYMENT_METHOD_SLUGS) { - paths.push(`/${locale}/pay-with/${method}`) - } - paths.push(`/${locale}/help`) - for (const slug of listContentSlugs('help')) { - paths.push(`/${locale}/help/${slug}`) - } +function writeState(urls: string[]) { + const state: State = { + sha: process.env.GITHUB_SHA || undefined, + contentSha: process.env.INDEXNOW_CONTENT_SHA || undefined, + submittedAt: new Date().toISOString(), + urls, } + fs.mkdirSync(path.dirname(STATE_FILE), { recursive: true }) + fs.writeFileSync(STATE_FILE, JSON.stringify(state)) +} - return paths +/** + * The sitemap is the single source of truth for which URLs exist. It is built with + * BASE_URL, which is env-dependent — rewrite onto the production origin so a stray + * NEXT_PUBLIC_BASE_URL can never make us submit preview URLs for peanut.me. + */ +async function listSitemapUrls(): Promise { + const entries = await generateSitemap() + const urls = entries.map((entry) => + entry.url.startsWith(BASE_URL) ? `${PRODUCTION_ORIGIN}${entry.url.slice(BASE_URL.length)}` : entry.url + ) + return [...new Set(urls)] } -async function main() { - const paths = cliPaths.length > 0 ? cliPaths : await getAllPaths() - const urlList = paths.map((p) => `${BASE_URL}${p}`) +/** + * Reduce a changed content file to the slugs that identify the page it renders. + * + * Paths are content/{intent}/{slug}/{lang}.md, content/{intent}/{lang}.md (singleton) or + * content/send-to/{dst}/from/{src}/{lang}.md (corridor). The intent segment is dropped + * when a slug follows it, since routes rename intents (`compare` → `/compare/peanut-vs-…`, + * `send-to` → `/send-money-to/…`) and matching on it would sweep in every sibling page. + * Matching on slugs alone needs no intent→route table, so there is nothing to drift. + */ +function changedSlugSets(files: string[]): string[][] { + const sets = new Map() + for (const file of files) { + const segments = file.split('/').filter(Boolean) + if (segments[0] === 'content') segments.shift() + segments.pop() // {lang}.md — every locale of a page maps to the same slugs + const slugs = (segments.length > 1 ? segments.slice(1) : segments).filter((s) => s !== 'from') + if (slugs.length > 0) sets.set(slugs.join('/'), slugs) + } + return [...sets.values()] +} - console.log(`Submitting ${urlList.length} URLs to IndexNow...`) +/** + * A URL is touched when one of a changed page's slugs is its leaf segment and the rest + * appear earlier in the path. Anchoring on the leaf is what keeps `help` (the singleton + * index) off every `/help/{article}`; the "rest appear earlier" half is what pins a + * corridor's {dst, src} pair to `/send-money-from/{src}/to/{dst}` alone. + */ +function urlTouchedBy(url: string, slugSets: string[][]): boolean { + const segments = new URL(url).pathname.split('/').filter(Boolean) + const leaf = segments[segments.length - 1] ?? '' + const isLeaf = (slug: string) => + leaf === slug || leaf === `peanut-vs-${slug}` || leaf === `from-${slug}` || leaf === `via-${slug}` + return slugSets.some( + (slugs) => slugs.some(isLeaf) && slugs.every((slug) => isLeaf(slug) || segments.includes(slug)) + ) +} - // IndexNow accepts up to 10,000 URLs per request - const batchSize = 10000 +/** Exits non-zero on any failure, which leaves the state file untouched so the next run retries. */ +async function submit(urls: string[]) { let failures = 0 - for (let i = 0; i < urlList.length; i += batchSize) { - const batch = urlList.slice(i, i + batchSize) - - const payload = { - host: 'peanut.me', - key: KEY, - keyLocation: `${BASE_URL}/${KEY}.txt`, - urlList: batch, - } + + for (let i = 0; i < urls.length; i += MAX_URLS_PER_REQUEST) { + const batch = urls.slice(i, i + MAX_URLS_PER_REQUEST) + const label = `Batch ${Math.floor(i / MAX_URLS_PER_REQUEST) + 1}` const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 30_000) - const res = await fetch(INDEXNOW_ENDPOINT, { - method: 'POST', - headers: { 'Content-Type': 'application/json; charset=utf-8' }, - body: JSON.stringify(payload), - signal: controller.signal, - }).finally(() => clearTimeout(timeout)) - - console.log(`Batch ${Math.floor(i / batchSize) + 1}: ${res.status} ${res.statusText} (${batch.length} URLs)`) - - if (res.status >= 400) { - const body = await res.text() - console.error(' Error:', body) + try { + const res = await fetch(INDEXNOW_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + body: JSON.stringify({ + host: 'peanut.me', + key: KEY, + keyLocation: `${PRODUCTION_ORIGIN}/${KEY}.txt`, + urlList: batch, + }), + signal: controller.signal, + }) + + console.log(`${label}: ${res.status} ${res.statusText} (${batch.length} URLs)`) + + if (res.status >= 400) { + console.error(' Error:', await res.text()) + failures++ + } + } catch (err) { + // A timeout or transport error is a failed batch, not a reason to abandon the rest. + console.error(`${label}: request failed (${batch.length} URLs) —`, err) failures++ + } finally { + clearTimeout(timeout) } } @@ -100,7 +158,57 @@ async function main() { console.error(`${failures} batch(es) failed.`) process.exit(1) } +} + +async function main() { + const cliPaths = process.argv.slice(2) + if (cliPaths.length > 0) { + const urls = cliPaths.map((p) => `${PRODUCTION_ORIGIN}${p}`) + console.log(`Submitting ${urls.length} explicitly requested URLs.`) + await submit(urls) + console.log('Done.') + return + } + + const current = await listSitemapUrls() + const previous = readState() + + if (!previous) { + console.log(`No previous submission on record — submitting all ${current.length} sitemap URLs.`) + await submit(current) + writeState(current) + console.log('Done.') + return + } + + if (process.env.INDEXNOW_FULL === 'true') { + console.log(`Full submission requested — submitting all ${current.length} sitemap URLs.`) + await submit(current) + writeState(current) + console.log('Done.') + return + } + + const known = new Set(previous.urls) + const added = current.filter((url) => !known.has(url)) + + const slugSets = changedSlugSets((process.env.INDEXNOW_CHANGED_FILES || '').split('\n').filter(Boolean)) + const isAdded = new Set(added) + const updated = slugSets.length > 0 ? current.filter((url) => !isAdded.has(url) && urlTouchedBy(url, slugSets)) : [] + + const urls = [...added, ...updated] + console.log( + `Sitemap has ${current.length} URLs: ${added.length} new, ${updated.length} touched by ${slugSets.length} changed page(s).` + ) + + if (urls.length === 0) { + console.log('Nothing changed — skipping IndexNow submission.') + writeState(current) + return + } + await submit(urls) + writeState(current) console.log('Done.') }