From d992db2fceceafea3c08d2c7419bf659f9674af3 Mon Sep 17 00:00:00 2001 From: innolove-dev Date: Thu, 30 Jul 2026 17:32:02 +0100 Subject: [PATCH 1/2] fix(seo): submit only changed URLs to IndexNow, and fix the workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workflow has never succeeded: it resolves Node from a `.node-version` file that does not exist in this repo, so every production deploy since February failed at setup-node before reaching the script. 0 of 2750 runs sent a ping. Behind that, the script resubmitted all ~900 URLs on every successful production deploy (~14/day). IndexNow is for URLs that were added, updated or deleted; resubmitting an unchanged site burns the daily quota and gets the host deprioritised. - Pin node-version: '20' so the job runs at all. - Take URLs from sitemap.ts instead of a second, drifted copy of the page list — the copy was missing blog posts, stories, use-cases, withdraw, deposit rails, pricing, supported-networks and the content hub, i.e. exactly the new-article pages worth submitting (916 vs 1121 URLs). - Default to a delta: diff the sitemap's URL set against the set submitted last time (cached between runs) and map content files changed since then back to the pages they render. A typical content deploy now submits ~15 URLs instead of ~1100, and a deploy that touches no content submits none. - Skip install entirely when nothing that feeds the sitemap has changed. - workflow_dispatch gains a `full` input for a deliberate full resubmission. Rewrite sitemap URLs onto the production origin, since BASE_URL is env-dependent and a stray NEXT_PUBLIC_BASE_URL would otherwise submit preview URLs under peanut.me. --- .github/workflows/indexnow.yml | 80 +++++++++++- .gitignore | 3 + scripts/ping-indexnow.ts | 220 ++++++++++++++++++++++++--------- 3 files changed, 242 insertions(+), 61 deletions(-) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index c288800735..86b5391ccb 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -6,10 +6,20 @@ on: deployment_status: # Allow manual trigger workflow_dispatch: + inputs: + full: + description: 'Submit every sitemap URL instead of only what changed' + type: boolean + default: false permissions: contents: read +# Deploys overlap; serialise so runs can't race on the submission-state cache. +concurrency: + group: indexnow + cancel-in-progress: false + jobs: ping: runs-on: ubuntu-latest @@ -24,16 +34,84 @@ jobs: with: submodules: true token: ${{ secrets.SUBMODULE_TOKEN }} + # Deep enough to diff against the content commit we last submitted for. + fetch-depth: 100 + + - uses: actions/cache/restore@v4 + with: + path: .indexnow-state + key: indexnow-state-${{ github.sha }} + restore-keys: indexnow-state- + + - name: Resolve what changed since the last submission + id: changes + run: | + state=.indexnow-state/urls.json + content_sha=$(git -C src/content rev-parse HEAD) + echo "content_sha=$content_sha" >> "$GITHUB_OUTPUT" + + if [ ! -f "$state" ]; then + echo "No submission state cached — the script will submit the full sitemap." + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + prev_content_sha=$(jq -r '.contentSha // empty' "$state") + if [ -n "$prev_content_sha" ] && git -C src/content cat-file -e "$prev_content_sha^{commit}" 2>/dev/null; then + changed=$(git -C src/content diff --name-only "$prev_content_sha" "$content_sha") + else + echo "Previous content commit unavailable — new URLs are still detected via the sitemap diff." + changed="" + fi + + { + echo 'changed_files<> "$GITHUB_OUTPUT" + + # Everything outside the submodule that can add or remove a sitemap URL. + prev_sha=$(jq -r '.sha // empty' "$state") + if [ -n "$prev_sha" ] && git cat-file -e "$prev_sha^{commit}" 2>/dev/null; then + sources_changed=$(git diff --name-only "$prev_sha" "$GITHUB_SHA" -- \ + src/app/sitemap.ts src/data/seo src/i18n src/lib/content.ts src/lib/blog.ts) + else + echo "Previous commit outside fetch depth — running rather than risk missing new URLs." + sources_changed="unknown" + fi + + if [ -z "$changed" ] && [ -z "$sources_changed" ] && [ "$prev_content_sha" = "$content_sha" ] \ + && [ "${{ inputs.full }}" != "true" ]; then + echo 'Nothing that affects the sitemap changed since the last submission — skipping.' + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi - uses: pnpm/action-setup@7088e561eb65bb68695d245aa206f005ef30921d # v4.1.0 + if: steps.changes.outputs.run == 'true' + - uses: actions/setup-node@v4 + if: steps.changes.outputs.run == 'true' with: - node-version-file: '.node-version' + node-version: '20' cache: 'pnpm' - run: pnpm install --frozen-lockfile + if: steps.changes.outputs.run == 'true' - name: Ping IndexNow + if: steps.changes.outputs.run == 'true' env: + # Not a secret by design — IndexNow requires it to be served at public/.txt. INDEXNOW_KEY: '054e10e6239a45cb2d06e92d669f5b6f' + INDEXNOW_FULL: ${{ inputs.full }} + INDEXNOW_CHANGED_FILES: ${{ steps.changes.outputs.changed_files }} + INDEXNOW_CONTENT_SHA: ${{ steps.changes.outputs.content_sha }} run: pnpm tsx scripts/ping-indexnow.ts + + - uses: actions/cache/save@v4 + if: steps.changes.outputs.run == 'true' + with: + path: .indexnow-state + key: indexnow-state-${{ github.sha }} diff --git a/.gitignore b/.gitignore index 70d2ea2ed6..09967530f8 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ coverage/ # See .github/workflows/tests.yml — `Fetch BE render-snapshot baseline` step. # Never commit; the canonical 49-entry baseline lives in the BE repo. src/components/TransactionDetails/__tests__/fixtures/be-entries.json + +# IndexNow submission state, restored from the Actions cache (see .github/workflows/indexnow.yml). +.indexnow-state/ diff --git a/scripts/ping-indexnow.ts b/scripts/ping-indexnow.ts index f0d0e0c1bd..93141207d7 100644 --- a/scripts/ping-indexnow.ts +++ b/scripts/ping-indexnow.ts @@ -1,97 +1,147 @@ /** - * Pings IndexNow (Bing, Yandex, etc.) with all sitemap URLs. + * Submits URLs to IndexNow (Bing, Yandex, Seznam, Naver). + * + * IndexNow is for URLs that were added, updated or deleted — resubmitting the whole + * site on every deploy burns the daily quota and gets the host deprioritised. So the + * default mode is a delta: the sitemap's URL set is diffed against the set submitted + * last time, and content files changed since then are mapped back to the pages they + * render. * * Usage: - * INDEXNOW_KEY=your-key-here tsx scripts/ping-indexnow.ts + * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts # delta vs. previous run + * INDEXNOW_KEY=xxx INDEXNOW_FULL=true tsx … # every sitemap URL + * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts /en/brazil # explicit paths * - * Or pass specific paths: - * INDEXNOW_KEY=xxx tsx scripts/ping-indexnow.ts /en/argentina /en/brazil + * Env: + * INDEXNOW_KEY required; must match public/.txt + * INDEXNOW_FULL 'true' to submit the full sitemap + * INDEXNOW_CHANGED_FILES newline-separated paths, relative to the content submodule + * INDEXNOW_CONTENT_SHA content submodule commit, recorded for the next run's diff + * GITHUB_SHA superproject commit, recorded for the next run's diff + * INDEXNOW_STATE_FILE defaults to .indexnow-state/urls.json */ -const BASE_URL = 'https://peanut.me' +import fs from 'fs' +import path from 'path' +import generateSitemap from '../src/app/sitemap' +import { BASE_URL } from '../src/constants/general.consts' + +const PRODUCTION_ORIGIN = 'https://peanut.me' const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/IndexNow' +const MAX_URLS_PER_REQUEST = 10_000 + const KEY = process.env.INDEXNOW_KEY +const STATE_FILE = process.env.INDEXNOW_STATE_FILE || path.join(process.cwd(), '.indexnow-state/urls.json') if (!KEY) { console.error('INDEXNOW_KEY environment variable is required') process.exit(1) } -// If CLI args provided, use those. Otherwise build full URL list. -const cliPaths = process.argv.slice(2) - -async function getAllPaths(): Promise { - // Dynamic import to reuse the same data sources as sitemap.ts - const { COUNTRIES_SEO, CORRIDORS, COMPETITORS, EXCHANGES, PAYMENT_METHOD_SLUGS } = - await import('../src/data/seo/index') - const { SUPPORTED_LOCALES } = await import('../src/i18n/types') - const { listContentSlugs } = await import('../src/lib/content') +interface State { + /** Commits the last submission accounted for, so the next run knows what to diff against. */ + sha?: string + contentSha?: string + submittedAt?: string + urls: string[] +} - const paths: string[] = ['/', '/lp/card', '/careers', '/exchange', '/privacy', '/terms'] +function readState(): State | null { + try { + const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')) as State + return Array.isArray(parsed.urls) ? parsed : null + } catch { + return null + } +} - for (const locale of SUPPORTED_LOCALES) { - for (const country of Object.keys(COUNTRIES_SEO)) { - paths.push(`/${locale}/${country}`) - paths.push(`/${locale}/send-money-to/${country}`) - } - for (const corridor of CORRIDORS) { - paths.push(`/${locale}/send-money-from/${corridor.from}/to/${corridor.to}`) - } - const receiveSources = [...new Set(CORRIDORS.map((c: { from: string }) => c.from))] - for (const source of receiveSources) { - paths.push(`/${locale}/receive-money-from/${source}`) - } - for (const slug of Object.keys(COMPETITORS)) { - paths.push(`/${locale}/compare/peanut-vs-${slug}`) - } - for (const slug of Object.keys(EXCHANGES)) { - paths.push(`/${locale}/deposit/from-${slug}`) - } - for (const method of PAYMENT_METHOD_SLUGS) { - paths.push(`/${locale}/pay-with/${method}`) - } - paths.push(`/${locale}/help`) - for (const slug of listContentSlugs('help')) { - paths.push(`/${locale}/help/${slug}`) - } +function writeState(urls: string[]) { + const state: State = { + sha: process.env.GITHUB_SHA || undefined, + contentSha: process.env.INDEXNOW_CONTENT_SHA || undefined, + submittedAt: new Date().toISOString(), + urls, } + fs.mkdirSync(path.dirname(STATE_FILE), { recursive: true }) + fs.writeFileSync(STATE_FILE, JSON.stringify(state)) +} - return paths +/** + * The sitemap is the single source of truth for which URLs exist. It is built with + * BASE_URL, which is env-dependent — rewrite onto the production origin so a stray + * NEXT_PUBLIC_BASE_URL can never make us submit preview URLs for peanut.me. + */ +async function listSitemapUrls(): Promise { + const entries = await generateSitemap() + const urls = entries.map((entry) => + entry.url.startsWith(BASE_URL) ? `${PRODUCTION_ORIGIN}${entry.url.slice(BASE_URL.length)}` : entry.url + ) + return [...new Set(urls)] } -async function main() { - const paths = cliPaths.length > 0 ? cliPaths : await getAllPaths() - const urlList = paths.map((p) => `${BASE_URL}${p}`) +/** + * Reduce a changed content file to the slugs that identify the page it renders. + * + * Paths are content/{intent}/{slug}/{lang}.md, content/{intent}/{lang}.md (singleton) or + * content/send-to/{dst}/from/{src}/{lang}.md (corridor). The intent segment is dropped + * when a slug follows it, since routes rename intents (`compare` → `/compare/peanut-vs-…`, + * `send-to` → `/send-money-to/…`) and matching on it would sweep in every sibling page. + * Matching on slugs alone needs no intent→route table, so there is nothing to drift. + */ +function changedSlugSets(files: string[]): string[][] { + const sets = new Map() + for (const file of files) { + const segments = file.split('/').filter(Boolean) + if (segments[0] === 'content') segments.shift() + segments.pop() // {lang}.md — every locale of a page maps to the same slugs + const slugs = (segments.length > 1 ? segments.slice(1) : segments).filter((s) => s !== 'from') + if (slugs.length > 0) sets.set(slugs.join('/'), slugs) + } + return [...sets.values()] +} - console.log(`Submitting ${urlList.length} URLs to IndexNow...`) +/** + * A URL is touched when one of a changed page's slugs is its leaf segment and the rest + * appear earlier in the path. Anchoring on the leaf is what keeps `help` (the singleton + * index) off every `/help/{article}`; the "rest appear earlier" half is what pins a + * corridor's {dst, src} pair to `/send-money-from/{src}/to/{dst}` alone. + */ +function urlTouchedBy(url: string, slugSets: string[][]): boolean { + const segments = new URL(url).pathname.split('/').filter(Boolean) + const leaf = segments[segments.length - 1] ?? '' + const isLeaf = (slug: string) => + leaf === slug || leaf === `peanut-vs-${slug}` || leaf === `from-${slug}` || leaf === `via-${slug}` + return slugSets.some( + (slugs) => slugs.some(isLeaf) && slugs.every((slug) => isLeaf(slug) || segments.includes(slug)) + ) +} - // IndexNow accepts up to 10,000 URLs per request - const batchSize = 10000 +async function submit(urls: string[]) { let failures = 0 - for (let i = 0; i < urlList.length; i += batchSize) { - const batch = urlList.slice(i, i + batchSize) - - const payload = { - host: 'peanut.me', - key: KEY, - keyLocation: `${BASE_URL}/${KEY}.txt`, - urlList: batch, - } + + for (let i = 0; i < urls.length; i += MAX_URLS_PER_REQUEST) { + const batch = urls.slice(i, i + MAX_URLS_PER_REQUEST) const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 30_000) const res = await fetch(INDEXNOW_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json; charset=utf-8' }, - body: JSON.stringify(payload), + body: JSON.stringify({ + host: 'peanut.me', + key: KEY, + keyLocation: `${PRODUCTION_ORIGIN}/${KEY}.txt`, + urlList: batch, + }), signal: controller.signal, }).finally(() => clearTimeout(timeout)) - console.log(`Batch ${Math.floor(i / batchSize) + 1}: ${res.status} ${res.statusText} (${batch.length} URLs)`) + console.log( + `Batch ${Math.floor(i / MAX_URLS_PER_REQUEST) + 1}: ${res.status} ${res.statusText} (${batch.length} URLs)` + ) if (res.status >= 400) { - const body = await res.text() - console.error(' Error:', body) + console.error(' Error:', await res.text()) failures++ } } @@ -100,7 +150,57 @@ async function main() { console.error(`${failures} batch(es) failed.`) process.exit(1) } +} + +async function main() { + const cliPaths = process.argv.slice(2) + if (cliPaths.length > 0) { + const urls = cliPaths.map((p) => `${PRODUCTION_ORIGIN}${p}`) + console.log(`Submitting ${urls.length} explicitly requested URLs.`) + await submit(urls) + console.log('Done.') + return + } + + const current = await listSitemapUrls() + const previous = readState() + + if (!previous) { + console.log(`No previous submission on record — submitting all ${current.length} sitemap URLs.`) + await submit(current) + writeState(current) + console.log('Done.') + return + } + + if (process.env.INDEXNOW_FULL === 'true') { + console.log(`Full submission requested — submitting all ${current.length} sitemap URLs.`) + await submit(current) + writeState(current) + console.log('Done.') + return + } + + const known = new Set(previous.urls) + const added = current.filter((url) => !known.has(url)) + + const slugSets = changedSlugSets((process.env.INDEXNOW_CHANGED_FILES || '').split('\n').filter(Boolean)) + const isAdded = new Set(added) + const updated = slugSets.length > 0 ? current.filter((url) => !isAdded.has(url) && urlTouchedBy(url, slugSets)) : [] + + const urls = [...added, ...updated] + console.log( + `Sitemap has ${current.length} URLs: ${added.length} new, ${updated.length} touched by ${slugSets.length} changed page(s).` + ) + + if (urls.length === 0) { + console.log('Nothing changed — skipping IndexNow submission.') + writeState(current) + return + } + await submit(urls) + writeState(current) console.log('Done.') } From c7d84d64ea0b10f2b5930d7c474442f35a89435b Mon Sep 17 00:00:00 2001 From: innolove-dev Date: Thu, 30 Jul 2026 17:55:58 +0100 Subject: [PATCH 2/2] fix(seo): harden IndexNow submission against transport errors and stale state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from CodeRabbit review on #2590. A rejected fetch — DNS failure, connection reset, or the 30s abort firing — propagated out of submit() and aborted the run mid-loop, skipping the remaining batches with a stack trace instead of a counted failure. Catch it, count it like an HTTP error, and keep going; the non-zero exit still leaves the state file untouched so the next run retries. When the content commit from the last submission could not be resolved, the diff came back empty and edits to existing pages were silently dropped (new pages were still caught by the sitemap diff). Unshallow the content submodule so the commit is almost always reachable, and fall back to a full submission when it still is not, rather than skipping. --- .github/workflows/indexnow.yml | 17 ++++++++++--- scripts/ping-indexnow.ts | 44 ++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index 86b5391ccb..c4a0c71899 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -56,13 +56,24 @@ jobs: exit 0 fi + # checkout clones submodules at the superproject's depth; the commit we last + # submitted for can sit outside it, so take the whole content history. + if [ "$(git -C src/content rev-parse --is-shallow-repository)" = 'true' ]; then + git -C src/content fetch --unshallow --quiet || true + fi + prev_content_sha=$(jq -r '.contentSha // empty' "$state") if [ -n "$prev_content_sha" ] && git -C src/content cat-file -e "$prev_content_sha^{commit}" 2>/dev/null; then changed=$(git -C src/content diff --name-only "$prev_content_sha" "$content_sha") + force_full=false else - echo "Previous content commit unavailable — new URLs are still detected via the sitemap diff." + # Without the diff we cannot tell which existing pages were edited, and the + # sitemap diff only surfaces new URLs. Resubmit everything rather than skip. + echo "Previous content commit unavailable — falling back to a full submission." changed="" + force_full=true fi + echo "force_full=$force_full" >> "$GITHUB_OUTPUT" { echo 'changed_files<> "$GITHUB_OUTPUT" else @@ -105,7 +116,7 @@ jobs: env: # Not a secret by design — IndexNow requires it to be served at public/.txt. INDEXNOW_KEY: '054e10e6239a45cb2d06e92d669f5b6f' - INDEXNOW_FULL: ${{ inputs.full }} + INDEXNOW_FULL: ${{ inputs.full == true || steps.changes.outputs.force_full == 'true' }} INDEXNOW_CHANGED_FILES: ${{ steps.changes.outputs.changed_files }} INDEXNOW_CONTENT_SHA: ${{ steps.changes.outputs.content_sha }} run: pnpm tsx scripts/ping-indexnow.ts diff --git a/scripts/ping-indexnow.ts b/scripts/ping-indexnow.ts index 93141207d7..9b737c85c0 100644 --- a/scripts/ping-indexnow.ts +++ b/scripts/ping-indexnow.ts @@ -116,33 +116,41 @@ function urlTouchedBy(url: string, slugSets: string[][]): boolean { ) } +/** Exits non-zero on any failure, which leaves the state file untouched so the next run retries. */ async function submit(urls: string[]) { let failures = 0 for (let i = 0; i < urls.length; i += MAX_URLS_PER_REQUEST) { const batch = urls.slice(i, i + MAX_URLS_PER_REQUEST) + const label = `Batch ${Math.floor(i / MAX_URLS_PER_REQUEST) + 1}` const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 30_000) - const res = await fetch(INDEXNOW_ENDPOINT, { - method: 'POST', - headers: { 'Content-Type': 'application/json; charset=utf-8' }, - body: JSON.stringify({ - host: 'peanut.me', - key: KEY, - keyLocation: `${PRODUCTION_ORIGIN}/${KEY}.txt`, - urlList: batch, - }), - signal: controller.signal, - }).finally(() => clearTimeout(timeout)) - - console.log( - `Batch ${Math.floor(i / MAX_URLS_PER_REQUEST) + 1}: ${res.status} ${res.statusText} (${batch.length} URLs)` - ) - - if (res.status >= 400) { - console.error(' Error:', await res.text()) + try { + const res = await fetch(INDEXNOW_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + body: JSON.stringify({ + host: 'peanut.me', + key: KEY, + keyLocation: `${PRODUCTION_ORIGIN}/${KEY}.txt`, + urlList: batch, + }), + signal: controller.signal, + }) + + console.log(`${label}: ${res.status} ${res.statusText} (${batch.length} URLs)`) + + if (res.status >= 400) { + console.error(' Error:', await res.text()) + failures++ + } + } catch (err) { + // A timeout or transport error is a failed batch, not a reason to abandon the rest. + console.error(`${label}: request failed (${batch.length} URLs) —`, err) failures++ + } finally { + clearTimeout(timeout) } }