diff --git a/scripts/seo/public-crawl-smoke.mjs b/scripts/seo/public-crawl-smoke.mjs new file mode 100644 index 0000000..88b07d6 --- /dev/null +++ b/scripts/seo/public-crawl-smoke.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node + +// Zero-dependency HTTP smoke test for search engine accessibility. +// Usage: node scripts/seo/public-crawl-smoke.mjs [https://insight.trigenys.com] +// Preview routes must still emit production canonical URLs. +const base = new URL(process.argv[2] || process.env.SEO_SMOKE_BASE_URL || 'https://insight.trigenys.com') +const canonicalHost = 'insight.trigenys.com' +const crawlerAgent = 'OAI-SearchBot/1.0 (+https://platform.openai.com/docs/bots)' +const failures = [] + +function assert(condition, message) { + if (!condition) { + failures.push(message) + console.error(`FAIL ${message}`) + } else { + console.log(`PASS ${message}`) + } +} + +async function request(path, userAgent = crawlerAgent) { + const url = new URL(path, base) + const response = await fetch(url, { + headers: { 'User-Agent': userAgent }, + signal: AbortSignal.timeout(20000), + redirect: 'follow', + }) + const text = await response.text() + assert(response.ok, `${url.pathname}: HTTP ${response.status}`) + return { text, response, url } +} + +function documentWithoutScripts(html) { + return html.replace(/]*>[\s\S]*?<\/script>/gi, '') +} + +function sitemapLocs(xml) { + return [...xml.matchAll(/\s*([^<]+?)\s*<\/loc>/gi)] + .map((match) => match[1].replace(/&/g, '&')) +} + +async function main() { + const { text: robots } = await request('/robots.txt') + assert(robots.includes('Sitemap:') && robots.includes('insight.trigenys.com'), 'robots.txt lists canonical sitemaps') + + const { text: xml } = await request('/posts-sitemap.xml') + assert(xml.includes(' 0, 'posts sitemap contains at least one article') + assert( + articleURLs.every((url) => { + try { + return new URL(url).host === canonicalHost && new URL(url).protocol === 'https:' + } catch { + return false + } + }), + 'all article sitemap URLs point to the canonical HTTPS host', + ) + + const { text: newsXML } = await request('/news-sitemap.xml') + assert(newsXML.includes(' new URL(url).host === canonicalHost), + 'News sitemap URLs point to the canonical host', + ) + + for (const locale of ['fr', 'en']) { + const { text: homeHTML } = await request(`/${locale}`) + const visibleHome = documentWithoutScripts(homeHTML) + assert(/ new URL(url).pathname.startsWith(`/${locale}/posts/`)) + if (!article) { + console.log(`SKIP ${locale} article check: no localized article in sitemap`) + continue + } + + const { text: articleHTML } = await request(new URL(article).pathname) + const visibleArticle = documentWithoutScripts(articleHTML) + assert(/ } diff --git a/src/components/insights/loadingPolicy.ts b/src/components/insights/loadingPolicy.ts new file mode 100644 index 0000000..a14023f --- /dev/null +++ b/src/components/insights/loadingPolicy.ts @@ -0,0 +1,13 @@ +/** + * Presentation only, never gate or vary the underlying editorial data. + * + * Next.js also detects HTML-limited bots for metadata on its own. This small + * list just suppresses the loading placeholder for common crawlers, while + * retaining the site's original skeleton for normal browsing. + */ +const KNOWN_CRAWLER_USER_AGENTS = + /Googlebot|Google-InspectionTool|Google-PageRenderer|bingbot|BingPreview|DuckDuckBot|Slurp|YandexBot|Baiduspider|OAI-SearchBot|GPTBot|ChatGPT-User|ClaudeBot|Claude-SearchBot|PerplexityBot|facebookexternalhit|Twitterbot|LinkedInBot/i + +export function isKnownEditorialCrawler(userAgent: string) { + return KNOWN_CRAWLER_USER_AGENTS.test(userAgent) +} diff --git a/tests/unit/seoPublicCrawl.spec.ts b/tests/unit/seoPublicCrawl.spec.ts new file mode 100644 index 0000000..da010f0 --- /dev/null +++ b/tests/unit/seoPublicCrawl.spec.ts @@ -0,0 +1,48 @@ +import { readFile } from 'node:fs/promises' +import path from 'node:path' + +import { describe, expect, it } from 'vitest' +import { isKnownEditorialCrawler } from '@/components/insights/loadingPolicy' + +describe('P0: public crawl entry points', () => { + it('preserves the original loading skeleton for human readers', async () => { + const filename = path.join(process.cwd(), 'src/app/(frontend)/loading.tsx') + const source = await readFile(filename, 'utf8') + + expect(source).toContain('return ') + expect(source).toContain('isKnownEditorialCrawler(userAgent)') + }) + + it('never varies editorial page content or auth based on crawler agent', async () => { + const policy = await readFile( + path.join(process.cwd(), 'src/components/insights/loadingPolicy.ts'), + 'utf8', + ) + expect(policy).not.toContain('notFound(') + expect(policy).not.toContain('redirect(') + expect(policy).not.toContain('payload.find(') + }) + + it('provides repeatable live crawler markup checks', async () => { + const script = await readFile( + path.join(process.cwd(), 'scripts/seo/public-crawl-smoke.mjs'), + 'utf8', + ) + expect(script).toContain('posts-sitemap.xml') + expect(script).toContain('article-reading-main') + expect(script).toContain('rel="canonical"') + }) +}) + +describe('human versus crawler — loading UI only', () => { + it('does not suppress skeleton for real readers', () => { + expect(isKnownEditorialCrawler('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/154.0.0.0')).toBe(false) + expect(isKnownEditorialCrawler('Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) Mobile')).toBe(false) + expect(isKnownEditorialCrawler('')).toBe(false) + }) + + it.each(['Googlebot/2.1', 'bingbot/2.0', 'OAI-SearchBot/1.0', 'ClaudeBot/1.0', 'PerplexityBot/1.0'])( + 'skips placeholder for known HTML crawler %s', + (userAgent) => expect(isKnownEditorialCrawler(userAgent)).toBe(true), + ) +})