fix(seo/P0): preserve human loading skeleton and crawler-readable HTML

scripts/seo/public-crawl-smoke.mjs

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+
+// Zero-dependency HTTP smoke test for search engine accessibility.
+// Usage: node scripts/seo/public-crawl-smoke.mjs [https://insight.trigenys.com]
+// Preview routes must still emit production canonical URLs.
+const base = new URL(process.argv[2] || process.env.SEO_SMOKE_BASE_URL || 'https://insight.trigenys.com')
+const canonicalHost = 'insight.trigenys.com'
+const crawlerAgent = 'OAI-SearchBot/1.0 (+https://platform.openai.com/docs/bots)'
+const failures = []
+
+function assert(condition, message) {
+  if (!condition) {
+    failures.push(message)
+    console.error(`FAIL ${message}`)
+  } else {
+    console.log(`PASS ${message}`)
+  }
+}
+
+async function request(path, userAgent = crawlerAgent) {
+  const url = new URL(path, base)
+  const response = await fetch(url, {
+    headers: { 'User-Agent': userAgent },
+    signal: AbortSignal.timeout(20000),
+    redirect: 'follow',
+  })
+  const text = await response.text()
+  assert(response.ok, `${url.pathname}: HTTP ${response.status}`)
+  return { text, response, url }
+}
+
+function documentWithoutScripts(html) {
+  return html.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
+}
+
+function sitemapLocs(xml) {
+  return [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)]
+    .map((match) => match[1].replace(/&amp;/g, '&'))
+}
+
+async function main() {
+  const { text: robots } = await request('/robots.txt')
+  assert(robots.includes('Sitemap:') && robots.includes('insight.trigenys.com'), 'robots.txt lists canonical sitemaps')
+
+  const { text: xml } = await request('/posts-sitemap.xml')
+  assert(xml.includes('<urlset'), 'posts sitemap returns a URL set')
+  const articleURLs = sitemapLocs(xml)
+  assert(articleURLs.length > 0, 'posts sitemap contains at least one article')
+  assert(
+    articleURLs.every((url) => {
+      try {
+        return new URL(url).host === canonicalHost && new URL(url).protocol === 'https:'
+      } catch {
+        return false
+      }
+    }),
+    'all article sitemap URLs point to the canonical HTTPS host',
+  )
+
+  const { text: newsXML } = await request('/news-sitemap.xml')
+  assert(newsXML.includes('<urlset'), 'News sitemap returns a URL set')
+  assert(
+    sitemapLocs(newsXML).every((url) => new URL(url).host === canonicalHost),
+    'News sitemap URLs point to the canonical host',
+  )
+
+  for (const locale of ['fr', 'en']) {
+    const { text: homeHTML } = await request(`/${locale}`)
+    const visibleHome = documentWithoutScripts(homeHTML)
+    assert(/<h1\b/i.test(visibleHome), `${locale} homepage has a server-rendered H1`)
+    assert(!/class="site-skeleton(?:\s|")/i.test(visibleHome), `${locale} crawler does not receive global placeholder markup`)
+
+    const article = articleURLs.find((url) => new URL(url).pathname.startsWith(`/${locale}/posts/`))
+    if (!article) {
+      console.log(`SKIP ${locale} article check: no localized article in sitemap`)
+      continue
+    }
+
+    const { text: articleHTML } = await request(new URL(article).pathname)
+    const visibleArticle = documentWithoutScripts(articleHTML)
+    assert(/<article\b/i.test(visibleArticle), `${locale} article body is in the HTML response`)
+    assert(/article-reading-main/.test(visibleArticle), `${locale} article reading content is in the HTML response`)
+    assert(/<h1\b/i.test(visibleArticle), `${locale} article heading is in the HTML response`)
+    assert(!/class="site-skeleton(?:\s|")/i.test(visibleArticle), `${locale} crawler does not receive global placeholder markup`)
+    assert(/rel="canonical"/.test(articleHTML) && articleHTML.includes(article), `${locale} article carries its canonical URL`)
+  }
+}
+
+try {
+  await main()
+} catch (error) {
+  failures.push(String(error))
+  console.error('FAIL crawl smoke:', error)
+}
+
+console.log(`\nPublic crawl smoke: ${failures.length ? failures.length + ' failure(s)' : 'all checks passed'}`)
+process.exitCode = failures.length ? 1 : 0

src/app/(frontend)/loading.tsx

@@ -1,5 +1,19 @@
+import { headers } from 'next/headers'
+
 import { SiteSkeleton } from '@/components/insights/SiteSkeleton'
+import { isKnownEditorialCrawler } from '@/components/insights/loadingPolicy'
+
+/**
+ * The loading UI is purely presentational: every request eventually receives
+ * the same published article and metadata. Suppressing the placeholder for
+ * known HTML-reading crawlers avoids placing a fake "loading" page ahead of
+ * the editorial HTML. Human readers retain the original skeleton on both
+ * initial page load and client-side transitions.
+ */
+export default async function Loading() {
+  const userAgent = (await headers()).get('user-agent') || ''
+
+  if (isKnownEditorialCrawler(userAgent)) return null
 
-export default function Loading() {
   return <SiteSkeleton />
 }

src/components/insights/loadingPolicy.ts

@@ -0,0 +1,13 @@
+/**
+ * Presentation only, never gate or vary the underlying editorial data.
+ *
+ * Next.js also detects HTML-limited bots for metadata on its own. This small
+ * list just suppresses the loading placeholder for common crawlers, while
+ * retaining the site's original skeleton for normal browsing.
+ */
+const KNOWN_CRAWLER_USER_AGENTS =
+  /Googlebot|Google-InspectionTool|Google-PageRenderer|bingbot|BingPreview|DuckDuckBot|Slurp|YandexBot|Baiduspider|OAI-SearchBot|GPTBot|ChatGPT-User|ClaudeBot|Claude-SearchBot|PerplexityBot|facebookexternalhit|Twitterbot|LinkedInBot/i
+
+export function isKnownEditorialCrawler(userAgent: string) {
+  return KNOWN_CRAWLER_USER_AGENTS.test(userAgent)
+}

tests/unit/seoPublicCrawl.spec.ts

@@ -0,0 +1,48 @@
+import { readFile } from 'node:fs/promises'
+import path from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+import { isKnownEditorialCrawler } from '@/components/insights/loadingPolicy'
+
+describe('P0: public crawl entry points', () => {
+  it('preserves the original loading skeleton for human readers', async () => {
+    const filename = path.join(process.cwd(), 'src/app/(frontend)/loading.tsx')
+    const source = await readFile(filename, 'utf8')
+
+    expect(source).toContain('return <SiteSkeleton />')
+    expect(source).toContain('isKnownEditorialCrawler(userAgent)')
+  })
+
+  it('never varies editorial page content or auth based on crawler agent', async () => {
+    const policy = await readFile(
+      path.join(process.cwd(), 'src/components/insights/loadingPolicy.ts'),
+      'utf8',
+    )
+    expect(policy).not.toContain('notFound(')
+    expect(policy).not.toContain('redirect(')
+    expect(policy).not.toContain('payload.find(')
+  })
+
+  it('provides repeatable live crawler markup checks', async () => {
+    const script = await readFile(
+      path.join(process.cwd(), 'scripts/seo/public-crawl-smoke.mjs'),
+      'utf8',
+    )
+    expect(script).toContain('posts-sitemap.xml')
+    expect(script).toContain('article-reading-main')
+    expect(script).toContain('rel="canonical"')
+  })
+})
+
+describe('human versus crawler — loading UI only', () => {
+  it('does not suppress skeleton for real readers', () => {
+    expect(isKnownEditorialCrawler('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/154.0.0.0')).toBe(false)
+    expect(isKnownEditorialCrawler('Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) Mobile')).toBe(false)
+    expect(isKnownEditorialCrawler('')).toBe(false)
+  })
+
+  it.each(['Googlebot/2.1', 'bingbot/2.0', 'OAI-SearchBot/1.0', 'ClaudeBot/1.0', 'PerplexityBot/1.0'])(
+    'skips placeholder for known HTML crawler %s',
+    (userAgent) => expect(isKnownEditorialCrawler(userAgent)).toBe(true),
+  )
+})