diff --git a/README.md b/README.md index 0cba792..d0cf92a 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ A compact Node.js application that audits a public webpage or pasted HTML, expla - Edits draft metadata with instant preview updates and generated `` tags. - Exposes the same URL analyzer through a versioned JSON response. - Copies shareable audit links and exports JSON or Markdown reports without storage. +- Audits up to 10 same-origin sitemap pages for scores, failures, and duplicate metadata. +- Validates JSON-LD syntax and reports detected Schema.org-style `@type` values. - Lists discovered Open Graph and Twitter metadata for inspection. - Resolves relative preview URLs against the audited page. - Returns clear errors for invalid, unreachable, oversized, slow, or non-HTML targets. @@ -26,6 +28,8 @@ Because the application fetches user-provided URLs, outbound requests are treate - Image requests reuse the SSRF boundary with a 7-second timeout and 5 MB limit. - Preview images are loaded through a bounded same-origin route instead of directly from metadata-controlled hosts. - Pasted HTML is processed in memory and capped at 256 KB. +- Sitemaps use the same SSRF-safe 2 MB fetch boundary and may include up to three same-origin child sitemaps. +- Site audits fetch at most 10 same-origin pages with three concurrent requests. - Non-HTML responses are rejected. - Metadata and URL attributes are escaped or sanitized before rendering. - Helmet security headers, a Content Security Policy, and per-IP rate limiting are enabled. @@ -85,6 +89,14 @@ GET /report.md?url=https%3A%2F%2Fexample.com URL results also include controls to copy the reproducible audit URL, open the JSON export, or download the Markdown report. Pasted HTML stays local to the request and therefore does not expose share/export controls. +Audit a sitemap through the UI at `/site?url=...` or as JSON: + +```http +GET /api/site?url=https%3A%2F%2Fexample.com%2Fsitemap.xml +``` + +The site response includes discovery limits, success/failure totals, average score, duplicate titles/descriptions/canonicals, structured-data summaries, and page-level checks. Sitemap indexes are followed one level deep, with at most three same-origin child sitemaps. + If the application runs behind a reverse proxy, set the number of trusted proxy hops so rate limiting uses the client address safely: ```bash @@ -114,6 +126,8 @@ The test suite covers: - Slack and WhatsApp preview rendering - URL and pasted-HTML analysis modes - Stable JSON analysis and Markdown report exports +- Bounded sitemap parsing, aggregation, and same-origin enforcement +- JSON-LD syntax and `@type` summaries - Deterministic audit rules and scoring - Bounded social-image inspection and safe same-origin previews - Generated metadata-tag escaping and unsafe URL omission @@ -130,7 +144,8 @@ The test suite covers: │ ├── image-inspector.js # Bounded social-image validation │ ├── metadata.js # Normalized metadata parsing │ ├── report.js # Public JSON and Markdown serializers -│ └── safe-fetch.js # URL validation and bounded page fetching +│ ├── safe-fetch.js # URL validation and bounded page fetching +│ └── site-audit.js # Bounded sitemap and cross-page analysis ├── public/ │ ├── app.js # Input tabs and live draft previews │ ├── tag-generator.js # Shared browser/server tag generation @@ -155,6 +170,8 @@ The previous Heroku demo is no longer linked because it currently returns 404. - Public websites may block automated requests or return different metadata based on geography or user agent. - Draft edits update previews and generated tags but do not recalculate the source-page audit. - Share links and exports rerun the live URL analysis; results can change when the target page changes. +- Site audits intentionally stop after 10 pages and one sitemap-index level; they are not a general-purpose crawler. +- Structured-data checks validate JSON-LD syntax and declared types, not full Schema.org vocabulary requirements. ## License diff --git a/lib/audit.js b/lib/audit.js index 6b372a5..f8c8ed0 100644 --- a/lib/audit.js +++ b/lib/audit.js @@ -191,6 +191,19 @@ function analyzeMetadata(metadata, targetUrl) { checks.push(socialField("twitter-description", "twitter", "twitter:description", metadata.twitter.description, metadata.openGraph.description)); checks.push(socialField("twitter-image", "twitter", "twitter:image", metadata.twitter.image.raw, metadata.openGraph.image.raw)); + const structuredData = metadata.structuredData || []; + const invalidStructuredData = structuredData.filter((item) => !item.valid).length; + const structuredTypes = [...new Set(structuredData.flatMap((item) => item.types || []))]; + checks.push( + !structuredData.length + ? result("structured-data", "structured-data", "info", "No JSON-LD structured data was found.") + : invalidStructuredData + ? result("structured-data", "structured-data", "error", `${invalidStructuredData} invalid JSON-LD block${invalidStructuredData === 1 ? " was" : "s were"} found.`) + : !structuredTypes.length + ? result("structured-data", "structured-data", "warning", "JSON-LD is valid but does not declare an @type.") + : result("structured-data", "structured-data", "pass", "JSON-LD structured data is valid.", structuredTypes.join(", ")), + ); + return { checks, score: summarizeChecks(checks), diff --git a/lib/metadata.js b/lib/metadata.js index 77b5b3c..c2d8eda 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -20,6 +20,46 @@ function urlValue(raw, baseUrl) { return { raw, url: safeWebUrl(raw, baseUrl) }; } +function structuredDataSummary($) { + return $('script[type="application/ld+json"]') + .map((_index, element) => { + try { + const value = JSON.parse($(element).text()); + const types = new Set(); + const queue = [value]; + let cursor = 0; + const enqueue = (values) => { + for (const child of values) { + if (queue.length >= 10_000) break; + queue.push(child); + } + }; + while (cursor < queue.length && cursor < 10_000 && types.size < 20) { + const node = queue[cursor]; + cursor += 1; + if (!node) continue; + if (Array.isArray(node)) { + enqueue(node); + continue; + } + if (typeof node !== "object") continue; + const nodeTypes = Array.isArray(node["@type"]) + ? node["@type"] + : [node["@type"]]; + nodeTypes.filter((type) => typeof type === "string" && type.trim()).forEach((type) => { + const normalizedType = type.trim(); + if (types.size < 20 && normalizedType.length <= 100) types.add(normalizedType); + }); + enqueue(Object.values(node)); + } + return { valid: true, types: [...types] }; + } catch { + return { valid: false, types: [], error: "Invalid JSON-LD." }; + } + }) + .get(); +} + function parseMetadata(html, baseUrl) { const $ = cheerio.load(html || ""); const canonicalRaw = ($('link[rel="canonical"]').first().attr("href") || "").trim(); @@ -69,6 +109,7 @@ function parseMetadata(html, baseUrl) { description: metaContent($, 'meta[name="twitter:description"]'), image: urlValue(twitterImageRaw, baseUrl), }, + structuredData: structuredDataSummary($), socialTags, }; } diff --git a/lib/safe-fetch.js b/lib/safe-fetch.js index 6b0c60a..c3ff4ac 100644 --- a/lib/safe-fetch.js +++ b/lib/safe-fetch.js @@ -101,17 +101,23 @@ function validateRedirect(options) { ); } -async function fetchPage( - value, - { client = axios, lookup = dns.lookup } = {}, -) { +async function fetchTextResource(value, { + accept, + allowedContentTypes, + client = axios, + fetchMessage, + invalidTypeMessage, + lookup = dns.lookup, + sizeMessage, + timeoutMessage, +}) { const url = normalizeTargetUrl(value); try { const response = await client.get(url, { beforeRedirect: validateRedirect, headers: { - Accept: "text/html,application/xhtml+xml", + Accept: accept, "User-Agent": "OGPreview/1.0 (+https://github.com/alianjum0/ogpreview-node)", }, lookup: createSafeLookup(lookup), @@ -127,8 +133,8 @@ async function fetchPage( .split(";", 1)[0] .trim() .toLowerCase(); - if (!["text/html", "application/xhtml+xml"].includes(contentType)) { - throw new TargetUrlError("The target URL did not return an HTML page."); + if (!allowedContentTypes.includes(contentType)) { + throw new TargetUrlError(invalidTypeMessage); } return response.data; @@ -136,23 +142,48 @@ async function fetchPage( if (error instanceof TargetUrlError) throw error; if (error.cause instanceof TargetUrlError) throw error.cause; if (error.code === "ECONNABORTED") { - throw new FetchPageError("The target website took too long to respond."); + throw new FetchPageError(timeoutMessage); } if ( error.code === "ERR_BAD_RESPONSE" && error.message.includes("maxContentLength") ) { - throw new FetchPageError("The target page is larger than 2 MB."); + throw new FetchPageError(sizeMessage); } - throw new FetchPageError("The target website could not be fetched."); + throw new FetchPageError(fetchMessage); } } +async function fetchPage(value, options = {}) { + return fetchTextResource(value, { + accept: "text/html,application/xhtml+xml", + allowedContentTypes: ["text/html", "application/xhtml+xml"], + fetchMessage: "The target website could not be fetched.", + invalidTypeMessage: "The target URL did not return an HTML page.", + sizeMessage: "The target page is larger than 2 MB.", + timeoutMessage: "The target website took too long to respond.", + ...options, + }); +} + +async function fetchSitemap(value, options = {}) { + return fetchTextResource(value, { + accept: "application/xml,text/xml,text/plain", + allowedContentTypes: ["application/xml", "text/xml", "text/plain"], + fetchMessage: "The sitemap could not be fetched.", + invalidTypeMessage: "The target URL did not return an XML sitemap.", + sizeMessage: "The sitemap is larger than 2 MB.", + timeoutMessage: "The sitemap took too long to respond.", + ...options, + }); +} + module.exports = { FetchPageError, TargetUrlError, createSafeLookup, fetchPage, + fetchSitemap, isBlockedAddress, normalizeTargetUrl, validateRedirect, diff --git a/lib/site-audit.js b/lib/site-audit.js new file mode 100644 index 0000000..c525896 --- /dev/null +++ b/lib/site-audit.js @@ -0,0 +1,190 @@ +const cheerio = require("cheerio"); +const { analyzeHtml } = require("./analyze"); +const { TargetUrlError, normalizeTargetUrl } = require("./safe-fetch"); + +const DEFAULT_MAX_PAGES = 10; +const DEFAULT_MAX_SITEMAPS = 3; +const DEFAULT_CONCURRENCY = 3; + +function parseSitemap(xml, sitemapUrl) { + const $ = cheerio.load(xml || "", { xmlMode: true }); + const type = $("urlset").length ? "urlset" : $("sitemapindex").length ? "index" : ""; + if (!type) throw new TargetUrlError("The target URL did not contain a valid sitemap."); + const selector = type === "urlset" ? "url > loc" : "sitemap > loc"; + const locations = $(selector) + .map((_index, element) => { + try { + const url = new URL($(element).text().trim(), sitemapUrl); + return ["http:", "https:"].includes(url.protocol) ? url.toString() : null; + } catch { + return null; + } + }) + .get() + .filter(Boolean); + return { type, locations }; +} + +function sameOriginLocations(locations, origin) { + const accepted = []; + let externalSkipped = 0; + let invalidSkipped = 0; + for (const value of locations) { + try { + const normalized = normalizeTargetUrl(value); + if (new URL(normalized).origin !== origin) { + externalSkipped += 1; + } else { + accepted.push(normalized); + } + } catch { + invalidSkipped += 1; + } + } + return { accepted, externalSkipped, invalidSkipped }; +} + +async function mapWithConcurrency(items, concurrency, mapper) { + const results = new Array(items.length); + let cursor = 0; + const worker = async () => { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + results[index] = await mapper(items[index], index); + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker)); + return results; +} + +function duplicateGroups(pages, field) { + const groups = new Map(); + for (const page of pages.filter((item) => item.status === "ok")) { + const value = page.metadata[field]; + if (!value) continue; + const key = value.trim().toLowerCase(); + if (!groups.has(key)) groups.set(key, { value, urls: [] }); + groups.get(key).urls.push(page.url); + } + return [...groups.values()].filter((group) => group.urls.length > 1); +} + +function pageResult(url, analysis) { + const structuredData = analysis.metadata.structuredData || []; + return { + url, + status: "ok", + score: analysis.score, + metadata: { + title: analysis.metadata.title, + description: analysis.metadata.description, + canonical: analysis.metadata.canonical.url || analysis.metadata.canonical.raw, + openGraphImage: analysis.metadata.openGraph.image.url || analysis.metadata.openGraph.image.raw, + twitterCard: analysis.metadata.twitter.card, + }, + structuredData: { + blocks: structuredData.length, + valid: structuredData.filter((item) => item.valid).length, + invalid: structuredData.filter((item) => !item.valid).length, + types: [...new Set(structuredData.flatMap((item) => item.types || []))], + }, + checks: analysis.checks, + }; +} + +async function auditSite(sitemapValue, { + concurrency = DEFAULT_CONCURRENCY, + maxPages = DEFAULT_MAX_PAGES, + maxSitemaps = DEFAULT_MAX_SITEMAPS, + pageFetcher, + sitemapFetcher, +} = {}) { + if (typeof sitemapFetcher !== "function" || typeof pageFetcher !== "function") { + throw new TypeError("Site audit fetchers are required."); + } + const sitemapUrl = normalizeTargetUrl(sitemapValue); + const origin = new URL(sitemapUrl).origin; + const root = parseSitemap(await sitemapFetcher(sitemapUrl), sitemapUrl); + let pageLocations = root.type === "urlset" ? root.locations : []; + let externalSkipped = 0; + let invalidSkipped = 0; + let childSitemaps = 0; + let childSitemapFailures = 0; + + if (root.type === "index") { + const filtered = sameOriginLocations(root.locations, origin); + externalSkipped += filtered.externalSkipped; + invalidSkipped += filtered.invalidSkipped; + const sitemapUrls = [...new Set(filtered.accepted)].slice(0, maxSitemaps); + childSitemaps = sitemapUrls.length; + for (const childUrl of sitemapUrls) { + try { + const child = parseSitemap(await sitemapFetcher(childUrl), childUrl); + if (child.type === "urlset") pageLocations.push(...child.locations); + } catch { + childSitemapFailures += 1; + } + } + } + + const filteredPages = sameOriginLocations(pageLocations, origin); + externalSkipped += filteredPages.externalSkipped; + invalidSkipped += filteredPages.invalidSkipped; + const discoveredPages = [...new Set(filteredPages.accepted)]; + if (!discoveredPages.length) { + throw new TargetUrlError("The sitemap did not contain any same-origin page URLs."); + } + const selectedPages = discoveredPages.slice(0, maxPages); + const pages = await mapWithConcurrency(selectedPages, concurrency, async (url) => { + try { + return pageResult(url, analyzeHtml(await pageFetcher(url), url)); + } catch (error) { + return { + url, + status: "error", + error: Number.isInteger(error.statusCode) + ? error.message + : "The page could not be analyzed.", + }; + } + }); + const successfulPages = pages.filter((page) => page.status === "ok"); + const averageScore = successfulPages.length + ? Math.round(successfulPages.reduce((total, page) => total + page.score.value, 0) / successfulPages.length) + : null; + + return { + schemaVersion: 1, + source: { sitemapUrl, sitemapType: root.type }, + limits: { maxPages, maxSitemaps, concurrency }, + discovery: { + discovered: discoveredPages.length, + truncated: Math.max(0, discoveredPages.length - selectedPages.length), + externalSkipped, + invalidSkipped, + childSitemaps, + childSitemapFailures, + }, + summary: { + total: pages.length, + succeeded: successfulPages.length, + failed: pages.length - successfulPages.length, + averageScore, + }, + duplicates: { + titles: duplicateGroups(pages, "title"), + descriptions: duplicateGroups(pages, "description"), + canonicals: duplicateGroups(pages, "canonical"), + }, + pages, + }; +} + +module.exports = { + DEFAULT_CONCURRENCY, + DEFAULT_MAX_PAGES, + DEFAULT_MAX_SITEMAPS, + auditSite, + parseSitemap, +}; diff --git a/package.json b/package.json index 31f9287..4c6ad48 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "A Node.js application that performs a basic SEO audit and displays social media previews based on a website's meta tags.", "main": "server.js", "scripts": { - "check:syntax": "node --check server.js && node --check lib/analyze.js && node --check lib/audit.js && node --check lib/image-inspector.js && node --check lib/metadata.js && node --check lib/report.js && node --check lib/safe-fetch.js && node --check public/app.js && node --check public/tag-generator.js", + "check:syntax": "node --check server.js && node --check lib/analyze.js && node --check lib/audit.js && node --check lib/image-inspector.js && node --check lib/metadata.js && node --check lib/report.js && node --check lib/safe-fetch.js && node --check lib/site-audit.js && node --check public/app.js && node --check public/tag-generator.js", "start": "node server.js", "test": "node --test", "test:ui": "node --test test/app.test.js", diff --git a/public/styles.css b/public/styles.css index 1281429..9808cb0 100644 --- a/public/styles.css +++ b/public/styles.css @@ -1003,6 +1003,87 @@ summary:focus-visible { margin: 0; } +.site-results { + padding: 10px 0 88px; +} + +.site-table { + overflow-x: auto; + border-radius: 10px; +} + +.site-table table { + width: 100%; + min-width: 840px; + border-collapse: collapse; +} + +.site-table th, +.site-table td { + padding: 14px 16px; + border-bottom: 1px solid var(--line); + font-size: 0.76rem; + text-align: left; + vertical-align: top; +} + +.site-table th { + color: var(--muted); + background: #f8f8fb; + font-family: var(--mono); + font-size: 0.66rem; + text-transform: uppercase; +} + +.site-table a { + display: block; + max-width: 280px; + overflow: hidden; + color: var(--violet-dark); + text-overflow: ellipsis; + white-space: nowrap; +} + +.duplicate-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 18px; + padding: 22px; + border-radius: 10px; +} + +.duplicate-group h3 { + margin: 0 0 12px; + font-size: 0.9rem; +} + +.duplicate-group p { + color: var(--muted); + font-size: 0.75rem; +} + +.duplicate-group__item { + display: grid; + gap: 5px; + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--line); +} + +.duplicate-group__item strong, +.duplicate-group__item span { + overflow-wrap: anywhere; +} + +.duplicate-group__item strong { + font-size: 0.76rem; +} + +.duplicate-group__item span { + color: var(--muted); + font-size: 0.65rem; +} + @media (max-width: 900px) { .hero { grid-template-columns: 1fr; @@ -1026,6 +1107,10 @@ summary:focus-visible { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .duplicate-grid { + grid-template-columns: 1fr; + } + .preview-layout { grid-template-columns: repeat(2, minmax(0, 1fr)); } @@ -1189,7 +1274,7 @@ summary:focus-visible { .input-tabs { display: grid; - grid-template-columns: 1fr 1fr; + grid-template-columns: repeat(3, 1fr); gap: 4px; margin-bottom: 22px; padding: 4px; diff --git a/server.js b/server.js index 7cbb861..9bcdf80 100644 --- a/server.js +++ b/server.js @@ -6,7 +6,8 @@ const { analyzeDocument, analyzeHtml } = require("./lib/analyze"); const { fetchImage, inspectImage } = require("./lib/image-inspector"); const { safeWebUrl } = require("./lib/metadata"); const { serializeAnalysis, toMarkdownReport } = require("./lib/report"); -const { fetchPage, normalizeTargetUrl } = require("./lib/safe-fetch"); +const { fetchPage, fetchSitemap, normalizeTargetUrl } = require("./lib/safe-fetch"); +const { auditSite } = require("./lib/site-audit"); const { generateTags } = require("./public/tag-generator"); const PLACEHOLDER_IMAGE = "/preview-placeholder.svg"; @@ -142,6 +143,7 @@ function renderAnalysis(htmlOrAnalysis, targetUrl) { "twitter-title": "Twitter title", "twitter-description": "Twitter description", "twitter-image": "Twitter image", + "structured-data": "Structured data", }; const seoAuditRows = checks @@ -329,7 +331,75 @@ function renderAnalysis(htmlOrAnalysis, targetUrl) { - `; + `; +} + +function renderDuplicateGroup(label, groups) { + return `
+

${escapeHtml(label)}

+ ${groups.length + ? groups.map((group) => `
+ ${escapeHtml(group.value)} + ${group.urls.map(escapeHtml).join(" · ")} +
`).join("") + : "

No duplicates found.

"} +
`; +} + +function renderSiteAudit(site) { + const rows = site.pages.map((page) => { + if (page.status === "error") { + return ` + ${escapeHtml(page.url)} + error + ${escapeHtml(page.error)} + `; + } + return ` + ${escapeHtml(page.url)} + ${page.score.value}/100 + ${page.score.errors} + ${page.score.warnings} + ${escapeHtml(page.metadata.title || "No title")} + ${escapeHtml(page.structuredData.types.join(", ") || "None")} + `; + }).join(""); + const average = site.summary.averageScore === null ? "—" : site.summary.averageScore; + + return `
+
+
+

Site audit

+

${site.summary.total} pages analyzed

+

${site.summary.succeeded} completed · ${site.summary.failed} failed · ${site.discovery.truncated} truncated

+ ${escapeHtml(site.source.sitemapUrl)} +
+
+ ${average}/ 100 +
+
+ +
+

Page inventory

Metadata health across the sitemap

+

Bounded to ${site.limits.maxPages} same-origin pages with ${site.limits.concurrency} concurrent requests.

+
+
+ + + ${rows} +
PageScoreErrorsWarningsTitleSchema types
+
+ +
+

Cross-page checks

Duplicate metadata

+

Repeated non-empty values found across successfully analyzed pages.

+
+
+ ${renderDuplicateGroup("Duplicate titles", site.duplicates.titles)} + ${renderDuplicateGroup("Duplicate descriptions", site.duplicates.descriptions)} + ${renderDuplicateGroup("Duplicate canonicals", site.duplicates.canonicals)} +
+
`; } function renderAnalyzerForm({ @@ -337,6 +407,7 @@ function renderAnalyzerForm({ baseUrl = "", pastedHtml = "", requestedUrl = "", + sitemapUrl = "", } = {}) { return `
@@ -344,12 +415,13 @@ function renderAnalyzerForm({

Analyze metadata

-

Inspect a public URL or paste HTML directly.

+

Inspect a page, paste HTML, or audit a bounded sitemap.

+
@@ -371,6 +443,15 @@ function renderAnalyzerForm({

processed in memory · 256 KB max · no page fetch

+
+ +
+ XML + +
+ +

10 same-origin pages max · 3 concurrent requests

+
`; } @@ -437,6 +518,7 @@ function createApp({ pageFetcher = fetchPage, imageFetcher = fetchImage, imageInspector = inspectImage, + sitemapFetcher = fetchSitemap, rateLimitOptions = {}, trustProxy = false, } = {}) { @@ -488,6 +570,8 @@ function createApp({ return analyzeDocument(html, requestedUrl, { imageInspector, mode: "url" }); }; + const analyzeSitemap = (value) => auditSite(value, { pageFetcher, sitemapFetcher }); + app.get( "/", (req, res, next) => @@ -543,6 +627,18 @@ function createApp({ } }); + app.get("/api/site", analysisLimiter, async (req, res) => { + try { + const requestedUrl = typeof req.query.url === "string" ? req.query.url : ""; + res.json(await analyzeSitemap(requestedUrl)); + } catch (error) { + const isPublicError = Number.isInteger(error.statusCode); + res.status(isPublicError ? error.statusCode : 502).json({ + error: isPublicError ? error.message : "The sitemap could not be analyzed.", + }); + } + }); + app.get("/report.md", analysisLimiter, async (req, res) => { try { const requestedUrl = typeof req.query.url === "string" ? req.query.url : ""; @@ -559,6 +655,19 @@ function createApp({ } }); + app.get("/site", analysisLimiter, async (req, res) => { + const requestedUrl = typeof req.query.url === "string" ? req.query.url : ""; + const form = renderAnalyzerForm({ activeMode: "sitemap", sitemapUrl: requestedUrl }); + try { + const site = await analyzeSitemap(requestedUrl); + res.type("html").send(renderPage(form, renderSiteAudit(site))); + } catch (error) { + const isPublicError = Number.isInteger(error.statusCode); + res.status(isPublicError ? error.statusCode : 502); + res.type("html").send(renderPage(form, renderError(error, "The sitemap could not be analyzed."))); + } + }); + app.post("/analyze/html", analysisLimiter, parseHtmlForm, async (req, res) => { const pastedHtml = typeof req.body.html === "string" ? req.body.html : ""; const suppliedBaseUrl = typeof req.body.baseUrl === "string" ? req.body.baseUrl.trim() : ""; diff --git a/test/app.test.js b/test/app.test.js index 860873b..f27f88a 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -154,6 +154,68 @@ test("downloads a Markdown audit report for a public URL", async () => { assert.match(response.text, /Fixture Open Graph title/); }); +test("offers a bounded sitemap audit and renders site-level findings", async () => { + const sitemap = ` + https://example.com/a + https://example.com/b + `; + const app = createApp({ + sitemapFetcher: async () => sitemap, + pageFetcher: async () => fixtureHtml, + }); + + const landing = await request(app).get("/").expect(200); + assert.match(landing.text, /data-input-mode="sitemap"/); + assert.match(landing.text, /action="\/site"/); + + const response = await request(app) + .get("/site") + .query({ url: "https://example.com/sitemap.xml" }) + .expect(200); + + assert.match(response.text, /Site audit/); + assert.match(response.text, /2 pages analyzed/); + assert.match(response.text, /Duplicate titles/); + assert.match(response.text, /https:\/\/example\.com\/a/); + assert.match(response.text, /https:\/\/example\.com\/b/); +}); + +test("exposes the bounded site audit as JSON", async () => { + const app = createApp({ + sitemapFetcher: async () => 'https://example.com/a', + pageFetcher: async () => fixtureHtml, + }); + const response = await request(app) + .get("/api/site") + .query({ url: "https://example.com/sitemap.xml" }) + .expect("Content-Type", /json/) + .expect(200); + + assert.equal(response.body.schemaVersion, 1); + assert.equal(response.body.source.sitemapUrl, "https://example.com/sitemap.xml"); + assert.equal(response.body.summary.total, 1); + assert.equal(response.body.pages[0].url, "https://example.com/a"); + assert.equal(response.body.limits.maxPages, 10); +}); + +test("returns safe site API validation and upstream errors", async () => { + const app = createApp({ + sitemapFetcher: async () => { + throw new Error("sensitive sitemap detail"); + }, + }); + + const missing = await request(app).get("/api/site").expect(400); + assert.deepEqual(missing.body, { error: "Enter a valid HTTP or HTTPS URL." }); + + const failed = await request(app) + .get("/api/site") + .query({ url: "https://example.com/sitemap.xml" }) + .expect(502); + assert.deepEqual(failed.body, { error: "The sitemap could not be analyzed." }); + assert.doesNotMatch(JSON.stringify(failed.body), /sensitive sitemap detail/); +}); + test("explains the product and next step on the landing page", async () => { const app = createApp(); const response = await request(app).get("/").expect(200); diff --git a/test/audit.test.js b/test/audit.test.js index 9136b94..2a74f8b 100644 --- a/test/audit.test.js +++ b/test/audit.test.js @@ -117,3 +117,16 @@ test("does not claim text consistency when the heuristic cannot tokenize it", () assert.equal(check(report, "og-title-consistency").status, "info"); assert.equal(check(report, "og-description-consistency").status, "info"); }); + +test("reports structured-data validity without penalizing absent optional markup", () => { + const absent = analyze("Only a short title

Heading

"); + assert.equal(check(absent, "structured-data").status, "info"); + + const valid = analyze(''); + assert.equal(check(valid, "structured-data").status, "pass"); + assert.match(check(valid, "structured-data").evidence, /Article/); + + const invalid = analyze(''); + assert.equal(check(invalid, "structured-data").status, "error"); + assert.match(check(invalid, "structured-data").message, /invalid JSON-LD/i); +}); diff --git a/test/metadata.test.js b/test/metadata.test.js index e63726e..7913f59 100644 --- a/test/metadata.test.js +++ b/test/metadata.test.js @@ -64,3 +64,18 @@ test("rejects unsafe metadata URLs without discarding the raw evidence", () => { assert.equal(metadata.openGraph.image.url, ""); assert.equal(safeWebUrl("/ok", "https://example.com/page"), "https://example.com/ok"); }); + +test("summarizes valid and invalid JSON-LD without exposing full payloads", () => { + const metadata = parseMetadata(` + + + `, "https://example.com/page"); + + assert.deepEqual(metadata.structuredData, [ + { valid: true, types: ["Article", "Person", "Author"] }, + { valid: false, types: [], error: "Invalid JSON-LD." }, + ]); + assert.doesNotMatch(JSON.stringify(metadata.structuredData), /@context/); +}); diff --git a/test/safe-fetch.test.js b/test/safe-fetch.test.js index de0a2fe..d224e83 100644 --- a/test/safe-fetch.test.js +++ b/test/safe-fetch.test.js @@ -3,6 +3,7 @@ const test = require("node:test"); const { createSafeLookup, fetchPage, + fetchSitemap, isBlockedAddress, normalizeTargetUrl, } = require("../lib/safe-fetch"); @@ -116,3 +117,36 @@ test("maps request timeouts to a safe public error", async () => { /took too long to respond/, ); }); + +test("fetches bounded XML sitemaps through the same safe request boundary", async () => { + let request; + const client = { + async get(url, options) { + request = { url, options }; + return { + data: "https://example.test/", + headers: { "content-type": "application/xml; charset=utf-8" }, + }; + }, + }; + + const xml = await fetchSitemap("https://example.test/sitemap.xml", { client }); + + assert.match(xml, /urlset/); + assert.match(request.options.headers.Accept, /application\/xml/); + assert.equal(request.options.maxContentLength, 2 * 1024 * 1024); + assert.equal(request.options.proxy, false); +}); + +test("rejects non-XML sitemap responses", async () => { + const client = { + async get() { + return { data: "{}", headers: { "content-type": "application/json" } }; + }, + }; + + await assert.rejects( + fetchSitemap("https://example.test/sitemap.xml", { client }), + /did not return an XML sitemap/, + ); +}); diff --git a/test/site-audit.test.js b/test/site-audit.test.js new file mode 100644 index 0000000..0d5b9a6 --- /dev/null +++ b/test/site-audit.test.js @@ -0,0 +1,103 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); +const { auditSite, parseSitemap } = require("../lib/site-audit"); + +const page = ({ title, description, canonical, structuredData = "" }) => ` + + ${title} + + + + ${structuredData} +

${title}

`; + +test("parses sitemap URL sets and indexes", () => { + assert.deepEqual( + parseSitemap(` + https://example.com/a + /b + `, "https://example.com/sitemap.xml"), + { type: "urlset", locations: ["https://example.com/a", "https://example.com/b"] }, + ); + assert.deepEqual( + parseSitemap(`/posts.xml`, "https://example.com/sitemap.xml"), + { type: "index", locations: ["https://example.com/posts.xml"] }, + ); + assert.throws(() => parseSitemap("", "https://example.com/sitemap.xml"), /valid sitemap/); +}); + +test("audits a bounded same-origin page set and reports duplicates", async () => { + const sitemap = ` + https://example.com/a + https://example.com/b + https://outside.example/x + https://example.com/c + https://example.com/d + `; + const pages = { + "https://example.com/a": page({ title: "Duplicate title", description: "Duplicate description long enough for audit checks on page a.", canonical: "https://example.com/shared" }), + "https://example.com/b": page({ title: "Duplicate title", description: "Duplicate description long enough for audit checks on page a.", canonical: "https://example.com/shared" }), + "https://example.com/c": page({ title: "Unique page title", description: "A separate description long enough for the audit checks on page c.", canonical: "https://example.com/c", structuredData: '' }), + }; + let activeRequests = 0; + let peakRequests = 0; + + const result = await auditSite("https://example.com/sitemap.xml", { + sitemapFetcher: async () => sitemap, + pageFetcher: async (url) => { + activeRequests += 1; + peakRequests = Math.max(peakRequests, activeRequests); + await new Promise((resolve) => setImmediate(resolve)); + activeRequests -= 1; + return pages[url]; + }, + maxPages: 3, + concurrency: 2, + }); + + assert.equal(result.schemaVersion, 1); + assert.equal(result.summary.total, 3); + assert.equal(result.summary.succeeded, 3); + assert.equal(result.discovery.externalSkipped, 1); + assert.equal(result.discovery.truncated, 1); + assert.equal(peakRequests, 2); + assert.deepEqual(result.pages.map((item) => item.url), [ + "https://example.com/a", + "https://example.com/b", + "https://example.com/c", + ]); + assert.deepEqual(result.duplicates.titles[0].urls, [ + "https://example.com/a", + "https://example.com/b", + ]); + assert.equal(result.pages[2].structuredData.types[0], "Article"); +}); + +test("supports one sitemap-index level and preserves safe page failures", async () => { + const fetchedSitemaps = []; + const result = await auditSite("https://example.com/sitemap.xml", { + sitemapFetcher: async (url) => { + fetchedSitemaps.push(url); + if (url.endsWith("sitemap.xml")) { + return ` + https://example.com/pages.xml + https://outside.example/private.xml + `; + } + return `https://example.com/okhttps://example.com/fail`; + }, + pageFetcher: async (url) => { + if (url.endsWith("/fail")) throw new Error("private upstream detail"); + return page({ title: "Working page title", description: "A working page description long enough for deterministic checks.", canonical: url }); + }, + }); + + assert.deepEqual(fetchedSitemaps, [ + "https://example.com/sitemap.xml", + "https://example.com/pages.xml", + ]); + assert.equal(result.summary.succeeded, 1); + assert.equal(result.summary.failed, 1); + assert.equal(result.pages[1].error, "The page could not be analyzed."); + assert.doesNotMatch(JSON.stringify(result), /private upstream detail/); +});