Implement Release 3 site auditing

README.md

@@ -11,6 +11,8 @@ A compact Node.js application that audits a public webpage or pasted HTML, expla
 - Edits draft metadata with instant preview updates and generated `<head>` tags.
 - Exposes the same URL analyzer through a versioned JSON response.
 - Copies shareable audit links and exports JSON or Markdown reports without storage.
+- Audits up to 10 same-origin sitemap pages for scores, failures, and duplicate metadata.
+- Validates JSON-LD syntax and reports detected Schema.org-style `@type` values.
 - Lists discovered Open Graph and Twitter metadata for inspection.
 - Resolves relative preview URLs against the audited page.
 - Returns clear errors for invalid, unreachable, oversized, slow, or non-HTML targets.
@@ -26,6 +28,8 @@ Because the application fetches user-provided URLs, outbound requests are treate
 - Image requests reuse the SSRF boundary with a 7-second timeout and 5 MB limit.
 - Preview images are loaded through a bounded same-origin route instead of directly from metadata-controlled hosts.
 - Pasted HTML is processed in memory and capped at 256 KB.
+- Sitemaps use the same SSRF-safe 2 MB fetch boundary and may include up to three same-origin child sitemaps.
+- Site audits fetch at most 10 same-origin pages with three concurrent requests.
 - Non-HTML responses are rejected.
 - Metadata and URL attributes are escaped or sanitized before rendering.
 - Helmet security headers, a Content Security Policy, and per-IP rate limiting are enabled.
@@ -85,6 +89,14 @@ GET /report.md?url=https%3A%2F%2Fexample.com
 
 URL results also include controls to copy the reproducible audit URL, open the JSON export, or download the Markdown report. Pasted HTML stays local to the request and therefore does not expose share/export controls.
 
+Audit a sitemap through the UI at `/site?url=...` or as JSON:
+
+```http
+GET /api/site?url=https%3A%2F%2Fexample.com%2Fsitemap.xml
+```
+
+The site response includes discovery limits, success/failure totals, average score, duplicate titles/descriptions/canonicals, structured-data summaries, and page-level checks. Sitemap indexes are followed one level deep, with at most three same-origin child sitemaps.
+
 If the application runs behind a reverse proxy, set the number of trusted proxy hops so rate limiting uses the client address safely:
 
 ```bash
@@ -114,6 +126,8 @@ The test suite covers:
 - Slack and WhatsApp preview rendering
 - URL and pasted-HTML analysis modes
 - Stable JSON analysis and Markdown report exports
+- Bounded sitemap parsing, aggregation, and same-origin enforcement
+- JSON-LD syntax and `@type` summaries
 - Deterministic audit rules and scoring
 - Bounded social-image inspection and safe same-origin previews
 - Generated metadata-tag escaping and unsafe URL omission
@@ -130,7 +144,8 @@ The test suite covers:
 │   ├── image-inspector.js    # Bounded social-image validation
 │   ├── metadata.js           # Normalized metadata parsing
 │   ├── report.js             # Public JSON and Markdown serializers
-│   └── safe-fetch.js         # URL validation and bounded page fetching
+│   ├── safe-fetch.js         # URL validation and bounded page fetching
+│   └── site-audit.js         # Bounded sitemap and cross-page analysis
 ├── public/
 │   ├── app.js                # Input tabs and live draft previews
 │   ├── tag-generator.js      # Shared browser/server tag generation
@@ -155,6 +170,8 @@ The previous Heroku demo is no longer linked because it currently returns 404.
 - Public websites may block automated requests or return different metadata based on geography or user agent.
 - Draft edits update previews and generated tags but do not recalculate the source-page audit.
 - Share links and exports rerun the live URL analysis; results can change when the target page changes.
+- Site audits intentionally stop after 10 pages and one sitemap-index level; they are not a general-purpose crawler.
+- Structured-data checks validate JSON-LD syntax and declared types, not full Schema.org vocabulary requirements.
 
 ## License
 

lib/audit.js

@@ -191,6 +191,19 @@ function analyzeMetadata(metadata, targetUrl) {
   checks.push(socialField("twitter-description", "twitter", "twitter:description", metadata.twitter.description, metadata.openGraph.description));
   checks.push(socialField("twitter-image", "twitter", "twitter:image", metadata.twitter.image.raw, metadata.openGraph.image.raw));
 
+  const structuredData = metadata.structuredData || [];
+  const invalidStructuredData = structuredData.filter((item) => !item.valid).length;
+  const structuredTypes = [...new Set(structuredData.flatMap((item) => item.types || []))];
+  checks.push(
+    !structuredData.length
+      ? result("structured-data", "structured-data", "info", "No JSON-LD structured data was found.")
+      : invalidStructuredData
+        ? result("structured-data", "structured-data", "error", `${invalidStructuredData} invalid JSON-LD block${invalidStructuredData === 1 ? " was" : "s were"} found.`)
+        : !structuredTypes.length
+          ? result("structured-data", "structured-data", "warning", "JSON-LD is valid but does not declare an @type.")
+          : result("structured-data", "structured-data", "pass", "JSON-LD structured data is valid.", structuredTypes.join(", ")),
+  );
+
   return {
     checks,
     score: summarizeChecks(checks),

lib/metadata.js

@@ -20,6 +20,46 @@ function urlValue(raw, baseUrl) {
   return { raw, url: safeWebUrl(raw, baseUrl) };
 }
 
+function structuredDataSummary($) {
+  return $('script[type="application/ld+json"]')
+    .map((_index, element) => {
+      try {
+        const value = JSON.parse($(element).text());
+        const types = new Set();
+        const queue = [value];
+        let cursor = 0;
+        const enqueue = (values) => {
+          for (const child of values) {
+            if (queue.length >= 10_000) break;
+            queue.push(child);
+          }
+        };
+        while (cursor < queue.length && cursor < 10_000 && types.size < 20) {
+          const node = queue[cursor];
+          cursor += 1;
+          if (!node) continue;
+          if (Array.isArray(node)) {
+            enqueue(node);
+            continue;
+          }
+          if (typeof node !== "object") continue;
+          const nodeTypes = Array.isArray(node["@type"])
+            ? node["@type"]
+            : [node["@type"]];
+          nodeTypes.filter((type) => typeof type === "string" && type.trim()).forEach((type) => {
+            const normalizedType = type.trim();
+            if (types.size < 20 && normalizedType.length <= 100) types.add(normalizedType);
+          });
+          enqueue(Object.values(node));
+        }
+        return { valid: true, types: [...types] };
+      } catch {
+        return { valid: false, types: [], error: "Invalid JSON-LD." };
+      }
+    })
+    .get();
+}
+
 function parseMetadata(html, baseUrl) {
   const $ = cheerio.load(html || "");
   const canonicalRaw = ($('link[rel="canonical"]').first().attr("href") || "").trim();
@@ -69,6 +109,7 @@ function parseMetadata(html, baseUrl) {
       description: metaContent($, 'meta[name="twitter:description"]'),
       image: urlValue(twitterImageRaw, baseUrl),
     },
+    structuredData: structuredDataSummary($),
     socialTags,
   };
 }

lib/safe-fetch.js

@@ -101,17 +101,23 @@ function validateRedirect(options) {
   );
 }
 
-async function fetchPage(
-  value,
-  { client = axios, lookup = dns.lookup } = {},
-) {
+async function fetchTextResource(value, {
+  accept,
+  allowedContentTypes,
+  client = axios,
+  fetchMessage,
+  invalidTypeMessage,
+  lookup = dns.lookup,
+  sizeMessage,
+  timeoutMessage,
+}) {
   const url = normalizeTargetUrl(value);
 
   try {
     const response = await client.get(url, {
       beforeRedirect: validateRedirect,
       headers: {
-        Accept: "text/html,application/xhtml+xml",
+        Accept: accept,
         "User-Agent": "OGPreview/1.0 (+https://github.com/alianjum0/ogpreview-node)",
       },
       lookup: createSafeLookup(lookup),
@@ -127,32 +133,57 @@ async function fetchPage(
       .split(";", 1)[0]
       .trim()
       .toLowerCase();
-    if (!["text/html", "application/xhtml+xml"].includes(contentType)) {
-      throw new TargetUrlError("The target URL did not return an HTML page.");
+    if (!allowedContentTypes.includes(contentType)) {
+      throw new TargetUrlError(invalidTypeMessage);
     }
 
     return response.data;
   } catch (error) {
     if (error instanceof TargetUrlError) throw error;
     if (error.cause instanceof TargetUrlError) throw error.cause;
     if (error.code === "ECONNABORTED") {
-      throw new FetchPageError("The target website took too long to respond.");
+      throw new FetchPageError(timeoutMessage);
     }
     if (
       error.code === "ERR_BAD_RESPONSE" &&
       error.message.includes("maxContentLength")
     ) {
-      throw new FetchPageError("The target page is larger than 2 MB.");
+      throw new FetchPageError(sizeMessage);
     }
-    throw new FetchPageError("The target website could not be fetched.");
+    throw new FetchPageError(fetchMessage);
   }
 }
 
+async function fetchPage(value, options = {}) {
+  return fetchTextResource(value, {
+    accept: "text/html,application/xhtml+xml",
+    allowedContentTypes: ["text/html", "application/xhtml+xml"],
+    fetchMessage: "The target website could not be fetched.",
+    invalidTypeMessage: "The target URL did not return an HTML page.",
+    sizeMessage: "The target page is larger than 2 MB.",
+    timeoutMessage: "The target website took too long to respond.",
+    ...options,
+  });
+}
+
+async function fetchSitemap(value, options = {}) {
+  return fetchTextResource(value, {
+    accept: "application/xml,text/xml,text/plain",
+    allowedContentTypes: ["application/xml", "text/xml", "text/plain"],
+    fetchMessage: "The sitemap could not be fetched.",
+    invalidTypeMessage: "The target URL did not return an XML sitemap.",
+    sizeMessage: "The sitemap is larger than 2 MB.",
+    timeoutMessage: "The sitemap took too long to respond.",
+    ...options,
+  });
+}
+
 module.exports = {
   FetchPageError,
   TargetUrlError,
   createSafeLookup,
   fetchPage,
+  fetchSitemap,
   isBlockedAddress,
   normalizeTargetUrl,
   validateRedirect,