From b98351fbe9157990de59bba44bf35fb7596372eb Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 01:04:47 +0000 Subject: [PATCH 01/10] Add markdown alternate links for LLM training data discovery - Add to page headers pointing to .md version - Improve MDX-to-markdown compilation to produce clean markdown output - Preserve code blocks and frontmatter while stripping JSX components Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 93 ++++++++++++++++++++++++++- app/layout.tsx | 5 ++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 61006c812..8cd890f21 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -7,6 +7,90 @@ export const dynamic = "force-dynamic"; // Regex pattern for removing .md extension const MD_EXTENSION_REGEX = /\.md$/; +// Regex patterns for MDX to Markdown compilation (top-level for performance) +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; +const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; +const IMPORT_DESTRUCTURE_REGEX = + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; +const EXPORT_REGEX = + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; +const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; +const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; +const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; +const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; +const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; + +/** + * Compiles MDX content to clean markdown by: + * - Preserving frontmatter + * - Removing import statements + * - Converting JSX components to their text content + * - Preserving standard markdown + */ +function compileMdxToMarkdown(content: string): string { + let result = content; + + // Extract and preserve frontmatter if present + let frontmatter = ""; + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove import statements (various formats) + result = result.replace(IMPORT_FROM_REGEX, ""); + result = result.replace(IMPORT_DIRECT_REGEX, ""); + result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); + + // Remove export statements (like export const metadata) + result = result.replace(EXPORT_REGEX, ""); + + // Process self-closing JSX components (e.g., or ) + // Handles components with dots like + result = result.replace(SELF_CLOSING_JSX_REGEX, ""); + + // Process JSX components with children - extract the text content + // Handles components with dots like content + // Keep processing until no more JSX components remain + let previousResult = ""; + while (previousResult !== result) { + previousResult = result; + // Match opening tag, capture tag name (with dots), and content until matching closing tag + result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => + innerContent.trim() + ); + } + + // Remove any remaining JSX expressions like {variable} or {expression} + // But preserve code blocks by temporarily replacing them + const codeBlocks: string[] = []; + result = result.replace(CODE_BLOCK_REGEX, (match) => { + codeBlocks.push(match); + return `__CODE_BLOCK_${codeBlocks.length - 1}__`; + }); + + // Now remove JSX expressions outside code blocks + result = result.replace(JSX_EXPRESSION_REGEX, ""); + + // Restore code blocks + result = result.replace( + CODE_BLOCK_PLACEHOLDER_REGEX, + (_, index) => codeBlocks[Number.parseInt(index, 10)] + ); + + // Clean up excessive blank lines (more than 2 consecutive) + result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); + + // Trim leading/trailing whitespace + result = result.trim(); + + // Reconstruct with frontmatter + return `${frontmatter}${result}\n`; +} + export async function GET( request: NextRequest, _context: { params: Promise<{ slug?: string[] }> } @@ -31,13 +115,16 @@ export async function GET( return new NextResponse("Markdown file not found", { status: 404 }); } - const content = await readFile(filePath, "utf-8"); + const rawContent = await readFile(filePath, "utf-8"); + + // Compile MDX to clean markdown + const content = compileMdxToMarkdown(rawContent); - // Return the raw markdown with proper headers + // Return the compiled markdown with proper headers return new NextResponse(content, { status: 200, headers: { - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/markdown; charset=utf-8", "Content-Disposition": "inline", }, }); diff --git a/app/layout.tsx b/app/layout.tsx index 56fdf950f..adad9f6ad 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -104,6 +104,11 @@ export default async function RootLayout({ + {lang !== "en" && ( From a79d46a9fde0cba73c912580eeb7cf4ebaa3fd89 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 01:09:28 +0000 Subject: [PATCH 02/10] Add fallback content for component-only pages in markdown API Pages that only contain React components (like the landing page) now return a helpful markdown response with the title, description, and a link to the full interactive page. Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 36 +++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 8cd890f21..71d0acb13 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -22,14 +22,34 @@ const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; +// Regex for extracting frontmatter fields +const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; +const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; + +/** + * Extracts title and description from frontmatter + */ +function extractFrontmatterMeta(frontmatter: string): { + title: string; + description: string; +} { + const titleMatch = frontmatter.match(TITLE_REGEX); + const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + return { + title: titleMatch?.[1] || "Arcade Documentation", + description: descriptionMatch?.[1] || "", + }; +} + /** * Compiles MDX content to clean markdown by: * - Preserving frontmatter * - Removing import statements * - Converting JSX components to their text content * - Preserving standard markdown + * - Providing fallback content for component-only pages */ -function compileMdxToMarkdown(content: string): string { +function compileMdxToMarkdown(content: string, pagePath: string): string { let result = content; // Extract and preserve frontmatter if present @@ -87,6 +107,18 @@ function compileMdxToMarkdown(content: string): string { // Trim leading/trailing whitespace result = result.trim(); + // If content is essentially empty (component-only page), provide fallback + if (!result || result.length < 10) { + const { title, description } = extractFrontmatterMeta(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + // Reconstruct with frontmatter return `${frontmatter}${result}\n`; } @@ -118,7 +150,7 @@ export async function GET( const rawContent = await readFile(filePath, "utf-8"); // Compile MDX to clean markdown - const content = compileMdxToMarkdown(rawContent); + const content = compileMdxToMarkdown(rawContent, pathWithoutMd); // Return the compiled markdown with proper headers return new NextResponse(content, { From 37a7fb1fe2ecdae2cf277fa2cf514e6372999b28 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 13:15:08 +0000 Subject: [PATCH 03/10] Fix indentation in compiled markdown output - Add dedent function to normalize indentation when extracting content from JSX components - Add normalizeIndentation function to clean up stray whitespace while preserving meaningful markdown indentation (nested lists, blockquotes) - Move list detection regex patterns to module top level for performance - Ensures code block markers (```) start at column 0 Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 90 ++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 71d0acb13..1b23977b9 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -22,10 +22,59 @@ const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; +// Regex for detecting markdown list items and numbered lists +const UNORDERED_LIST_REGEX = /^[-*+]\s/; +const ORDERED_LIST_REGEX = /^\d+[.)]\s/; + // Regex for extracting frontmatter fields const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; +// Regex for detecting leading whitespace on lines +const LEADING_WHITESPACE_REGEX = /^[ \t]+/; + +/** + * Removes consistent leading indentation from all lines of text. + * This normalizes content that was indented inside JSX components. + * Code block markers (```) are ignored when calculating minimum indent + * since they typically start at column 0 in MDX files. + */ +function dedent(text: string): string { + const lines = text.split("\n"); + + // Find minimum indentation, ignoring: + // - Empty lines + // - Code block markers (lines starting with ```) + let minIndent = Number.POSITIVE_INFINITY; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === "" || trimmed.startsWith("```")) { + continue; // Ignore empty lines and code block markers + } + const match = line.match(LEADING_WHITESPACE_REGEX); + const indent = match ? match[0].length : 0; + if (indent < minIndent) { + minIndent = indent; + } + } + + // If no indentation found, return as-is + if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { + return text; + } + + // Remove the minimum indentation from each line (except code block content) + return lines + .map((line) => { + // Don't modify empty lines or lines with less indentation than min + if (line.trim() === "" || line.length < minIndent) { + return line.trimStart(); + } + return line.slice(minIndent); + }) + .join("\n"); +} + /** * Extracts title and description from frontmatter */ @@ -41,6 +90,41 @@ function extractFrontmatterMeta(frontmatter: string): { }; } +/** + * Normalizes indentation in the final output. + * Removes stray leading whitespace outside code blocks while preserving + * meaningful markdown indentation (nested lists, blockquotes). + */ +function normalizeIndentation(text: string): string { + const finalLines: string[] = []; + let inCodeBlock = false; + + for (const line of text.split("\n")) { + if (line.trim().startsWith("```")) { + inCodeBlock = !inCodeBlock; + finalLines.push(line.trimStart()); // Code block markers should start at column 0 + } else if (inCodeBlock) { + finalLines.push(line); // Preserve indentation inside code blocks + } else { + const trimmed = line.trimStart(); + // Preserve indentation for nested list items and blockquotes + const isListItem = + UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); + const isBlockquote = trimmed.startsWith(">"); + if ((isListItem || isBlockquote) && line.startsWith(" ")) { + // Keep markdown-meaningful indentation (but normalize to 2-space increments) + const leadingSpaces = line.length - line.trimStart().length; + const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); + finalLines.push(normalizedIndent + trimmed); + } else { + finalLines.push(trimmed); // Remove leading whitespace for other lines + } + } + } + + return finalLines.join("\n"); +} + /** * Compiles MDX content to clean markdown by: * - Preserving frontmatter @@ -79,8 +163,9 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { while (previousResult !== result) { previousResult = result; // Match opening tag, capture tag name (with dots), and content until matching closing tag + // Apply dedent to each extracted piece to normalize indentation result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => - innerContent.trim() + dedent(innerContent.trim()) ); } @@ -101,6 +186,9 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { (_, index) => codeBlocks[Number.parseInt(index, 10)] ); + // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) + result = normalizeIndentation(result); + // Clean up excessive blank lines (more than 2 consecutive) result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); From 020d3635f5f1b23b6feab770d63f0f78eed13a02 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:14:41 +0000 Subject: [PATCH 04/10] mindent fix --- app/api/markdown/[[...slug]]/route.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 1b23977b9..b0b3e8dc9 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -66,10 +66,16 @@ function dedent(text: string): string { // Remove the minimum indentation from each line (except code block content) return lines .map((line) => { + const trimmed = line.trim(); // Don't modify empty lines or lines with less indentation than min - if (line.trim() === "" || line.length < minIndent) { + if (trimmed === "" || line.length < minIndent) { return line.trimStart(); } + // Preserve code block markers - just remove leading whitespace + // This matches the logic that ignores them when calculating minIndent + if (trimmed.startsWith("```")) { + return trimmed; + } return line.slice(minIndent); }) .join("\n"); From c6e4fb8fd666a65063362d82bd45b6c1d11fdff6 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:43:20 +0000 Subject: [PATCH 05/10] Fix frontmatter regex to handle apostrophes in quoted strings The previous regex patterns `["']?([^"'\n]+)["']?` would truncate text at the first apostrophe (e.g., "Arcade's" became "Arcade"). This fix: - Uses separate patterns for double-quoted, single-quoted, and unquoted values - Requires closing quotes to be at end of line to prevent apostrophes from being misinterpreted as closing delimiters - Adds stripSurroundingQuotes helper for fallback cases Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 43 +++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index b0b3e8dc9..10812365f 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -27,8 +27,12 @@ const UNORDERED_LIST_REGEX = /^[-*+]\s/; const ORDERED_LIST_REGEX = /^\d+[.)]\s/; // Regex for extracting frontmatter fields -const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; -const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; +// Handles: "double quoted", 'single quoted', or unquoted values +// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback +// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters +const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; +const DESCRIPTION_REGEX = + /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; // Regex for detecting leading whitespace on lines const LEADING_WHITESPACE_REGEX = /^[ \t]+/; @@ -82,7 +86,23 @@ function dedent(text: string): string { } /** - * Extracts title and description from frontmatter + * Strips surrounding quotes from a value if present. + * Used for unquoted fallback values that may contain quotes due to apostrophe handling. + */ +function stripSurroundingQuotes(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +/** + * Extracts title and description from frontmatter. + * Handles double-quoted, single-quoted, and unquoted YAML values. */ function extractFrontmatterMeta(frontmatter: string): { title: string; @@ -90,9 +110,22 @@ function extractFrontmatterMeta(frontmatter: string): { } { const titleMatch = frontmatter.match(TITLE_REGEX); const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + + // Extract from whichever capture group matched: + // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback + // For group 3 (fallback), strip surrounding quotes if present + const title = + titleMatch?.[1] ?? + titleMatch?.[2] ?? + stripSurroundingQuotes(titleMatch?.[3] ?? ""); + const description = + descriptionMatch?.[1] ?? + descriptionMatch?.[2] ?? + stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); + return { - title: titleMatch?.[1] || "Arcade Documentation", - description: descriptionMatch?.[1] || "", + title: title || "Arcade Documentation", + description, }; } From fd16de7dcb787e805ace673729aa9bc232b29956 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:59:26 +0000 Subject: [PATCH 06/10] Skip alternate link for root pathname fallback When x-pathname header is not set, pathname defaults to "/" which would produce an invalid alternate link "https://docs.arcade.dev/.md". Only render the alternate link when we have a real page path. Co-Authored-By: Claude Opus 4.5 --- app/layout.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/layout.tsx b/app/layout.tsx index adad9f6ad..bc3a1f8b9 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -104,11 +104,13 @@ export default async function RootLayout({ - + {pathname !== "/" && ( + + )} {lang !== "en" && ( From d7b7c711018863a34b02b9b998b3711904439f98 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 17:51:28 +0000 Subject: [PATCH 07/10] Generate static markdown files at build time - Add scripts/generate-markdown.ts to pre-render MDX to markdown - Update proxy.ts to serve static .md files from public/ - Delete API route in favor of static file serving - Add link rewriting to add /en/ prefix and .md extension - Add markdown-friendly component implementations - Fix localhost URL in gmail integration page Co-Authored-By: Claude Opus 4.5 --- .gitignore | 1 + app/api/markdown/[[...slug]]/route.ts | 295 --- .../integrations/productivity/gmail/page.mdx | 2 +- lib/mdx-to-markdown.tsx | 2314 +++++++++++++++++ package.json | 12 +- pnpm-lock.yaml | 181 +- proxy.ts | 25 +- scripts/generate-markdown.ts | 155 ++ 8 files changed, 2663 insertions(+), 322 deletions(-) delete mode 100644 app/api/markdown/[[...slug]]/route.ts create mode 100644 lib/mdx-to-markdown.tsx create mode 100644 scripts/generate-markdown.ts diff --git a/.gitignore b/.gitignore index c714c9723..5968906cb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules .DS_Store .env.local public/sitemap*.xml +public/en/**/*.md .env _pagefind/ diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts deleted file mode 100644 index 10812365f..000000000 --- a/app/api/markdown/[[...slug]]/route.ts +++ /dev/null @@ -1,295 +0,0 @@ -import { access, readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { type NextRequest, NextResponse } from "next/server"; - -export const dynamic = "force-dynamic"; - -// Regex pattern for removing .md extension -const MD_EXTENSION_REGEX = /\.md$/; - -// Regex patterns for MDX to Markdown compilation (top-level for performance) -const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; -const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; -const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; -const IMPORT_DESTRUCTURE_REGEX = - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; -const EXPORT_REGEX = - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; -const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; -const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; -const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; -const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; -const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; - -// Regex for detecting markdown list items and numbered lists -const UNORDERED_LIST_REGEX = /^[-*+]\s/; -const ORDERED_LIST_REGEX = /^\d+[.)]\s/; - -// Regex for extracting frontmatter fields -// Handles: "double quoted", 'single quoted', or unquoted values -// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback -// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters -const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; -const DESCRIPTION_REGEX = - /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; - -// Regex for detecting leading whitespace on lines -const LEADING_WHITESPACE_REGEX = /^[ \t]+/; - -/** - * Removes consistent leading indentation from all lines of text. - * This normalizes content that was indented inside JSX components. - * Code block markers (```) are ignored when calculating minimum indent - * since they typically start at column 0 in MDX files. - */ -function dedent(text: string): string { - const lines = text.split("\n"); - - // Find minimum indentation, ignoring: - // - Empty lines - // - Code block markers (lines starting with ```) - let minIndent = Number.POSITIVE_INFINITY; - for (const line of lines) { - const trimmed = line.trim(); - if (trimmed === "" || trimmed.startsWith("```")) { - continue; // Ignore empty lines and code block markers - } - const match = line.match(LEADING_WHITESPACE_REGEX); - const indent = match ? match[0].length : 0; - if (indent < minIndent) { - minIndent = indent; - } - } - - // If no indentation found, return as-is - if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { - return text; - } - - // Remove the minimum indentation from each line (except code block content) - return lines - .map((line) => { - const trimmed = line.trim(); - // Don't modify empty lines or lines with less indentation than min - if (trimmed === "" || line.length < minIndent) { - return line.trimStart(); - } - // Preserve code block markers - just remove leading whitespace - // This matches the logic that ignores them when calculating minIndent - if (trimmed.startsWith("```")) { - return trimmed; - } - return line.slice(minIndent); - }) - .join("\n"); -} - -/** - * Strips surrounding quotes from a value if present. - * Used for unquoted fallback values that may contain quotes due to apostrophe handling. - */ -function stripSurroundingQuotes(value: string): string { - const trimmed = value.trim(); - if ( - (trimmed.startsWith('"') && trimmed.endsWith('"')) || - (trimmed.startsWith("'") && trimmed.endsWith("'")) - ) { - return trimmed.slice(1, -1); - } - return trimmed; -} - -/** - * Extracts title and description from frontmatter. - * Handles double-quoted, single-quoted, and unquoted YAML values. - */ -function extractFrontmatterMeta(frontmatter: string): { - title: string; - description: string; -} { - const titleMatch = frontmatter.match(TITLE_REGEX); - const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); - - // Extract from whichever capture group matched: - // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback - // For group 3 (fallback), strip surrounding quotes if present - const title = - titleMatch?.[1] ?? - titleMatch?.[2] ?? - stripSurroundingQuotes(titleMatch?.[3] ?? ""); - const description = - descriptionMatch?.[1] ?? - descriptionMatch?.[2] ?? - stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); - - return { - title: title || "Arcade Documentation", - description, - }; -} - -/** - * Normalizes indentation in the final output. - * Removes stray leading whitespace outside code blocks while preserving - * meaningful markdown indentation (nested lists, blockquotes). - */ -function normalizeIndentation(text: string): string { - const finalLines: string[] = []; - let inCodeBlock = false; - - for (const line of text.split("\n")) { - if (line.trim().startsWith("```")) { - inCodeBlock = !inCodeBlock; - finalLines.push(line.trimStart()); // Code block markers should start at column 0 - } else if (inCodeBlock) { - finalLines.push(line); // Preserve indentation inside code blocks - } else { - const trimmed = line.trimStart(); - // Preserve indentation for nested list items and blockquotes - const isListItem = - UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); - const isBlockquote = trimmed.startsWith(">"); - if ((isListItem || isBlockquote) && line.startsWith(" ")) { - // Keep markdown-meaningful indentation (but normalize to 2-space increments) - const leadingSpaces = line.length - line.trimStart().length; - const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); - finalLines.push(normalizedIndent + trimmed); - } else { - finalLines.push(trimmed); // Remove leading whitespace for other lines - } - } - } - - return finalLines.join("\n"); -} - -/** - * Compiles MDX content to clean markdown by: - * - Preserving frontmatter - * - Removing import statements - * - Converting JSX components to their text content - * - Preserving standard markdown - * - Providing fallback content for component-only pages - */ -function compileMdxToMarkdown(content: string, pagePath: string): string { - let result = content; - - // Extract and preserve frontmatter if present - let frontmatter = ""; - const frontmatterMatch = result.match(FRONTMATTER_REGEX); - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - result = result.slice(frontmatterMatch[0].length); - } - - // Remove import statements (various formats) - result = result.replace(IMPORT_FROM_REGEX, ""); - result = result.replace(IMPORT_DIRECT_REGEX, ""); - result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); - - // Remove export statements (like export const metadata) - result = result.replace(EXPORT_REGEX, ""); - - // Process self-closing JSX components (e.g., or ) - // Handles components with dots like - result = result.replace(SELF_CLOSING_JSX_REGEX, ""); - - // Process JSX components with children - extract the text content - // Handles components with dots like content - // Keep processing until no more JSX components remain - let previousResult = ""; - while (previousResult !== result) { - previousResult = result; - // Match opening tag, capture tag name (with dots), and content until matching closing tag - // Apply dedent to each extracted piece to normalize indentation - result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => - dedent(innerContent.trim()) - ); - } - - // Remove any remaining JSX expressions like {variable} or {expression} - // But preserve code blocks by temporarily replacing them - const codeBlocks: string[] = []; - result = result.replace(CODE_BLOCK_REGEX, (match) => { - codeBlocks.push(match); - return `__CODE_BLOCK_${codeBlocks.length - 1}__`; - }); - - // Now remove JSX expressions outside code blocks - result = result.replace(JSX_EXPRESSION_REGEX, ""); - - // Restore code blocks - result = result.replace( - CODE_BLOCK_PLACEHOLDER_REGEX, - (_, index) => codeBlocks[Number.parseInt(index, 10)] - ); - - // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) - result = normalizeIndentation(result); - - // Clean up excessive blank lines (more than 2 consecutive) - result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); - - // Trim leading/trailing whitespace - result = result.trim(); - - // If content is essentially empty (component-only page), provide fallback - if (!result || result.length < 10) { - const { title, description } = extractFrontmatterMeta(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - // Reconstruct with frontmatter - return `${frontmatter}${result}\n`; -} - -export async function GET( - request: NextRequest, - _context: { params: Promise<{ slug?: string[] }> } -) { - try { - // Get the original pathname from the request - const url = new URL(request.url); - // Remove /api/markdown prefix to get the original path - const originalPath = url.pathname.replace("/api/markdown", ""); - - // Remove .md extension - const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, ""); - - // Map URL to file path - // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx - const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`); - - // Check if file exists - try { - await access(filePath); - } catch { - return new NextResponse("Markdown file not found", { status: 404 }); - } - - const rawContent = await readFile(filePath, "utf-8"); - - // Compile MDX to clean markdown - const content = compileMdxToMarkdown(rawContent, pathWithoutMd); - - // Return the compiled markdown with proper headers - return new NextResponse(content, { - status: 200, - headers: { - "Content-Type": "text/markdown; charset=utf-8", - "Content-Disposition": "inline", - }, - }); - } catch (error) { - return new NextResponse(`Internal server error: ${error}`, { - status: 500, - }); - } -} diff --git a/app/en/resources/integrations/productivity/gmail/page.mdx b/app/en/resources/integrations/productivity/gmail/page.mdx index f20dc6303..7e68ebc22 100644 --- a/app/en/resources/integrations/productivity/gmail/page.mdx +++ b/app/en/resources/integrations/productivity/gmail/page.mdx @@ -272,7 +272,7 @@ Delete a draft email using the Gmail API. The `TrashEmail` tool is currently only available on a self-hosted instance of the Arcade Engine. To learn more about self-hosting, see the [self-hosting - documentation](http://localhost:3000/en/home/deployment/engine-configuration). + documentation](/guides/deployment-hosting/configure-engine).
diff --git a/lib/mdx-to-markdown.tsx b/lib/mdx-to-markdown.tsx new file mode 100644 index 000000000..0e8345df4 --- /dev/null +++ b/lib/mdx-to-markdown.tsx @@ -0,0 +1,2314 @@ +/** + * MDX to Markdown converter + * + * Compiles MDX content, renders it with markdown-friendly components, + * then converts the resulting HTML to clean Markdown using the unified ecosystem. + */ + +import { pathToFileURL } from "node:url"; +import { compile, run } from "@mdx-js/mdx"; +import type { ReactNode } from "react"; +import { createElement, Fragment } from "react"; +import { Fragment as JsxFragment, jsx, jsxs } from "react/jsx-runtime"; +import rehypeParse from "rehype-parse"; +import rehypeRemark from "rehype-remark"; +import remarkGfm from "remark-gfm"; +import remarkStringify from "remark-stringify"; +import { unified } from "unified"; + +// Regex patterns (at top level for performance) +const FILE_EXTENSION_REGEX = /\.[^.]+$/; +const UNDERSCORE_DASH_REGEX = /[_-]/g; +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const TITLE_REGEX = /title:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; +const DESCRIPTION_REGEX = /description:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; + +// Types for mdast table nodes +type MdastTableCell = { type: "tableCell"; children: unknown[] }; +type MdastTableRow = { type: "tableRow"; children: MdastTableCell[] }; +type HtmlNode = { + type: string; + tagName?: string; + children?: HtmlNode[]; + properties?: Record; +}; +type StateAll = (node: HtmlNode) => unknown[]; + +/** Extract cells from a table row element */ +function extractCellsFromRow( + state: { all: StateAll }, + row: HtmlNode +): MdastTableCell[] { + const cells: MdastTableCell[] = []; + for (const cell of row.children || []) { + if ( + cell.type === "element" && + (cell.tagName === "th" || cell.tagName === "td") + ) { + cells.push({ type: "tableCell", children: state.all(cell) }); + } + } + return cells; +} + +/** Extract rows from a table section (thead, tbody, tfoot) or direct tr children */ +function extractRowsFromTableSection( + state: { all: StateAll }, + section: HtmlNode +): MdastTableRow[] { + const rows: MdastTableRow[] = []; + for (const child of section.children || []) { + if (child.type === "element" && child.tagName === "tr") { + const cells = extractCellsFromRow(state, child); + if (cells.length > 0) { + rows.push({ type: "tableRow", children: cells }); + } + } + } + return rows; +} + +/** Check if element is a table section (thead, tbody, tfoot) */ +function isTableSection(tagName: string | undefined): boolean { + return tagName === "thead" || tagName === "tbody" || tagName === "tfoot"; +} + +// Dynamic import to avoid Next.js RSC restrictions +let renderToStaticMarkup: typeof import("react-dom/server").renderToStaticMarkup; +async function getRenderer() { + if (!renderToStaticMarkup) { + const reactDomServer = await import("react-dom/server"); + renderToStaticMarkup = reactDomServer.renderToStaticMarkup; + } + return renderToStaticMarkup; +} + +/** + * Convert HTML to Markdown using unified ecosystem (rehype-remark) + * This is more reliable than turndown for complex HTML structures + */ +async function htmlToMarkdown(html: string): Promise { + const result = await unified() + .use(rehypeParse, { fragment: true }) + .use(rehypeRemark, { + handlers: { + // Custom handler for video elements + video: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; + const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: `Video: ${title}` }], + }, + ], + }; + }, + // Custom handler for audio elements + audio: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; + const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: `Audio: ${title}` }], + }, + ], + }; + }, + // Custom handler for iframe elements + iframe: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const title = + (node.properties?.title as string) || "Embedded content"; + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: title }], + }, + ], + }; + }, + // Custom handler for HTML tables - convert to markdown tables + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Table parsing requires nested logic + table: (state, node) => { + const rows: MdastTableRow[] = []; + + for (const child of node.children || []) { + if (child.type !== "element") { + continue; + } + if (isTableSection(child.tagName)) { + rows.push( + ...extractRowsFromTableSection(state, child as HtmlNode) + ); + } else if (child.tagName === "tr") { + const cells = extractCellsFromRow(state, child as HtmlNode); + if (cells.length > 0) { + rows.push({ type: "tableRow", children: cells }); + } + } + } + + const colCount = rows[0]?.children?.length || 0; + return { + type: "table", + align: new Array(colCount).fill(null), + children: rows, + }; + }, + // These are handled by the table handler above, but we still need to define them + // to prevent "unknown node" errors when encountered outside tables + thead: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tbody: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tfoot: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tr: (state, node) => { + const cells: { type: "tableCell"; children: unknown[] }[] = []; + for (const child of node.children || []) { + if ( + child.type === "element" && + (child.tagName === "th" || child.tagName === "td") + ) { + cells.push({ + type: "tableCell", + children: state.all(child), + }); + } + } + return { + type: "tableRow", + children: cells, + }; + }, + th: (state, node) => ({ + type: "tableCell", + children: state.all(node), + }), + td: (state, node) => ({ + type: "tableCell", + children: state.all(node), + }), + // Custom handler for callout divs - render as paragraph with bold label + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Callout parsing logic + div: (state, node) => { + const className = + (node.properties?.className as string[])?.join(" ") || ""; + + // Check if this is a callout + if ( + className.includes("callout") || + className.includes("admonition") || + className.includes("warning") || + className.includes("info") || + className.includes("error") || + className.includes("tip") + ) { + let label = ""; + if (className.includes("warning")) { + label = "Warning"; + } else if (className.includes("error")) { + label = "Error"; + } else if (className.includes("tip")) { + label = "Tip"; + } else if (className.includes("info")) { + label = "Note"; + } + + // Process children and prepend bold label + const children = state.all(node); + if (label && children.length > 0) { + // Add bold label to the first paragraph's children + const firstChild = children[0]; + if (firstChild && firstChild.type === "paragraph") { + return [ + { + type: "paragraph", + children: [ + { + type: "strong", + children: [{ type: "text", value: `${label}:` }], + }, + { type: "text", value: " " }, + ...(firstChild.children || []), + ], + }, + ...children.slice(1), + ]; + } + } + return children; + } + + // Default: just return children (strip the div wrapper) + return state.all(node); + }, + }, + }) + .use(remarkGfm) // Enable GFM for tables, strikethrough, etc. + .use(remarkStringify, { + bullet: "-", + fences: true, + listItemIndent: "one", + }) + .process(html); + + return String(result); +} + +// ============================================ +// Markdown-Friendly Component Implementations +// ============================================ + +// Simple wrapper that just renders children +function PassThrough({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +// Tabs - render all tab content with headers +function MarkdownTabs({ + children, + items, +}: { + children?: ReactNode; + items?: string[]; +}) { + // If we have items array, children are the tab panels + if (items && Array.isArray(items)) { + const childArray = Array.isArray(children) ? children : [children]; + return createElement( + "div", + null, + childArray.map((child, i) => + createElement( + "div", + { key: i }, + createElement("h4", null, items[i] || `Option ${i + 1}`), + child + ) + ) + ); + } + return createElement("div", null, children); +} + +// Tab content - just render the content +function MarkdownTab({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// Assign Tab to Tabs for Tabs.Tab syntax +MarkdownTabs.Tab = MarkdownTab; + +// Steps - render as numbered sections +function MarkdownSteps({ children }: { children?: ReactNode }) { + return createElement("div", { className: "steps" }, children); +} + +// Callout - render as a styled div that turndown will convert +function MarkdownCallout({ + children, + type = "info", +}: { + children?: ReactNode; + type?: string; +}) { + return createElement("div", { className: `callout ${type}` }, children); +} + +// Cards - render as sections +function MarkdownCards({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCard({ + children, + title, + href, +}: { + children?: ReactNode; + title?: string; + href?: string; +}) { + if (href) { + return createElement( + "div", + null, + title && createElement("h4", null, createElement("a", { href }, title)), + children + ); + } + return createElement( + "div", + null, + title && createElement("h4", null, title), + children + ); +} + +MarkdownCards.Card = MarkdownCard; + +// FileTree - render as a code block +function MarkdownFileTree({ children }: { children?: ReactNode }) { + return createElement("pre", null, createElement("code", null, children)); +} + +// Link components - render as standard links +function _MarkdownLink({ + children, + href, +}: { + children?: ReactNode; + href?: string; +}) { + return createElement("a", { href }, children); +} + +// ============================================ +// HTML Element Handlers +// ============================================ + +// Video - convert to a descriptive link +function MarkdownVideo({ + src, + title, + children, +}: { + src?: string; + title?: string; + children?: ReactNode; +}) { + if (!src) { + return createElement(Fragment, null, children); + } + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; + const videoTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); + return createElement("p", null, `[Video: ${videoTitle}](${src})`); +} + +// Audio - convert to a descriptive link +function MarkdownAudio({ + src, + title, + children, +}: { + src?: string; + title?: string; + children?: ReactNode; +}) { + if (!src) { + return createElement(Fragment, null, children); + } + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; + const audioTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); + return createElement("p", null, `[Audio: ${audioTitle}](${src})`); +} + +// Image - keep as img for turndown to handle +function MarkdownImage({ + src, + alt, + title, +}: { + src?: string; + alt?: string; + title?: string; +}) { + return createElement("img", { src, alt: alt || title || "" }); +} + +// Iframe - convert to link +function MarkdownIframe({ src, title }: { src?: string; title?: string }) { + if (!src) { + return null; + } + const label = title || "Embedded content"; + return createElement("p", null, `[${label}](${src})`); +} + +// HR - render as markdown horizontal rule +function MarkdownHr() { + return createElement("hr", null); +} + +// BR - render as line break +function MarkdownBr() { + return createElement("br", null); +} + +// Container elements - just pass through children (strips the wrapper) +function MarkdownPassthrough({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +// Figure/Figcaption - extract content +function MarkdownFigure({ children }: { children?: ReactNode }) { + return createElement("figure", null, children); +} + +function MarkdownFigcaption({ children }: { children?: ReactNode }) { + return createElement("figcaption", null, children); +} + +// Details/Summary - convert to blockquote-style +function MarkdownDetails({ children }: { children?: ReactNode }) { + return createElement("blockquote", null, children); +} + +function MarkdownSummary({ children }: { children?: ReactNode }) { + return createElement("strong", null, children); +} + +// Table elements - pass through for turndown +function MarkdownTable({ children }: { children?: ReactNode }) { + return createElement("table", null, children); +} + +function MarkdownThead({ children }: { children?: ReactNode }) { + return createElement("thead", null, children); +} + +function MarkdownTbody({ children }: { children?: ReactNode }) { + return createElement("tbody", null, children); +} + +function MarkdownTr({ children }: { children?: ReactNode }) { + return createElement("tr", null, children); +} + +function MarkdownTh({ children }: { children?: ReactNode }) { + return createElement("th", null, children); +} + +function MarkdownTd({ children }: { children?: ReactNode }) { + return createElement("td", null, children); +} + +// Definition lists +function MarkdownDl({ children }: { children?: ReactNode }) { + return createElement("dl", null, children); +} + +function MarkdownDt({ children }: { children?: ReactNode }) { + return createElement("dt", null, createElement("strong", null, children)); +} + +function MarkdownDd({ children }: { children?: ReactNode }) { + return createElement("dd", null, children); +} + +// Code block elements - preserve language and content +function MarkdownPre({ + children, + ...props +}: { + children?: ReactNode; + [key: string]: unknown; +}) { + // Extract data-language if present (MDX sometimes uses this) + const lang = props["data-language"] || ""; + return createElement("pre", { "data-language": lang }, children); +} + +function MarkdownCode({ + children, + className, + ...props +}: { + children?: ReactNode; + className?: string; + [key: string]: unknown; +}) { + // Preserve the language class for turndown + return createElement( + "code", + { className: className || "", ...props }, + children + ); +} + +// GuideOverview custom component +function MarkdownGuideOverview({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownGuideOverviewItem({ + children, + title, + href, +}: { + children?: ReactNode; + title?: string; + href?: string; +}) { + return createElement( + "div", + null, + title && + createElement( + "h4", + null, + href ? createElement("a", { href }, title) : title + ), + children + ); +} + +function MarkdownGuideOverviewOutcomes({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +function MarkdownGuideOverviewPrerequisites({ + children, +}: { + children?: ReactNode; +}) { + return createElement( + "div", + null, + createElement("strong", null, "Prerequisites:"), + children + ); +} + +MarkdownGuideOverview.Item = MarkdownGuideOverviewItem; +MarkdownGuideOverview.Outcomes = MarkdownGuideOverviewOutcomes; +MarkdownGuideOverview.Prerequisites = MarkdownGuideOverviewPrerequisites; + +// CheatSheet components +function MarkdownCheatSheetGrid({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +function MarkdownCheatSheetSection({ + children, + title, + icon, +}: { + children?: ReactNode; + title?: string; + icon?: string; +}) { + return createElement( + "div", + null, + title && createElement("h3", null, icon ? `${icon} ${title}` : title), + children + ); +} + +function MarkdownInfoBox({ + children, + type = "info", +}: { + children?: ReactNode; + type?: string; +}) { + // Use a div with callout class so the callout handler processes it + // The div handler in htmlToMarkdown will add the bold label + return createElement("div", { className: `callout ${type}` }, children); +} + +function MarkdownCommandItem({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCommandList({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCommandBlock({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// GlossaryTerm - just render the text +function MarkdownGlossaryTerm({ + children, + term, +}: { + children?: ReactNode; + term?: string; +}) { + return createElement("strong", { title: term }, children); +} + +// DashboardLink - render as a link +function MarkdownDashboardLink({ + children, + href, +}: { + children?: ReactNode; + href?: string; +}) { + const dashboardUrl = href || "https://api.arcade.dev/dashboard"; + return createElement("a", { href: dashboardUrl }, children); +} + +// SignupLink - render as a link +function MarkdownSignupLink({ + children, +}: { + children?: ReactNode; + linkLocation?: string; +}) { + return createElement( + "a", + { href: "https://api.arcade.dev/dashboard/signup" }, + children + ); +} + +// Generic catch-all for unknown components +function _MarkdownGeneric({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// ToolCard - render as a link with summary +function MarkdownToolCard({ + name, + summary, + link, + category: _category, +}: { + name?: string; + image?: string; + summary?: string; + link?: string; + category?: string; +}) { + return createElement( + "li", + null, + createElement("a", { href: link }, name), + summary ? ` - ${summary}` : "" + ); +} + +// MCPClientGrid - render as a list of MCP clients +function MarkdownMCPClientGrid() { + const mcpClients = [ + { + key: "cursor", + name: "Cursor", + description: "AI-powered code editor with built-in MCP support", + }, + { + key: "claude-desktop", + name: "Claude Desktop", + description: "Anthropic's desktop app for Claude with MCP integration", + }, + { + key: "visual-studio-code", + name: "Visual Studio Code", + description: "Microsoft's code editor with MCP extensions", + }, + ]; + + return createElement( + "ul", + null, + ...mcpClients.map((client, i) => + createElement( + "li", + { key: i }, + createElement( + "a", + { href: `/guides/tool-calling/mcp-clients/${client.key}` }, + client.name + ), + ` - ${client.description}` + ) + ) + ); +} + +// ContactCards - render contact information +function MarkdownContactCards() { + const contacts = [ + { + title: "Email Support", + description: + "Get help with technical issues, account questions, and general inquiries.", + href: "mailto:support@arcade.dev", + }, + { + title: "Contact Sales", + description: + "Discuss enterprise plans, pricing, and how Arcade can help your organization.", + href: "mailto:sales@arcade.dev", + }, + { + title: "Discord Community", + description: + "Join for real-time help, connect with developers, and stay updated.", + href: "https://discord.gg/GUZEMpEZ9p", + }, + { + title: "GitHub Issues & Discussions", + description: "Report bugs, request features, and contribute to Arcade.", + href: "https://github.com/ArcadeAI/arcade-mcp", + }, + { + title: "Security Research", + description: "Report security vulnerabilities responsibly.", + href: "/guides/security/security-research-program", + }, + { + title: "System Status", + description: "Check the current status of Arcade's services.", + href: "https://status.arcade.dev", + }, + ]; + + return createElement( + "ul", + null, + ...contacts.map((contact, i) => + createElement( + "li", + { key: i }, + createElement("a", { href: contact.href }, contact.title), + ` - ${contact.description}` + ) + ) + ); +} + +// SubpageList - render list of subpages (uses provided meta) +function MarkdownSubpageList({ + basePath, + meta, +}: { + basePath?: string; + meta?: Record; +}) { + if (!(meta && basePath)) { + return null; + } + + const subpages = Object.entries(meta).filter( + ([key]) => key !== "index" && key !== "*" + ); + + return createElement( + "ul", + null, + ...subpages.map(([key, title], i) => { + let linkText: string; + if (typeof title === "string") { + linkText = title; + } else if ( + typeof title === "object" && + title !== null && + "title" in title + ) { + linkText = (title as { title: string }).title; + } else { + linkText = String(title); + } + return createElement( + "li", + { key: i }, + createElement("a", { href: `${basePath}/${key}` }, linkText) + ); + }) + ); +} + +// ============================================ +// Landing Page Component - renders as markdown-friendly content +// ============================================ +function MarkdownLandingPage() { + return createElement( + "div", + null, + // Hero + createElement("h1", null, "MCP Runtime for AI agents that get things done"), + createElement( + "p", + null, + "Arcade handles OAuth, manages user tokens, and gives you 100+ pre-built integrations so your agents can take real action in production." + ), + createElement("hr", null), + + // Get Started links + createElement("h2", null, "Get Started"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/get-started/quickstarts/call-tool-agent" }, + "Get Started with Arcade" + ) + ), + createElement( + "li", + null, + createElement("a", { href: "/resources/integrations" }, "Explore Tools") + ) + ), + + // Get Tools Section + createElement("h2", null, "Get Tools"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations" }, + "Pre-built Integrations" + ), + " - Browse 100+ ready-to-use integrations for Gmail, Slack, GitHub, and more." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/create-tools/tool-basics/build-mcp-server" }, + "Build Custom Tools" + ), + " - Create your own MCP servers and custom tools with our SDK." + ) + ), + + // Use Arcade Section + createElement("h2", null, "Use Arcade"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/tool-calling/mcp-clients" }, + "Connect to Your IDE" + ), + " - Add tools to Cursor, VS Code, Claude Desktop, or any MCP client." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks" }, + "Power Your Agent" + ), + " - Integrate with LangChain, OpenAI Agents, CrewAI, Vercel AI, and more." + ) + ), + + // Popular Integrations + createElement("h2", null, "Popular Integrations"), + createElement( + "p", + null, + "Pre-built MCP servers ready to use with your agents. ", + createElement("a", { href: "/resources/integrations" }, "See all 100+") + ), + createElement( + "ul", + null, + // Row 1 + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-sheets" }, + "Google Sheets" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/jira" }, + "Jira" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/gmail" }, + "Gmail" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/confluence" }, + "Confluence" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/slack" }, + "Slack" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-docs" }, + "Google Docs" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-slides" }, + "Google Slides" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/crm/hubspot" }, + "HubSpot" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/linear" }, + "Linear" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-drive" }, + "Google Drive" + ) + ), + // Row 2 + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/development/github" }, + "GitHub" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/x" }, + "X" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { + href: "/resources/integrations/social-communication/microsoft-teams", + }, + "MS Teams" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/outlook-mail" }, + "Outlook" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/payments/stripe" }, + "Stripe" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/notion" }, + "Notion" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/asana" }, + "Asana" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/reddit" }, + "Reddit" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/search/youtube" }, + "YouTube" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/dropbox" }, + "Dropbox" + ) + ) + ), + + // Framework Compatibility + createElement("h2", null, "Works With Your Stack"), + createElement( + "p", + null, + "Arcade integrates with popular agent frameworks and LLM providers." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/langchain/use-arcade-tools" }, + "LangChain" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/openai-agents/use-arcade-tools" }, + "OpenAI Agents" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/crewai/use-arcade-tools" }, + "CrewAI" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/vercelai" }, + "Vercel AI" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/google-adk/use-arcade-tools" }, + "Google ADK" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/mastra/use-arcade-tools" }, + "Mastra" + ) + ) + ), + + // How Arcade Works + createElement("h2", null, "How Arcade Works"), + createElement( + "p", + null, + "Three core components that power your AI agents." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "strong", + null, + createElement("a", { href: "/get-started/about-arcade" }, "Runtime") + ), + " - Your MCP server and agentic tool provider. Manages authentication, tool registration, and execution." + ), + createElement( + "li", + null, + createElement( + "strong", + null, + createElement( + "a", + { href: "/resources/integrations" }, + "Tool Catalog" + ) + ), + " - Catalog of pre-built tools and integrations. Browse 100+ ready-to-use MCP servers." + ), + createElement( + "li", + null, + createElement( + "strong", + null, + createElement( + "a", + { href: "/guides/tool-calling/custom-apps/auth-tool-calling" }, + "Agent Authorization" + ) + ), + " - Let agents act on behalf of users. Handle OAuth, API keys, and tokens for tools like Gmail and Google Drive." + ) + ), + + // Sample Applications + createElement("h2", null, "Sample Applications"), + createElement( + "p", + null, + "See Arcade in action with these example applications." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement("a", { href: "https://chat.arcade.dev/" }, "Arcade Chat"), + " - A chatbot that can help you with your daily tasks." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "https://github.com/ArcadeAI/ArcadeSlackAgent" }, + "Archer" + ), + " - A bot for Slack that can act on your behalf." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "https://github.com/dforwardfeed/slack-AIpodcast-summaries" }, + "YouTube Podcast Summarizer" + ), + " - A Slack bot that extracts and summarizes YouTube transcripts." + ) + ), + createElement( + "p", + null, + createElement("a", { href: "/resources/examples" }, "See all examples") + ), + + // Connect IDE callout + createElement("h2", null, "Connect Your IDE with Arcade's LLMs.txt"), + createElement( + "p", + null, + "Give Cursor, Claude Code, and other AI IDEs access to Arcade's documentation so they can write integration code for you. ", + createElement( + "a", + { href: "/get-started/setup/connect-arcade-docs" }, + "Learn more" + ) + ), + + // Quick Links + createElement("h2", null, "Quick Links"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement("a", { href: "/references/api" }, "API Reference") + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/references/cli-cheat-sheet" }, + "CLI Cheat Sheet" + ) + ), + createElement( + "li", + null, + createElement("a", { href: "/resources/faq" }, "FAQ") + ), + createElement( + "li", + null, + createElement("a", { href: "/references/changelog" }, "Changelog") + ) + ) + ); +} + +// ============================================ +// MCP Servers Component - renders MCP server list as markdown +// ============================================ +function MarkdownMcpServers() { + // All available MCP servers (alphabetically sorted) + const allMcpServers = [ + { + label: "Airtable API", + href: "/resources/integrations/productivity/airtable-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Arcade Engine API", + href: "/resources/integrations/development/arcade-engine-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Asana", + href: "/resources/integrations/productivity/asana", + type: "arcade", + category: "productivity", + }, + { + label: "Asana API", + href: "/resources/integrations/productivity/asana-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Ashby API", + href: "/resources/integrations/productivity/ashby-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Box API", + href: "/resources/integrations/productivity/box-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Bright Data", + href: "/resources/integrations/development/brightdata", + type: "community", + category: "development", + }, + { + label: "Calendly API", + href: "/resources/integrations/productivity/calendly-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Clickhouse", + href: "/resources/integrations/databases/clickhouse", + type: "community", + category: "databases", + }, + { + label: "ClickUp", + href: "/resources/integrations/productivity/clickup", + type: "arcade", + category: "productivity", + }, + { + label: "ClickUp API", + href: "/resources/integrations/productivity/clickup-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Close.io", + href: "/resources/integrations/productivity/closeio", + type: "community", + category: "productivity", + }, + { + label: "Confluence", + href: "/resources/integrations/productivity/confluence", + type: "arcade", + category: "productivity", + }, + { + label: "Cursor Agents API", + href: "/resources/integrations/development/cursor-agents-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Customer.io API", + href: "/resources/integrations/customer-support/customerio-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Customer.io Pipelines API", + href: "/resources/integrations/customer-support/customerio-pipelines-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Customer.io Track API", + href: "/resources/integrations/customer-support/customerio-track-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Datadog API", + href: "/resources/integrations/development/datadog-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Discord", + href: "/resources/integrations/social-communication/discord", + type: "auth", + category: "social", + }, + { + label: "Dropbox", + href: "/resources/integrations/productivity/dropbox", + type: "arcade", + category: "productivity", + }, + { + label: "E2B", + href: "/resources/integrations/development/e2b", + type: "arcade", + category: "development", + }, + { + label: "Exa API", + href: "/resources/integrations/search/exa-api", + type: "arcade_starter", + category: "search", + }, + { + label: "Figma API", + href: "/resources/integrations/productivity/figma-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Firecrawl", + href: "/resources/integrations/development/firecrawl", + type: "arcade", + category: "development", + }, + { + label: "Freshservice API", + href: "/resources/integrations/customer-support/freshservice-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "GitHub", + href: "/resources/integrations/development/github", + type: "arcade", + category: "development", + }, + { + label: "GitHub API", + href: "/resources/integrations/development/github-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Gmail", + href: "/resources/integrations/productivity/gmail", + type: "arcade", + category: "productivity", + }, + { + label: "Google Calendar", + href: "/resources/integrations/productivity/google-calendar", + type: "arcade", + category: "productivity", + }, + { + label: "Google Contacts", + href: "/resources/integrations/productivity/google-contacts", + type: "arcade", + category: "productivity", + }, + { + label: "Google Docs", + href: "/resources/integrations/productivity/google-docs", + type: "arcade", + category: "productivity", + }, + { + label: "Google Drive", + href: "/resources/integrations/productivity/google-drive", + type: "arcade", + category: "productivity", + }, + { + label: "Google Finance", + href: "/resources/integrations/search/google_finance", + type: "arcade", + category: "search", + }, + { + label: "Google Flights", + href: "/resources/integrations/search/google_flights", + type: "arcade", + category: "search", + }, + { + label: "Google Hotels", + href: "/resources/integrations/search/google_hotels", + type: "arcade", + category: "search", + }, + { + label: "Google Jobs", + href: "/resources/integrations/search/google_jobs", + type: "arcade", + category: "search", + }, + { + label: "Google Maps", + href: "/resources/integrations/search/google_maps", + type: "arcade", + category: "search", + }, + { + label: "Google News", + href: "/resources/integrations/search/google_news", + type: "arcade", + category: "search", + }, + { + label: "Google Search", + href: "/resources/integrations/search/google_search", + type: "arcade", + category: "search", + }, + { + label: "Google Sheets", + href: "/resources/integrations/productivity/google-sheets", + type: "arcade", + category: "productivity", + }, + { + label: "Google Shopping", + href: "/resources/integrations/search/google_shopping", + type: "arcade", + category: "search", + }, + { + label: "Google Slides", + href: "/resources/integrations/productivity/google-slides", + type: "arcade", + category: "productivity", + }, + { + label: "HubSpot", + href: "/resources/integrations/sales/hubspot", + type: "arcade", + category: "sales", + }, + { + label: "HubSpot Automation API", + href: "/resources/integrations/sales/hubspot-automation-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot CMS API", + href: "/resources/integrations/sales/hubspot-cms-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Conversations API", + href: "/resources/integrations/sales/hubspot-conversations-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot CRM API", + href: "/resources/integrations/sales/hubspot-crm-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Events API", + href: "/resources/integrations/sales/hubspot-events-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Marketing API", + href: "/resources/integrations/sales/hubspot-marketing-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Meetings API", + href: "/resources/integrations/sales/hubspot-meetings-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Users API", + href: "/resources/integrations/sales/hubspot-users-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "Imgflip", + href: "/resources/integrations/entertainment/imgflip", + type: "arcade", + category: "entertainment", + }, + { + label: "Intercom API", + href: "/resources/integrations/customer-support/intercom-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Jira", + href: "/resources/integrations/productivity/jira", + type: "auth", + category: "productivity", + }, + { + label: "Linear", + href: "/resources/integrations/productivity/linear", + type: "arcade", + category: "productivity", + }, + { + label: "LinkedIn", + href: "/resources/integrations/social-communication/linkedin", + type: "arcade", + category: "social", + }, + { + label: "Luma API", + href: "/resources/integrations/productivity/luma-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Mailchimp API", + href: "/resources/integrations/productivity/mailchimp-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Microsoft SharePoint", + href: "/resources/integrations/productivity/sharepoint", + type: "arcade", + category: "productivity", + }, + { + label: "Microsoft Teams", + href: "/resources/integrations/social-communication/microsoft-teams", + type: "arcade", + category: "social", + }, + { + label: "Miro API", + href: "/resources/integrations/productivity/miro-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "MongoDB", + href: "/resources/integrations/databases/mongodb", + type: "community", + category: "databases", + }, + { + label: "Notion", + href: "/resources/integrations/productivity/notion", + type: "arcade", + category: "productivity", + }, + { + label: "Obsidian", + href: "/resources/integrations/productivity/obsidian", + type: "community", + category: "productivity", + }, + { + label: "Outlook Calendar", + href: "/resources/integrations/productivity/outlook-calendar", + type: "arcade", + category: "productivity", + }, + { + label: "Outlook Mail", + href: "/resources/integrations/productivity/outlook-mail", + type: "arcade", + category: "productivity", + }, + { + label: "PagerDuty API", + href: "/resources/integrations/development/pagerduty-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Postgres", + href: "/resources/integrations/databases/postgres", + type: "community", + category: "databases", + }, + { + label: "PostHog API", + href: "/resources/integrations/development/posthog-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Reddit", + href: "/resources/integrations/social-communication/reddit", + type: "arcade", + category: "social", + }, + { + label: "Salesforce", + href: "/resources/integrations/sales/salesforce", + type: "arcade", + category: "sales", + }, + { + label: "Slack", + href: "/resources/integrations/social-communication/slack", + type: "arcade", + category: "social", + }, + { + label: "Slack API", + href: "/resources/integrations/social-communication/slack-api", + type: "arcade_starter", + category: "social", + }, + { + label: "Spotify", + href: "/resources/integrations/entertainment/spotify", + type: "arcade", + category: "entertainment", + }, + { + label: "SquareUp API", + href: "/resources/integrations/productivity/squareup-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Stripe", + href: "/resources/integrations/payments/stripe", + type: "arcade", + category: "payments", + }, + { + label: "Stripe API", + href: "/resources/integrations/payments/stripe_api", + type: "arcade_starter", + category: "payments", + }, + { + label: "TickTick API", + href: "/resources/integrations/productivity/ticktick-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Trello API", + href: "/resources/integrations/productivity/trello-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Twilio", + href: "/resources/integrations/social-communication/twilio", + type: "verified", + category: "social", + }, + { + label: "Twitch", + href: "/resources/integrations/entertainment/twitch", + type: "auth", + category: "entertainment", + }, + { + label: "Vercel API", + href: "/resources/integrations/development/vercel-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Walmart", + href: "/resources/integrations/search/walmart", + type: "arcade", + category: "search", + }, + { + label: "Weaviate API", + href: "/resources/integrations/development/weaviate-api", + type: "arcade_starter", + category: "development", + }, + { + label: "X", + href: "/resources/integrations/social-communication/x", + type: "arcade", + category: "social", + }, + { + label: "Xero API", + href: "/resources/integrations/productivity/xero-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "YouTube", + href: "/resources/integrations/search/youtube", + type: "arcade", + category: "search", + }, + { + label: "Zendesk", + href: "/resources/integrations/customer-support/zendesk", + type: "arcade", + category: "customer-support", + }, + { + label: "Zoho Books API", + href: "/resources/integrations/payments/zoho-books-api", + type: "arcade_starter", + category: "payments", + }, + { + label: "Zoho Creator API", + href: "/resources/integrations/productivity/zoho-creator-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Zoom", + href: "/resources/integrations/social-communication/zoom", + type: "arcade", + category: "social", + }, + ]; + + const typeLabels: Record = { + arcade: "Arcade Optimized", + arcade_starter: "Arcade Starter", + verified: "Verified", + community: "Community", + auth: "Auth Provider", + }; + + const categoryLabels: Record = { + productivity: "Productivity", + development: "Development", + social: "Social", + search: "Search", + sales: "Sales", + payments: "Payments", + entertainment: "Entertainment", + databases: "Databases", + "customer-support": "Customer Support", + }; + + return createElement( + "div", + null, + createElement( + "p", + null, + "Registry of all MCP Servers available in the Arcade ecosystem. ", + createElement( + "a", + { href: "/guides/create-tools/tool-basics/build-mcp-server" }, + "Build your own MCP Server" + ), + "." + ), + createElement( + "ul", + null, + ...allMcpServers.map((server, i) => + createElement( + "li", + { key: i }, + createElement("a", { href: server.href }, server.label), + ` - ${typeLabels[server.type] || server.type}, ${categoryLabels[server.category] || server.category}` + ) + ) + ) + ); +} + +// All available markdown-friendly components +const markdownComponents: Record< + string, + React.ComponentType> +> = { + // Nextra components + Tabs: MarkdownTabs, + "Tabs.Tab": MarkdownTab, + Tab: MarkdownTab, + Steps: MarkdownSteps, + Callout: MarkdownCallout, + Cards: MarkdownCards, + "Cards.Card": MarkdownCard, + Card: MarkdownCard, + FileTree: MarkdownFileTree, + + // Custom components + GuideOverview: MarkdownGuideOverview, + "GuideOverview.Item": MarkdownGuideOverviewItem, + "GuideOverview.Outcomes": MarkdownGuideOverviewOutcomes, + "GuideOverview.Prerequisites": MarkdownGuideOverviewPrerequisites, + CheatSheetGrid: MarkdownCheatSheetGrid, + CheatSheetSection: MarkdownCheatSheetSection, + InfoBox: MarkdownInfoBox, + CommandItem: MarkdownCommandItem, + CommandList: MarkdownCommandList, + CommandBlock: MarkdownCommandBlock, + GlossaryTerm: MarkdownGlossaryTerm, + DashboardLink: MarkdownDashboardLink, + SignupLink: MarkdownSignupLink, + ToolCard: MarkdownToolCard, + MCPClientGrid: MarkdownMCPClientGrid, + ContactCards: MarkdownContactCards, + SubpageList: MarkdownSubpageList, + + // Page-level components + LandingPage: MarkdownLandingPage, + Toolkits: MarkdownMcpServers, + + // HTML-like components (uppercase - for custom components) + Video: MarkdownVideo, + Audio: MarkdownAudio, + Image: MarkdownImage, + + // ============================================ + // Lowercase HTML element overrides + // MDX passes these through as intrinsic elements + // ============================================ + + // Media elements - convert to links + video: MarkdownVideo, + audio: MarkdownAudio, + img: MarkdownImage, + iframe: MarkdownIframe, + embed: MarkdownPassthrough, + object: MarkdownPassthrough, + source: MarkdownPassthrough, + track: MarkdownPassthrough, + picture: MarkdownPassthrough, + + // Structural/semantic elements - strip wrapper, keep children + div: MarkdownPassthrough, + span: MarkdownPassthrough, + section: MarkdownPassthrough, + article: MarkdownPassthrough, + aside: MarkdownPassthrough, + header: MarkdownPassthrough, + footer: MarkdownPassthrough, + main: MarkdownPassthrough, + nav: MarkdownPassthrough, + address: MarkdownPassthrough, + hgroup: MarkdownPassthrough, + + // Figure elements + figure: MarkdownFigure, + figcaption: MarkdownFigcaption, + + // Interactive elements + details: MarkdownDetails, + summary: MarkdownSummary, + dialog: MarkdownPassthrough, + + // Self-closing elements + hr: MarkdownHr, + br: MarkdownBr, + wbr: MarkdownPassthrough, + + // Table elements - pass through for turndown + table: MarkdownTable, + thead: MarkdownThead, + tbody: MarkdownTbody, + tfoot: MarkdownTbody, + tr: MarkdownTr, + th: MarkdownTh, + td: MarkdownTd, + caption: MarkdownPassthrough, + colgroup: MarkdownPassthrough, + col: MarkdownPassthrough, + + // Code elements - preserve language info + pre: MarkdownPre, + code: MarkdownCode, + + // Definition lists + dl: MarkdownDl, + dt: MarkdownDt, + dd: MarkdownDd, + + // Form elements - strip (not useful in markdown) + form: MarkdownPassthrough, + input: MarkdownPassthrough, + button: MarkdownPassthrough, + select: MarkdownPassthrough, + option: MarkdownPassthrough, + optgroup: MarkdownPassthrough, + textarea: MarkdownPassthrough, + label: MarkdownPassthrough, + fieldset: MarkdownPassthrough, + legend: MarkdownPassthrough, + datalist: MarkdownPassthrough, + output: MarkdownPassthrough, + progress: MarkdownPassthrough, + meter: MarkdownPassthrough, + + // Script/style elements - remove entirely + script: () => null, + style: () => null, + noscript: MarkdownPassthrough, + template: MarkdownPassthrough, + slot: MarkdownPassthrough, + + // Canvas/SVG - remove (not representable in markdown) + canvas: () => null, + svg: () => null, + + // Misc inline elements - pass through + abbr: MarkdownPassthrough, + bdi: MarkdownPassthrough, + bdo: MarkdownPassthrough, + cite: MarkdownPassthrough, + data: MarkdownPassthrough, + dfn: MarkdownPassthrough, + kbd: MarkdownPassthrough, + mark: MarkdownPassthrough, + q: MarkdownPassthrough, + rb: MarkdownPassthrough, + rp: MarkdownPassthrough, + rt: MarkdownPassthrough, + rtc: MarkdownPassthrough, + ruby: MarkdownPassthrough, + s: MarkdownPassthrough, + samp: MarkdownPassthrough, + small: MarkdownPassthrough, + sub: MarkdownPassthrough, + sup: MarkdownPassthrough, + time: MarkdownPassthrough, + u: MarkdownPassthrough, + var: MarkdownPassthrough, + + // Map/area elements + map: MarkdownPassthrough, + area: MarkdownPassthrough, + + // Fallbacks + wrapper: PassThrough, +}; + +/** + * Strip import and export statements from MDX content + * This allows us to compile MDX without needing to resolve external modules + */ +function stripImportsAndExports(content: string): string { + let result = content; + + // Remove import statements (various formats) + // import X from 'module' + result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); + // import 'module' + result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); + // import { X } from 'module' (multiline) + result = result.replace( + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, + "" + ); + + // Remove export statements + result = result.replace( + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, + "" + ); + + return result; +} + +/** + * Convert MDX content to clean Markdown + */ +export async function mdxToMarkdown( + mdxContent: string, + pagePath: string +): Promise { + // Extract frontmatter first + const frontmatterMatch = mdxContent.match(FRONTMATTER_REGEX); + let frontmatter = ""; + let contentWithoutFrontmatter = mdxContent; + + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + contentWithoutFrontmatter = mdxContent.slice(frontmatterMatch[0].length); + } + + try { + // Strip imports before compilation so MDX doesn't try to resolve them + const strippedContent = stripImportsAndExports(contentWithoutFrontmatter); + + // Compile MDX to JavaScript + // Include remarkGfm to properly parse GFM tables, strikethrough, etc. in the MDX source + const compiled = await compile(strippedContent, { + outputFormat: "function-body", + development: false, + remarkPlugins: [remarkGfm], + }); + + // Run the compiled code to get the component + // Use process.cwd() as baseUrl since we're not resolving any imports + const { default: MDXContent } = await run(String(compiled), { + Fragment: JsxFragment, + jsx, + jsxs, + baseUrl: pathToFileURL(process.cwd()).href, + }); + + // Render with markdown-friendly components + const element = createElement(MDXContent, { + components: markdownComponents, + }); + + // Convert React element to HTML string + const render = await getRenderer(); + const html = render(element); + + // Convert HTML to Markdown using unified ecosystem + let markdown = await htmlToMarkdown(html); + + // Clean up excessive whitespace + markdown = markdown.replace(/\n{3,}/g, "\n\n").trim(); + + // If result is empty, provide fallback + if (!markdown || markdown.length < 10) { + const title = extractTitle(frontmatter); + const description = extractDescription(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + return `${frontmatter}${markdown}\n`; + } catch (_error) { + return fallbackMdxToMarkdown(mdxContent, pagePath); + } +} + +/** + * Extract title from frontmatter + */ +function extractTitle(frontmatter: string): string { + const match = frontmatter.match(TITLE_REGEX); + return ( + match?.[1] || match?.[2] || match?.[3]?.trim() || "Arcade Documentation" + ); +} + +/** + * Extract description from frontmatter + */ +function extractDescription(frontmatter: string): string { + const match = frontmatter.match(DESCRIPTION_REGEX); + return match?.[1] || match?.[2] || match?.[3]?.trim() || ""; +} + +/** + * Fallback: Simple regex-based MDX to Markdown conversion + * Used when MDX compilation fails + */ +function fallbackMdxToMarkdown(content: string, pagePath: string): string { + let result = content; + + // Extract frontmatter + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + let frontmatter = ""; + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove imports + result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); + result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); + result = result.replace( + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, + "" + ); + + // Remove exports + result = result.replace( + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, + "" + ); + + // Remove self-closing JSX components (uppercase) + result = result.replace(/<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g, ""); + + // Remove self-closing HTML elements (lowercase) - but convert video/img to links + result = result.replace( + /]*src=["']([^"']+)["'][^>]*\/?>/gi, + "\n\n[Video]($1)\n\n" + ); + result = result.replace( + /]*src=["']([^"']+)["'][^>]*\/?>/gi, + "![]($1)" + ); + result = result.replace(/<[a-z][a-zA-Z0-9]*[^>]*\/>/g, ""); + + // Extract content from JSX with children (process iteratively for nesting) + let prev = ""; + while (prev !== result) { + prev = result; + // Uppercase components + result = result.replace( + /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g, + "$2" + ); + // Lowercase HTML elements (div, span, etc.) + result = result.replace( + /<(div|span|section|article|aside|header|footer|main|nav|video|figure|figcaption)[^>]*>([\s\S]*?)<\/\1>/gi, + "$2" + ); + } + + // Remove JSX expressions + result = result.replace(/\{[^}]+\}/g, ""); + + // Clean up + result = result.replace(/\n{3,}/g, "\n\n").trim(); + + if (!result || result.length < 10) { + const title = extractTitle(frontmatter); + const description = extractDescription(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + return `${frontmatter}${result}\n`; +} diff --git a/package.json b/package.json index ba7953195..f2316f47a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "dev": "next dev --webpack", - "build": "next build --webpack", + "build": "pnpm run generate:markdown && next build --webpack", + "generate:markdown": "pnpm exec tsx scripts/generate-markdown.ts", "start": "next start", "lint": "pnpm dlx ultracite check", "format": "pnpm dlx ultracite fix", @@ -39,6 +40,7 @@ "homepage": "https://arcade.dev/", "dependencies": { "@arcadeai/design-system": "^3.26.0", + "@mdx-js/mdx": "^3.1.1", "@next/third-parties": "16.0.1", "@ory/client": "1.22.7", "@theguild/remark-mermaid": "0.3.0", @@ -54,8 +56,14 @@ "react-dom": "19.2.3", "react-hook-form": "7.65.0", "react-syntax-highlighter": "16.1.0", + "rehype-parse": "^9.0.1", + "rehype-remark": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-stringify": "^11.0.0", "swagger-ui-react": "^5.30.0", "tailwindcss-animate": "1.0.7", + "turndown": "^7.2.2", + "unified": "^11.0.5", "unist-util-visit": "5.0.0", "unist-util-visit-parents": "6.0.2", "zustand": "5.0.8" @@ -71,6 +79,7 @@ "@types/react": "19.2.7", "@types/react-dom": "19.2.3", "@types/react-syntax-highlighter": "15.5.13", + "@types/turndown": "^5.0.6", "@types/unist": "3.0.3", "commander": "14.0.2", "dotenv": "^17.2.3", @@ -87,6 +96,7 @@ "remark": "^15.0.1", "remark-rehype": "^11.1.2", "tailwindcss": "4.1.16", + "tsx": "^4.21.0", "typescript": "5.9.3", "ultracite": "6.1.0", "vitest": "4.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 436ac536b..7923f5808 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,7 +10,10 @@ importers: dependencies: '@arcadeai/design-system': specifier: ^3.26.0 - version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@mdx-js/mdx': + specifier: ^3.1.1 + version: 3.1.1 '@next/third-parties': specifier: 16.0.1 version: 16.0.1(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -56,12 +59,30 @@ importers: react-syntax-highlighter: specifier: 16.1.0 version: 16.1.0(react@19.2.3) + rehype-parse: + specifier: ^9.0.1 + version: 9.0.1 + rehype-remark: + specifier: ^10.0.1 + version: 10.0.1 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-stringify: + specifier: ^11.0.0 + version: 11.0.0 swagger-ui-react: specifier: ^5.30.0 version: 5.31.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) tailwindcss-animate: specifier: 1.0.7 version: 1.0.7(tailwindcss@4.1.16) + turndown: + specifier: ^7.2.2 + version: 7.2.2 + unified: + specifier: ^11.0.5 + version: 11.0.5 unist-util-visit: specifier: 5.0.0 version: 5.0.0 @@ -102,6 +123,9 @@ importers: '@types/react-syntax-highlighter': specifier: 15.5.13 version: 15.5.13 + '@types/turndown': + specifier: ^5.0.6 + version: 5.0.6 '@types/unist': specifier: 3.0.3 version: 3.0.3 @@ -150,6 +174,9 @@ importers: tailwindcss: specifier: 4.1.16 version: 4.1.16 + tsx: + specifier: ^4.21.0 + version: 4.21.0 typescript: specifier: 5.9.3 version: 5.9.3 @@ -158,7 +185,7 @@ importers: version: 6.1.0(typescript@5.9.3) vitest: specifier: 4.0.5 - version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) zod: specifier: 4.1.12 version: 4.1.12 @@ -653,6 +680,9 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@napi-rs/simple-git-android-arm-eabi@0.1.22': resolution: {integrity: sha512-JQZdnDNm8o43A5GOzwN/0Tz3CDBQtBUNqzVwEopm32uayjdjxev1Csp1JeaqF3v9djLDIvsSE39ecsN2LhCKKQ==} engines: {node: '>= 10'} @@ -2253,6 +2283,9 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + '@types/turndown@5.0.6': + resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -3015,6 +3048,9 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} + get-tsconfig@4.13.0: + resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} + github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -3051,6 +3087,9 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hast-util-embedded@3.0.0: + resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} + hast-util-from-dom@5.0.1: resolution: {integrity: sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q==} @@ -3063,12 +3102,24 @@ packages: hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} + hast-util-has-property@3.0.0: + resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} + + hast-util-is-body-ok-link@3.0.1: + resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} + hast-util-is-element@3.0.0: resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} + hast-util-minify-whitespace@1.0.1: + resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} + hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + hast-util-phrasing@3.0.1: + resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} + hast-util-raw@9.1.0: resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} @@ -3081,6 +3132,9 @@ packages: hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} + hast-util-to-mdast@10.1.2: + resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} + hast-util-to-parse5@8.0.1: resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==} @@ -4165,6 +4219,9 @@ packages: rehype-katex@7.0.1: resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==} + rehype-minify-whitespace@6.0.2: + resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} + rehype-parse@9.0.1: resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} @@ -4180,6 +4237,9 @@ packages: rehype-recma@1.0.0: resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} + rehype-remark@10.0.1: + resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} + rehype-stringify@10.0.1: resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} @@ -4229,6 +4289,9 @@ packages: reselect@5.1.1: resolution: {integrity: sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -4521,6 +4584,9 @@ packages: trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + trim-trailing-lines@2.1.0: + resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} + trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -4568,6 +4634,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + turndown@7.2.2: + resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} + twoslash-protocol@0.3.4: resolution: {integrity: sha512-HHd7lzZNLUvjPzG/IE6js502gEzLC1x7HaO1up/f72d8G8ScWAs9Yfa97igelQRDl5h9tGcdFsRp+lNVre1EeQ==} @@ -4918,7 +4992,7 @@ snapshots: transitivePeerDependencies: - encoding - '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@arcadeai/arcadejs': 1.15.0 '@hookform/resolvers': 5.2.2(react-hook-form@7.65.0(react@19.2.3)) @@ -4943,7 +5017,7 @@ snapshots: '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-toggle': 1.1.10(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-tooltip': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) class-variance-authority: 0.7.1 clsx: 2.1.1 cmdk: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -5327,6 +5401,8 @@ snapshots: dependencies: langium: 3.3.1 + '@mixmark-io/domino@2.2.0': {} + '@napi-rs/simple-git-android-arm-eabi@0.1.22': optional: true @@ -6926,12 +7002,12 @@ snapshots: postcss: 8.5.6 tailwindcss: 4.1.16 - '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@tailwindcss/node': 4.1.14 '@tailwindcss/oxide': 4.1.14 tailwindcss: 4.1.14 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) '@tanstack/react-virtual@3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: @@ -7158,6 +7234,8 @@ snapshots: '@types/trusted-types@2.0.7': optional: true + '@types/turndown@5.0.6': {} + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -7187,13 +7265,13 @@ snapshots: chai: 6.2.1 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) '@vitest/pretty-format@4.0.5': dependencies: @@ -7934,6 +8012,10 @@ snapshots: get-stream@8.0.1: {} + get-tsconfig@4.13.0: + dependencies: + resolve-pkg-maps: 1.0.0 + github-slugger@2.0.0: {} glob-parent@5.1.2: @@ -7967,6 +8049,11 @@ snapshots: dependencies: function-bind: 1.1.2 + hast-util-embedded@3.0.0: + dependencies: + '@types/hast': 3.0.4 + hast-util-is-element: 3.0.0 + hast-util-from-dom@5.0.1: dependencies: '@types/hast': 3.0.4 @@ -8000,14 +8087,38 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 + hast-util-has-property@3.0.0: + dependencies: + '@types/hast': 3.0.4 + + hast-util-is-body-ok-link@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-is-element@3.0.0: dependencies: '@types/hast': 3.0.4 + hast-util-minify-whitespace@1.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-is-element: 3.0.0 + hast-util-whitespace: 3.0.0 + unist-util-is: 6.0.1 + hast-util-parse-selector@4.0.0: dependencies: '@types/hast': 3.0.4 + hast-util-phrasing@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-has-property: 3.0.0 + hast-util-is-body-ok-link: 3.0.1 + hast-util-is-element: 3.0.0 + hast-util-raw@9.1.0: dependencies: '@types/hast': 3.0.4 @@ -8079,6 +8190,23 @@ snapshots: transitivePeerDependencies: - supports-color + hast-util-to-mdast@10.1.2: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.0 + hast-util-phrasing: 3.0.1 + hast-util-to-html: 9.0.5 + hast-util-to-text: 4.0.2 + hast-util-whitespace: 3.0.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-hast: 13.2.1 + mdast-util-to-string: 4.0.0 + rehype-minify-whitespace: 6.0.2 + trim-trailing-lines: 2.1.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + hast-util-to-parse5@8.0.1: dependencies: '@types/hast': 3.0.4 @@ -9509,6 +9637,11 @@ snapshots: unist-util-visit-parents: 6.0.2 vfile: 6.0.3 + rehype-minify-whitespace@6.0.2: + dependencies: + '@types/hast': 3.0.4 + hast-util-minify-whitespace: 1.0.1 + rehype-parse@9.0.1: dependencies: '@types/hast': 3.0.4 @@ -9539,6 +9672,14 @@ snapshots: transitivePeerDependencies: - supports-color + rehype-remark@10.0.1: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + hast-util-to-mdast: 10.1.2 + unified: 11.0.5 + vfile: 6.0.3 + rehype-stringify@10.0.1: dependencies: '@types/hast': 3.0.4 @@ -9638,6 +9779,8 @@ snapshots: reselect@5.1.1: {} + resolve-pkg-maps@1.0.0: {} + restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -10029,6 +10172,8 @@ snapshots: trim-lines@3.0.1: {} + trim-trailing-lines@2.1.0: {} + trough@2.2.0: {} trpc-cli@0.12.1(@trpc/server@11.8.0(typescript@5.9.3))(zod@4.1.12): @@ -10053,6 +10198,17 @@ snapshots: tslib@2.8.1: {} + tsx@4.21.0: + dependencies: + esbuild: 0.27.1 + get-tsconfig: 4.13.0 + optionalDependencies: + fsevents: 2.3.3 + + turndown@7.2.2: + dependencies: + '@mixmark-io/domino': 2.2.0 + twoslash-protocol@0.3.4: {} twoslash@0.3.4(typescript@5.9.3): @@ -10238,7 +10394,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): + vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): dependencies: esbuild: 0.27.1 fdir: 6.5.0(picomatch@4.0.3) @@ -10251,12 +10407,13 @@ snapshots: fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 + tsx: 4.21.0 yaml: 2.8.2 - vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): + vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.5 - '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.5 '@vitest/runner': 4.0.5 '@vitest/snapshot': 4.0.5 @@ -10273,7 +10430,7 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 diff --git a/proxy.ts b/proxy.ts index df8e7d83a..f9cd4b515 100644 --- a/proxy.ts +++ b/proxy.ts @@ -63,19 +63,18 @@ function pathnameIsMissingLocale(pathname: string): boolean { export function proxy(request: NextRequest) { const pathname = request.nextUrl.pathname; - // Handle .md requests without locale - redirect to add locale first - if (pathname.endsWith(".md") && pathnameIsMissingLocale(pathname)) { - const locale = getPreferredLocale(request); - const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); - const redirectPath = `/${locale}${pathWithoutMd}.md`; - return NextResponse.redirect(new URL(redirectPath, request.url)); - } - - // Rewrite .md requests (with locale) to the markdown API route - if (pathname.endsWith(".md") && !pathname.startsWith("/api/")) { - const url = request.nextUrl.clone(); - url.pathname = `/api/markdown${pathname}`; - return NextResponse.rewrite(url); + // .md requests are served as static files from public/ + // (generated at build time by scripts/generate-markdown.ts) + if (pathname.endsWith(".md")) { + // Add locale prefix if missing, then let Next.js serve from public/ + if (pathnameIsMissingLocale(pathname)) { + const locale = getPreferredLocale(request); + const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); + const redirectPath = `/${locale}${pathWithoutMd}.md`; + return NextResponse.redirect(new URL(redirectPath, request.url)); + } + // Let Next.js serve the static file from public/ + return NextResponse.next(); } // Redirect if there is no locale diff --git a/scripts/generate-markdown.ts b/scripts/generate-markdown.ts new file mode 100644 index 000000000..618f9953f --- /dev/null +++ b/scripts/generate-markdown.ts @@ -0,0 +1,155 @@ +/** + * Static Markdown Generation Script + * + * Generates pre-rendered markdown files from MDX pages for LLM consumption. + * Outputs to public/en/ so files are served directly by Next.js/CDN. + * + * Usage: pnpm dlx tsx scripts/generate-markdown.ts + */ + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import fastGlob from "fast-glob"; +import { mdxToMarkdown } from "../lib/mdx-to-markdown"; + +const OUTPUT_DIR = join(process.cwd(), "public"); +const SEPARATOR_WIDTH = 50; +const RESOURCE_FILE_REGEX = + /\.(png|jpg|jpeg|gif|svg|webp|mp4|webm|pdf|zip|tar|gz)$/i; + +/** + * Rewrite internal links to point to .md files + * - /references/foo → /en/references/foo.md + * - /en/references/foo → /en/references/foo.md + * - External links, anchors, and resource links are unchanged + */ +function rewriteLinksToMarkdown(markdown: string): string { + // Match markdown links: [text](url) + return markdown.replace( + /\[([^\]]*)\]\(([^)]+)\)/g, + (match, text, url: string) => { + // Skip external links + if ( + url.startsWith("http://") || + url.startsWith("https://") || + url.startsWith("mailto:") + ) { + return match; + } + + // Skip anchor-only links + if (url.startsWith("#")) { + return match; + } + + // Skip resource links (images, videos, files) + if ( + RESOURCE_FILE_REGEX.test(url) || + url.startsWith("/images/") || + url.startsWith("/videos/") || + url.startsWith("/files/") + ) { + return match; + } + + // Skip if already has .md extension + if (url.endsWith(".md")) { + return match; + } + + // Handle anchor in URL + let anchor = ""; + let pathPart = url; + const anchorIndex = url.indexOf("#"); + if (anchorIndex !== -1) { + anchor = url.slice(anchorIndex); + pathPart = url.slice(0, anchorIndex); + } + + // Add /en prefix if not present (internal doc links) + if (pathPart.startsWith("/") && !pathPart.startsWith("/en/")) { + pathPart = `/en${pathPart}`; + } + + // Add .md extension + const newUrl = `${pathPart}.md${anchor}`; + return `[${text}](${newUrl})`; + } + ); +} + +async function generateMarkdownFiles() { + console.log("Generating static markdown files...\n"); + + // Find all MDX pages + const mdxFiles = await fastGlob("app/en/**/page.mdx", { + cwd: process.cwd(), + absolute: false, + }); + + console.log(`Found ${mdxFiles.length} MDX files\n`); + + let successCount = 0; + let errorCount = 0; + const errors: { file: string; error: string }[] = []; + + for (const mdxFile of mdxFiles) { + try { + // Read MDX content + const mdxPath = join(process.cwd(), mdxFile); + const mdxContent = await readFile(mdxPath, "utf-8"); + + // Compute paths + // app/en/references/auth-providers/page.mdx → /en/references/auth-providers + const relativePath = mdxFile + .replace("app/", "/") + .replace("/page.mdx", ""); + + // Convert to markdown + let markdown = await mdxToMarkdown(mdxContent, relativePath); + + // Rewrite links to point to .md files + markdown = rewriteLinksToMarkdown(markdown); + + // Output path: public/en/references/auth-providers.md + const outputPath = join(OUTPUT_DIR, `${relativePath}.md`); + + // Ensure directory exists + await mkdir(dirname(outputPath), { recursive: true }); + + // Write markdown file + await writeFile(outputPath, markdown, "utf-8"); + + successCount += 1; + process.stdout.write(`✓ ${relativePath}.md\n`); + } catch (error) { + errorCount += 1; + const errorMessage = + error instanceof Error ? error.message : String(error); + errors.push({ file: mdxFile, error: errorMessage }); + process.stdout.write(`✗ ${mdxFile}: ${errorMessage}\n`); + } + } + + console.log(`\n${"=".repeat(SEPARATOR_WIDTH)}`); + console.log(`Generated: ${successCount} files`); + if (errorCount > 0) { + console.log(`Errors: ${errorCount} files`); + console.log("\nFailed files:"); + for (const { file, error } of errors) { + console.log(` - ${file}: ${error}`); + } + } + console.log("=".repeat(SEPARATOR_WIDTH)); + + // Exit with error code if any files failed + if (errorCount > 0) { + process.exit(1); + } +} + +// Run the script +generateMarkdownFiles().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); From 0ea4dbb5d666cc97ff25f26552e027f5fd8f612e Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 17:54:06 +0000 Subject: [PATCH 08/10] Revert "Generate static markdown files at build time" This reverts commit d7b7c711018863a34b02b9b998b3711904439f98. --- .gitignore | 1 - app/api/markdown/[[...slug]]/route.ts | 295 +++ .../integrations/productivity/gmail/page.mdx | 2 +- lib/mdx-to-markdown.tsx | 2314 ----------------- package.json | 12 +- pnpm-lock.yaml | 181 +- proxy.ts | 25 +- scripts/generate-markdown.ts | 155 -- 8 files changed, 322 insertions(+), 2663 deletions(-) create mode 100644 app/api/markdown/[[...slug]]/route.ts delete mode 100644 lib/mdx-to-markdown.tsx delete mode 100644 scripts/generate-markdown.ts diff --git a/.gitignore b/.gitignore index 5968906cb..c714c9723 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ node_modules .DS_Store .env.local public/sitemap*.xml -public/en/**/*.md .env _pagefind/ diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts new file mode 100644 index 000000000..10812365f --- /dev/null +++ b/app/api/markdown/[[...slug]]/route.ts @@ -0,0 +1,295 @@ +import { access, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type NextRequest, NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +// Regex pattern for removing .md extension +const MD_EXTENSION_REGEX = /\.md$/; + +// Regex patterns for MDX to Markdown compilation (top-level for performance) +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; +const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; +const IMPORT_DESTRUCTURE_REGEX = + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; +const EXPORT_REGEX = + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; +const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; +const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; +const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; +const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; +const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; + +// Regex for detecting markdown list items and numbered lists +const UNORDERED_LIST_REGEX = /^[-*+]\s/; +const ORDERED_LIST_REGEX = /^\d+[.)]\s/; + +// Regex for extracting frontmatter fields +// Handles: "double quoted", 'single quoted', or unquoted values +// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback +// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters +const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; +const DESCRIPTION_REGEX = + /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; + +// Regex for detecting leading whitespace on lines +const LEADING_WHITESPACE_REGEX = /^[ \t]+/; + +/** + * Removes consistent leading indentation from all lines of text. + * This normalizes content that was indented inside JSX components. + * Code block markers (```) are ignored when calculating minimum indent + * since they typically start at column 0 in MDX files. + */ +function dedent(text: string): string { + const lines = text.split("\n"); + + // Find minimum indentation, ignoring: + // - Empty lines + // - Code block markers (lines starting with ```) + let minIndent = Number.POSITIVE_INFINITY; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === "" || trimmed.startsWith("```")) { + continue; // Ignore empty lines and code block markers + } + const match = line.match(LEADING_WHITESPACE_REGEX); + const indent = match ? match[0].length : 0; + if (indent < minIndent) { + minIndent = indent; + } + } + + // If no indentation found, return as-is + if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { + return text; + } + + // Remove the minimum indentation from each line (except code block content) + return lines + .map((line) => { + const trimmed = line.trim(); + // Don't modify empty lines or lines with less indentation than min + if (trimmed === "" || line.length < minIndent) { + return line.trimStart(); + } + // Preserve code block markers - just remove leading whitespace + // This matches the logic that ignores them when calculating minIndent + if (trimmed.startsWith("```")) { + return trimmed; + } + return line.slice(minIndent); + }) + .join("\n"); +} + +/** + * Strips surrounding quotes from a value if present. + * Used for unquoted fallback values that may contain quotes due to apostrophe handling. + */ +function stripSurroundingQuotes(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +/** + * Extracts title and description from frontmatter. + * Handles double-quoted, single-quoted, and unquoted YAML values. + */ +function extractFrontmatterMeta(frontmatter: string): { + title: string; + description: string; +} { + const titleMatch = frontmatter.match(TITLE_REGEX); + const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + + // Extract from whichever capture group matched: + // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback + // For group 3 (fallback), strip surrounding quotes if present + const title = + titleMatch?.[1] ?? + titleMatch?.[2] ?? + stripSurroundingQuotes(titleMatch?.[3] ?? ""); + const description = + descriptionMatch?.[1] ?? + descriptionMatch?.[2] ?? + stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); + + return { + title: title || "Arcade Documentation", + description, + }; +} + +/** + * Normalizes indentation in the final output. + * Removes stray leading whitespace outside code blocks while preserving + * meaningful markdown indentation (nested lists, blockquotes). + */ +function normalizeIndentation(text: string): string { + const finalLines: string[] = []; + let inCodeBlock = false; + + for (const line of text.split("\n")) { + if (line.trim().startsWith("```")) { + inCodeBlock = !inCodeBlock; + finalLines.push(line.trimStart()); // Code block markers should start at column 0 + } else if (inCodeBlock) { + finalLines.push(line); // Preserve indentation inside code blocks + } else { + const trimmed = line.trimStart(); + // Preserve indentation for nested list items and blockquotes + const isListItem = + UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); + const isBlockquote = trimmed.startsWith(">"); + if ((isListItem || isBlockquote) && line.startsWith(" ")) { + // Keep markdown-meaningful indentation (but normalize to 2-space increments) + const leadingSpaces = line.length - line.trimStart().length; + const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); + finalLines.push(normalizedIndent + trimmed); + } else { + finalLines.push(trimmed); // Remove leading whitespace for other lines + } + } + } + + return finalLines.join("\n"); +} + +/** + * Compiles MDX content to clean markdown by: + * - Preserving frontmatter + * - Removing import statements + * - Converting JSX components to their text content + * - Preserving standard markdown + * - Providing fallback content for component-only pages + */ +function compileMdxToMarkdown(content: string, pagePath: string): string { + let result = content; + + // Extract and preserve frontmatter if present + let frontmatter = ""; + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove import statements (various formats) + result = result.replace(IMPORT_FROM_REGEX, ""); + result = result.replace(IMPORT_DIRECT_REGEX, ""); + result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); + + // Remove export statements (like export const metadata) + result = result.replace(EXPORT_REGEX, ""); + + // Process self-closing JSX components (e.g., or ) + // Handles components with dots like + result = result.replace(SELF_CLOSING_JSX_REGEX, ""); + + // Process JSX components with children - extract the text content + // Handles components with dots like content + // Keep processing until no more JSX components remain + let previousResult = ""; + while (previousResult !== result) { + previousResult = result; + // Match opening tag, capture tag name (with dots), and content until matching closing tag + // Apply dedent to each extracted piece to normalize indentation + result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => + dedent(innerContent.trim()) + ); + } + + // Remove any remaining JSX expressions like {variable} or {expression} + // But preserve code blocks by temporarily replacing them + const codeBlocks: string[] = []; + result = result.replace(CODE_BLOCK_REGEX, (match) => { + codeBlocks.push(match); + return `__CODE_BLOCK_${codeBlocks.length - 1}__`; + }); + + // Now remove JSX expressions outside code blocks + result = result.replace(JSX_EXPRESSION_REGEX, ""); + + // Restore code blocks + result = result.replace( + CODE_BLOCK_PLACEHOLDER_REGEX, + (_, index) => codeBlocks[Number.parseInt(index, 10)] + ); + + // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) + result = normalizeIndentation(result); + + // Clean up excessive blank lines (more than 2 consecutive) + result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); + + // Trim leading/trailing whitespace + result = result.trim(); + + // If content is essentially empty (component-only page), provide fallback + if (!result || result.length < 10) { + const { title, description } = extractFrontmatterMeta(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + // Reconstruct with frontmatter + return `${frontmatter}${result}\n`; +} + +export async function GET( + request: NextRequest, + _context: { params: Promise<{ slug?: string[] }> } +) { + try { + // Get the original pathname from the request + const url = new URL(request.url); + // Remove /api/markdown prefix to get the original path + const originalPath = url.pathname.replace("/api/markdown", ""); + + // Remove .md extension + const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, ""); + + // Map URL to file path + // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx + const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`); + + // Check if file exists + try { + await access(filePath); + } catch { + return new NextResponse("Markdown file not found", { status: 404 }); + } + + const rawContent = await readFile(filePath, "utf-8"); + + // Compile MDX to clean markdown + const content = compileMdxToMarkdown(rawContent, pathWithoutMd); + + // Return the compiled markdown with proper headers + return new NextResponse(content, { + status: 200, + headers: { + "Content-Type": "text/markdown; charset=utf-8", + "Content-Disposition": "inline", + }, + }); + } catch (error) { + return new NextResponse(`Internal server error: ${error}`, { + status: 500, + }); + } +} diff --git a/app/en/resources/integrations/productivity/gmail/page.mdx b/app/en/resources/integrations/productivity/gmail/page.mdx index 7e68ebc22..f20dc6303 100644 --- a/app/en/resources/integrations/productivity/gmail/page.mdx +++ b/app/en/resources/integrations/productivity/gmail/page.mdx @@ -272,7 +272,7 @@ Delete a draft email using the Gmail API. The `TrashEmail` tool is currently only available on a self-hosted instance of the Arcade Engine. To learn more about self-hosting, see the [self-hosting - documentation](/guides/deployment-hosting/configure-engine). + documentation](http://localhost:3000/en/home/deployment/engine-configuration).
diff --git a/lib/mdx-to-markdown.tsx b/lib/mdx-to-markdown.tsx deleted file mode 100644 index 0e8345df4..000000000 --- a/lib/mdx-to-markdown.tsx +++ /dev/null @@ -1,2314 +0,0 @@ -/** - * MDX to Markdown converter - * - * Compiles MDX content, renders it with markdown-friendly components, - * then converts the resulting HTML to clean Markdown using the unified ecosystem. - */ - -import { pathToFileURL } from "node:url"; -import { compile, run } from "@mdx-js/mdx"; -import type { ReactNode } from "react"; -import { createElement, Fragment } from "react"; -import { Fragment as JsxFragment, jsx, jsxs } from "react/jsx-runtime"; -import rehypeParse from "rehype-parse"; -import rehypeRemark from "rehype-remark"; -import remarkGfm from "remark-gfm"; -import remarkStringify from "remark-stringify"; -import { unified } from "unified"; - -// Regex patterns (at top level for performance) -const FILE_EXTENSION_REGEX = /\.[^.]+$/; -const UNDERSCORE_DASH_REGEX = /[_-]/g; -const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; -const TITLE_REGEX = /title:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; -const DESCRIPTION_REGEX = /description:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; - -// Types for mdast table nodes -type MdastTableCell = { type: "tableCell"; children: unknown[] }; -type MdastTableRow = { type: "tableRow"; children: MdastTableCell[] }; -type HtmlNode = { - type: string; - tagName?: string; - children?: HtmlNode[]; - properties?: Record; -}; -type StateAll = (node: HtmlNode) => unknown[]; - -/** Extract cells from a table row element */ -function extractCellsFromRow( - state: { all: StateAll }, - row: HtmlNode -): MdastTableCell[] { - const cells: MdastTableCell[] = []; - for (const cell of row.children || []) { - if ( - cell.type === "element" && - (cell.tagName === "th" || cell.tagName === "td") - ) { - cells.push({ type: "tableCell", children: state.all(cell) }); - } - } - return cells; -} - -/** Extract rows from a table section (thead, tbody, tfoot) or direct tr children */ -function extractRowsFromTableSection( - state: { all: StateAll }, - section: HtmlNode -): MdastTableRow[] { - const rows: MdastTableRow[] = []; - for (const child of section.children || []) { - if (child.type === "element" && child.tagName === "tr") { - const cells = extractCellsFromRow(state, child); - if (cells.length > 0) { - rows.push({ type: "tableRow", children: cells }); - } - } - } - return rows; -} - -/** Check if element is a table section (thead, tbody, tfoot) */ -function isTableSection(tagName: string | undefined): boolean { - return tagName === "thead" || tagName === "tbody" || tagName === "tfoot"; -} - -// Dynamic import to avoid Next.js RSC restrictions -let renderToStaticMarkup: typeof import("react-dom/server").renderToStaticMarkup; -async function getRenderer() { - if (!renderToStaticMarkup) { - const reactDomServer = await import("react-dom/server"); - renderToStaticMarkup = reactDomServer.renderToStaticMarkup; - } - return renderToStaticMarkup; -} - -/** - * Convert HTML to Markdown using unified ecosystem (rehype-remark) - * This is more reliable than turndown for complex HTML structures - */ -async function htmlToMarkdown(html: string): Promise { - const result = await unified() - .use(rehypeParse, { fragment: true }) - .use(rehypeRemark, { - handlers: { - // Custom handler for video elements - video: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; - const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: `Video: ${title}` }], - }, - ], - }; - }, - // Custom handler for audio elements - audio: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; - const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: `Audio: ${title}` }], - }, - ], - }; - }, - // Custom handler for iframe elements - iframe: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const title = - (node.properties?.title as string) || "Embedded content"; - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: title }], - }, - ], - }; - }, - // Custom handler for HTML tables - convert to markdown tables - // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Table parsing requires nested logic - table: (state, node) => { - const rows: MdastTableRow[] = []; - - for (const child of node.children || []) { - if (child.type !== "element") { - continue; - } - if (isTableSection(child.tagName)) { - rows.push( - ...extractRowsFromTableSection(state, child as HtmlNode) - ); - } else if (child.tagName === "tr") { - const cells = extractCellsFromRow(state, child as HtmlNode); - if (cells.length > 0) { - rows.push({ type: "tableRow", children: cells }); - } - } - } - - const colCount = rows[0]?.children?.length || 0; - return { - type: "table", - align: new Array(colCount).fill(null), - children: rows, - }; - }, - // These are handled by the table handler above, but we still need to define them - // to prevent "unknown node" errors when encountered outside tables - thead: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tbody: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tfoot: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tr: (state, node) => { - const cells: { type: "tableCell"; children: unknown[] }[] = []; - for (const child of node.children || []) { - if ( - child.type === "element" && - (child.tagName === "th" || child.tagName === "td") - ) { - cells.push({ - type: "tableCell", - children: state.all(child), - }); - } - } - return { - type: "tableRow", - children: cells, - }; - }, - th: (state, node) => ({ - type: "tableCell", - children: state.all(node), - }), - td: (state, node) => ({ - type: "tableCell", - children: state.all(node), - }), - // Custom handler for callout divs - render as paragraph with bold label - // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Callout parsing logic - div: (state, node) => { - const className = - (node.properties?.className as string[])?.join(" ") || ""; - - // Check if this is a callout - if ( - className.includes("callout") || - className.includes("admonition") || - className.includes("warning") || - className.includes("info") || - className.includes("error") || - className.includes("tip") - ) { - let label = ""; - if (className.includes("warning")) { - label = "Warning"; - } else if (className.includes("error")) { - label = "Error"; - } else if (className.includes("tip")) { - label = "Tip"; - } else if (className.includes("info")) { - label = "Note"; - } - - // Process children and prepend bold label - const children = state.all(node); - if (label && children.length > 0) { - // Add bold label to the first paragraph's children - const firstChild = children[0]; - if (firstChild && firstChild.type === "paragraph") { - return [ - { - type: "paragraph", - children: [ - { - type: "strong", - children: [{ type: "text", value: `${label}:` }], - }, - { type: "text", value: " " }, - ...(firstChild.children || []), - ], - }, - ...children.slice(1), - ]; - } - } - return children; - } - - // Default: just return children (strip the div wrapper) - return state.all(node); - }, - }, - }) - .use(remarkGfm) // Enable GFM for tables, strikethrough, etc. - .use(remarkStringify, { - bullet: "-", - fences: true, - listItemIndent: "one", - }) - .process(html); - - return String(result); -} - -// ============================================ -// Markdown-Friendly Component Implementations -// ============================================ - -// Simple wrapper that just renders children -function PassThrough({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -// Tabs - render all tab content with headers -function MarkdownTabs({ - children, - items, -}: { - children?: ReactNode; - items?: string[]; -}) { - // If we have items array, children are the tab panels - if (items && Array.isArray(items)) { - const childArray = Array.isArray(children) ? children : [children]; - return createElement( - "div", - null, - childArray.map((child, i) => - createElement( - "div", - { key: i }, - createElement("h4", null, items[i] || `Option ${i + 1}`), - child - ) - ) - ); - } - return createElement("div", null, children); -} - -// Tab content - just render the content -function MarkdownTab({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// Assign Tab to Tabs for Tabs.Tab syntax -MarkdownTabs.Tab = MarkdownTab; - -// Steps - render as numbered sections -function MarkdownSteps({ children }: { children?: ReactNode }) { - return createElement("div", { className: "steps" }, children); -} - -// Callout - render as a styled div that turndown will convert -function MarkdownCallout({ - children, - type = "info", -}: { - children?: ReactNode; - type?: string; -}) { - return createElement("div", { className: `callout ${type}` }, children); -} - -// Cards - render as sections -function MarkdownCards({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCard({ - children, - title, - href, -}: { - children?: ReactNode; - title?: string; - href?: string; -}) { - if (href) { - return createElement( - "div", - null, - title && createElement("h4", null, createElement("a", { href }, title)), - children - ); - } - return createElement( - "div", - null, - title && createElement("h4", null, title), - children - ); -} - -MarkdownCards.Card = MarkdownCard; - -// FileTree - render as a code block -function MarkdownFileTree({ children }: { children?: ReactNode }) { - return createElement("pre", null, createElement("code", null, children)); -} - -// Link components - render as standard links -function _MarkdownLink({ - children, - href, -}: { - children?: ReactNode; - href?: string; -}) { - return createElement("a", { href }, children); -} - -// ============================================ -// HTML Element Handlers -// ============================================ - -// Video - convert to a descriptive link -function MarkdownVideo({ - src, - title, - children, -}: { - src?: string; - title?: string; - children?: ReactNode; -}) { - if (!src) { - return createElement(Fragment, null, children); - } - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; - const videoTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); - return createElement("p", null, `[Video: ${videoTitle}](${src})`); -} - -// Audio - convert to a descriptive link -function MarkdownAudio({ - src, - title, - children, -}: { - src?: string; - title?: string; - children?: ReactNode; -}) { - if (!src) { - return createElement(Fragment, null, children); - } - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; - const audioTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); - return createElement("p", null, `[Audio: ${audioTitle}](${src})`); -} - -// Image - keep as img for turndown to handle -function MarkdownImage({ - src, - alt, - title, -}: { - src?: string; - alt?: string; - title?: string; -}) { - return createElement("img", { src, alt: alt || title || "" }); -} - -// Iframe - convert to link -function MarkdownIframe({ src, title }: { src?: string; title?: string }) { - if (!src) { - return null; - } - const label = title || "Embedded content"; - return createElement("p", null, `[${label}](${src})`); -} - -// HR - render as markdown horizontal rule -function MarkdownHr() { - return createElement("hr", null); -} - -// BR - render as line break -function MarkdownBr() { - return createElement("br", null); -} - -// Container elements - just pass through children (strips the wrapper) -function MarkdownPassthrough({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -// Figure/Figcaption - extract content -function MarkdownFigure({ children }: { children?: ReactNode }) { - return createElement("figure", null, children); -} - -function MarkdownFigcaption({ children }: { children?: ReactNode }) { - return createElement("figcaption", null, children); -} - -// Details/Summary - convert to blockquote-style -function MarkdownDetails({ children }: { children?: ReactNode }) { - return createElement("blockquote", null, children); -} - -function MarkdownSummary({ children }: { children?: ReactNode }) { - return createElement("strong", null, children); -} - -// Table elements - pass through for turndown -function MarkdownTable({ children }: { children?: ReactNode }) { - return createElement("table", null, children); -} - -function MarkdownThead({ children }: { children?: ReactNode }) { - return createElement("thead", null, children); -} - -function MarkdownTbody({ children }: { children?: ReactNode }) { - return createElement("tbody", null, children); -} - -function MarkdownTr({ children }: { children?: ReactNode }) { - return createElement("tr", null, children); -} - -function MarkdownTh({ children }: { children?: ReactNode }) { - return createElement("th", null, children); -} - -function MarkdownTd({ children }: { children?: ReactNode }) { - return createElement("td", null, children); -} - -// Definition lists -function MarkdownDl({ children }: { children?: ReactNode }) { - return createElement("dl", null, children); -} - -function MarkdownDt({ children }: { children?: ReactNode }) { - return createElement("dt", null, createElement("strong", null, children)); -} - -function MarkdownDd({ children }: { children?: ReactNode }) { - return createElement("dd", null, children); -} - -// Code block elements - preserve language and content -function MarkdownPre({ - children, - ...props -}: { - children?: ReactNode; - [key: string]: unknown; -}) { - // Extract data-language if present (MDX sometimes uses this) - const lang = props["data-language"] || ""; - return createElement("pre", { "data-language": lang }, children); -} - -function MarkdownCode({ - children, - className, - ...props -}: { - children?: ReactNode; - className?: string; - [key: string]: unknown; -}) { - // Preserve the language class for turndown - return createElement( - "code", - { className: className || "", ...props }, - children - ); -} - -// GuideOverview custom component -function MarkdownGuideOverview({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownGuideOverviewItem({ - children, - title, - href, -}: { - children?: ReactNode; - title?: string; - href?: string; -}) { - return createElement( - "div", - null, - title && - createElement( - "h4", - null, - href ? createElement("a", { href }, title) : title - ), - children - ); -} - -function MarkdownGuideOverviewOutcomes({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -function MarkdownGuideOverviewPrerequisites({ - children, -}: { - children?: ReactNode; -}) { - return createElement( - "div", - null, - createElement("strong", null, "Prerequisites:"), - children - ); -} - -MarkdownGuideOverview.Item = MarkdownGuideOverviewItem; -MarkdownGuideOverview.Outcomes = MarkdownGuideOverviewOutcomes; -MarkdownGuideOverview.Prerequisites = MarkdownGuideOverviewPrerequisites; - -// CheatSheet components -function MarkdownCheatSheetGrid({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -function MarkdownCheatSheetSection({ - children, - title, - icon, -}: { - children?: ReactNode; - title?: string; - icon?: string; -}) { - return createElement( - "div", - null, - title && createElement("h3", null, icon ? `${icon} ${title}` : title), - children - ); -} - -function MarkdownInfoBox({ - children, - type = "info", -}: { - children?: ReactNode; - type?: string; -}) { - // Use a div with callout class so the callout handler processes it - // The div handler in htmlToMarkdown will add the bold label - return createElement("div", { className: `callout ${type}` }, children); -} - -function MarkdownCommandItem({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCommandList({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCommandBlock({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// GlossaryTerm - just render the text -function MarkdownGlossaryTerm({ - children, - term, -}: { - children?: ReactNode; - term?: string; -}) { - return createElement("strong", { title: term }, children); -} - -// DashboardLink - render as a link -function MarkdownDashboardLink({ - children, - href, -}: { - children?: ReactNode; - href?: string; -}) { - const dashboardUrl = href || "https://api.arcade.dev/dashboard"; - return createElement("a", { href: dashboardUrl }, children); -} - -// SignupLink - render as a link -function MarkdownSignupLink({ - children, -}: { - children?: ReactNode; - linkLocation?: string; -}) { - return createElement( - "a", - { href: "https://api.arcade.dev/dashboard/signup" }, - children - ); -} - -// Generic catch-all for unknown components -function _MarkdownGeneric({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// ToolCard - render as a link with summary -function MarkdownToolCard({ - name, - summary, - link, - category: _category, -}: { - name?: string; - image?: string; - summary?: string; - link?: string; - category?: string; -}) { - return createElement( - "li", - null, - createElement("a", { href: link }, name), - summary ? ` - ${summary}` : "" - ); -} - -// MCPClientGrid - render as a list of MCP clients -function MarkdownMCPClientGrid() { - const mcpClients = [ - { - key: "cursor", - name: "Cursor", - description: "AI-powered code editor with built-in MCP support", - }, - { - key: "claude-desktop", - name: "Claude Desktop", - description: "Anthropic's desktop app for Claude with MCP integration", - }, - { - key: "visual-studio-code", - name: "Visual Studio Code", - description: "Microsoft's code editor with MCP extensions", - }, - ]; - - return createElement( - "ul", - null, - ...mcpClients.map((client, i) => - createElement( - "li", - { key: i }, - createElement( - "a", - { href: `/guides/tool-calling/mcp-clients/${client.key}` }, - client.name - ), - ` - ${client.description}` - ) - ) - ); -} - -// ContactCards - render contact information -function MarkdownContactCards() { - const contacts = [ - { - title: "Email Support", - description: - "Get help with technical issues, account questions, and general inquiries.", - href: "mailto:support@arcade.dev", - }, - { - title: "Contact Sales", - description: - "Discuss enterprise plans, pricing, and how Arcade can help your organization.", - href: "mailto:sales@arcade.dev", - }, - { - title: "Discord Community", - description: - "Join for real-time help, connect with developers, and stay updated.", - href: "https://discord.gg/GUZEMpEZ9p", - }, - { - title: "GitHub Issues & Discussions", - description: "Report bugs, request features, and contribute to Arcade.", - href: "https://github.com/ArcadeAI/arcade-mcp", - }, - { - title: "Security Research", - description: "Report security vulnerabilities responsibly.", - href: "/guides/security/security-research-program", - }, - { - title: "System Status", - description: "Check the current status of Arcade's services.", - href: "https://status.arcade.dev", - }, - ]; - - return createElement( - "ul", - null, - ...contacts.map((contact, i) => - createElement( - "li", - { key: i }, - createElement("a", { href: contact.href }, contact.title), - ` - ${contact.description}` - ) - ) - ); -} - -// SubpageList - render list of subpages (uses provided meta) -function MarkdownSubpageList({ - basePath, - meta, -}: { - basePath?: string; - meta?: Record; -}) { - if (!(meta && basePath)) { - return null; - } - - const subpages = Object.entries(meta).filter( - ([key]) => key !== "index" && key !== "*" - ); - - return createElement( - "ul", - null, - ...subpages.map(([key, title], i) => { - let linkText: string; - if (typeof title === "string") { - linkText = title; - } else if ( - typeof title === "object" && - title !== null && - "title" in title - ) { - linkText = (title as { title: string }).title; - } else { - linkText = String(title); - } - return createElement( - "li", - { key: i }, - createElement("a", { href: `${basePath}/${key}` }, linkText) - ); - }) - ); -} - -// ============================================ -// Landing Page Component - renders as markdown-friendly content -// ============================================ -function MarkdownLandingPage() { - return createElement( - "div", - null, - // Hero - createElement("h1", null, "MCP Runtime for AI agents that get things done"), - createElement( - "p", - null, - "Arcade handles OAuth, manages user tokens, and gives you 100+ pre-built integrations so your agents can take real action in production." - ), - createElement("hr", null), - - // Get Started links - createElement("h2", null, "Get Started"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/get-started/quickstarts/call-tool-agent" }, - "Get Started with Arcade" - ) - ), - createElement( - "li", - null, - createElement("a", { href: "/resources/integrations" }, "Explore Tools") - ) - ), - - // Get Tools Section - createElement("h2", null, "Get Tools"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations" }, - "Pre-built Integrations" - ), - " - Browse 100+ ready-to-use integrations for Gmail, Slack, GitHub, and more." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/create-tools/tool-basics/build-mcp-server" }, - "Build Custom Tools" - ), - " - Create your own MCP servers and custom tools with our SDK." - ) - ), - - // Use Arcade Section - createElement("h2", null, "Use Arcade"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/tool-calling/mcp-clients" }, - "Connect to Your IDE" - ), - " - Add tools to Cursor, VS Code, Claude Desktop, or any MCP client." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks" }, - "Power Your Agent" - ), - " - Integrate with LangChain, OpenAI Agents, CrewAI, Vercel AI, and more." - ) - ), - - // Popular Integrations - createElement("h2", null, "Popular Integrations"), - createElement( - "p", - null, - "Pre-built MCP servers ready to use with your agents. ", - createElement("a", { href: "/resources/integrations" }, "See all 100+") - ), - createElement( - "ul", - null, - // Row 1 - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-sheets" }, - "Google Sheets" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/jira" }, - "Jira" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/gmail" }, - "Gmail" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/confluence" }, - "Confluence" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/slack" }, - "Slack" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-docs" }, - "Google Docs" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-slides" }, - "Google Slides" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/crm/hubspot" }, - "HubSpot" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/linear" }, - "Linear" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-drive" }, - "Google Drive" - ) - ), - // Row 2 - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/development/github" }, - "GitHub" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/x" }, - "X" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { - href: "/resources/integrations/social-communication/microsoft-teams", - }, - "MS Teams" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/outlook-mail" }, - "Outlook" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/payments/stripe" }, - "Stripe" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/notion" }, - "Notion" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/asana" }, - "Asana" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/reddit" }, - "Reddit" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/search/youtube" }, - "YouTube" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/dropbox" }, - "Dropbox" - ) - ) - ), - - // Framework Compatibility - createElement("h2", null, "Works With Your Stack"), - createElement( - "p", - null, - "Arcade integrates with popular agent frameworks and LLM providers." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/langchain/use-arcade-tools" }, - "LangChain" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/openai-agents/use-arcade-tools" }, - "OpenAI Agents" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/crewai/use-arcade-tools" }, - "CrewAI" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/vercelai" }, - "Vercel AI" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/google-adk/use-arcade-tools" }, - "Google ADK" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/mastra/use-arcade-tools" }, - "Mastra" - ) - ) - ), - - // How Arcade Works - createElement("h2", null, "How Arcade Works"), - createElement( - "p", - null, - "Three core components that power your AI agents." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "strong", - null, - createElement("a", { href: "/get-started/about-arcade" }, "Runtime") - ), - " - Your MCP server and agentic tool provider. Manages authentication, tool registration, and execution." - ), - createElement( - "li", - null, - createElement( - "strong", - null, - createElement( - "a", - { href: "/resources/integrations" }, - "Tool Catalog" - ) - ), - " - Catalog of pre-built tools and integrations. Browse 100+ ready-to-use MCP servers." - ), - createElement( - "li", - null, - createElement( - "strong", - null, - createElement( - "a", - { href: "/guides/tool-calling/custom-apps/auth-tool-calling" }, - "Agent Authorization" - ) - ), - " - Let agents act on behalf of users. Handle OAuth, API keys, and tokens for tools like Gmail and Google Drive." - ) - ), - - // Sample Applications - createElement("h2", null, "Sample Applications"), - createElement( - "p", - null, - "See Arcade in action with these example applications." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement("a", { href: "https://chat.arcade.dev/" }, "Arcade Chat"), - " - A chatbot that can help you with your daily tasks." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "https://github.com/ArcadeAI/ArcadeSlackAgent" }, - "Archer" - ), - " - A bot for Slack that can act on your behalf." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "https://github.com/dforwardfeed/slack-AIpodcast-summaries" }, - "YouTube Podcast Summarizer" - ), - " - A Slack bot that extracts and summarizes YouTube transcripts." - ) - ), - createElement( - "p", - null, - createElement("a", { href: "/resources/examples" }, "See all examples") - ), - - // Connect IDE callout - createElement("h2", null, "Connect Your IDE with Arcade's LLMs.txt"), - createElement( - "p", - null, - "Give Cursor, Claude Code, and other AI IDEs access to Arcade's documentation so they can write integration code for you. ", - createElement( - "a", - { href: "/get-started/setup/connect-arcade-docs" }, - "Learn more" - ) - ), - - // Quick Links - createElement("h2", null, "Quick Links"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement("a", { href: "/references/api" }, "API Reference") - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/references/cli-cheat-sheet" }, - "CLI Cheat Sheet" - ) - ), - createElement( - "li", - null, - createElement("a", { href: "/resources/faq" }, "FAQ") - ), - createElement( - "li", - null, - createElement("a", { href: "/references/changelog" }, "Changelog") - ) - ) - ); -} - -// ============================================ -// MCP Servers Component - renders MCP server list as markdown -// ============================================ -function MarkdownMcpServers() { - // All available MCP servers (alphabetically sorted) - const allMcpServers = [ - { - label: "Airtable API", - href: "/resources/integrations/productivity/airtable-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Arcade Engine API", - href: "/resources/integrations/development/arcade-engine-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Asana", - href: "/resources/integrations/productivity/asana", - type: "arcade", - category: "productivity", - }, - { - label: "Asana API", - href: "/resources/integrations/productivity/asana-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Ashby API", - href: "/resources/integrations/productivity/ashby-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Box API", - href: "/resources/integrations/productivity/box-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Bright Data", - href: "/resources/integrations/development/brightdata", - type: "community", - category: "development", - }, - { - label: "Calendly API", - href: "/resources/integrations/productivity/calendly-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Clickhouse", - href: "/resources/integrations/databases/clickhouse", - type: "community", - category: "databases", - }, - { - label: "ClickUp", - href: "/resources/integrations/productivity/clickup", - type: "arcade", - category: "productivity", - }, - { - label: "ClickUp API", - href: "/resources/integrations/productivity/clickup-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Close.io", - href: "/resources/integrations/productivity/closeio", - type: "community", - category: "productivity", - }, - { - label: "Confluence", - href: "/resources/integrations/productivity/confluence", - type: "arcade", - category: "productivity", - }, - { - label: "Cursor Agents API", - href: "/resources/integrations/development/cursor-agents-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Customer.io API", - href: "/resources/integrations/customer-support/customerio-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Customer.io Pipelines API", - href: "/resources/integrations/customer-support/customerio-pipelines-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Customer.io Track API", - href: "/resources/integrations/customer-support/customerio-track-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Datadog API", - href: "/resources/integrations/development/datadog-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Discord", - href: "/resources/integrations/social-communication/discord", - type: "auth", - category: "social", - }, - { - label: "Dropbox", - href: "/resources/integrations/productivity/dropbox", - type: "arcade", - category: "productivity", - }, - { - label: "E2B", - href: "/resources/integrations/development/e2b", - type: "arcade", - category: "development", - }, - { - label: "Exa API", - href: "/resources/integrations/search/exa-api", - type: "arcade_starter", - category: "search", - }, - { - label: "Figma API", - href: "/resources/integrations/productivity/figma-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Firecrawl", - href: "/resources/integrations/development/firecrawl", - type: "arcade", - category: "development", - }, - { - label: "Freshservice API", - href: "/resources/integrations/customer-support/freshservice-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "GitHub", - href: "/resources/integrations/development/github", - type: "arcade", - category: "development", - }, - { - label: "GitHub API", - href: "/resources/integrations/development/github-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Gmail", - href: "/resources/integrations/productivity/gmail", - type: "arcade", - category: "productivity", - }, - { - label: "Google Calendar", - href: "/resources/integrations/productivity/google-calendar", - type: "arcade", - category: "productivity", - }, - { - label: "Google Contacts", - href: "/resources/integrations/productivity/google-contacts", - type: "arcade", - category: "productivity", - }, - { - label: "Google Docs", - href: "/resources/integrations/productivity/google-docs", - type: "arcade", - category: "productivity", - }, - { - label: "Google Drive", - href: "/resources/integrations/productivity/google-drive", - type: "arcade", - category: "productivity", - }, - { - label: "Google Finance", - href: "/resources/integrations/search/google_finance", - type: "arcade", - category: "search", - }, - { - label: "Google Flights", - href: "/resources/integrations/search/google_flights", - type: "arcade", - category: "search", - }, - { - label: "Google Hotels", - href: "/resources/integrations/search/google_hotels", - type: "arcade", - category: "search", - }, - { - label: "Google Jobs", - href: "/resources/integrations/search/google_jobs", - type: "arcade", - category: "search", - }, - { - label: "Google Maps", - href: "/resources/integrations/search/google_maps", - type: "arcade", - category: "search", - }, - { - label: "Google News", - href: "/resources/integrations/search/google_news", - type: "arcade", - category: "search", - }, - { - label: "Google Search", - href: "/resources/integrations/search/google_search", - type: "arcade", - category: "search", - }, - { - label: "Google Sheets", - href: "/resources/integrations/productivity/google-sheets", - type: "arcade", - category: "productivity", - }, - { - label: "Google Shopping", - href: "/resources/integrations/search/google_shopping", - type: "arcade", - category: "search", - }, - { - label: "Google Slides", - href: "/resources/integrations/productivity/google-slides", - type: "arcade", - category: "productivity", - }, - { - label: "HubSpot", - href: "/resources/integrations/sales/hubspot", - type: "arcade", - category: "sales", - }, - { - label: "HubSpot Automation API", - href: "/resources/integrations/sales/hubspot-automation-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot CMS API", - href: "/resources/integrations/sales/hubspot-cms-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Conversations API", - href: "/resources/integrations/sales/hubspot-conversations-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot CRM API", - href: "/resources/integrations/sales/hubspot-crm-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Events API", - href: "/resources/integrations/sales/hubspot-events-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Marketing API", - href: "/resources/integrations/sales/hubspot-marketing-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Meetings API", - href: "/resources/integrations/sales/hubspot-meetings-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Users API", - href: "/resources/integrations/sales/hubspot-users-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "Imgflip", - href: "/resources/integrations/entertainment/imgflip", - type: "arcade", - category: "entertainment", - }, - { - label: "Intercom API", - href: "/resources/integrations/customer-support/intercom-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Jira", - href: "/resources/integrations/productivity/jira", - type: "auth", - category: "productivity", - }, - { - label: "Linear", - href: "/resources/integrations/productivity/linear", - type: "arcade", - category: "productivity", - }, - { - label: "LinkedIn", - href: "/resources/integrations/social-communication/linkedin", - type: "arcade", - category: "social", - }, - { - label: "Luma API", - href: "/resources/integrations/productivity/luma-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Mailchimp API", - href: "/resources/integrations/productivity/mailchimp-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Microsoft SharePoint", - href: "/resources/integrations/productivity/sharepoint", - type: "arcade", - category: "productivity", - }, - { - label: "Microsoft Teams", - href: "/resources/integrations/social-communication/microsoft-teams", - type: "arcade", - category: "social", - }, - { - label: "Miro API", - href: "/resources/integrations/productivity/miro-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "MongoDB", - href: "/resources/integrations/databases/mongodb", - type: "community", - category: "databases", - }, - { - label: "Notion", - href: "/resources/integrations/productivity/notion", - type: "arcade", - category: "productivity", - }, - { - label: "Obsidian", - href: "/resources/integrations/productivity/obsidian", - type: "community", - category: "productivity", - }, - { - label: "Outlook Calendar", - href: "/resources/integrations/productivity/outlook-calendar", - type: "arcade", - category: "productivity", - }, - { - label: "Outlook Mail", - href: "/resources/integrations/productivity/outlook-mail", - type: "arcade", - category: "productivity", - }, - { - label: "PagerDuty API", - href: "/resources/integrations/development/pagerduty-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Postgres", - href: "/resources/integrations/databases/postgres", - type: "community", - category: "databases", - }, - { - label: "PostHog API", - href: "/resources/integrations/development/posthog-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Reddit", - href: "/resources/integrations/social-communication/reddit", - type: "arcade", - category: "social", - }, - { - label: "Salesforce", - href: "/resources/integrations/sales/salesforce", - type: "arcade", - category: "sales", - }, - { - label: "Slack", - href: "/resources/integrations/social-communication/slack", - type: "arcade", - category: "social", - }, - { - label: "Slack API", - href: "/resources/integrations/social-communication/slack-api", - type: "arcade_starter", - category: "social", - }, - { - label: "Spotify", - href: "/resources/integrations/entertainment/spotify", - type: "arcade", - category: "entertainment", - }, - { - label: "SquareUp API", - href: "/resources/integrations/productivity/squareup-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Stripe", - href: "/resources/integrations/payments/stripe", - type: "arcade", - category: "payments", - }, - { - label: "Stripe API", - href: "/resources/integrations/payments/stripe_api", - type: "arcade_starter", - category: "payments", - }, - { - label: "TickTick API", - href: "/resources/integrations/productivity/ticktick-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Trello API", - href: "/resources/integrations/productivity/trello-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Twilio", - href: "/resources/integrations/social-communication/twilio", - type: "verified", - category: "social", - }, - { - label: "Twitch", - href: "/resources/integrations/entertainment/twitch", - type: "auth", - category: "entertainment", - }, - { - label: "Vercel API", - href: "/resources/integrations/development/vercel-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Walmart", - href: "/resources/integrations/search/walmart", - type: "arcade", - category: "search", - }, - { - label: "Weaviate API", - href: "/resources/integrations/development/weaviate-api", - type: "arcade_starter", - category: "development", - }, - { - label: "X", - href: "/resources/integrations/social-communication/x", - type: "arcade", - category: "social", - }, - { - label: "Xero API", - href: "/resources/integrations/productivity/xero-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "YouTube", - href: "/resources/integrations/search/youtube", - type: "arcade", - category: "search", - }, - { - label: "Zendesk", - href: "/resources/integrations/customer-support/zendesk", - type: "arcade", - category: "customer-support", - }, - { - label: "Zoho Books API", - href: "/resources/integrations/payments/zoho-books-api", - type: "arcade_starter", - category: "payments", - }, - { - label: "Zoho Creator API", - href: "/resources/integrations/productivity/zoho-creator-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Zoom", - href: "/resources/integrations/social-communication/zoom", - type: "arcade", - category: "social", - }, - ]; - - const typeLabels: Record = { - arcade: "Arcade Optimized", - arcade_starter: "Arcade Starter", - verified: "Verified", - community: "Community", - auth: "Auth Provider", - }; - - const categoryLabels: Record = { - productivity: "Productivity", - development: "Development", - social: "Social", - search: "Search", - sales: "Sales", - payments: "Payments", - entertainment: "Entertainment", - databases: "Databases", - "customer-support": "Customer Support", - }; - - return createElement( - "div", - null, - createElement( - "p", - null, - "Registry of all MCP Servers available in the Arcade ecosystem. ", - createElement( - "a", - { href: "/guides/create-tools/tool-basics/build-mcp-server" }, - "Build your own MCP Server" - ), - "." - ), - createElement( - "ul", - null, - ...allMcpServers.map((server, i) => - createElement( - "li", - { key: i }, - createElement("a", { href: server.href }, server.label), - ` - ${typeLabels[server.type] || server.type}, ${categoryLabels[server.category] || server.category}` - ) - ) - ) - ); -} - -// All available markdown-friendly components -const markdownComponents: Record< - string, - React.ComponentType> -> = { - // Nextra components - Tabs: MarkdownTabs, - "Tabs.Tab": MarkdownTab, - Tab: MarkdownTab, - Steps: MarkdownSteps, - Callout: MarkdownCallout, - Cards: MarkdownCards, - "Cards.Card": MarkdownCard, - Card: MarkdownCard, - FileTree: MarkdownFileTree, - - // Custom components - GuideOverview: MarkdownGuideOverview, - "GuideOverview.Item": MarkdownGuideOverviewItem, - "GuideOverview.Outcomes": MarkdownGuideOverviewOutcomes, - "GuideOverview.Prerequisites": MarkdownGuideOverviewPrerequisites, - CheatSheetGrid: MarkdownCheatSheetGrid, - CheatSheetSection: MarkdownCheatSheetSection, - InfoBox: MarkdownInfoBox, - CommandItem: MarkdownCommandItem, - CommandList: MarkdownCommandList, - CommandBlock: MarkdownCommandBlock, - GlossaryTerm: MarkdownGlossaryTerm, - DashboardLink: MarkdownDashboardLink, - SignupLink: MarkdownSignupLink, - ToolCard: MarkdownToolCard, - MCPClientGrid: MarkdownMCPClientGrid, - ContactCards: MarkdownContactCards, - SubpageList: MarkdownSubpageList, - - // Page-level components - LandingPage: MarkdownLandingPage, - Toolkits: MarkdownMcpServers, - - // HTML-like components (uppercase - for custom components) - Video: MarkdownVideo, - Audio: MarkdownAudio, - Image: MarkdownImage, - - // ============================================ - // Lowercase HTML element overrides - // MDX passes these through as intrinsic elements - // ============================================ - - // Media elements - convert to links - video: MarkdownVideo, - audio: MarkdownAudio, - img: MarkdownImage, - iframe: MarkdownIframe, - embed: MarkdownPassthrough, - object: MarkdownPassthrough, - source: MarkdownPassthrough, - track: MarkdownPassthrough, - picture: MarkdownPassthrough, - - // Structural/semantic elements - strip wrapper, keep children - div: MarkdownPassthrough, - span: MarkdownPassthrough, - section: MarkdownPassthrough, - article: MarkdownPassthrough, - aside: MarkdownPassthrough, - header: MarkdownPassthrough, - footer: MarkdownPassthrough, - main: MarkdownPassthrough, - nav: MarkdownPassthrough, - address: MarkdownPassthrough, - hgroup: MarkdownPassthrough, - - // Figure elements - figure: MarkdownFigure, - figcaption: MarkdownFigcaption, - - // Interactive elements - details: MarkdownDetails, - summary: MarkdownSummary, - dialog: MarkdownPassthrough, - - // Self-closing elements - hr: MarkdownHr, - br: MarkdownBr, - wbr: MarkdownPassthrough, - - // Table elements - pass through for turndown - table: MarkdownTable, - thead: MarkdownThead, - tbody: MarkdownTbody, - tfoot: MarkdownTbody, - tr: MarkdownTr, - th: MarkdownTh, - td: MarkdownTd, - caption: MarkdownPassthrough, - colgroup: MarkdownPassthrough, - col: MarkdownPassthrough, - - // Code elements - preserve language info - pre: MarkdownPre, - code: MarkdownCode, - - // Definition lists - dl: MarkdownDl, - dt: MarkdownDt, - dd: MarkdownDd, - - // Form elements - strip (not useful in markdown) - form: MarkdownPassthrough, - input: MarkdownPassthrough, - button: MarkdownPassthrough, - select: MarkdownPassthrough, - option: MarkdownPassthrough, - optgroup: MarkdownPassthrough, - textarea: MarkdownPassthrough, - label: MarkdownPassthrough, - fieldset: MarkdownPassthrough, - legend: MarkdownPassthrough, - datalist: MarkdownPassthrough, - output: MarkdownPassthrough, - progress: MarkdownPassthrough, - meter: MarkdownPassthrough, - - // Script/style elements - remove entirely - script: () => null, - style: () => null, - noscript: MarkdownPassthrough, - template: MarkdownPassthrough, - slot: MarkdownPassthrough, - - // Canvas/SVG - remove (not representable in markdown) - canvas: () => null, - svg: () => null, - - // Misc inline elements - pass through - abbr: MarkdownPassthrough, - bdi: MarkdownPassthrough, - bdo: MarkdownPassthrough, - cite: MarkdownPassthrough, - data: MarkdownPassthrough, - dfn: MarkdownPassthrough, - kbd: MarkdownPassthrough, - mark: MarkdownPassthrough, - q: MarkdownPassthrough, - rb: MarkdownPassthrough, - rp: MarkdownPassthrough, - rt: MarkdownPassthrough, - rtc: MarkdownPassthrough, - ruby: MarkdownPassthrough, - s: MarkdownPassthrough, - samp: MarkdownPassthrough, - small: MarkdownPassthrough, - sub: MarkdownPassthrough, - sup: MarkdownPassthrough, - time: MarkdownPassthrough, - u: MarkdownPassthrough, - var: MarkdownPassthrough, - - // Map/area elements - map: MarkdownPassthrough, - area: MarkdownPassthrough, - - // Fallbacks - wrapper: PassThrough, -}; - -/** - * Strip import and export statements from MDX content - * This allows us to compile MDX without needing to resolve external modules - */ -function stripImportsAndExports(content: string): string { - let result = content; - - // Remove import statements (various formats) - // import X from 'module' - result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); - // import 'module' - result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); - // import { X } from 'module' (multiline) - result = result.replace( - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, - "" - ); - - // Remove export statements - result = result.replace( - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, - "" - ); - - return result; -} - -/** - * Convert MDX content to clean Markdown - */ -export async function mdxToMarkdown( - mdxContent: string, - pagePath: string -): Promise { - // Extract frontmatter first - const frontmatterMatch = mdxContent.match(FRONTMATTER_REGEX); - let frontmatter = ""; - let contentWithoutFrontmatter = mdxContent; - - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - contentWithoutFrontmatter = mdxContent.slice(frontmatterMatch[0].length); - } - - try { - // Strip imports before compilation so MDX doesn't try to resolve them - const strippedContent = stripImportsAndExports(contentWithoutFrontmatter); - - // Compile MDX to JavaScript - // Include remarkGfm to properly parse GFM tables, strikethrough, etc. in the MDX source - const compiled = await compile(strippedContent, { - outputFormat: "function-body", - development: false, - remarkPlugins: [remarkGfm], - }); - - // Run the compiled code to get the component - // Use process.cwd() as baseUrl since we're not resolving any imports - const { default: MDXContent } = await run(String(compiled), { - Fragment: JsxFragment, - jsx, - jsxs, - baseUrl: pathToFileURL(process.cwd()).href, - }); - - // Render with markdown-friendly components - const element = createElement(MDXContent, { - components: markdownComponents, - }); - - // Convert React element to HTML string - const render = await getRenderer(); - const html = render(element); - - // Convert HTML to Markdown using unified ecosystem - let markdown = await htmlToMarkdown(html); - - // Clean up excessive whitespace - markdown = markdown.replace(/\n{3,}/g, "\n\n").trim(); - - // If result is empty, provide fallback - if (!markdown || markdown.length < 10) { - const title = extractTitle(frontmatter); - const description = extractDescription(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - return `${frontmatter}${markdown}\n`; - } catch (_error) { - return fallbackMdxToMarkdown(mdxContent, pagePath); - } -} - -/** - * Extract title from frontmatter - */ -function extractTitle(frontmatter: string): string { - const match = frontmatter.match(TITLE_REGEX); - return ( - match?.[1] || match?.[2] || match?.[3]?.trim() || "Arcade Documentation" - ); -} - -/** - * Extract description from frontmatter - */ -function extractDescription(frontmatter: string): string { - const match = frontmatter.match(DESCRIPTION_REGEX); - return match?.[1] || match?.[2] || match?.[3]?.trim() || ""; -} - -/** - * Fallback: Simple regex-based MDX to Markdown conversion - * Used when MDX compilation fails - */ -function fallbackMdxToMarkdown(content: string, pagePath: string): string { - let result = content; - - // Extract frontmatter - const frontmatterMatch = result.match(FRONTMATTER_REGEX); - let frontmatter = ""; - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - result = result.slice(frontmatterMatch[0].length); - } - - // Remove imports - result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); - result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); - result = result.replace( - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, - "" - ); - - // Remove exports - result = result.replace( - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, - "" - ); - - // Remove self-closing JSX components (uppercase) - result = result.replace(/<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g, ""); - - // Remove self-closing HTML elements (lowercase) - but convert video/img to links - result = result.replace( - /]*src=["']([^"']+)["'][^>]*\/?>/gi, - "\n\n[Video]($1)\n\n" - ); - result = result.replace( - /]*src=["']([^"']+)["'][^>]*\/?>/gi, - "![]($1)" - ); - result = result.replace(/<[a-z][a-zA-Z0-9]*[^>]*\/>/g, ""); - - // Extract content from JSX with children (process iteratively for nesting) - let prev = ""; - while (prev !== result) { - prev = result; - // Uppercase components - result = result.replace( - /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g, - "$2" - ); - // Lowercase HTML elements (div, span, etc.) - result = result.replace( - /<(div|span|section|article|aside|header|footer|main|nav|video|figure|figcaption)[^>]*>([\s\S]*?)<\/\1>/gi, - "$2" - ); - } - - // Remove JSX expressions - result = result.replace(/\{[^}]+\}/g, ""); - - // Clean up - result = result.replace(/\n{3,}/g, "\n\n").trim(); - - if (!result || result.length < 10) { - const title = extractTitle(frontmatter); - const description = extractDescription(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - return `${frontmatter}${result}\n`; -} diff --git a/package.json b/package.json index f2316f47a..ba7953195 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,7 @@ "type": "module", "scripts": { "dev": "next dev --webpack", - "build": "pnpm run generate:markdown && next build --webpack", - "generate:markdown": "pnpm exec tsx scripts/generate-markdown.ts", + "build": "next build --webpack", "start": "next start", "lint": "pnpm dlx ultracite check", "format": "pnpm dlx ultracite fix", @@ -40,7 +39,6 @@ "homepage": "https://arcade.dev/", "dependencies": { "@arcadeai/design-system": "^3.26.0", - "@mdx-js/mdx": "^3.1.1", "@next/third-parties": "16.0.1", "@ory/client": "1.22.7", "@theguild/remark-mermaid": "0.3.0", @@ -56,14 +54,8 @@ "react-dom": "19.2.3", "react-hook-form": "7.65.0", "react-syntax-highlighter": "16.1.0", - "rehype-parse": "^9.0.1", - "rehype-remark": "^10.0.1", - "remark-gfm": "^4.0.1", - "remark-stringify": "^11.0.0", "swagger-ui-react": "^5.30.0", "tailwindcss-animate": "1.0.7", - "turndown": "^7.2.2", - "unified": "^11.0.5", "unist-util-visit": "5.0.0", "unist-util-visit-parents": "6.0.2", "zustand": "5.0.8" @@ -79,7 +71,6 @@ "@types/react": "19.2.7", "@types/react-dom": "19.2.3", "@types/react-syntax-highlighter": "15.5.13", - "@types/turndown": "^5.0.6", "@types/unist": "3.0.3", "commander": "14.0.2", "dotenv": "^17.2.3", @@ -96,7 +87,6 @@ "remark": "^15.0.1", "remark-rehype": "^11.1.2", "tailwindcss": "4.1.16", - "tsx": "^4.21.0", "typescript": "5.9.3", "ultracite": "6.1.0", "vitest": "4.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7923f5808..436ac536b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,10 +10,7 @@ importers: dependencies: '@arcadeai/design-system': specifier: ^3.26.0 - version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) - '@mdx-js/mdx': - specifier: ^3.1.1 - version: 3.1.1 + version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) '@next/third-parties': specifier: 16.0.1 version: 16.0.1(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -59,30 +56,12 @@ importers: react-syntax-highlighter: specifier: 16.1.0 version: 16.1.0(react@19.2.3) - rehype-parse: - specifier: ^9.0.1 - version: 9.0.1 - rehype-remark: - specifier: ^10.0.1 - version: 10.0.1 - remark-gfm: - specifier: ^4.0.1 - version: 4.0.1 - remark-stringify: - specifier: ^11.0.0 - version: 11.0.0 swagger-ui-react: specifier: ^5.30.0 version: 5.31.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) tailwindcss-animate: specifier: 1.0.7 version: 1.0.7(tailwindcss@4.1.16) - turndown: - specifier: ^7.2.2 - version: 7.2.2 - unified: - specifier: ^11.0.5 - version: 11.0.5 unist-util-visit: specifier: 5.0.0 version: 5.0.0 @@ -123,9 +102,6 @@ importers: '@types/react-syntax-highlighter': specifier: 15.5.13 version: 15.5.13 - '@types/turndown': - specifier: ^5.0.6 - version: 5.0.6 '@types/unist': specifier: 3.0.3 version: 3.0.3 @@ -174,9 +150,6 @@ importers: tailwindcss: specifier: 4.1.16 version: 4.1.16 - tsx: - specifier: ^4.21.0 - version: 4.21.0 typescript: specifier: 5.9.3 version: 5.9.3 @@ -185,7 +158,7 @@ importers: version: 6.1.0(typescript@5.9.3) vitest: specifier: 4.0.5 - version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) zod: specifier: 4.1.12 version: 4.1.12 @@ -680,9 +653,6 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} - '@mixmark-io/domino@2.2.0': - resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': resolution: {integrity: sha512-JQZdnDNm8o43A5GOzwN/0Tz3CDBQtBUNqzVwEopm32uayjdjxev1Csp1JeaqF3v9djLDIvsSE39ecsN2LhCKKQ==} engines: {node: '>= 10'} @@ -2283,9 +2253,6 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} - '@types/turndown@5.0.6': - resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} - '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -3048,9 +3015,6 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} - get-tsconfig@4.13.0: - resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} - github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -3087,9 +3051,6 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hast-util-embedded@3.0.0: - resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} - hast-util-from-dom@5.0.1: resolution: {integrity: sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q==} @@ -3102,24 +3063,12 @@ packages: hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} - hast-util-has-property@3.0.0: - resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} - - hast-util-is-body-ok-link@3.0.1: - resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} - hast-util-is-element@3.0.0: resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} - hast-util-minify-whitespace@1.0.1: - resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} - hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} - hast-util-phrasing@3.0.1: - resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} - hast-util-raw@9.1.0: resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} @@ -3132,9 +3081,6 @@ packages: hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} - hast-util-to-mdast@10.1.2: - resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} - hast-util-to-parse5@8.0.1: resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==} @@ -4219,9 +4165,6 @@ packages: rehype-katex@7.0.1: resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==} - rehype-minify-whitespace@6.0.2: - resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} - rehype-parse@9.0.1: resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} @@ -4237,9 +4180,6 @@ packages: rehype-recma@1.0.0: resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} - rehype-remark@10.0.1: - resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} - rehype-stringify@10.0.1: resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} @@ -4289,9 +4229,6 @@ packages: reselect@5.1.1: resolution: {integrity: sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==} - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -4584,9 +4521,6 @@ packages: trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} - trim-trailing-lines@2.1.0: - resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} - trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -4634,14 +4568,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} - engines: {node: '>=18.0.0'} - hasBin: true - - turndown@7.2.2: - resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} - twoslash-protocol@0.3.4: resolution: {integrity: sha512-HHd7lzZNLUvjPzG/IE6js502gEzLC1x7HaO1up/f72d8G8ScWAs9Yfa97igelQRDl5h9tGcdFsRp+lNVre1EeQ==} @@ -4992,7 +4918,7 @@ snapshots: transitivePeerDependencies: - encoding - '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@arcadeai/arcadejs': 1.15.0 '@hookform/resolvers': 5.2.2(react-hook-form@7.65.0(react@19.2.3)) @@ -5017,7 +4943,7 @@ snapshots: '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-toggle': 1.1.10(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-tooltip': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) class-variance-authority: 0.7.1 clsx: 2.1.1 cmdk: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -5401,8 +5327,6 @@ snapshots: dependencies: langium: 3.3.1 - '@mixmark-io/domino@2.2.0': {} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': optional: true @@ -7002,12 +6926,12 @@ snapshots: postcss: 8.5.6 tailwindcss: 4.1.16 - '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@tailwindcss/node': 4.1.14 '@tailwindcss/oxide': 4.1.14 tailwindcss: 4.1.14 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) '@tanstack/react-virtual@3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: @@ -7234,8 +7158,6 @@ snapshots: '@types/trusted-types@2.0.7': optional: true - '@types/turndown@5.0.6': {} - '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -7265,13 +7187,13 @@ snapshots: chai: 6.2.1 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) '@vitest/pretty-format@4.0.5': dependencies: @@ -8012,10 +7934,6 @@ snapshots: get-stream@8.0.1: {} - get-tsconfig@4.13.0: - dependencies: - resolve-pkg-maps: 1.0.0 - github-slugger@2.0.0: {} glob-parent@5.1.2: @@ -8049,11 +7967,6 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-embedded@3.0.0: - dependencies: - '@types/hast': 3.0.4 - hast-util-is-element: 3.0.0 - hast-util-from-dom@5.0.1: dependencies: '@types/hast': 3.0.4 @@ -8087,38 +8000,14 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 - hast-util-has-property@3.0.0: - dependencies: - '@types/hast': 3.0.4 - - hast-util-is-body-ok-link@3.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-is-element@3.0.0: dependencies: '@types/hast': 3.0.4 - hast-util-minify-whitespace@1.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-embedded: 3.0.0 - hast-util-is-element: 3.0.0 - hast-util-whitespace: 3.0.0 - unist-util-is: 6.0.1 - hast-util-parse-selector@4.0.0: dependencies: '@types/hast': 3.0.4 - hast-util-phrasing@3.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-embedded: 3.0.0 - hast-util-has-property: 3.0.0 - hast-util-is-body-ok-link: 3.0.1 - hast-util-is-element: 3.0.0 - hast-util-raw@9.1.0: dependencies: '@types/hast': 3.0.4 @@ -8190,23 +8079,6 @@ snapshots: transitivePeerDependencies: - supports-color - hast-util-to-mdast@10.1.2: - dependencies: - '@types/hast': 3.0.4 - '@types/mdast': 4.0.4 - '@ungap/structured-clone': 1.3.0 - hast-util-phrasing: 3.0.1 - hast-util-to-html: 9.0.5 - hast-util-to-text: 4.0.2 - hast-util-whitespace: 3.0.0 - mdast-util-phrasing: 4.1.0 - mdast-util-to-hast: 13.2.1 - mdast-util-to-string: 4.0.0 - rehype-minify-whitespace: 6.0.2 - trim-trailing-lines: 2.1.0 - unist-util-position: 5.0.0 - unist-util-visit: 5.0.0 - hast-util-to-parse5@8.0.1: dependencies: '@types/hast': 3.0.4 @@ -9637,11 +9509,6 @@ snapshots: unist-util-visit-parents: 6.0.2 vfile: 6.0.3 - rehype-minify-whitespace@6.0.2: - dependencies: - '@types/hast': 3.0.4 - hast-util-minify-whitespace: 1.0.1 - rehype-parse@9.0.1: dependencies: '@types/hast': 3.0.4 @@ -9672,14 +9539,6 @@ snapshots: transitivePeerDependencies: - supports-color - rehype-remark@10.0.1: - dependencies: - '@types/hast': 3.0.4 - '@types/mdast': 4.0.4 - hast-util-to-mdast: 10.1.2 - unified: 11.0.5 - vfile: 6.0.3 - rehype-stringify@10.0.1: dependencies: '@types/hast': 3.0.4 @@ -9779,8 +9638,6 @@ snapshots: reselect@5.1.1: {} - resolve-pkg-maps@1.0.0: {} - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -10172,8 +10029,6 @@ snapshots: trim-lines@3.0.1: {} - trim-trailing-lines@2.1.0: {} - trough@2.2.0: {} trpc-cli@0.12.1(@trpc/server@11.8.0(typescript@5.9.3))(zod@4.1.12): @@ -10198,17 +10053,6 @@ snapshots: tslib@2.8.1: {} - tsx@4.21.0: - dependencies: - esbuild: 0.27.1 - get-tsconfig: 4.13.0 - optionalDependencies: - fsevents: 2.3.3 - - turndown@7.2.2: - dependencies: - '@mixmark-io/domino': 2.2.0 - twoslash-protocol@0.3.4: {} twoslash@0.3.4(typescript@5.9.3): @@ -10394,7 +10238,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): + vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): dependencies: esbuild: 0.27.1 fdir: 6.5.0(picomatch@4.0.3) @@ -10407,13 +10251,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 - tsx: 4.21.0 yaml: 2.8.2 - vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): + vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.5 - '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.5 '@vitest/runner': 4.0.5 '@vitest/snapshot': 4.0.5 @@ -10430,7 +10273,7 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 diff --git a/proxy.ts b/proxy.ts index f9cd4b515..df8e7d83a 100644 --- a/proxy.ts +++ b/proxy.ts @@ -63,18 +63,19 @@ function pathnameIsMissingLocale(pathname: string): boolean { export function proxy(request: NextRequest) { const pathname = request.nextUrl.pathname; - // .md requests are served as static files from public/ - // (generated at build time by scripts/generate-markdown.ts) - if (pathname.endsWith(".md")) { - // Add locale prefix if missing, then let Next.js serve from public/ - if (pathnameIsMissingLocale(pathname)) { - const locale = getPreferredLocale(request); - const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); - const redirectPath = `/${locale}${pathWithoutMd}.md`; - return NextResponse.redirect(new URL(redirectPath, request.url)); - } - // Let Next.js serve the static file from public/ - return NextResponse.next(); + // Handle .md requests without locale - redirect to add locale first + if (pathname.endsWith(".md") && pathnameIsMissingLocale(pathname)) { + const locale = getPreferredLocale(request); + const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); + const redirectPath = `/${locale}${pathWithoutMd}.md`; + return NextResponse.redirect(new URL(redirectPath, request.url)); + } + + // Rewrite .md requests (with locale) to the markdown API route + if (pathname.endsWith(".md") && !pathname.startsWith("/api/")) { + const url = request.nextUrl.clone(); + url.pathname = `/api/markdown${pathname}`; + return NextResponse.rewrite(url); } // Redirect if there is no locale diff --git a/scripts/generate-markdown.ts b/scripts/generate-markdown.ts deleted file mode 100644 index 618f9953f..000000000 --- a/scripts/generate-markdown.ts +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Static Markdown Generation Script - * - * Generates pre-rendered markdown files from MDX pages for LLM consumption. - * Outputs to public/en/ so files are served directly by Next.js/CDN. - * - * Usage: pnpm dlx tsx scripts/generate-markdown.ts - */ - -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import fastGlob from "fast-glob"; -import { mdxToMarkdown } from "../lib/mdx-to-markdown"; - -const OUTPUT_DIR = join(process.cwd(), "public"); -const SEPARATOR_WIDTH = 50; -const RESOURCE_FILE_REGEX = - /\.(png|jpg|jpeg|gif|svg|webp|mp4|webm|pdf|zip|tar|gz)$/i; - -/** - * Rewrite internal links to point to .md files - * - /references/foo → /en/references/foo.md - * - /en/references/foo → /en/references/foo.md - * - External links, anchors, and resource links are unchanged - */ -function rewriteLinksToMarkdown(markdown: string): string { - // Match markdown links: [text](url) - return markdown.replace( - /\[([^\]]*)\]\(([^)]+)\)/g, - (match, text, url: string) => { - // Skip external links - if ( - url.startsWith("http://") || - url.startsWith("https://") || - url.startsWith("mailto:") - ) { - return match; - } - - // Skip anchor-only links - if (url.startsWith("#")) { - return match; - } - - // Skip resource links (images, videos, files) - if ( - RESOURCE_FILE_REGEX.test(url) || - url.startsWith("/images/") || - url.startsWith("/videos/") || - url.startsWith("/files/") - ) { - return match; - } - - // Skip if already has .md extension - if (url.endsWith(".md")) { - return match; - } - - // Handle anchor in URL - let anchor = ""; - let pathPart = url; - const anchorIndex = url.indexOf("#"); - if (anchorIndex !== -1) { - anchor = url.slice(anchorIndex); - pathPart = url.slice(0, anchorIndex); - } - - // Add /en prefix if not present (internal doc links) - if (pathPart.startsWith("/") && !pathPart.startsWith("/en/")) { - pathPart = `/en${pathPart}`; - } - - // Add .md extension - const newUrl = `${pathPart}.md${anchor}`; - return `[${text}](${newUrl})`; - } - ); -} - -async function generateMarkdownFiles() { - console.log("Generating static markdown files...\n"); - - // Find all MDX pages - const mdxFiles = await fastGlob("app/en/**/page.mdx", { - cwd: process.cwd(), - absolute: false, - }); - - console.log(`Found ${mdxFiles.length} MDX files\n`); - - let successCount = 0; - let errorCount = 0; - const errors: { file: string; error: string }[] = []; - - for (const mdxFile of mdxFiles) { - try { - // Read MDX content - const mdxPath = join(process.cwd(), mdxFile); - const mdxContent = await readFile(mdxPath, "utf-8"); - - // Compute paths - // app/en/references/auth-providers/page.mdx → /en/references/auth-providers - const relativePath = mdxFile - .replace("app/", "/") - .replace("/page.mdx", ""); - - // Convert to markdown - let markdown = await mdxToMarkdown(mdxContent, relativePath); - - // Rewrite links to point to .md files - markdown = rewriteLinksToMarkdown(markdown); - - // Output path: public/en/references/auth-providers.md - const outputPath = join(OUTPUT_DIR, `${relativePath}.md`); - - // Ensure directory exists - await mkdir(dirname(outputPath), { recursive: true }); - - // Write markdown file - await writeFile(outputPath, markdown, "utf-8"); - - successCount += 1; - process.stdout.write(`✓ ${relativePath}.md\n`); - } catch (error) { - errorCount += 1; - const errorMessage = - error instanceof Error ? error.message : String(error); - errors.push({ file: mdxFile, error: errorMessage }); - process.stdout.write(`✗ ${mdxFile}: ${errorMessage}\n`); - } - } - - console.log(`\n${"=".repeat(SEPARATOR_WIDTH)}`); - console.log(`Generated: ${successCount} files`); - if (errorCount > 0) { - console.log(`Errors: ${errorCount} files`); - console.log("\nFailed files:"); - for (const { file, error } of errors) { - console.log(` - ${file}: ${error}`); - } - } - console.log("=".repeat(SEPARATOR_WIDTH)); - - // Exit with error code if any files failed - if (errorCount > 0) { - process.exit(1); - } -} - -// Run the script -generateMarkdownFiles().catch((error) => { - console.error("Fatal error:", error); - process.exit(1); -}); From 6054f7612a187bb8f68f76715ca66958d38f0a59 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 18:27:35 +0000 Subject: [PATCH 09/10] Resolve merge conflict in markdown route --- app/api/markdown/[[...slug]]/route.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 10812365f..8e84e8a4f 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -15,8 +15,17 @@ const IMPORT_DESTRUCTURE_REGEX = /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; const EXPORT_REGEX = /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; -const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +// JSX attribute pattern that properly handles quoted strings containing ">" characters +// Matches: non-quote/non-angle chars, OR complete double-quoted strings, OR complete single-quoted strings +const JSX_ATTRS_PATTERN = "(?:[^>\"'\\n]|\"[^\"]*\"|'[^']*')*"; +const SELF_CLOSING_JSX_REGEX = new RegExp( + `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}\\/>`, + "g" +); +const JSX_WITH_CHILDREN_REGEX = new RegExp( + `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}>([\\s\\S]*?)<\\/\\1>`, + "g" +); const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; @@ -71,8 +80,11 @@ function dedent(text: string): string { return lines .map((line) => { const trimmed = line.trim(); + // Calculate leading whitespace length for this line + const leadingMatch = line.match(LEADING_WHITESPACE_REGEX); + const leadingLength = leadingMatch ? leadingMatch[0].length : 0; // Don't modify empty lines or lines with less indentation than min - if (trimmed === "" || line.length < minIndent) { + if (trimmed === "" || leadingLength < minIndent) { return line.trimStart(); } // Preserve code block markers - just remove leading whitespace @@ -219,10 +231,10 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Now remove JSX expressions outside code blocks result = result.replace(JSX_EXPRESSION_REGEX, ""); - // Restore code blocks + // Restore code blocks (return original placeholder if index doesn't exist) result = result.replace( CODE_BLOCK_PLACEHOLDER_REGEX, - (_, index) => codeBlocks[Number.parseInt(index, 10)] + (match, index) => codeBlocks[Number.parseInt(index, 10)] ?? match ); // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) From b0682e8bb3345cc4852b096759b4c6ad078792e5 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 19:49:47 +0000 Subject: [PATCH 10/10] Update markdown route --- app/api/markdown/[[...slug]]/route.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 8e84e8a4f..753af3d59 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -15,9 +15,17 @@ const IMPORT_DESTRUCTURE_REGEX = /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; const EXPORT_REGEX = /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -// JSX attribute pattern that properly handles quoted strings containing ">" characters -// Matches: non-quote/non-angle chars, OR complete double-quoted strings, OR complete single-quoted strings -const JSX_ATTRS_PATTERN = "(?:[^>\"'\\n]|\"[^\"]*\"|'[^']*')*"; +// JSX attribute pattern that properly handles: +// - Quoted strings containing ">" characters +// - JSX expressions in curly braces containing ">" (arrow functions, comparisons) +// - Multiline attributes (newlines allowed between attributes) +// - Up to 3 levels of brace nesting for style={{outer: {inner: 1}}} patterns +// The brace pattern uses a recursive-like structure to handle nested braces +const BRACE_CONTENT_L0 = "[^{}]*"; // Innermost: no braces +const BRACE_CONTENT_L1 = `(?:${BRACE_CONTENT_L0}|\\{${BRACE_CONTENT_L0}\\})*`; // 1 level +const BRACE_CONTENT_L2 = `(?:${BRACE_CONTENT_L0}|\\{${BRACE_CONTENT_L1}\\})*`; // 2 levels +const BRACE_PATTERN = `\\{${BRACE_CONTENT_L2}\\}`; // Full brace expression (supports 3 levels) +const JSX_ATTRS_PATTERN = `(?:[^>"'{}]|"[^"]*"|'[^']*'|${BRACE_PATTERN})*`; const SELF_CLOSING_JSX_REGEX = new RegExp( `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}\\/>`, "g"