diff --git a/apps/web/lib/api.ts b/apps/web/lib/api.ts index 49075ef..8eb99e3 100644 --- a/apps/web/lib/api.ts +++ b/apps/web/lib/api.ts @@ -1,4 +1,6 @@ import "server-only"; +import { outboundApiUrl } from "./outbound-api-url"; +export { outboundApiUrl } from "./outbound-api-url"; import { cookies } from "next/headers"; export interface Session { @@ -3919,10 +3921,6 @@ export async function archiveChannelCampaign( }); } -export function outboundApiUrl(pathname: string): URL { - return new URL(pathname, process.env.OUTBOUND_API_URL ?? "http://127.0.0.1:3001"); -} - async function apiFetch( pathname: string, options: { diff --git a/apps/web/lib/outbound-api-url.ts b/apps/web/lib/outbound-api-url.ts new file mode 100644 index 0000000..19ffd98 --- /dev/null +++ b/apps/web/lib/outbound-api-url.ts @@ -0,0 +1,27 @@ +export function outboundApiUrl(pathname: string): URL { + if ( + !pathname.startsWith("/api/") + || pathname.startsWith("//") + || pathname.includes("#") + || /[\\\u0000-\u001f\u007f]/.test(pathname) + ) { + throw new Error("INVALID_OUTBOUND_API_PATH"); + } + + const base = new URL(process.env.OUTBOUND_API_URL ?? "http://127.0.0.1:3001"); + if ( + (base.protocol !== "http:" && base.protocol !== "https:") + || base.username + || base.password + || base.search + || base.hash + ) { + throw new Error("INVALID_OUTBOUND_API_URL"); + } + + const queryOffset = pathname.indexOf("?"); + const target = new URL(base.origin); + target.pathname = queryOffset === -1 ? pathname : pathname.slice(0, queryOffset); + target.search = queryOffset === -1 ? "" : pathname.slice(queryOffset + 1); + return target; +} diff --git a/bun.lock b/bun.lock index 09c7df2..4cb0c29 100644 --- a/bun.lock +++ b/bun.lock @@ -26,6 +26,7 @@ "mammoth": "^1.12.1", "next": "^16.2.11", "node-html-markdown": "^2.0.0", + "node-html-parser": "^7.0.1", "pdf-lib": "^1.17.1", "postgres": "^3.4.9", "react": "^19.2.8", @@ -657,7 +658,7 @@ "node-html-markdown": ["node-html-markdown@2.0.0", "", { "dependencies": { "node-html-parser": "^6.1.13" } }, "sha512-DqUC3GGP7pwSYxS93SwHoP+qCw78xcMP6C6H2DuC8rPD2AweJRjBzQb5SdXpKtDlqAQ7hVotJcfhgU7hU5Gthw=="], - "node-html-parser": ["node-html-parser@6.1.13", "", { "dependencies": { "css-select": "^5.1.0", "he": "1.2.0" } }, "sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg=="], + "node-html-parser": ["node-html-parser@7.1.0", "", { "dependencies": { "css-select": "^5.1.0", "he": "1.2.0" } }, "sha512-iJo8b2uYGT40Y8BTyy5ufL6IVbN8rbm/1QK2xffXU/1a/v3AAa0d1YAoqBNYqaS4R/HajkWIpIfdE6KcyFh1AQ=="], "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], @@ -845,6 +846,8 @@ "next/sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "node-html-markdown/node-html-parser": ["node-html-parser@6.1.13", "", { "dependencies": { "css-select": "^5.1.0", "he": "1.2.0" } }, "sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg=="], + "pdf-lib/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], diff --git a/package.json b/package.json index b35329f..8f10f53 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "lucide-react": "^1.26.0", "mammoth": "^1.12.1", "next": "^16.2.11", + "node-html-parser": "^7.0.1", "node-html-markdown": "^2.0.0", "pdf-lib": "^1.17.1", "postgres": "^3.4.9", diff --git a/packages/infrastructure/src/documents/document-extractor-process.ts b/packages/infrastructure/src/documents/document-extractor-process.ts index eb4b109..635224e 100644 --- a/packages/infrastructure/src/documents/document-extractor-process.ts +++ b/packages/infrastructure/src/documents/document-extractor-process.ts @@ -11,6 +11,7 @@ import mammoth from "mammoth"; import { unzipSync, type UnzipFileInfo, type Unzipped } from "fflate"; import { XMLParser } from "fast-xml-parser"; import { NodeHtmlMarkdown } from "node-html-markdown"; +import { parse } from "node-html-parser"; import { extractText } from "unpdf"; import type { DocumentExtractionSection, @@ -444,10 +445,27 @@ function normalizeZipPath(base: string, target: string): string { } function stripUnsafeHtml(value: string): string { - return value - .replace(/<(script|style|iframe|object|embed)[^>]*>[\s\S]*?<\/\1>/gi, "") - .replace(/\son\w+\s*=\s*(["']).*?\1/gi, "") - .replace(/\s(href|src)\s*=\s*(["'])\s*(javascript|data):.*?\2/gi, ""); + const root = parse(value, { + comment: false, + blockTextElements: { script: false, style: false, pre: true }, + }); + for (const element of root.querySelectorAll("script,style,iframe,object,embed,svg,math,template")) { + element.remove(); + } + for (const element of root.querySelectorAll("*")) { + for (const [name, rawValue] of Object.entries(element.attributes)) { + const attribute = name.toLowerCase(); + if (attribute.startsWith("on") || attribute === "style") { + element.removeAttribute(name); + continue; + } + if (["href", "src", "xlink:href", "formaction"].includes(attribute)) { + const normalized = rawValue.replace(/[\u0000-\u0020\u007f]+/g, "").toLowerCase(); + if (/^(javascript|data|vbscript):/.test(normalized)) element.removeAttribute(name); + } + } + } + return root.toString(); } function baseMetrics(bytes: Uint8Array, markdown: string, sections: readonly DocumentExtractionSection[]) { @@ -466,7 +484,21 @@ function assertHasText(value: string): void { } function visibleText(value: string): string { - return value.replace(//gs, "").replace(/[#|*_`\s-]+/g, " ").trim(); + return stripHtmlComments(value).replace(/[#|*_`\s-]+/g, " ").trim(); +} + +function stripHtmlComments(value: string): string { + let output = ""; + let offset = 0; + while (offset < value.length) { + const start = value.indexOf("", start + 4); + if (end === -1) return output; + offset = end + 3; + } + return output; } function normalize(value: string): string { @@ -474,7 +506,7 @@ function normalize(value: string): string { } function escapeTableCell(value: string): string { - return value.replace(/\|/g, "\\|").replace(/\r?\n/g, "
"); + return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r\n?|\n/g, "
"); } function escapeRegExp(value: string): string { diff --git a/packages/infrastructure/src/inbox/html-to-text.ts b/packages/infrastructure/src/inbox/html-to-text.ts new file mode 100644 index 0000000..2dbffb8 --- /dev/null +++ b/packages/infrastructure/src/inbox/html-to-text.ts @@ -0,0 +1,18 @@ +import { parse } from "node-html-parser"; + +export function htmlToText(value: string | null): string | null { + if (!value) return null; + const root = parse(value, { + comment: false, + blockTextElements: { script: false, style: false, pre: true }, + }); + for (const element of root.querySelectorAll("script,style,iframe,object,embed,svg,math,template")) { + element.remove(); + } + const text = root.structuredText + .replace(/\u00a0/g, " ") + .replace(/[ \t]+/g, " ") + .replace(/\n{3,}/g, "\n\n") + .trim(); + return text || null; +} diff --git a/packages/infrastructure/src/inbox/unipile-account-inbox-synchronizer.ts b/packages/infrastructure/src/inbox/unipile-account-inbox-synchronizer.ts index 59fc53a..a461f34 100644 --- a/packages/infrastructure/src/inbox/unipile-account-inbox-synchronizer.ts +++ b/packages/infrastructure/src/inbox/unipile-account-inbox-synchronizer.ts @@ -3,6 +3,7 @@ import type { ProspectingChannel } from "@outbound/domain/campaigns/prospecting- import { normalizeEmail } from "@outbound/domain/crm/normalization"; import type { Database } from "@outbound/infrastructure/database/client"; import { captureProspectMemoryMutation } from "@outbound/infrastructure/prospect-memory/capture-prospect-memory-mutation"; +import { htmlToText } from "@outbound/infrastructure/inbox/html-to-text"; import { automatedReplies, connectedAccounts, @@ -807,26 +808,6 @@ function recordList(value: unknown): Record[] { : []; } -function htmlToText(value: string | null): string | null { - if (!value) return null; - const text = value - .replace(/]*>[\s\S]*?<\/style>/gi, " ") - .replace(/]*>[\s\S]*?<\/script>/gi, " ") - .replace(//gi, "\n") - .replace(/<\/p>/gi, "\n") - .replace(/<[^>]+>/g, " ") - .replace(/ /gi, " ") - .replace(/&/gi, "&") - .replace(/</gi, "<") - .replace(/>/gi, ">") - .replace(/"/gi, '"') - .replace(/'/gi, "'") - .replace(/[ \t]+/g, " ") - .replace(/\n{3,}/g, "\n\n") - .trim(); - return text || null; -} - function batches(items: readonly T[], size: number): T[][] { const result: T[][] = []; for (let offset = 0; offset < items.length; offset += size) result.push(items.slice(offset, offset + size)); diff --git a/tests/unit/html-to-text.test.ts b/tests/unit/html-to-text.test.ts new file mode 100644 index 0000000..15ed923 --- /dev/null +++ b/tests/unit/html-to-text.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "bun:test"; +import { htmlToText } from "@outbound/infrastructure/inbox/html-to-text"; + +describe("inbox HTML to text", () => { + test("keeps readable structure while discarding executable elements", () => { + expect(htmlToText("

Bonjour
Salim

Suite

")) + .toBe("Bonjour\nSalim\nSuite"); + }); + + test("decodes entities once without turning encoded markup into HTML", () => { + expect(htmlToText("&lt;script&gt;preuve&lt;/script&gt;")) + .toBe("<script>preuve</script>"); + }); + + test("fails closed on malformed executable markup", () => { + expect(htmlToText("

visible

")) + .toBeNull(); + }); +}); diff --git a/tests/unit/outbound-api-url.test.ts b/tests/unit/outbound-api-url.test.ts new file mode 100644 index 0000000..8e30944 --- /dev/null +++ b/tests/unit/outbound-api-url.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { outboundApiUrl } from "../../apps/web/lib/outbound-api-url"; + +const originalUrl = process.env.OUTBOUND_API_URL; + +afterEach(() => { + if (originalUrl === undefined) delete process.env.OUTBOUND_API_URL; + else process.env.OUTBOUND_API_URL = originalUrl; +}); + +describe("outbound API URL", () => { + test("keeps API paths and queries on the configured internal origin", () => { + process.env.OUTBOUND_API_URL = "https://api.internal.example:3443"; + expect(outboundApiUrl("/api/v1/conversations?cursor=next").href) + .toBe("https://api.internal.example:3443/api/v1/conversations?cursor=next"); + }); + + test("rejects paths that could override or escape the internal origin", () => { + for (const pathname of [ + "https://attacker.example/api/v1/data", + "//attacker.example/api/v1/data", + "/api\\\\attacker.example/data", + "/health", + "/api/v1/data#fragment", + "/api/v1/data\nX-Test: injected", + ]) { + expect(() => outboundApiUrl(pathname)).toThrow("INVALID_OUTBOUND_API_PATH"); + } + }); + + test("rejects unsafe backend base URLs", () => { + for (const base of [ + "file:///tmp/socket", + "https://user:secret@api.internal.example", + "https://api.internal.example?redirect=1", + "https://api.internal.example#fragment", + ]) { + process.env.OUTBOUND_API_URL = base; + expect(() => outboundApiUrl("/api/v1/data")).toThrow("INVALID_OUTBOUND_API_URL"); + } + }); +}); diff --git a/tests/unit/structured-document-text-extractor.test.ts b/tests/unit/structured-document-text-extractor.test.ts index cc6b4d3..55ea1d5 100644 --- a/tests/unit/structured-document-text-extractor.test.ts +++ b/tests/unit/structured-document-text-extractor.test.ts @@ -22,6 +22,19 @@ describe("structured document text extractor", () => { expect(result.sections[0]?.locator).toBe("section:1"); }); + test("removes active HTML attributes and unsafe links before Markdown conversion", async () => { + const result = await extractor.extract({ + filename: "hostile.html", + contentType: "text/html", + bytes: new TextEncoder().encode('

Titre

Lien

Preuve sûre

'), + }); + expect(result.markdown).toContain("Titre"); + expect(result.markdown).toContain("Preuve sûre"); + expect(result.markdown).not.toContain("javascript:"); + expect(result.markdown).not.toContain("data:text"); + expect(result.markdown).not.toContain("steal()"); + }); + test("preserves physical PDF pages and marks image-only PDFs for OCR", async () => { const textPdf = await createTextPdf(); const text = await extractor.extract({ filename: "offre.pdf", contentType: "application/pdf", bytes: textPdf });