From 8d783827e9a7f12d6d928e9b8d2dea87a81bffe8 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 08:29:36 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Inbox:=20read=20Word=20(.docx)=20do?= =?UTF-8?q?cuments=20dropped=20or=20linked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headings, paragraphs, lists and tables become markdown; the title comes from docProps/core.xml or the first heading. Read in Go with archive/zip and encoding/xml: entry-count, inflated-size (declared and actual) and compression-ratio caps against zip bombs; DTDs and unknown entities are refused (no XXE). Signed-off-by: Andy Anderson --- changelog.d/added-inbox-upload-docx.md | 1 + dashboard/openapi.json | 26 +- src/docs/api-reference/knowledge.md | 2 +- src/docs/inbox-uploads.md | 6 +- src/docs/security-model.md | 5 +- src/pkg/inbox/link_fetch.go | 1 + src/pkg/inbox/upload_docx.go | 369 +++++++++++++++++++++++++ src/pkg/inbox/upload_docx_test.go | 153 ++++++++++ src/pkg/inbox/upload_extract.go | 3 + src/pkg/inbox/upload_types.go | 8 +- 10 files changed, 560 insertions(+), 14 deletions(-) create mode 100644 changelog.d/added-inbox-upload-docx.md create mode 100644 src/pkg/inbox/upload_docx.go create mode 100644 src/pkg/inbox/upload_docx_test.go diff --git a/changelog.d/added-inbox-upload-docx.md b/changelog.d/added-inbox-upload-docx.md new file mode 100644 index 0000000000..915e15fe56 --- /dev/null +++ b/changelog.d/added-inbox-upload-docx.md @@ -0,0 +1 @@ +- Word documents (`.docx`) can be dropped onto the Inbox or linked: headings, paragraphs, bulleted and numbered lists and tables become markdown, and the document title names the item. The file is read in Go (`archive/zip`, `encoding/xml`) with zip-bomb caps (1000 entries, 200 MiB inflated, 50 MiB per part, 100:1 ratio) and an XML decoder that refuses DTDs and unknown entities, so XXE payloads are rejected; legacy `.doc` is still refused with "save it as .docx or PDF". diff --git a/dashboard/openapi.json b/dashboard/openapi.json index d8b9bdc365..a4cbabf858 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -38633,7 +38633,7 @@ "Knowledge" ], "summary": "Add dropped files to the Inbox", - "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV and TSV become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", + "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV, TSV, HTML and Word (.docx, read with zip-bomb caps and no DTDs or external entities) become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", "requestBody": { "required": true, "content": { @@ -38694,7 +38694,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -38760,7 +38761,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -38826,7 +38828,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -38898,7 +38901,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -38964,7 +38968,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -39030,7 +39035,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -39096,7 +39102,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { @@ -39865,7 +39872,8 @@ "yaml", "csv", "tsv", - "html" + "html", + "docx" ] }, "code": { diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 8d9e6dfb22..c969935b82 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -76,7 +76,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/inbox/transcriptions` | Dashboard auth | Live progress of queued and running transcriptions in queue order (`{active: [{source, id, state}]}` with `phase`, `progress`, `estimated`, `started_at`, `audio_seconds`, `processed_seconds`, `eta_seconds`, `queue_position`); reads no source, so the Inbox polls it every 3 s while a job runs; `Cache-Control: no-store`; see [inbox-transcription.md](../inbox-transcription.md) | `pkg/dashboard/api_inbox.go:34` | | `GET` | `/api/inbox/{source}/{id}/audio` | Owner or read-write | Mint a fresh short-lived audio/video link for one item (`{url, expires_at}`; Plaud: presigned, 24h; Zoom: audio-only or video `download_url` with a one-hour download token; Webex: the recording's temporary direct audio (else MP4) download link, valid 3 hours; Teams and Google Meet: 404, no browser-playable link; Slack huddles: 404, Slack keeps no huddle audio; Otter: the conversation's MP3 download link (`GET /conversations/{id}/audio`), minted per click; Granola: 404, Granola keeps no meeting audio; Limitless: 404, Pendant audio is served only behind the API key; Fathom: the signed url of a recording download (`POST /recordings/{id}/download`, audio-only recordings complete at once, video is polled), valid about 24 hours; Fireflies: the transcript's hashed `audio_url` (else `video_url`), valid 24 hours, Pro plan or higher); never stored by the hive, replied with `Cache-Control: no-store`, audited as `inbox_audio` (403 role, 404 unknown item or no audio, 502 source unreachable) | `pkg/dashboard/api_inbox.go:30` | | `GET` | `/api/inbox/{source}/{id}/audio/stream` | Owner or read-write | Stream one item's audio from the dashboard origin, for the Inbox player (the CSP refuses cross-origin media). The hive mints the link as above, fetches it server-side and copies the bytes with the media `Content-Type` (from storage, else the link's extension); a single `Range: bytes=…` is passed through (206 + `Content-Range`; 200 whole file when storage ignores it; 416 passed through). The link stays in memory for one playback's Range requests (10 min, never past expiry), is never logged, returned or stored, and is re-minted once when storage refuses it; nothing is written to disk. `Cache-Control: no-store`; audited as `inbox_audio_stream` once per minted link (403 role, 404 unknown item or no audio, 413 over the 2 GiB cap, 502 source or storage unreachable, or link refused/expired) | `pkg/dashboard/api_inbox.go:31` | -| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv and html become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | +| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv, html and docx become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | | `POST` | `/api/inbox/uploads/url` | Owner or read-write | Add a link: `{url}`. The hive fetches it under the SSRF guard (http/https only, no credentials, its own DNS lookup with the connection pinned to the checked address, private/loopback/link-local/CGNAT/multicast/ULA/metadata ranges refused, every redirect re-checked, at most 3, body capped at `inbox.uploads.max_file_bytes`, 60 s) and runs the body through the upload type pipeline (an HTML page becomes readable markdown titled by its ``). 201 `{results: [{filename: host, ok, source, item_id, title, kind}]}`; 400 `link_refused`, 413, 415, 422, 502 `fetch_failed`, 507; audited as `inbox_upload` with `origin=link` and the host (403 role, 404 no upload store) | `pkg/dashboard/api_inbox_uploads.go:28` | | `POST` | `/api/inbox/recordings` | Owner or read-write | Start a dashboard recording (● Record): `{title, mime: audio/webm\|audio/mp4, consent_version, includes_tab_audio}`. `consent_version` must name the current consent notice; it is stored with the user in the recording's metadata. One recording per user at a time. 201 `{id, max_chunk_bytes, max_seconds, retention_days, transcribes}`; audited as `inbox_recording_start` (400 consent missing or stale, 403 role, 409 `{active_id}` already recording, 412 `inbox.recording.enabled` off, 415 other media types, 507 `inbox.recording.quota_bytes` used up); see [inbox-recording.md](../inbox-recording.md) | `pkg/dashboard/api_inbox_recordings.go:20` | | `POST` | `/api/inbox/recordings/{id}/chunks` | The recorder (owner or read-write) | Store one MediaRecorder chunk `?seq=N`, the raw bytes as the body (at most `max_chunk_bytes`, 512 KiB). In order only; each part is fsynced before the 200 `{id, seq, next_seq, bytes, duplicate?}`. The first chunk must start with a WebM or MP4 signature; the hive never decodes audio. A re-send with the same bytes is acknowledged again (`duplicate: true`) (400 bad seq, 403 someone else's, 409 `{expected_seq}` gap, other bytes for a stored seq, or already ended, 413 chunk over 512 KiB or past `max_bytes` / `max_seconds`, 415 not WebM/MP4, 507 quota) | `pkg/dashboard/api_inbox_recordings.go:21` | diff --git a/src/docs/inbox-uploads.md b/src/docs/inbox-uploads.md index 2f6e1f6832..9d8ddaf83d 100644 --- a/src/docs/inbox-uploads.md +++ b/src/docs/inbox-uploads.md @@ -19,7 +19,8 @@ the browser's MIME type alone: it checks magic bytes first (an executable, archive, PDF, Office or media file renamed to `.txt` is refused), then that the file is UTF-8 text, then that it parses as the format its extension asks for. A file without an extension is read as JSON when it parses as JSON and -as plain text otherwise. +as plain text otherwise. A zip archive is refused unless it is named `.docx` +and opens as a Word document. | Type | Extensions | What the item shows | | --- | --- | --- | @@ -29,6 +30,7 @@ as plain text otherwise. | JSON Lines | `.jsonl`, `.ndjson` | the record count and the first 20 records in a fenced block; every line is validated | | YAML | `.yaml`, `.yml` | a one-line summary and the document in a fenced `yaml` block (parsed with `gopkg.in/yaml.v3`) | | CSV / TSV | `.csv`, `.tsv`, `.tab` | row and column counts and a markdown table of the first 20 rows (20 columns at most) | +| Word | `.docx` | headings (Title and Heading 1 to 6 styles) as `#`, paragraphs, bulleted and numbered lists (by level) as `-` items, and tables as markdown tables; deleted text and field codes are left out; the title comes from the document properties, else the first heading. Read in Go with `archive/zip` and `encoding/xml` (standard library): at most 1000 zip entries, 200 MiB declared uncompressed in all, 50 MiB per part (checked while inflating, not only from the header) and a 100:1 compression ratio per entry; a part with a DOCTYPE or an entity the decoder does not know is refused (XXE), and nothing is fetched | | HTML | `.html`, `.htm`, and web pages behind a link | readable markdown: the `<title>` names the item; the `<article>` or `<main>` (else `<body>`) is kept with headings, paragraphs, lists, quotes, code, tables, link targets and image alt text; scripts, styles, forms, navigation, footers, asides and hidden elements are dropped. Parsed with `golang.org/x/net/html` (BSD-3-Clause); nothing is executed and nothing the page references is loaded | Text is validated as UTF-8 (a UTF-8 BOM is dropped; UTF-16 is refused with @@ -40,7 +42,7 @@ Not supported (the item is refused with a message saying what to do): | File | Message | | --- | --- | -| PDF, Word `.docx` | not supported yet | +| PDF | not supported yet | | Legacy Word `.doc` (and `.xls`, `.ppt`) | save it as .docx or PDF and try again | | RTF, OpenDocument `.odt` | save it as .docx, PDF or plain text | | PowerPoint `.pptx` | export the slides as PDF | diff --git a/src/docs/security-model.md b/src/docs/security-model.md index daf1b52907..2e7f3fd599 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -168,7 +168,10 @@ timeout and size caps; the original, the extracted text and the meta are stored 0600 in a 0700 directory per upload, never executed or served raw, and bounded by a per-hive quota. The extracted text is fenced in the same `<untrusted-inbox-data>` envelope in its Knowledge fact and before any agent -prompt ([inbox-uploads.md](inbox-uploads.md)). Links dropped onto the Inbox are +prompt ([inbox-uploads.md](inbox-uploads.md)). Word documents are read in +Go with zip-bomb caps (entry count, declared and actual inflated size, +compression ratio) and an XML decoder that refuses DTDs and unknown entities +(no XXE). Links dropped onto the Inbox are fetched by the hive under an SSRF guard: http/https only, no credentials in the URL, the hive's own DNS lookup with the connection pinned to the checked address (any private, loopback, link-local, CGNAT, multicast, ULA or metadata diff --git a/src/pkg/inbox/link_fetch.go b/src/pkg/inbox/link_fetch.go index 93fbbcf5be..6377e0d852 100644 --- a/src/pkg/inbox/link_fetch.go +++ b/src/pkg/inbox/link_fetch.go @@ -373,6 +373,7 @@ var linkTypeExtensions = map[string]string{ "text/x-yaml": ".yaml", "text/csv": ".csv", "text/tab-separated-values": ".tsv", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", } // linkPipelineName picks the file name the type pipeline sniffs a fetched diff --git a/src/pkg/inbox/upload_docx.go b/src/pkg/inbox/upload_docx.go new file mode 100644 index 0000000000..2bc15c1f43 --- /dev/null +++ b/src/pkg/inbox/upload_docx.go @@ -0,0 +1,369 @@ +package inbox + +import ( + "archive/zip" + "bytes" + "context" + "encoding/xml" + "errors" + "io" + "strconv" + "strings" +) + +// Word (.docx) extraction: the zip is opened in memory with archive/zip and +// word/document.xml is read with encoding/xml, both in Go and in-process. +// Zip bombs are refused by entry count, declared and actual uncompressed +// size and compression ratio; XML with a DOCTYPE or an entity the decoder +// does not know (XXE) is refused, and encoding/xml never fetches anything. +const ( + // docxMaxEntries bounds the zip's entries. + docxMaxEntries = 1000 + // docxMaxPartBytes bounds one XML part once inflated. + docxMaxPartBytes = 50 << 20 + // docxMaxTotalBytes bounds the declared uncompressed size of all entries. + docxMaxTotalBytes = 200 << 20 + // docxMaxRatio bounds one entry's compression ratio. + docxMaxRatio = 100 + // docxMaxListLevel caps list indentation. + docxMaxListLevel = 8 + // docxTableMaxRows caps a rendered table. + docxTableMaxRows = 200 + docxDocumentPart = "word/document.xml" + docxCorePart = "docProps/core.xml" + docxNamespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +) + +// UploadKindDOCX is a Word document. +const UploadKindDOCX = "docx" + +func docxErr(format string, args ...any) *UploadError { + return uploadErr(UploadErrParse, "the file is not a readable Word document: "+format, args...) +} + +// openDocx validates the archive and returns its parts by name. +func openDocx(data []byte) (map[string]*zip.File, *UploadError) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, docxErr("%s", err) + } + if len(zr.File) > docxMaxEntries { + return nil, docxErr("the archive has %d entries (at most %d)", len(zr.File), docxMaxEntries) + } + parts := make(map[string]*zip.File, len(zr.File)) + var total uint64 + for _, f := range zr.File { + total += f.UncompressedSize64 + if total > docxMaxTotalBytes { + return nil, docxErr("the archive inflates past %s", humanBytes(docxMaxTotalBytes)) + } + if f.CompressedSize64 > 0 && f.UncompressedSize64/f.CompressedSize64 > docxMaxRatio { + return nil, docxErr("%s is compressed more than %d:1 (a zip bomb?)", f.Name, docxMaxRatio) + } + if f.CompressedSize64 == 0 && f.UncompressedSize64 > 0 { + return nil, docxErr("%s has an impossible size", f.Name) + } + parts[f.Name] = f + } + if parts[docxDocumentPart] == nil { + return nil, docxErr("%s is missing", docxDocumentPart) + } + return parts, nil +} + +// readDocxPart inflates one part, refusing more than docxMaxPartBytes even +// when the zip header claims less. +func readDocxPart(f *zip.File) ([]byte, *UploadError) { + if f.UncompressedSize64 > docxMaxPartBytes { + return nil, docxErr("%s is larger than %s", f.Name, humanBytes(docxMaxPartBytes)) + } + rc, err := f.Open() + if err != nil { + return nil, docxErr("%s", err) + } + defer rc.Close() + data, err := io.ReadAll(io.LimitReader(rc, docxMaxPartBytes+1)) + if err != nil { + return nil, docxErr("%s", err) + } + if len(data) > docxMaxPartBytes { + return nil, docxErr("%s is larger than %s", f.Name, humanBytes(docxMaxPartBytes)) + } + return data, nil +} + +// newSafeXMLDecoder is strict and knows no entities beyond the predefined +// five, so &xxe; fails; DOCTYPE directives are refused by the callers. +func newSafeXMLDecoder(data []byte) *xml.Decoder { + d := xml.NewDecoder(bytes.NewReader(data)) + d.Strict = true + d.Entity = nil + return d +} + +// errDocxDTD is returned for a part that declares a DTD. +var errDocxDTD = errors.New("a DOCTYPE (DTD) is not allowed") + +func xmlDirectiveIsDTD(d xml.Directive) bool { + return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(string(d))), "DOCTYPE") +} + +func extractDOCX(ctx context.Context, data []byte) (Extracted, *UploadError) { + parts, uerr := openDocx(data) + if uerr != nil { + return Extracted{}, uerr + } + doc, uerr := readDocxPart(parts[docxDocumentPart]) + if uerr != nil { + return Extracted{}, uerr + } + md, heading, err := docxMarkdown(ctx, doc) + if err != nil { + return Extracted{}, docxErr("%s", err) + } + if md == "" { + return Extracted{}, uploadErr(UploadErrEmpty, "the Word document has no text") + } + title := heading + if f := parts[docxCorePart]; f != nil { + if core, uerr := readDocxPart(f); uerr == nil { + if t := docxCoreTitle(core); t != "" { + title = t + } + } + } + return Extracted{Markdown: md, Title: truncateRunes(title, uploadTitleRunes)}, nil +} + +// docxCoreTitle reads dc:title from docProps/core.xml. +func docxCoreTitle(core []byte) string { + d := newSafeXMLDecoder(core) + in := false + var b strings.Builder + for { + tok, err := d.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.Directive: + if xmlDirectiveIsDTD(t) { + return "" + } + case xml.StartElement: + in = t.Name.Local == "title" + case xml.CharData: + if in { + b.Write(t) + } + case xml.EndElement: + if in && t.Name.Local == "title" { + return oneLine(b.String()) + } + } + } + return "" +} + +type docxTable struct { + rows [][]string + row []string + cell []string +} + +type docxPara struct { + text strings.Builder + heading int + list bool + level int +} + +// docxMarkdown streams document.xml into markdown: headings from Heading +// and Title styles, list items from numbering, tables as markdown tables. +// It returns the first heading too. +func docxMarkdown(ctx context.Context, doc []byte) (string, string, error) { + d := newSafeXMLDecoder(doc) + var out strings.Builder + var tables []*docxTable + var p *docxPara + firstHeading := "" + inText := false + // afterList ends a list with a blank line before the next block. + afterList := false + endList := func() { + if afterList { + out.WriteString("\n") + afterList = false + } + } + tokens := 0 + for { + tok, err := d.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", "", err + } + tokens++ + if tokens%4096 == 0 && ctx.Err() != nil { + return "", "", errors.New("reading the document took too long") + } + switch t := tok.(type) { + case xml.Directive: + if xmlDirectiveIsDTD(t) { + return "", "", errDocxDTD + } + case xml.StartElement: + if t.Name.Space != docxNamespace { + continue + } + switch t.Name.Local { + case "p": + p = &docxPara{} + case "pStyle": + if p != nil { + p.heading = docxHeadingLevel(docxAttr(t, "val")) + } + case "numPr": + if p != nil { + p.list = true + } + case "ilvl": + if p != nil { + if n, err := strconv.Atoi(docxAttr(t, "val")); err == nil && n > 0 { + p.level = min(n, docxMaxListLevel) + } + } + case "t": + inText = true + case "tab": + if p != nil { + p.text.WriteByte(' ') + } + case "br", "cr": + if p != nil { + p.text.WriteByte(' ') + } + case "tbl": + tables = append(tables, &docxTable{}) + case "tr": + if len(tables) > 0 { + tables[len(tables)-1].row = nil + } + case "tc": + if len(tables) > 0 { + tables[len(tables)-1].cell = nil + } + } + case xml.CharData: + if inText && p != nil { + p.text.Write(t) + } + case xml.EndElement: + if t.Name.Space != docxNamespace { + continue + } + switch t.Name.Local { + case "t": + inText = false + case "p": + if p == nil { + continue + } + text := oneLine(p.text.String()) + if len(tables) > 0 { + if text != "" { + tb := tables[len(tables)-1] + tb.cell = append(tb.cell, text) + } + } else if text != "" { + if p.heading > 0 && firstHeading == "" { + firstHeading = text + } + if !p.list || p.heading > 0 { + endList() + } + out.WriteString(docxBlock(p, text)) + afterList = p.list && p.heading == 0 + } + p = nil + case "tc": + if len(tables) > 0 { + tb := tables[len(tables)-1] + tb.row = append(tb.row, strings.Join(tb.cell, " ")) + } + case "tr": + if len(tables) > 0 { + tb := tables[len(tables)-1] + if len(tb.row) > 0 && len(tb.rows) < docxTableMaxRows { + tb.rows = append(tb.rows, tb.row) + } + } + case "tbl": + if len(tables) == 0 { + continue + } + tb := tables[len(tables)-1] + tables = tables[:len(tables)-1] + if len(tb.rows) == 0 { + continue + } + if len(tables) > 0 { + // A nested table becomes text in its parent's cell. + parent := tables[len(tables)-1] + for _, r := range tb.rows { + parent.cell = append(parent.cell, strings.Join(r, " ")) + } + continue + } + cols := 0 + for _, r := range tb.rows { + cols = max(cols, len(r)) + } + endList() + out.WriteString(markdownTable(tb.rows, cols) + "\n\n") + } + } + } + return strings.TrimSpace(out.String()), firstHeading, nil +} + +func docxBlock(p *docxPara, text string) string { + switch { + case p.heading > 0: + return strings.Repeat("#", p.heading) + " " + text + "\n\n" + case p.list: + return strings.Repeat(" ", p.level) + "- " + text + "\n" + } + return text + "\n\n" +} + +// docxHeadingLevel maps a paragraph style id to a heading level: Title is +// 1, Heading1..Heading6 are 1..6. +func docxHeadingLevel(style string) int { + s := strings.ToLower(strings.ReplaceAll(style, " ", "")) + if s == "title" { + return 1 + } + if rest, ok := strings.CutPrefix(s, "heading"); ok { + if n, err := strconv.Atoi(rest); err == nil && n >= 1 { + return min(n, htmlHeadingLevels) + } + } + return 0 +} + +func docxAttr(t xml.StartElement, local string) string { + for _, a := range t.Attr { + if a.Name.Local == local { + return a.Value + } + } + return "" +} + +// docxLooksLikeZip reports a zip signature. +func docxLooksLikeZip(data []byte) bool { + return bytes.HasPrefix(data, []byte("PK\x03\x04")) +} diff --git a/src/pkg/inbox/upload_docx_test.go b/src/pkg/inbox/upload_docx_test.go new file mode 100644 index 0000000000..0682ed98ab --- /dev/null +++ b/src/pkg/inbox/upload_docx_test.go @@ -0,0 +1,153 @@ +package inbox + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "strings" + "testing" +) + +const docxNS = `xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"` + +// makeDocx builds a minimal Word document in memory; extra entries are +// added as given. +func makeDocx(t *testing.T, documentXML string, extra map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + files := map[string]string{ + "[Content_Types].xml": `<?xml version="1.0"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>`, + docxDocumentPart: documentXML, + } + for k, v := range extra { + files[k] = v + } + for name, body := range files { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + _, _ = w.Write([]byte(body)) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func para(style, text string) string { + ppr := "" + if style != "" { + ppr = `<w:pPr><w:pStyle w:val="` + style + `"/></w:pPr>` + } + return `<w:p>` + ppr + `<w:r><w:t>` + text + `</w:t></w:r></w:p>` +} + +func listPara(level int, text string) string { + return fmt.Sprintf(`<w:p><w:pPr><w:numPr><w:ilvl w:val="%d"/><w:numId w:val="1"/></w:numPr></w:pPr><w:r><w:t>%s</w:t></w:r></w:p>`, level, text) +} + +func cell(text string) string { return `<w:tc>` + para("", text) + `</w:tc>` } + +func TestExtractDOCX(t *testing.T) { + body := `<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document ` + docxNS + `><w:body>` + + para("Title", "Quarterly plan") + + para("Heading2", "Goals") + + `<w:p><w:r><w:t xml:space="preserve">Ship the </w:t></w:r><w:r><w:t>drop zone</w:t></w:r><w:r><w:tab/><w:t>first & fast.</w:t></w:r></w:p>` + + `<w:p><w:r><w:delText>deleted words</w:delText><w:instrText>HYPERLINK x</w:instrText></w:r></w:p>` + + listPara(0, "docs") + listPara(1, "api reference") + listPara(0, "tests") + + `<w:tbl><w:tr>` + cell("Owner") + cell("Task") + `</w:tr><w:tr>` + cell("Ana") + cell("a|b") + `</w:tr></w:tbl>` + + `</w:body></w:document>` + data := makeDocx(t, body, map[string]string{docxCorePart: `<cp:coreProperties xmlns:cp="x" xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Plan 2027</dc:title></cp:coreProperties>`}) + kind, uerr := SniffUpload("plan.docx", data) + if uerr != nil || kind != UploadKindDOCX { + t.Fatalf("sniff = %q, %v", kind, uerr) + } + ex, uerr := ExtractUpload(context.Background(), kind, data) + if uerr != nil { + t.Fatal(uerr) + } + if ex.Title != "Plan 2027" { + t.Fatalf("title = %q", ex.Title) + } + for _, want := range []string{"# Quarterly plan", "## Goals", "Ship the drop zone first & fast.", "- docs\n - api reference\n- tests", "| Owner | Task |", `| Ana | a\|b |`} { + if !strings.Contains(ex.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, ex.Markdown) + } + } + for _, gone := range []string{"deleted words", "HYPERLINK"} { + if strings.Contains(ex.Markdown, gone) { + t.Errorf("markdown kept %q", gone) + } + } + // Without core properties the first heading names it. + ex, uerr = ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, body, nil)) + if uerr != nil || ex.Title != "Quarterly plan" { + t.Fatalf("heading title = %q, %v", ex.Title, uerr) + } +} + +func TestExtractDOCXRefusesXXE(t *testing.T) { + xxe := `<?xml version="1.0"?><!DOCTYPE w:document [<!ENTITY xxe SYSTEM "file:///etc/passwd">]><w:document ` + docxNS + `><w:body>` + para("", "&xxe;") + `</w:body></w:document>` + _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, xxe, nil)) + if uerr == nil || uerr.Code != UploadErrParse || !strings.Contains(uerr.Message, "DOCTYPE") { + t.Fatalf("xxe = %v", uerr) + } + // An undeclared entity is refused by the strict decoder too. + ent := `<w:document ` + docxNS + `><w:body>` + para("", "&secret;") + `</w:body></w:document>` + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, ent, nil)); uerr == nil || uerr.Code != UploadErrParse { + t.Fatalf("entity = %v", uerr) + } + // A DTD in core.xml only drops the title. + ok := `<w:document ` + docxNS + `><w:body>` + para("", "plain") + `</w:body></w:document>` + ex, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, ok, map[string]string{docxCorePart: `<!DOCTYPE x [<!ENTITY e SYSTEM "file:///etc/passwd">]><x><title>&e;`})) + if uerr != nil || ex.Title != "" || ex.Markdown != "plain" { + t.Fatalf("core dtd = %+v, %v", ex, uerr) + } +} + +func TestExtractDOCXRefusesZipBombs(t *testing.T) { + // A highly compressible document.xml: refused by the ratio check before + // it is inflated. + bomb := `` + para("", strings.Repeat("a", 2<<20)) + `` + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, bomb, nil)); uerr == nil || !strings.Contains(uerr.Message, "zip bomb") { + t.Fatalf("ratio bomb = %v", uerr) + } + // Too many entries. + extra := map[string]string{} + for i := range docxMaxEntries + 1 { + extra[fmt.Sprintf("x/%d.xml", i)] = "x" + } + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, ``, extra)); uerr == nil || !strings.Contains(uerr.Message, "entries") { + t.Fatalf("entry bomb = %v", uerr) + } + // Not a Word document: a zip without word/document.xml, and garbage. + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("readme.txt") + _, _ = w.Write([]byte("hi")) + _ = zw.Close() + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, buf.Bytes()); uerr == nil || !strings.Contains(uerr.Message, "missing") { + t.Fatalf("no document = %v", uerr) + } + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, []byte("PK\x03\x04garbage")); uerr == nil || uerr.Code != UploadErrParse { + t.Fatalf("garbage = %v", uerr) + } + if _, uerr := ExtractUpload(context.Background(), UploadKindDOCX, makeDocx(t, ``, nil)); uerr == nil || uerr.Code != UploadErrEmpty { + t.Fatalf("empty = %v", uerr) + } + // A zip renamed to anything else is still refused as an archive. + if _, uerr := SniffUpload("notes.md", makeDocx(t, ``, nil)); uerr == nil || uerr.Code != UploadErrUnsupported { + t.Fatalf("zip as md = %v", uerr) + } +} + +func TestDocxHeadingLevel(t *testing.T) { + for in, want := range map[string]int{"Title": 1, "Heading1": 1, "heading 3": 3, "Heading9": htmlHeadingLevels, "Normal": 0, "HeadingX": 0} { + if got := docxHeadingLevel(in); got != want { + t.Errorf("docxHeadingLevel(%q) = %d, want %d", in, got, want) + } + } +} diff --git a/src/pkg/inbox/upload_extract.go b/src/pkg/inbox/upload_extract.go index ddc67d939f..868d4cbb4a 100644 --- a/src/pkg/inbox/upload_extract.go +++ b/src/pkg/inbox/upload_extract.go @@ -82,6 +82,9 @@ func ExtractUpload(ctx context.Context, kind string, data []byte) (Extracted, *U } func extractUpload(ctx context.Context, kind string, data []byte) (Extracted, *UploadError) { + if kind == UploadKindDOCX { + return extractDOCX(ctx, data) + } text := normalizeText(data) switch kind { case UploadKindMarkdown: diff --git a/src/pkg/inbox/upload_types.go b/src/pkg/inbox/upload_types.go index 99e883796d..55cd9f89bb 100644 --- a/src/pkg/inbox/upload_types.go +++ b/src/pkg/inbox/upload_types.go @@ -76,6 +76,7 @@ var uploadTextExtensions = map[string]string{ ".tab": UploadKindTSV, ".html": UploadKindHTML, ".htm": UploadKindHTML, + ".docx": UploadKindDOCX, } // uploadUnsupportedExtensions names formats people commonly drop that this @@ -102,7 +103,7 @@ func UploadSupportedExtensions() []string { } func uploadSupportedList() string { - return "md, txt, json, jsonl, yaml, csv, tsv, html" + return "md, txt, json, jsonl, yaml, csv, tsv, html, docx" } // uploadMagic is a binary signature that is never a text document. @@ -164,6 +165,11 @@ func SniffUpload(name string, data []byte) (string, *UploadError) { if len(data) == 0 { return "", uploadErr(UploadErrEmpty, "the file is empty") } + // A .docx is a zip: it skips the archive refusal and must then open as a + // Word document (guarded against zip bombs and XXE) in extraction. + if strings.EqualFold(filepath.Ext(name), ".docx") && docxLooksLikeZip(data) { + return UploadKindDOCX, nil + } for _, m := range uploadBinaryMagic { if len(data) >= m.offset+len(m.prefix) && bytes.Equal(data[m.offset:m.offset+len(m.prefix)], m.prefix) { return "", uploadErr(UploadErrUnsupported, "unsupported file type: this is %s; supported: %s", m.label, uploadSupportedList())