From fe0b14fa4a770820560d6072faf3028e83a65f65 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 08:35:36 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Inbox:=20read=20PDFs=20in=20a?= =?UTF-8?q?=20sandboxed=20hive-pdftext=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PDF text is extracted out of process by the new hive-pdftext command (pure Go, github.com/ledongthuc/pdf, BSD-3-Clause, already in go.mod): as the hive-transcribe uid via su-exec, empty environment, prlimit + nice, empty network namespace where available, private 0700 work dir, page and character caps, and a wall-clock timeout that kills the process group. Scanned PDFs without a text layer are refused with a clear message. Signed-off-by: Andy Anderson --- changelog.d/added-inbox-upload-pdf.md | 1 + dashboard/openapi.json | 26 ++- src/Dockerfile | 6 +- src/cmd/hive-pdftext/main.go | 59 +++++ src/cmd/hive-pdftext/main_test.go | 42 ++++ src/docs/api-reference/knowledge.md | 2 +- src/docs/env-vars.md | 3 + src/docs/inbox-uploads.md | 24 +- src/docs/security-model.md | 7 +- src/pkg/inbox/link_fetch.go | 1 + src/pkg/inbox/pdfbox/pdfbox.go | 259 ++++++++++++++++++++ src/pkg/inbox/pdfbox/pdfbox_test.go | 125 ++++++++++ src/pkg/inbox/pdfbox/testdata/scanned.pdf | Bin 0 -> 833 bytes src/pkg/inbox/pdfbox/testdata/text.pdf | Bin 0 -> 968 bytes src/pkg/inbox/upload_pdf.go | 272 ++++++++++++++++++++++ src/pkg/inbox/upload_pdf_test.go | 138 +++++++++++ src/pkg/inbox/upload_source.go | 38 ++- src/pkg/inbox/upload_types.go | 8 +- 18 files changed, 993 insertions(+), 18 deletions(-) create mode 100644 changelog.d/added-inbox-upload-pdf.md create mode 100644 src/cmd/hive-pdftext/main.go create mode 100644 src/cmd/hive-pdftext/main_test.go create mode 100644 src/pkg/inbox/pdfbox/pdfbox.go create mode 100644 src/pkg/inbox/pdfbox/pdfbox_test.go create mode 100644 src/pkg/inbox/pdfbox/testdata/scanned.pdf create mode 100644 src/pkg/inbox/pdfbox/testdata/text.pdf create mode 100644 src/pkg/inbox/upload_pdf.go create mode 100644 src/pkg/inbox/upload_pdf_test.go diff --git a/changelog.d/added-inbox-upload-pdf.md b/changelog.d/added-inbox-upload-pdf.md new file mode 100644 index 0000000000..c05e7e6633 --- /dev/null +++ b/changelog.d/added-inbox-upload-pdf.md @@ -0,0 +1 @@ +- PDFs can be dropped onto the Inbox or linked: each page with text becomes a `## Page N` section and the document title names the item, while a scanned PDF without a text layer is refused with "no text layer (scanned PDF); OCR not supported yet". PDFs are never parsed in the hive process: the new `hive-pdftext` sandbox (pure Go, `github.com/ledongthuc/pdf`, BSD-3-Clause, already a dependency) runs as the `hive-transcribe` uid through su-exec with an empty environment, under prlimit and nice, in an empty network namespace where available, from a private 0700 work dir, capped at 200 pages and 1 MiB of text and killed with its process group after 60 seconds (`HIVE_PDF_SANDBOX`, `HIVE_PDF_RUN_AS`, `HIVE_PDF_MEMORY_MB`). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index a4cbabf858..650972dcc3 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -38633,7 +38633,7 @@ "Knowledge" ], "summary": "Add dropped files to the Inbox", - "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV, TSV, HTML and Word (.docx, read with zip-bomb caps and no DTDs or external entities) become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", + "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV, TSV, HTML, Word (.docx, read with zip-bomb caps and no DTDs or external entities) and PDF (parsed out of process in the hive-pdftext sandbox; a PDF without a text layer is refused) become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", "requestBody": { "required": true, "content": { @@ -38695,7 +38695,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -38762,7 +38763,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -38829,7 +38831,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -38902,7 +38905,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -38969,7 +38973,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -39036,7 +39041,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -39103,7 +39109,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { @@ -39873,7 +39880,8 @@ "csv", "tsv", "html", - "docx" + "docx", + "pdf" ] }, "code": { diff --git a/src/Dockerfile b/src/Dockerfile index f29564a2f2..bc05fa9081 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -84,7 +84,8 @@ RUN --mount=type=cache,id=hive-go-build,target=/tmp/gobuild \ GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /hive-backup ./cmd/hive-backup && \ GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /hivectl ./cmd/hivectl && \ GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /crux ./cmd/crux && \ - GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /hive-whisper ./cmd/hive-whisper + GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /hive-whisper ./cmd/hive-whisper && \ + GOCACHE=/tmp/gobuild GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o /hive-pdftext ./cmd/hive-pdftext # --- Go security scanners for the sec-check lane (#18192) --- @@ -738,6 +739,9 @@ COPY --from=builder /crux /usr/local/bin/crux # whisper.cpp + LGPL FFmpeg engine from the whisper-builder stage. They run as # the dedicated hive-transcribe uid (created below), never as dev. COPY --from=builder /hive-whisper /usr/local/bin/hive-whisper +# Inbox PDF uploads are parsed out of process by the hive-pdftext sandbox +# (pure Go, github.com/ledongthuc/pdf), as the same hive-transcribe uid. +COPY --from=builder /hive-pdftext /usr/local/bin/hive-pdftext COPY --from=whisper-builder /opt/hive-whisper /opt/hive-whisper # Go security scanners for the sec-check lane (#18192); built in the diff --git a/src/cmd/hive-pdftext/main.go b/src/cmd/hive-pdftext/main.go new file mode 100644 index 0000000000..6cc9115808 --- /dev/null +++ b/src/cmd/hive-pdftext/main.go @@ -0,0 +1,59 @@ +// Command hive-pdftext is the sandbox the Inbox reads one uploaded PDF in. +// The hive starts it as the hive-transcribe uid through su-exec, with an +// empty environment, under prlimit and nice and, where the kernel allows it, +// in an empty network namespace; the PDF arrives on stdin and the text +// leaves as JSON on stdout. See pkg/inbox/pdfbox and +// src/docs/inbox-uploads.md. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/hivecommons/hive/pkg/inbox/pdfbox" +) + +// version is the sandbox protocol version printed by -version. +const version = "hive-pdftext 1" + +func main() { os.Exit(runMain(os.Args[1:])) } + +func runMain(args []string) int { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) + defer stop() + return run(ctx, args, os.Stdin, os.Stdout, os.Stderr) +} + +// run parses args and reads one PDF. +func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + var o pdfbox.Options + fs := flag.NewFlagSet("hive-pdftext", flag.ContinueOnError) + fs.SetOutput(stderr) + showVersion := fs.Bool("version", false, "print the version and exit") + fs.Int64Var(&o.MaxBytes, "max-bytes", 0, "largest PDF accepted on stdin (0: no cap)") + fs.IntVar(&o.MaxPages, "max-pages", pdfbox.DefaultMaxPages, "pages read at most") + fs.IntVar(&o.MaxChars, "max-chars", pdfbox.DefaultMaxChars, "characters returned at most") + fs.StringVar(&o.WorkRoot, "work-root", "", "where the private work dir is created (default: $TMPDIR or /tmp)") + if err := fs.Parse(args); err != nil { + return pdfbox.ExitUsage + } + if *showVersion { + fmt.Fprintln(stdout, version) + return pdfbox.ExitOK + } + if err := pdfbox.Run(ctx, o, stdin, stdout); err != nil { + fmt.Fprintln(stderr, "hive-pdftext: "+err.Error()) + var pe *pdfbox.Error + if errors.As(err, &pe) { + return pe.Code + } + return pdfbox.ExitSetup + } + return pdfbox.ExitOK +} diff --git a/src/cmd/hive-pdftext/main_test.go b/src/cmd/hive-pdftext/main_test.go new file mode 100644 index 0000000000..e225a164e8 --- /dev/null +++ b/src/cmd/hive-pdftext/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/inbox/pdfbox" +) + +func TestRunVersionUsageAndExtract(t *testing.T) { + var out, errb bytes.Buffer + if code := run(context.Background(), []string{"-version"}, nil, &out, &errb); code != pdfbox.ExitOK || !strings.HasPrefix(out.String(), "hive-pdftext ") { + t.Fatalf("-version = %d %q", code, out.String()) + } + if code := run(context.Background(), []string{"-nope"}, nil, &out, &errb); code != pdfbox.ExitUsage { + t.Fatalf("bad flag = %d", code) + } + data, err := os.ReadFile(filepath.Join("..", "..", "pkg", "inbox", "pdfbox", "testdata", "text.pdf")) + if err != nil { + t.Fatal(err) + } + out.Reset() + if code := run(context.Background(), []string{"-work-root", t.TempDir()}, bytes.NewReader(data), &out, &errb); code != pdfbox.ExitOK { + t.Fatalf("extract = %d %s", code, errb.String()) + } + var res pdfbox.Result + if err := json.Unmarshal(out.Bytes(), &res); err != nil || res.Pages != 2 || !res.HasText() { + t.Fatalf("result = %+v, %v", res, err) + } + errb.Reset() + if code := run(context.Background(), []string{"-max-bytes", "10", "-work-root", t.TempDir()}, bytes.NewReader(data), &out, &errb); code != pdfbox.ExitTooLarge || !strings.Contains(errb.String(), "larger") { + t.Fatalf("too large = %d %q", code, errb.String()) + } + if code := run(context.Background(), []string{"-work-root", t.TempDir()}, strings.NewReader("%PDF-1.4 garbage"), &out, &errb); code != pdfbox.ExitParse { + t.Fatalf("garbage = %d", code) + } +} diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index c969935b82..711b5ede2d 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -76,7 +76,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/inbox/transcriptions` | Dashboard auth | Live progress of queued and running transcriptions in queue order (`{active: [{source, id, state}]}` with `phase`, `progress`, `estimated`, `started_at`, `audio_seconds`, `processed_seconds`, `eta_seconds`, `queue_position`); reads no source, so the Inbox polls it every 3 s while a job runs; `Cache-Control: no-store`; see [inbox-transcription.md](../inbox-transcription.md) | `pkg/dashboard/api_inbox.go:34` | | `GET` | `/api/inbox/{source}/{id}/audio` | Owner or read-write | Mint a fresh short-lived audio/video link for one item (`{url, expires_at}`; Plaud: presigned, 24h; Zoom: audio-only or video `download_url` with a one-hour download token; Webex: the recording's temporary direct audio (else MP4) download link, valid 3 hours; Teams and Google Meet: 404, no browser-playable link; Slack huddles: 404, Slack keeps no huddle audio; Otter: the conversation's MP3 download link (`GET /conversations/{id}/audio`), minted per click; Granola: 404, Granola keeps no meeting audio; Limitless: 404, Pendant audio is served only behind the API key; Fathom: the signed url of a recording download (`POST /recordings/{id}/download`, audio-only recordings complete at once, video is polled), valid about 24 hours; Fireflies: the transcript's hashed `audio_url` (else `video_url`), valid 24 hours, Pro plan or higher); never stored by the hive, replied with `Cache-Control: no-store`, audited as `inbox_audio` (403 role, 404 unknown item or no audio, 502 source unreachable) | `pkg/dashboard/api_inbox.go:30` | | `GET` | `/api/inbox/{source}/{id}/audio/stream` | Owner or read-write | Stream one item's audio from the dashboard origin, for the Inbox player (the CSP refuses cross-origin media). The hive mints the link as above, fetches it server-side and copies the bytes with the media `Content-Type` (from storage, else the link's extension); a single `Range: bytes=…` is passed through (206 + `Content-Range`; 200 whole file when storage ignores it; 416 passed through). The link stays in memory for one playback's Range requests (10 min, never past expiry), is never logged, returned or stored, and is re-minted once when storage refuses it; nothing is written to disk. `Cache-Control: no-store`; audited as `inbox_audio_stream` once per minted link (403 role, 404 unknown item or no audio, 413 over the 2 GiB cap, 502 source or storage unreachable, or link refused/expired) | `pkg/dashboard/api_inbox.go:31` | -| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv, html and docx become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | +| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv, html, docx and pdf (parsed in the `hive-pdftext` sandbox) become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | | `POST` | `/api/inbox/uploads/url` | Owner or read-write | Add a link: `{url}`. The hive fetches it under the SSRF guard (http/https only, no credentials, its own DNS lookup with the connection pinned to the checked address, private/loopback/link-local/CGNAT/multicast/ULA/metadata ranges refused, every redirect re-checked, at most 3, body capped at `inbox.uploads.max_file_bytes`, 60 s) and runs the body through the upload type pipeline (an HTML page becomes readable markdown titled by its ``). 201 `{results: [{filename: host, ok, source, item_id, title, kind}]}`; 400 `link_refused`, 413, 415, 422, 502 `fetch_failed`, 507; audited as `inbox_upload` with `origin=link` and the host (403 role, 404 no upload store) | `pkg/dashboard/api_inbox_uploads.go:28` | | `POST` | `/api/inbox/recordings` | Owner or read-write | Start a dashboard recording (● Record): `{title, mime: audio/webm\|audio/mp4, consent_version, includes_tab_audio}`. `consent_version` must name the current consent notice; it is stored with the user in the recording's metadata. One recording per user at a time. 201 `{id, max_chunk_bytes, max_seconds, retention_days, transcribes}`; audited as `inbox_recording_start` (400 consent missing or stale, 403 role, 409 `{active_id}` already recording, 412 `inbox.recording.enabled` off, 415 other media types, 507 `inbox.recording.quota_bytes` used up); see [inbox-recording.md](../inbox-recording.md) | `pkg/dashboard/api_inbox_recordings.go:20` | | `POST` | `/api/inbox/recordings/{id}/chunks` | The recorder (owner or read-write) | Store one MediaRecorder chunk `?seq=N`, the raw bytes as the body (at most `max_chunk_bytes`, 512 KiB). In order only; each part is fsynced before the 200 `{id, seq, next_seq, bytes, duplicate?}`. The first chunk must start with a WebM or MP4 signature; the hive never decodes audio. A re-send with the same bytes is acknowledged again (`duplicate: true`) (400 bad seq, 403 someone else's, 409 `{expected_seq}` gap, other bytes for a stored seq, or already ended, 413 chunk over 512 KiB or past `max_bytes` / `max_seconds`, 415 not WebM/MP4, 507 quota) | `pkg/dashboard/api_inbox_recordings.go:21` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 9614921c4e..dc48b6ef92 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -139,6 +139,9 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_INBOX_UPLOADS_MAX_FILES_PER_DROP` | No | `20` | Overrides `inbox.uploads.max_files_per_drop`: files in one drop or one upload request. | | `HIVE_INBOX_UPLOADS_DOCUMENT_RETENTION_DAYS` | No | `0` | Overrides `inbox.uploads.document_retention_days`: days after which a text document's original is deleted (its extracted text and item stay); unset or `0` keeps originals. | | `HIVE_INBOX_UPLOADS_MEDIA_RETENTION_DAYS` | No | `5` | Overrides `inbox.uploads.media_retention_days`: days after which an audio or video upload's original is deleted. | +| `HIVE_PDF_SANDBOX` | No | `/usr/local/bin/hive-pdftext` | The sandbox that reads PDFs dropped onto the Inbox out of process ([inbox-uploads.md](inbox-uploads.md)). | +| `HIVE_PDF_RUN_AS` | No | `hive-transcribe` | The uid the PDF sandbox runs as through su-exec; `self` runs it as the hive's own uid (development only). | +| `HIVE_PDF_MEMORY_MB` | No | `1024` | Address-space limit (prlimit `--as`) of the PDF sandbox, in MiB. | | `HIVE_INBOX_TRANSCRIPTION_DAILY_MAX_ITEMS` | No | `20` | Overrides `inbox.transcription.daily_max_items`: transcription runs per UTC day. | | `GEMINI_API_KEY` / `GOOGLE_API_KEY` / `GOOGLE_GENAI_API_KEY` | No | none | The hive's own Gemini API key, read in this order. Gemini model discovery already uses it; Inbox transcription with `inbox.transcription.provider: gemini` (or `auto` when whisper is unavailable) sends recordings to the Gemini API with it, in the `x-goog-api-key` header. Never stored in `hive.yaml`, never shown ([inbox-transcription.md](inbox-transcription.md)). | | `HIVE_WHISPER_MODEL_DIR` | No | `/data/models/whisper` | Where the built-in whisper provider keeps its ggml model (`ggml-<model>.bin`, mode 0644). Air-gapped hives pre-seed the file here; it is checked against its pinned size and SHA-256 before use ([inbox-transcription.md](inbox-transcription.md)). | diff --git a/src/docs/inbox-uploads.md b/src/docs/inbox-uploads.md index 9d8ddaf83d..714a8e25f8 100644 --- a/src/docs/inbox-uploads.md +++ b/src/docs/inbox-uploads.md @@ -31,6 +31,7 @@ and opens as a Word document. | YAML | `.yaml`, `.yml` | a one-line summary and the document in a fenced `yaml` block (parsed with `gopkg.in/yaml.v3`) | | CSV / TSV | `.csv`, `.tsv`, `.tab` | row and column counts and a markdown table of the first 20 rows (20 columns at most) | | Word | `.docx` | headings (Title and Heading 1 to 6 styles) as `#`, paragraphs, bulleted and numbered lists (by level) as `-` items, and tables as markdown tables; deleted text and field codes are left out; the title comes from the document properties, else the first heading. Read in Go with `archive/zip` and `encoding/xml` (standard library): at most 1000 zip entries, 200 MiB declared uncompressed in all, 50 MiB per part (checked while inflating, not only from the header) and a 100:1 compression ratio per entry; a part with a DOCTYPE or an entity the decoder does not know is refused (XXE), and nothing is fetched | +| PDF | `.pdf` (or a file without an extension that starts with `%PDF-`) | one `## Page N` section per page with text, a page-count summary, and the document title from its Info dictionary. Parsed **out of process** in the `hive-pdftext` sandbox with `github.com/ledongthuc/pdf` (BSD-3-Clause, pure Go): at most 200 pages and 1 MiB of text; a PDF without a text layer is refused with "no text layer (scanned PDF); OCR not supported yet" | | HTML | `.html`, `.htm`, and web pages behind a link | readable markdown: the `<title>` names the item; the `<article>` or `<main>` (else `<body>`) is kept with headings, paragraphs, lists, quotes, code, tables, link targets and image alt text; scripts, styles, forms, navigation, footers, asides and hidden elements are dropped. Parsed with `golang.org/x/net/html` (BSD-3-Clause); nothing is executed and nothing the page references is loaded | Text is validated as UTF-8 (a UTF-8 BOM is dropped; UTF-16 is refused with @@ -42,7 +43,6 @@ Not supported (the item is refused with a message saying what to do): | File | Message | | --- | --- | -| PDF | not supported yet | | Legacy Word `.doc` (and `.xls`, `.ppt`) | save it as .docx or PDF and try again | | RTF, OpenDocument `.odt` | save it as .docx, PDF or plain text | | PowerPoint `.pptx` | export the slides as PDF | @@ -72,6 +72,28 @@ Every file, accepted or refused, is audited as `inbox_upload` with the user, the item id, the detected kind, the size and the outcome or refusal code; the file's content and name are not written to the audit log. +## PDF sandbox + +PDF parsers have a long history of bugs, so the hive never parses an +uploaded PDF in its own process. It runs `/usr/local/bin/hive-pdftext` the +same way it runs the whisper transcription sandbox: + +- as the separate `hive-transcribe` uid (no home, shell or supplementary + groups), reached through `su-exec`, with an environment of only `PATH` + and `LC_ALL`; +- under `prlimit` (1 GiB address space, CPU seconds = timeout + 10, file + size = the PDF's size, 32 open files, no core dumps) and `nice -n 10`; +- in an empty network namespace (`unshare --user --net`) where the kernel + allows unprivileged user namespaces; +- with the PDF on stdin, copied into a private 0700 work dir (file 0600) + that is removed on exit, and only JSON on stdout (capped); +- with a 60-second wall clock, after which the whole process group is + killed. + +`HIVE_PDF_SANDBOX`, `HIVE_PDF_RUN_AS` and `HIVE_PDF_MEMORY_MB` override the +path, the uid and the memory cap. When the sandbox is not in the image, PDF +uploads are refused with "PDF reading is not available on this hive". + ## Links A link is fetched by the hive itself, once, when it is added, and its body diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 2e7f3fd599..8ceab9a3e7 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -168,7 +168,12 @@ timeout and size caps; the original, the extracted text and the meta are stored 0600 in a 0700 directory per upload, never executed or served raw, and bounded by a per-hive quota. The extracted text is fenced in the same `<untrusted-inbox-data>` envelope in its Knowledge fact and before any agent -prompt ([inbox-uploads.md](inbox-uploads.md)). Word documents are read in +prompt ([inbox-uploads.md](inbox-uploads.md)). Uploaded PDFs are never parsed in the hive process: the +`hive-pdftext` sandbox reads them as the `hive-transcribe` uid through +su-exec with an empty environment, under prlimit (address space, CPU, file +size, open files, no core dumps) and nice, in an empty network namespace +where the kernel allows it, from a private 0700 work dir, with a wall-clock +timeout that kills the process group. Word documents are read in Go with zip-bomb caps (entry count, declared and actual inflated size, compression ratio) and an XML decoder that refuses DTDs and unknown entities (no XXE). Links dropped onto the Inbox are diff --git a/src/pkg/inbox/link_fetch.go b/src/pkg/inbox/link_fetch.go index 6377e0d852..0e877dedc6 100644 --- a/src/pkg/inbox/link_fetch.go +++ b/src/pkg/inbox/link_fetch.go @@ -373,6 +373,7 @@ var linkTypeExtensions = map[string]string{ "text/x-yaml": ".yaml", "text/csv": ".csv", "text/tab-separated-values": ".tsv", + "application/pdf": ".pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", } diff --git a/src/pkg/inbox/pdfbox/pdfbox.go b/src/pkg/inbox/pdfbox/pdfbox.go new file mode 100644 index 0000000000..49f7c7caa8 --- /dev/null +++ b/src/pkg/inbox/pdfbox/pdfbox.go @@ -0,0 +1,259 @@ +// Package pdfbox is the sandbox side of Inbox PDF uploads: the hive-pdftext +// command that reads one untrusted PDF and prints its text as JSON. +// +// PDF parsers have a long history of bugs, so the hive never parses an +// uploaded PDF in its own process. It runs hive-pdftext the way it runs the +// whisper sandbox: as the low-privilege hive-transcribe uid (through +// su-exec), with an empty environment, under prlimit (address space, CPU +// seconds, file size, open files, no core dumps) and nice, in an empty +// network namespace where the kernel allows it, with the PDF on stdin and a +// wall-clock timeout after which the whole process group is killed. +// hive-pdftext keeps the PDF in a private 0700 work dir that it removes on +// exit, caps pages and characters, and writes only JSON to stdout. +// +// Text is extracted with github.com/ledongthuc/pdf (BSD-3-Clause, pure Go), +// the library the knowledge document parser already uses. +package pdfbox + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/ledongthuc/pdf" +) + +// Defaults shared by the sandbox and the hive side. +const ( + // DefaultSandbox is the hive-pdftext path in the hive image. + DefaultSandbox = "/usr/local/bin/hive-pdftext" + // DefaultMaxPages caps the pages read. + DefaultMaxPages = 200 + // DefaultMaxChars caps the characters returned. + DefaultMaxChars = 1 << 20 + // DefaultMemoryMB caps the sandbox's address space. + DefaultMemoryMB = 1024 + // DefaultTimeout bounds one extraction. + DefaultTimeout = 60 * time.Second + // maxOpenFiles / niceness for the sandbox. + maxOpenFiles = 32 + niceness = "10" + // cpuLimitMargin is added to RLIMIT_CPU on top of the timeout, so the + // wall-clock timeout fires first in normal operation. + cpuLimitMargin = 10 + // dirMode / fileMode for the private work dir and the PDF in it. + dirMode = 0o700 + fileMode = 0o600 + // umask makes everything the sandbox creates private. + umask = 0o077 +) + +// Exit codes of hive-pdftext. +const ( + ExitOK = 0 + ExitUsage = 2 + ExitTooLarge = 3 + ExitParse = 5 + ExitSetup = 7 +) + +// Result is what hive-pdftext prints: the page count, how many pages were +// read, each read page's text, the document title, and whether a cap cut it. +type Result struct { + Pages int `json:"pages"` + Read int `json:"read"` + Text []string `json:"text"` + Title string `json:"title,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// HasText reports whether any read page has text (a scanned PDF has none). +func (r Result) HasText() bool { + for _, t := range r.Text { + if strings.TrimSpace(t) != "" { + return true + } + } + return false +} + +// Options configure one sandbox run. +type Options struct { + // MaxBytes caps the PDF read from stdin (0 disables). + MaxBytes int64 + // MaxPages and MaxChars cap the output (0 uses the defaults). + MaxPages int + MaxChars int + // WorkRoot is where the private work dir is created ("" = os.TempDir). + WorkRoot string + // OnWorkDir, when set, is called with the work dir once it exists (tests). + OnWorkDir func(string) +} + +// Error is a sandbox failure with its exit code. +type Error struct { + Code int + Msg string +} + +func (e *Error) Error() string { return e.Msg } + +func fail(code int, format string, args ...any) *Error { + return &Error{Code: code, Msg: fmt.Sprintf(format, args...)} +} + +// Run reads one PDF from stdin into a private work dir, extracts its text +// within the caps and writes the Result as JSON to stdout. The work dir is +// removed whatever happens. +func Run(ctx context.Context, o Options, stdin io.Reader, stdout io.Writer) error { + if o.MaxPages <= 0 { + o.MaxPages = DefaultMaxPages + } + if o.MaxChars <= 0 { + o.MaxChars = DefaultMaxChars + } + syscall.Umask(umask) + work, err := os.MkdirTemp(o.WorkRoot, "hive-pdftext-") + if err != nil { + return fail(ExitSetup, "work dir: %v", err) + } + defer os.RemoveAll(work) + if err := os.Chmod(work, dirMode); err != nil { + return fail(ExitSetup, "work dir mode: %v", err) + } + if o.OnWorkDir != nil { + o.OnWorkDir(work) + } + input := filepath.Join(work, "input.pdf") + size, err := copyCapped(input, stdin, o.MaxBytes) + if err != nil { + return err + } + f, err := os.Open(input) + if err != nil { + return fail(ExitSetup, "open input: %v", err) + } + defer f.Close() + res, err := extract(ctx, f, size, o) + if err != nil { + return err + } + if err := json.NewEncoder(stdout).Encode(res); err != nil { + return fail(ExitSetup, "write result: %v", err) + } + return nil +} + +// copyCapped writes stdin to path (0600), refusing more than limit bytes. +func copyCapped(path string, r io.Reader, limit int64) (int64, error) { + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileMode) + if err != nil { + return 0, fail(ExitSetup, "input file: %v", err) + } + defer out.Close() + src := r + if limit > 0 { + src = io.LimitReader(r, limit+1) + } + n, err := io.Copy(out, src) + if err != nil { + return 0, fail(ExitSetup, "read input: %v", err) + } + if limit > 0 && n > limit { + return 0, fail(ExitTooLarge, "the PDF is larger than %d bytes", limit) + } + if n == 0 { + return 0, fail(ExitParse, "the PDF is empty") + } + return n, nil +} + +// extract reads the pages; a panic inside the library is a parse failure. +func extract(ctx context.Context, f io.ReaderAt, size int64, o Options) (res Result, err error) { + defer func() { + if r := recover(); r != nil { + err = fail(ExitParse, "the PDF could not be read") + } + }() + reader, rerr := pdf.NewReader(f, size) + if rerr != nil { + if errors.Is(rerr, pdf.ErrInvalidPassword) { + return Result{}, fail(ExitParse, "the PDF is encrypted") + } + return Result{}, fail(ExitParse, "the PDF could not be read: %v", rerr) + } + res.Pages = reader.NumPage() + res.Title = strings.Join(strings.Fields(reader.Trailer().Key("Info").Key("Title").Text()), " ") + chars := 0 + for i := 1; i <= res.Pages; i++ { + if ctx.Err() != nil { + return Result{}, fail(ExitParse, "reading the PDF took too long") + } + if i > o.MaxPages { + res.Truncated = true + break + } + text := pageText(reader.Page(i)) + if chars+len(text) > o.MaxChars { + text = strings.ToValidUTF8(text[:max(0, o.MaxChars-chars)], "") + res.Truncated = true + } + chars += len(text) + res.Text = append(res.Text, text) + res.Read++ + if res.Truncated { + break + } + } + return res, nil +} + +// pageText is one page's plain text with whitespace normalised per line; a +// page the library cannot read has none. +func pageText(p pdf.Page) (out string) { + defer func() { + if recover() != nil { + out = "" + } + }() + if p.V.IsNull() { + return "" + } + text, err := p.GetPlainText(nil) + if err != nil { + return "" + } + text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n") + var lines []string + for _, l := range strings.Split(text, "\n") { + lines = append(lines, strings.Join(strings.Fields(strings.ToValidUTF8(l, "")), " ")) + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + +// LimitPrefix is the prlimit + nice argv for one sandbox run. +func LimitPrefix(memoryMB int, timeout time.Duration, maxBytes int64) []string { + return []string{ + "prlimit", + "--as=" + strconv.FormatInt(int64(memoryMB)<<20, 10), + "--cpu=" + strconv.Itoa(int(timeout.Seconds())+cpuLimitMargin), + "--fsize=" + strconv.FormatInt(maxBytes+1, 10), + "--nofile=" + strconv.Itoa(maxOpenFiles), + "--core=0", + "--", "nice", "-n", niceness, + } +} + +// NetIsolationPrefix runs a command in a fresh user and network namespace +// (only a down loopback), the same isolation the whisper sandbox uses. +func NetIsolationPrefix() []string { + return []string{"unshare", "--user", "--map-current-user", "--net", "--"} +} diff --git a/src/pkg/inbox/pdfbox/pdfbox_test.go b/src/pkg/inbox/pdfbox/pdfbox_test.go new file mode 100644 index 0000000000..9928ad087c --- /dev/null +++ b/src/pkg/inbox/pdfbox/pdfbox_test.go @@ -0,0 +1,125 @@ +package pdfbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The fixtures are hand-built PDF 1.4 files: text.pdf has two pages of +// WinAnsi Helvetica text and an Info title; scanned.pdf has two pages that +// only paint rectangles (no text layer), like a scan. +func fixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + return data +} + +func runPDF(t *testing.T, o Options, data []byte) (Result, error) { + t.Helper() + if o.WorkRoot == "" { + o.WorkRoot = t.TempDir() + } + var out bytes.Buffer + err := Run(context.Background(), o, bytes.NewReader(data), &out) + if err != nil { + return Result{}, err + } + var res Result + if err := json.Unmarshal(out.Bytes(), &res); err != nil { + t.Fatalf("output is not JSON: %q", out.String()) + } + return res, nil +} + +func TestRunExtractsTextPDF(t *testing.T) { + res, err := runPDF(t, Options{}, fixture(t, "text.pdf")) + if err != nil { + t.Fatal(err) + } + if res.Pages != 2 || res.Read != 2 || res.Truncated || !res.HasText() || res.Title != "Inbox test PDF" { + t.Fatalf("result = %+v", res) + } + if !strings.Contains(res.Text[0], "Hello from the Inbox") || !strings.Contains(res.Text[1], "Second page text") { + t.Fatalf("text = %q", res.Text) + } +} + +func TestRunScannedPDFHasNoText(t *testing.T) { + res, err := runPDF(t, Options{}, fixture(t, "scanned.pdf")) + if err != nil { + t.Fatal(err) + } + if res.Pages != 2 || res.HasText() { + t.Fatalf("scanned = %+v", res) + } +} + +func TestRunCapsPagesCharsAndBytes(t *testing.T) { + data := fixture(t, "text.pdf") + res, err := runPDF(t, Options{MaxPages: 1}, data) + if err != nil || res.Read != 1 || !res.Truncated || res.Pages != 2 { + t.Fatalf("page cap = %+v, %v", res, err) + } + res, err = runPDF(t, Options{MaxChars: 5}, data) + if err != nil || !res.Truncated || len(strings.Join(res.Text, "")) > 5 { + t.Fatalf("char cap = %+v, %v", res, err) + } + var pe *Error + if _, err := runPDF(t, Options{MaxBytes: 100}, data); !errors.As(err, &pe) || pe.Code != ExitTooLarge { + t.Fatalf("byte cap = %v", err) + } + if _, err := runPDF(t, Options{}, nil); !errors.As(err, &pe) || pe.Code != ExitParse { + t.Fatalf("empty = %v", err) + } + for _, bad := range [][]byte{[]byte("%PDF-1.7\nnot really"), bytes.Repeat([]byte{0xff}, 512), data[:len(data)/2]} { + if _, err := runPDF(t, Options{}, bad); !errors.As(err, &pe) || pe.Code != ExitParse { + t.Fatalf("malformed = %v", err) + } + } +} + +func TestRunWorkDirIsPrivateAndRemoved(t *testing.T) { + root := t.TempDir() + var seen string + _, err := runPDF(t, Options{WorkRoot: root, OnWorkDir: func(dir string) { + seen = dir + info, err := os.Stat(dir) + if err != nil || info.Mode().Perm() != dirMode { + t.Errorf("work dir mode = %v, %v", info.Mode().Perm(), err) + } + }}, fixture(t, "text.pdf")) + if err != nil { + t.Fatal(err) + } + if seen == "" || !strings.HasPrefix(seen, root) { + t.Fatalf("work dir = %q", seen) + } + if _, err := os.Stat(seen); !os.IsNotExist(err) { + t.Fatal("the work dir must be removed after the run") + } +} + +func TestPrefixes(t *testing.T) { + p := strings.Join(LimitPrefix(DefaultMemoryMB, 30*time.Second, 1000), " ") + for _, want := range []string{"prlimit", "--as=1073741824", "--cpu=40", "--fsize=1001", "--core=0", "-- nice -n 10"} { + if !strings.Contains(p, want) { + t.Errorf("limit prefix %q missing %q", p, want) + } + } + if NetIsolationPrefix()[0] != "unshare" { + t.Fatal("isolation prefix") + } + if (&Error{Msg: "m"}).Error() != "m" { + t.Fatal("Error()") + } +} diff --git a/src/pkg/inbox/pdfbox/testdata/scanned.pdf b/src/pkg/inbox/pdfbox/testdata/scanned.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c3019b18acee65adecea48f26675beb05cde2936 GIT binary patch literal 833 zcmcIjQBT4!5Pt8kxECI!>)6If2#F}fM2!SYd>}p)R%Q*hrY#2j_4c}rA@FLmE$iKP zci(rtc4oKJE7na2g9!BMoeYOSpT9o^&>QafR+m6$yc8B<517MfM1(33i2J_*6^`D^ z!oo5M@C?7NK9}Ap=To%dy|eJC_7J+*tz0ONBpA>!x1#dgiS3tga?OEG)LIu(d2TOK zT`MaOhPD6_IOm6Ij0jv%4t*4b<YT>uW#sW0i=jV=J>j{qdS}*NH#Q^?2tN#vIQ9V_ zuS+?f%|cpRo9^Jgss12T>E0ktP`-6W@DCDoQz)VD%f*mnIb=x=M!*K+Ye?N^Y+;kB zIvDKpf9dF-=)iqbz1G39V;$blqUycOe{aNwM39r{upmp*UJ4s%Z375iP^~S^&ulFA zV~Xnt7qfF*FY%{;f=e#o(qo)6TyBL4H(()u1mu1Uw03}3BTT+^+_<18V-e|eCQnoH E3%1zWegFUf literal 0 HcmV?d00001 diff --git a/src/pkg/inbox/pdfbox/testdata/text.pdf b/src/pkg/inbox/pdfbox/testdata/text.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f250957444f3baed0b1a3a0f457baaf6ee68524e GIT binary patch literal 968 zcmcIjO>5gg5WVYHyo(_@w9?A@h#>^(#HOJw39%rD(8JnVIUAJSpj{WYzrHhTId&|1 ztPnywJ2UUS`54{b%s+|GStO9aE?=U{OT^XhH--3GdfD1F;(NJP4k<Z2;OZ(;re;Sz ze+=09#ou)8@RSAjg1<j0m)#goDGuS|LHOK~OT5@DeSl{Y0`XKjwY&MM+F#1+l|(!< zD_d(r+#a?0Vw`?A8~~WWkRPk*RN%sL;vcHka%#7DO31$uDauJogddc%n{GwAxgdc+ z_z_^@TnBzhm+_R%A@8mG>hO7w{#}{1e?~4?zVn@uuhF!ktO99<szFJpX@aW88HsEy zn$Et$KP%jtrQM#RdcmyX)O~c{0<P{JWY;rp!T*w2o=D{L9=(^Cl|zYC(vK1sYGqB0 zH>T><*8fx9#SuWIeXDR5>OoP?g9i7U(fV3!JJm!9CQ<VKpvd!ajs^$Y1mPCuV9N=H zpCg+{X&X*#sTlf=Ga9lJzbvEHpV;zpn6D^~^7&5cR&`<d7W$9EgeK5h58g_e(p%G5 Wa5=&PIxjmP^p;{AjYhM(dGr@5cmUr3 literal 0 HcmV?d00001 diff --git a/src/pkg/inbox/upload_pdf.go b/src/pkg/inbox/upload_pdf.go new file mode 100644 index 0000000000..d0146d42d9 --- /dev/null +++ b/src/pkg/inbox/upload_pdf.go @@ -0,0 +1,272 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "os/user" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/hivecommons/hive/pkg/inbox/pdfbox" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// UploadKindPDF is a PDF document. +const UploadKindPDF = "pdf" + +// Environment overrides for the PDF sandbox. +const ( + EnvPDFSandbox = "HIVE_PDF_SANDBOX" + EnvPDFRunAs = "HIVE_PDF_RUN_AS" + EnvPDFMemoryMB = "HIVE_PDF_MEMORY_MB" +) + +const ( + // pdfWaitDelay bounds how long a killed sandbox's pipes may linger. + pdfWaitDelay = 5 * time.Second + // pdfStderrBytes keeps the tail of the sandbox's diagnostics. + pdfStderrBytes = 4 << 10 + // pdfStdoutSlack is allowed on top of the character cap for JSON framing. + pdfStdoutSlack = 1 << 20 + // pdfScannedMessage is what a PDF without a text layer is refused with. + pdfScannedMessage = "no text layer (scanned PDF); OCR not supported yet" +) + +// PDFRuntime runs the hive-pdftext sandbox. The zero value is filled from +// the environment by DefaultPDFRuntime. +type PDFRuntime struct { + // Sandbox is the hive-pdftext path. + Sandbox string + // RunAs is the sandbox uid (default hive-transcribe; "self" runs as the + // hive's own uid, for development only). SuExec is the su-exec path + // ("" = looked up on PATH). + RunAs string + SuExec string + // MemoryMB caps the sandbox address space; Timeout the whole run. + MemoryMB int + Timeout time.Duration + // MaxPages and MaxChars cap the extraction. + MaxPages int + MaxChars int + // Limits wraps the sandbox in prlimit and nice; NoNetwork in an empty + // network namespace when the kernel allows it (production: both true). + Limits bool + NoNetwork bool + + isoOnce sync.Once + iso bool +} + +// DefaultPDFRuntime reads the HIVE_PDF_* overrides. +func DefaultPDFRuntime(getenv func(string) string) *PDFRuntime { + env := func(k, def string) string { + if v := strings.TrimSpace(getenv(k)); v != "" { + return v + } + return def + } + mem, err := strconv.Atoi(env(EnvPDFMemoryMB, "")) + if err != nil || mem <= 0 { + mem = pdfbox.DefaultMemoryMB + } + return &PDFRuntime{ + Sandbox: env(EnvPDFSandbox, pdfbox.DefaultSandbox), + RunAs: env(EnvPDFRunAs, transcribe.DefaultWhisperRunAs), + MemoryMB: mem, + Timeout: pdfbox.DefaultTimeout, + MaxPages: pdfbox.DefaultMaxPages, + MaxChars: pdfbox.DefaultMaxChars, + Limits: true, + NoNetwork: true, + } +} + +// Available reports whether the sandbox can run on this hive, or why not. +func (r *PDFRuntime) Available() (bool, string) { + if r == nil || !executable(r.Sandbox) { + return false, "PDF reading is not available on this hive (the hive-pdftext sandbox is not installed)" + } + if r.RunAs != transcribe.WhisperRunAsSelf { + if _, err := user.Lookup(r.RunAs); err != nil { + return false, "PDF reading is not available on this hive (the sandbox user " + r.RunAs + " does not exist)" + } + if r.SuExec == "" { + if _, err := exec.LookPath("su-exec"); err != nil { + return false, "PDF reading is not available on this hive (su-exec is missing)" + } + } + } + return true, "" +} + +// userPrefix is the su-exec argv that drops to RunAs (empty for "self"). +func (r *PDFRuntime) userPrefix() ([]string, error) { + if r.RunAs == transcribe.WhisperRunAsSelf { + return nil, nil + } + suexec := r.SuExec + if suexec == "" { + p, err := exec.LookPath("su-exec") + if err != nil { + return nil, err + } + suexec = p + } + return []string{suexec, r.RunAs}, nil +} + +// isolation reports, once, whether an empty network namespace can be made +// for the sandbox user. +func (r *PDFRuntime) isolation(ctx context.Context, drop []string) bool { + if !r.NoNetwork { + return false + } + r.isoOnce.Do(func() { + argv := append(append([]string{}, drop...), append(pdfbox.NetIsolationPrefix(), "true")...) + probe := exec.CommandContext(ctx, argv[0], argv[1:]...) + probe.Env = []string{} + r.iso = probe.Run() == nil + }) + return r.iso +} + +// argv is su-exec user, prlimit + nice, unshare, then hive-pdftext. +func (r *PDFRuntime) argv(ctx context.Context, maxBytes int64) ([]string, error) { + drop, err := r.userPrefix() + if err != nil { + return nil, err + } + argv := append([]string{}, drop...) + if r.Limits { + argv = append(argv, pdfbox.LimitPrefix(r.MemoryMB, r.Timeout, maxBytes)...) + } + if r.isolation(ctx, drop) { + argv = append(argv, pdfbox.NetIsolationPrefix()...) + } + return append(argv, r.Sandbox, + "-max-bytes", strconv.FormatInt(maxBytes, 10), + "-max-pages", strconv.Itoa(r.MaxPages), + "-max-chars", strconv.Itoa(r.MaxChars)), nil +} + +// tailBuffer keeps the last n bytes written. +type tailBuffer struct { + n int + b []byte +} + +func (t *tailBuffer) Write(p []byte) (int, error) { + t.b = append(t.b, p...) + if len(t.b) > t.n { + t.b = t.b[len(t.b)-t.n:] + } + return len(p), nil +} + +// capBuffer refuses output past n bytes. +type capBuffer struct { + n int + bytes.Buffer +} + +var errPDFOutputTooLarge = errors.New("the sandbox wrote more than expected") + +func (c *capBuffer) Write(p []byte) (int, error) { + if c.Len()+len(p) > c.n { + return 0, errPDFOutputTooLarge + } + return c.Buffer.Write(p) +} + +// Run extracts one PDF in the sandbox. The process group is killed at the +// timeout. +func (r *PDFRuntime) Run(ctx context.Context, data []byte) (pdfbox.Result, *UploadError) { + if ok, why := r.Available(); !ok { + return pdfbox.Result{}, uploadErr(UploadErrUnsupported, "%s", why) + } + timeout := r.Timeout + if timeout <= 0 { + timeout = pdfbox.DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + argv, err := r.argv(ctx, int64(len(data))) + if err != nil { + return pdfbox.Result{}, uploadErr(UploadErrUnsupported, "PDF reading is not available on this hive (%s)", err) + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = []string{"PATH=/usr/local/bin:/usr/bin:/bin", "LC_ALL=C"} + cmd.Stdin = bytes.NewReader(data) + out := &capBuffer{n: r.maxChars() + pdfStdoutSlack} + errTail := &tailBuffer{n: pdfStderrBytes} + cmd.Stdout, cmd.Stderr = out, errTail + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process == nil { + return nil + } + // The whole group: nice, unshare and the sandbox itself. + return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + cmd.WaitDelay = pdfWaitDelay + runErr := cmd.Run() + if ctx.Err() != nil { + return pdfbox.Result{}, uploadErr(UploadErrParse, "reading the PDF took longer than %s", timeout) + } + if runErr != nil { + var ee *exec.ExitError + if errors.As(runErr, &ee) && ee.ExitCode() == pdfbox.ExitTooLarge { + return pdfbox.Result{}, uploadErr(UploadErrTooLarge, "the PDF is too large") + } + reason := lastLine(string(errTail.b)) + if reason == "" { + reason = runErr.Error() + } + return pdfbox.Result{}, uploadErr(UploadErrParse, "the PDF could not be read (%s)", strings.TrimPrefix(reason, "hive-pdftext: ")) + } + var res pdfbox.Result + if err := json.Unmarshal(out.Bytes(), &res); err != nil { + return pdfbox.Result{}, uploadErr(UploadErrParse, "the PDF sandbox answered something unreadable") + } + return res, nil +} + +func (r *PDFRuntime) maxChars() int { + if r.MaxChars > 0 { + return r.MaxChars + } + return pdfbox.DefaultMaxChars +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} + +// pdfMarkdown renders a sandbox result: one "## Page N" section per page +// with text. A PDF without any text is refused as scanned. +func pdfMarkdown(res pdfbox.Result) (Extracted, *UploadError) { + if !res.HasText() { + return Extracted{}, uploadErr(UploadErrParse, "%s", pdfScannedMessage) + } + var b strings.Builder + for i, t := range res.Text { + if strings.TrimSpace(t) == "" { + continue + } + fmt.Fprintf(&b, "## Page %d\n\n%s\n\n", i+1, strings.TrimSpace(t)) + } + md := strings.TrimSpace(b.String()) + if res.Truncated { + md += fmt.Sprintf("\n\n_Read %d of %d pages (the page or character limit was reached). The whole file is stored on this hive._", res.Read, res.Pages) + } + summary := fmt.Sprintf("PDF, %d %s", res.Pages, plural(res.Pages, "page", "pages")) + return Extracted{Markdown: md, Summary: summary, Title: truncateRunes(res.Title, uploadTitleRunes)}, nil +} diff --git a/src/pkg/inbox/upload_pdf_test.go b/src/pkg/inbox/upload_pdf_test.go new file mode 100644 index 0000000000..241e868580 --- /dev/null +++ b/src/pkg/inbox/upload_pdf_test.go @@ -0,0 +1,138 @@ +package inbox + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox/pdfbox" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// fakePDFSandbox writes a shell script standing in for hive-pdftext. +func fakePDFSandbox(t *testing.T, body string) *PDFRuntime { + t.Helper() + path := filepath.Join(t.TempDir(), "hive-pdftext") + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return &PDFRuntime{Sandbox: path, RunAs: transcribe.WhisperRunAsSelf, Timeout: 5 * time.Second} +} + +const pdfMagic = "%PDF-1.4\n" + +func TestPDFRuntimeRunsTheSandbox(t *testing.T) { + rt := fakePDFSandbox(t, `cat >/dev/null; echo '{"pages":3,"read":2,"text":["Hello page one",""],"title":"Deck","truncated":true}'`) + res, uerr := rt.Run(context.Background(), []byte(pdfMagic)) + if uerr != nil || res.Pages != 3 || res.Title != "Deck" { + t.Fatalf("run = %+v, %v", res, uerr) + } + ex, uerr := pdfMarkdown(res) + if uerr != nil || !strings.Contains(ex.Markdown, "## Page 1\n\nHello page one") || strings.Contains(ex.Markdown, "## Page 2") || !strings.Contains(ex.Markdown, "Read 2 of 3 pages") || ex.Summary != "PDF, 3 pages" || ex.Title != "Deck" { + t.Fatalf("markdown = %+v, %v", ex, uerr) + } + // The sandbox gets the configured caps and the PDF on stdin. + argvFile := filepath.Join(t.TempDir(), "argv") + rt = fakePDFSandbox(t, `echo "$@" > `+argvFile+`; wc -c | tr -d ' ' >> `+argvFile+`; echo '{"pages":1,"read":1,"text":["x"]}'`) + rt.MaxPages, rt.MaxChars = 7, 99 + if _, uerr := rt.Run(context.Background(), []byte(pdfMagic+"body")); uerr != nil { + t.Fatal(uerr) + } + got, _ := os.ReadFile(argvFile) + if !strings.Contains(string(got), "-max-bytes 13 -max-pages 7 -max-chars 99") || !strings.Contains(string(got), "\n13") { + t.Fatalf("argv/stdin = %q", got) + } +} + +func TestPDFRuntimeScannedAndFailures(t *testing.T) { + rt := fakePDFSandbox(t, `cat >/dev/null; echo '{"pages":2,"read":2,"text":["",""]}'`) + res, uerr := rt.Run(context.Background(), []byte(pdfMagic)) + if uerr != nil { + t.Fatal(uerr) + } + if _, uerr := pdfMarkdown(res); uerr == nil || uerr.Message != pdfScannedMessage { + t.Fatalf("scanned = %v", uerr) + } + for body, code := range map[string]string{ + `cat >/dev/null; echo "hive-pdftext: the PDF could not be read" >&2; exit 5`: UploadErrParse, + `cat >/dev/null; exit 3`: UploadErrTooLarge, + `cat >/dev/null; echo 'not json'`: UploadErrParse, + `cat >/dev/null; head -c 3000000 /dev/zero`: UploadErrParse, + } { + rt := fakePDFSandbox(t, body) + rt.MaxChars = 10 + if _, uerr := rt.Run(context.Background(), []byte(pdfMagic)); uerr == nil || uerr.Code != code { + t.Errorf("%s = %v, want %s", body, uerr, code) + } + } + missing := &PDFRuntime{Sandbox: filepath.Join(t.TempDir(), "nope"), RunAs: transcribe.WhisperRunAsSelf} + if _, uerr := missing.Run(context.Background(), []byte(pdfMagic)); uerr == nil || uerr.Code != UploadErrUnsupported || !strings.Contains(uerr.Message, "not installed") { + t.Fatalf("missing sandbox = %v", uerr) + } + nouser := fakePDFSandbox(t, "true") + nouser.RunAs = "hive-no-such-user-xyz" + if ok, why := nouser.Available(); ok || !strings.Contains(why, "does not exist") { + t.Fatalf("no user = %v %q", ok, why) + } +} + +// TestPDFRuntimeKillsOnTimeout pins the wall-clock limit: a sandbox that +// hangs (and a child it started) is killed, and the upload fails fast. +func TestPDFRuntimeKillsOnTimeout(t *testing.T) { + rt := fakePDFSandbox(t, `sleep 30 & sleep 30`) + rt.Timeout = 300 * time.Millisecond + start := time.Now() + _, uerr := rt.Run(context.Background(), []byte(pdfMagic)) + if uerr == nil || !strings.Contains(uerr.Message, "took longer") { + t.Fatalf("timeout = %v", uerr) + } + if elapsed := time.Since(start); elapsed > pdfWaitDelay+5*time.Second { + t.Fatalf("the sandbox was not killed in time: %s", elapsed) + } +} + +func TestPDFRuntimeArgvAndDefaults(t *testing.T) { + env := map[string]string{EnvPDFSandbox: "/x/hive-pdftext", EnvPDFRunAs: "pdfuser", EnvPDFMemoryMB: "256"} + rt := DefaultPDFRuntime(func(k string) string { return env[k] }) + if rt.Sandbox != "/x/hive-pdftext" || rt.RunAs != "pdfuser" || rt.MemoryMB != 256 || !rt.Limits || !rt.NoNetwork || rt.Timeout != pdfbox.DefaultTimeout { + t.Fatalf("runtime = %+v", rt) + } + rt = DefaultPDFRuntime(func(string) string { return "" }) + if rt.Sandbox != pdfbox.DefaultSandbox || rt.RunAs != transcribe.DefaultWhisperRunAs || rt.MemoryMB != pdfbox.DefaultMemoryMB { + t.Fatalf("defaults = %+v", rt) + } + rt.SuExec, rt.NoNetwork = "/sbin/su-exec", false + argv, err := rt.argv(context.Background(), 1234) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(argv, " ") + if !strings.HasPrefix(joined, "/sbin/su-exec hive-transcribe prlimit --as=1073741824 ") || !strings.Contains(joined, "--fsize=1235") || !strings.HasSuffix(joined, pdfbox.DefaultSandbox+" -max-bytes 1234 -max-pages 200 -max-chars 1048576") { + t.Fatalf("argv = %q", joined) + } + tb := &tailBuffer{n: 4} + _, _ = tb.Write([]byte("abcdef")) + if string(tb.b) != "cdef" || lastLine("a\nb\n") != "b" { + t.Fatal("tail") + } +} + +func TestUploadSourceReadsPDFInTheSandbox(t *testing.T) { + e := newUploadEnv(t, config.InboxConfig{}) + e.src.PDF = fakePDFSandbox(t, `cat >/dev/null; echo '{"pages":1,"read":1,"text":["Quarterly numbers"],"title":"Q3 report"}'`) + it, err := e.svc.AddUpload(context.Background(), UploadInput{Filename: "q3.pdf", Data: []byte(pdfMagic + "rest"), AddedBy: "ana"}) + if err != nil || it.UploadKind != UploadKindPDF || it.Title != "Q3 report" || it.Summary != "PDF, 1 page" { + t.Fatalf("pdf item = %+v, %v", it, err) + } + e.src.PDF = fakePDFSandbox(t, `cat >/dev/null; echo '{"pages":1,"read":1,"text":[""]}'`) + if _, err := e.svc.AddUpload(context.Background(), UploadInput{Filename: "scan.pdf", Data: []byte(pdfMagic)}); err == nil || !strings.Contains(err.Error(), "OCR not supported yet") { + t.Fatalf("scanned upload = %v", err) + } + if k, uerr := SniffUpload("noext", []byte(pdfMagic)); uerr != nil || k != UploadKindPDF { + t.Fatalf("pdf without extension = %q, %v", k, uerr) + } +} diff --git a/src/pkg/inbox/upload_source.go b/src/pkg/inbox/upload_source.go index 67a8e76931..f0954abde6 100644 --- a/src/pkg/inbox/upload_source.go +++ b/src/pkg/inbox/upload_source.go @@ -66,8 +66,11 @@ type UploadSource struct { FactWritten func(dir string) // Fetcher fetches links; nil refuses links. Fetcher *LinkFetcher - Logger *slog.Logger - Now func() time.Time + // PDF runs the hive-pdftext sandbox; nil reads HIVE_PDF_* on first use. + PDF *PDFRuntime + pdfMu sync.Mutex + Logger *slog.Logger + Now func() time.Time factMu sync.Mutex } @@ -217,7 +220,7 @@ func (u *UploadSource) Add(ctx context.Context, in UploadInput) (UploadMeta, err if uerr != nil { return UploadMeta{}, uerr } - ex, uerr := ExtractUpload(ctx, kind, in.Data) + ex, uerr := u.extract(ctx, kind, in.Data) if uerr != nil { return UploadMeta{}, uerr } @@ -241,6 +244,33 @@ func (u *UploadSource) Add(ctx context.Context, in UploadInput) (UploadMeta, err return u.store(m, in.Data, ex.Markdown) } +// extract runs the type's extractor: PDFs in the sandbox, everything else +// in-process. +func (u *UploadSource) extract(ctx context.Context, kind string, data []byte) (Extracted, *UploadError) { + if kind != UploadKindPDF { + return ExtractUpload(ctx, kind, data) + } + res, uerr := u.pdfRuntime().Run(ctx, data) + if uerr != nil { + return Extracted{}, uerr + } + ex, uerr := pdfMarkdown(res) + if uerr != nil { + return Extracted{}, uerr + } + ex.Markdown = capMarkdown(ex.Markdown) + return ex, nil +} + +func (u *UploadSource) pdfRuntime() *PDFRuntime { + u.pdfMu.Lock() + defer u.pdfMu.Unlock() + if u.PDF == nil { + u.PDF = DefaultPDFRuntime(os.Getenv) + } + return u.PDF +} + // LinkInput is one dropped or pasted URL. type LinkInput struct { URL string @@ -279,7 +309,7 @@ func (u *UploadSource) AddLink(ctx context.Context, in LinkInput) (UploadMeta, e if uerr != nil { return UploadMeta{}, uerr } - ex, uerr := ExtractUpload(ctx, kind, fl.Body) + ex, uerr := u.extract(ctx, kind, fl.Body) if uerr != nil { return UploadMeta{}, uerr } diff --git a/src/pkg/inbox/upload_types.go b/src/pkg/inbox/upload_types.go index 55cd9f89bb..38f3ffe014 100644 --- a/src/pkg/inbox/upload_types.go +++ b/src/pkg/inbox/upload_types.go @@ -77,6 +77,7 @@ var uploadTextExtensions = map[string]string{ ".html": UploadKindHTML, ".htm": UploadKindHTML, ".docx": UploadKindDOCX, + ".pdf": UploadKindPDF, } // uploadUnsupportedExtensions names formats people commonly drop that this @@ -103,7 +104,7 @@ func UploadSupportedExtensions() []string { } func uploadSupportedList() string { - return "md, txt, json, jsonl, yaml, csv, tsv, html, docx" + return "md, txt, json, jsonl, yaml, csv, tsv, html, docx, pdf" } // uploadMagic is a binary signature that is never a text document. @@ -170,6 +171,11 @@ func SniffUpload(name string, data []byte) (string, *UploadError) { if strings.EqualFold(filepath.Ext(name), ".docx") && docxLooksLikeZip(data) { return UploadKindDOCX, nil } + // A PDF is read only when it says so by content and is not named as + // something else; it is then parsed in the hive-pdftext sandbox. + if pext := strings.ToLower(filepath.Ext(name)); bytes.HasPrefix(data, []byte("%PDF-")) && (pext == ".pdf" || pext == "") { + return UploadKindPDF, nil + } for _, m := range uploadBinaryMagic { if len(data) >= m.offset+len(m.prefix) && bytes.Equal(data[m.offset:m.offset+len(m.prefix)], m.prefix) { return "", uploadErr(UploadErrUnsupported, "unsupported file type: this is %s; supported: %s", m.label, uploadSupportedList()) From 605f7d7983c4a084586ec00c7d5b5be052235a20 Mon Sep 17 00:00:00 2001 From: Andy Anderson <andy@clubanderson.com> Date: Mon, 16 Nov 2026 09:34:33 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20Inbox:=20cover=20the=20PDF?= =?UTF-8?q?=20sandbox's=20setup=20and=20I/O=20failure=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson <andy@clubanderson.com> --- src/cmd/hive-pdftext/main_test.go | 6 ++++++ src/pkg/inbox/pdfbox/pdfbox_test.go | 30 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/cmd/hive-pdftext/main_test.go b/src/cmd/hive-pdftext/main_test.go index e225a164e8..d2dbea91e4 100644 --- a/src/cmd/hive-pdftext/main_test.go +++ b/src/cmd/hive-pdftext/main_test.go @@ -40,3 +40,9 @@ func TestRunVersionUsageAndExtract(t *testing.T) { t.Fatalf("garbage = %d", code) } } + +func TestRunMainVersion(t *testing.T) { + if code := runMain([]string{"-version"}); code != pdfbox.ExitOK { + t.Fatalf("runMain -version = %d", code) + } +} diff --git a/src/pkg/inbox/pdfbox/pdfbox_test.go b/src/pkg/inbox/pdfbox/pdfbox_test.go index 9928ad087c..f04fc6dcf6 100644 --- a/src/pkg/inbox/pdfbox/pdfbox_test.go +++ b/src/pkg/inbox/pdfbox/pdfbox_test.go @@ -123,3 +123,33 @@ func TestPrefixes(t *testing.T) { t.Fatal("Error()") } } + +type failingReader struct{} + +func (failingReader) Read([]byte) (int, error) { return 0, errors.New("broken pipe") } + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("closed") } + +func TestRunSetupAndIOFailures(t *testing.T) { + var pe *Error + missing := filepath.Join(t.TempDir(), "does", "not", "exist") + if err := Run(context.Background(), Options{WorkRoot: missing}, bytes.NewReader(fixture(t, "text.pdf")), &bytes.Buffer{}); !errors.As(err, &pe) || pe.Code != ExitSetup { + t.Fatalf("missing work root = %v", err) + } + if err := Run(context.Background(), Options{WorkRoot: t.TempDir()}, failingReader{}, &bytes.Buffer{}); !errors.As(err, &pe) || pe.Code != ExitSetup || !strings.Contains(pe.Msg, "read input") { + t.Fatalf("stdin error = %v", err) + } + if err := Run(context.Background(), Options{WorkRoot: t.TempDir()}, bytes.NewReader(fixture(t, "text.pdf")), failingWriter{}); !errors.As(err, &pe) || pe.Code != ExitSetup { + t.Fatalf("stdout error = %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := Run(ctx, Options{WorkRoot: t.TempDir()}, bytes.NewReader(fixture(t, "text.pdf")), &bytes.Buffer{}); !errors.As(err, &pe) || pe.Code != ExitParse || !strings.Contains(pe.Msg, "too long") { + t.Fatalf("cancelled = %v", err) + } + if (Result{Text: []string{" ", ""}}).HasText() { + t.Fatal("whitespace is not text") + } +}