From f203b19c61e5f2066acd7758363ccbe9c8ba2d33 Mon Sep 17 00:00:00 2001 From: Armin Fauland Date: Tue, 4 Aug 2026 12:22:23 +0000 Subject: [PATCH] feat(api): let download verify the file starts with an expected signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a session expires or a bot check kicks in, the other side usually answers with HTTP 200 and an HTML page rather than an error status. The download action wrote that page under the requested name, producing an "invoice.pdf" whose content is "". Downstream processing notices late or not at all — in our case three such files reached a document management system before anyone spotted them. Adds an optional `expectMagic` parameter: "params": { "url": "…", "path": "./downloads", "filename": "invoice.pdf", "expectMagic": "%PDF" } If the content does not start with the given signature, the file is not written, the actual bytes are logged, and the action returns null so a skipIf can react. Both download paths are covered — the remote fetch and the local file:// copy, where only the first bytes are read rather than the whole file. Without the parameter behaviour is unchanged, so existing configurations are unaffected. Tests: 4 new cases (match, mismatch, parameter absent, local file). Docs: parameter table and a section in EN and DE. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LixHBPkhb8h5oDdMqSG4se --- .../actions/download.action.spec.ts | 76 +++++++++++++++++++ .../actions/download.actions.ts | 55 ++++++++++++++ .../docs/de/user-guide/actions/download.mdx | 27 +++++++ .../docs/en/user-guide/actions/download.mdx | 27 +++++++ 4 files changed, 185 insertions(+) diff --git a/apps/api/src/action-handler/actions/download.action.spec.ts b/apps/api/src/action-handler/actions/download.action.spec.ts index 3c440e5..65cbc7b 100644 --- a/apps/api/src/action-handler/actions/download.action.spec.ts +++ b/apps/api/src/action-handler/actions/download.action.spec.ts @@ -13,6 +13,9 @@ vi.mock('fs', async (importOriginal) => { copyFileSync: vi.fn(), writeFileSync: vi.fn(), statSync: vi.fn().mockReturnValue({ size: 1024 }), + openSync: vi.fn().mockReturnValue(3), + readSync: vi.fn().mockReturnValue(0), + closeSync: vi.fn(), }; }); @@ -48,6 +51,79 @@ describe('DownloadAction', () => { vi.clearAllMocks(); }); + describe('expectMagic', () => { + function bytesOf(text: string) { + return Array.from(new Uint8Array(Buffer.from(text, 'latin1'))); + } + + it('writes the file when the remote content starts with the signature', async () => { + const action = createAction({ + url: 'https://example.com/invoice.pdf', + path: './downloads', + filename: 'invoice.pdf', + expectMagic: '%PDF', + }); + action.page.evaluate.mockResolvedValue(bytesOf('%PDF-1.7 …')); + + const result = await action.run(); + + expect(result).toBeDefined(); + expect(fs.writeFileSync).toHaveBeenCalled(); + }); + + it('refuses to write an HTML error page served as a PDF', async () => { + const action = createAction({ + url: 'https://example.com/invoice.pdf', + path: './downloads', + filename: 'invoice.pdf', + expectMagic: '%PDF', + }); + action.page.evaluate.mockResolvedValue(bytesOf('')); + + const result = await action.run(); + + expect(result).toBeNull(); + expect(fs.writeFileSync).not.toHaveBeenCalled(); + expect(action.logger.error).toHaveBeenCalled(); + }); + + it('leaves behaviour unchanged when expectMagic is not set', async () => { + const action = createAction({ + url: 'https://example.com/invoice.pdf', + path: './downloads', + filename: 'invoice.pdf', + }); + action.page.evaluate.mockResolvedValue(bytesOf('')); + + const result = await action.run(); + + expect(result).toBeDefined(); + expect(fs.writeFileSync).toHaveBeenCalled(); + }); + + it('does not copy a local file whose signature does not match', async () => { + const action = createAction({ + url: 'file:///tmp/invoice.pdf', + path: './downloads', + filename: 'invoice.pdf', + expectMagic: '%PDF', + }); + vi.mocked(fs.openSync).mockReturnValue(7 as never); + vi.mocked(fs.readSync).mockImplementation((( + _fd: number, + buffer: Buffer, + ) => { + Buffer.from(' { const action = createAction({ url: 'https://example.com/file.pdf', diff --git a/apps/api/src/action-handler/actions/download.actions.ts b/apps/api/src/action-handler/actions/download.actions.ts index 5abcad8..5a90c37 100644 --- a/apps/api/src/action-handler/actions/download.actions.ts +++ b/apps/api/src/action-handler/actions/download.actions.ts @@ -9,6 +9,14 @@ export type DownloadActionParams = { url: string; // Die URL, die heruntergeladen werden soll path: string; filename: string; + /** + * Optional: Signatur, mit der die Datei beginnen muss, z. B. "%PDF". + * + * Ohne diesen Parameter aendert sich nichts. Ist er gesetzt und passt der + * Anfang nicht, wird die Datei NICHT geschrieben und die Aktion liefert + * null — statt eine Fehlerseite unter dem erwarteten Namen abzulegen. + */ + expectMagic?: string; }; @Action('download', { @@ -156,6 +164,36 @@ export class DownloadAction extends BaseAction { /** * Kopiert eine lokale Datei (file:// URL) */ + /** + * Prueft, ob der Dateianfang der erwarteten Signatur entspricht. + * + * Hintergrund: Laeuft eine Sitzung ab oder greift eine Bot-Erkennung, liefert + * die Gegenstelle HTTP 200 mit einer HTML-Seite. Die wurde bisher unter dem + * erwarteten Namen abgelegt — eine "Rechnung.pdf", die in Wahrheit + * "" enthaelt. Nachgelagerte Verarbeitung merkt das erst spaet + * oder gar nicht. + */ + private magicMatches(head: Buffer): boolean { + const expected = this.params.expectMagic; + if (!expected) { + return true; + } + + const actual = head + .subarray(0, Buffer.byteLength(expected)) + .toString('latin1'); + if (actual === expected) { + return true; + } + + this.logger.error( + `❌ ${this.params.filename}: expected content to start with ` + + `${JSON.stringify(expected)} but found ${JSON.stringify(actual)} — ` + + `file not written`, + ); + return false; + } + private async downloadLocalFile( url: string, fullPath: string, @@ -178,6 +216,20 @@ export class DownloadAction extends BaseAction { return null; } + if (this.params.expectMagic) { + const length = Buffer.byteLength(this.params.expectMagic); + const head = Buffer.alloc(length); + const fd = fs.openSync(decodedPath, 'r'); + try { + fs.readSync(fd, head, 0, length, 0); + } finally { + fs.closeSync(fd); + } + if (!this.magicMatches(head)) { + return null; + } + } + // Kopiere die Datei fs.copyFileSync(decodedPath, fullPath); @@ -220,6 +272,9 @@ export class DownloadAction extends BaseAction { } const buffer = Buffer.from(fileBuffer); + if (!this.magicMatches(buffer)) { + return null; + } fs.writeFileSync(fullPath, buffer); const fileSizeKB = Math.round(buffer.length / 1024); diff --git a/apps/docs/src/content/docs/de/user-guide/actions/download.mdx b/apps/docs/src/content/docs/de/user-guide/actions/download.mdx index 3ed9446..d042715 100644 --- a/apps/docs/src/content/docs/de/user-guide/actions/download.mdx +++ b/apps/docs/src/content/docs/de/user-guide/actions/download.mdx @@ -18,11 +18,38 @@ Das Beste: Relative URLs werden automatisch aufgelöst und die Dateien werden sa | `url` | `string` | Ja | Die URL der Datei. Unterstützt absolute URLs, relative URLs und `file://`-Pfade. Handlebars-Templates möglich. | | `path` | `string` | Ja | Zielverzeichnis für den Download (z.B. `./downloads/invoices`). | | `filename` | `string` | Ja | Dateiname für die gespeicherte Datei. Handlebars-Templates möglich. | +| `expectMagic` | `string` | Nein | Signatur, mit der die Datei beginnen muss, z.B. `%PDF`. Passt sie nicht, wird nichts geschrieben und die Aktion liefert `null`. | +## 🛡️ Prüfen, ob die Datei wirklich das ist, was du wolltest + +Läuft eine Sitzung ab oder greift eine Bot-Erkennung, antwortet die Gegenstelle oft mit **HTTP 200 und einer HTML-Seite**. Ohne Prüfung landet die unter dem erwarteten Namen -- eine `rechnung.pdf`, die in Wahrheit `` enthält. Die nachgelagerte Verarbeitung merkt das spät oder gar nicht. + +Mit `expectMagic` gibst du an, womit die Datei beginnen muss: + +```jsonc +{ + "action": "download", + "params": { + "url": "{{previousData.invoiceUrl}}", + "path": "./downloads", + "filename": "rechnung-{{previousData.orderId}}.pdf", + "expectMagic": "%PDF" + } +} +``` + +Beginnt der Inhalt nicht mit `%PDF`, wird die Datei **nicht geschrieben**, die Aktion protokolliert, was tatsächlich kam, und liefert `null` -- darauf kann ein `skipIf` reagieren. + +Übliche Signaturen: `%PDF` (PDF), `PK` (ZIP, XLSX, DOCX), `\x89PNG` (PNG), `GIF8` (GIF). + + + ## 📂 Ordnerstruktur Die Download-Action organisiert Dateien automatisch in Unterordner basierend auf der **scrapeId** (der Dateiname deiner `.jsonc`-Konfiguration ohne Endung). Die `scrapeId` wird als Unterordner nach dem ersten "echten" Verzeichnissegment in deinem `path` eingefügt. diff --git a/apps/docs/src/content/docs/en/user-guide/actions/download.mdx b/apps/docs/src/content/docs/en/user-guide/actions/download.mdx index f98f051..9aa4fee 100644 --- a/apps/docs/src/content/docs/en/user-guide/actions/download.mdx +++ b/apps/docs/src/content/docs/en/user-guide/actions/download.mdx @@ -18,11 +18,38 @@ The best part: relative URLs are resolved automatically and files are neatly sor | `url` | `string` | Yes | The file URL. Supports absolute URLs, relative URLs, and `file://` paths. Handlebars templates supported. | | `path` | `string` | Yes | Target directory for the download (e.g. `./downloads/invoices`). | | `filename` | `string` | Yes | Filename for the saved file. Handlebars templates supported. | +| `expectMagic` | `string` | No | Signature the file must start with, e.g. `%PDF`. If it does not match, nothing is written and the action returns `null`. | +## 🛡️ Verifying the file really is what you asked for + +When a session expires or a bot check kicks in, the other side often answers with **HTTP 200 and an HTML page**. Without a check that page is saved under the name you expected -- an `invoice.pdf` that actually contains ``. Whatever processes the file downstream notices late, or not at all. + +Set `expectMagic` to the signature the file has to start with: + +```jsonc +{ + "action": "download", + "params": { + "url": "{{previousData.invoiceUrl}}", + "path": "./downloads", + "filename": "invoice-{{previousData.orderId}}.pdf", + "expectMagic": "%PDF" + } +} +``` + +If the content does not start with `%PDF`, the file is **not written**, the action logs what it actually received, and returns `null` -- so a `skipIf` can react to it. + +Common signatures: `%PDF` (PDF), `PK` (ZIP, XLSX, DOCX), `\x89PNG` (PNG), `GIF8` (GIF). + + + ## 📂 Folder Structure The download action automatically organizes files into subfolders based on the **scrapeId** (the filename of your `.jsonc` config without the extension). The `scrapeId` is inserted as a subfolder after the first "real" directory segment in your `path`.