Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions apps/api/src/action-handler/actions/download.action.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ vi.mock('fs', async (importOriginal) => {
copyFileSync: vi.fn(),
writeFileSync: vi.fn(),
statSync: vi.fn().mockReturnValue({ size: 1024 }),
openSync: vi.fn().mockReturnValue(3),
readSync: vi.fn().mockReturnValue(0),
closeSync: vi.fn(),
};
});

Expand Down Expand Up @@ -48,6 +51,79 @@ describe('DownloadAction', () => {
vi.clearAllMocks();
});

describe('expectMagic', () => {
function bytesOf(text: string) {
return Array.from(new Uint8Array(Buffer.from(text, 'latin1')));
}

it('writes the file when the remote content starts with the signature', async () => {
const action = createAction({
url: 'https://example.com/invoice.pdf',
path: './downloads',
filename: 'invoice.pdf',
expectMagic: '%PDF',
});
action.page.evaluate.mockResolvedValue(bytesOf('%PDF-1.7 …'));

const result = await action.run();

expect(result).toBeDefined();
expect(fs.writeFileSync).toHaveBeenCalled();
});

it('refuses to write an HTML error page served as a PDF', async () => {
const action = createAction({
url: 'https://example.com/invoice.pdf',
path: './downloads',
filename: 'invoice.pdf',
expectMagic: '%PDF',
});
action.page.evaluate.mockResolvedValue(bytesOf('<!DOCTYPE html><html>'));

const result = await action.run();

expect(result).toBeNull();
expect(fs.writeFileSync).not.toHaveBeenCalled();
expect(action.logger.error).toHaveBeenCalled();
});

it('leaves behaviour unchanged when expectMagic is not set', async () => {
const action = createAction({
url: 'https://example.com/invoice.pdf',
path: './downloads',
filename: 'invoice.pdf',
});
action.page.evaluate.mockResolvedValue(bytesOf('<!DOCTYPE html>'));

const result = await action.run();

expect(result).toBeDefined();
expect(fs.writeFileSync).toHaveBeenCalled();
});

it('does not copy a local file whose signature does not match', async () => {
const action = createAction({
url: 'file:///tmp/invoice.pdf',
path: './downloads',
filename: 'invoice.pdf',
expectMagic: '%PDF',
});
vi.mocked(fs.openSync).mockReturnValue(7 as never);
vi.mocked(fs.readSync).mockImplementation(((
_fd: number,
buffer: Buffer,
) => {
Buffer.from('<htm', 'latin1').copy(buffer);
return 4;
}) as never);

const result = await action.run();

expect(result).toBeNull();
expect(fs.copyFileSync).not.toHaveBeenCalled();
});
});

it('should return null for empty filename', async () => {
const action = createAction({
url: 'https://example.com/file.pdf',
Expand Down
55 changes: 55 additions & 0 deletions apps/api/src/action-handler/actions/download.actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ export type DownloadActionParams = {
url: string; // Die URL, die heruntergeladen werden soll
path: string;
filename: string;
/**
* Optional: Signatur, mit der die Datei beginnen muss, z. B. "%PDF".
*
* Ohne diesen Parameter aendert sich nichts. Ist er gesetzt und passt der
* Anfang nicht, wird die Datei NICHT geschrieben und die Aktion liefert
* null — statt eine Fehlerseite unter dem erwarteten Namen abzulegen.
*/
expectMagic?: string;
};

@Action('download', {
Expand Down Expand Up @@ -156,6 +164,36 @@ export class DownloadAction extends BaseAction<DownloadActionParams> {
/**
* Kopiert eine lokale Datei (file:// URL)
*/
/**
* Prueft, ob der Dateianfang der erwarteten Signatur entspricht.
*
* Hintergrund: Laeuft eine Sitzung ab oder greift eine Bot-Erkennung, liefert
* die Gegenstelle HTTP 200 mit einer HTML-Seite. Die wurde bisher unter dem
* erwarteten Namen abgelegt — eine "Rechnung.pdf", die in Wahrheit
* "<!DOCTYPE html>" enthaelt. Nachgelagerte Verarbeitung merkt das erst spaet
* oder gar nicht.
*/
private magicMatches(head: Buffer): boolean {
const expected = this.params.expectMagic;
if (!expected) {
return true;
}

const actual = head
.subarray(0, Buffer.byteLength(expected))
.toString('latin1');
if (actual === expected) {
return true;
}

this.logger.error(
`❌ ${this.params.filename}: expected content to start with ` +
`${JSON.stringify(expected)} but found ${JSON.stringify(actual)} — ` +
`file not written`,
);
return false;
}

private async downloadLocalFile(
url: string,
fullPath: string,
Expand All @@ -178,6 +216,20 @@ export class DownloadAction extends BaseAction<DownloadActionParams> {
return null;
}

if (this.params.expectMagic) {
const length = Buffer.byteLength(this.params.expectMagic);
const head = Buffer.alloc(length);
const fd = fs.openSync(decodedPath, 'r');
try {
fs.readSync(fd, head, 0, length, 0);
} finally {
fs.closeSync(fd);
}
if (!this.magicMatches(head)) {
return null;
}
}

// Kopiere die Datei
fs.copyFileSync(decodedPath, fullPath);

Expand Down Expand Up @@ -220,6 +272,9 @@ export class DownloadAction extends BaseAction<DownloadActionParams> {
}

const buffer = Buffer.from(fileBuffer);
if (!this.magicMatches(buffer)) {
return null;
}
fs.writeFileSync(fullPath, buffer);

const fileSizeKB = Math.round(buffer.length / 1024);
Expand Down
27 changes: 27 additions & 0 deletions apps/docs/src/content/docs/de/user-guide/actions/download.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,38 @@ Das Beste: Relative URLs werden automatisch aufgelöst und die Dateien werden sa
| `url` | `string` | Ja | Die URL der Datei. Unterstützt absolute URLs, relative URLs und `file://`-Pfade. Handlebars-Templates möglich. |
| `path` | `string` | Ja | Zielverzeichnis für den Download (z.B. `./downloads/invoices`). |
| `filename` | `string` | Ja | Dateiname für die gespeicherte Datei. Handlebars-Templates möglich. |
| `expectMagic` | `string` | Nein | Signatur, mit der die Datei beginnen muss, z.B. `%PDF`. Passt sie nicht, wird nichts geschrieben und die Aktion liefert `null`. |

<Aside type="note">
Der `path` wird automatisch um einen Unterordner mit der `scrapeId` erweitert. Wenn du `./downloads` angibst und dein Scrape `amazon-invoices` heißt, landen die Dateien in `./downloads/amazon-invoices/`.
</Aside>

## 🛡️ Prüfen, ob die Datei wirklich das ist, was du wolltest

Läuft eine Sitzung ab oder greift eine Bot-Erkennung, antwortet die Gegenstelle oft mit **HTTP 200 und einer HTML-Seite**. Ohne Prüfung landet die unter dem erwarteten Namen -- eine `rechnung.pdf`, die in Wahrheit `<!DOCTYPE html>` enthält. Die nachgelagerte Verarbeitung merkt das spät oder gar nicht.

Mit `expectMagic` gibst du an, womit die Datei beginnen muss:

```jsonc
{
"action": "download",
"params": {
"url": "{{previousData.invoiceUrl}}",
"path": "./downloads",
"filename": "rechnung-{{previousData.orderId}}.pdf",
"expectMagic": "%PDF"
}
}
```

Beginnt der Inhalt nicht mit `%PDF`, wird die Datei **nicht geschrieben**, die Aktion protokolliert, was tatsächlich kam, und liefert `null` -- darauf kann ein `skipIf` reagieren.

Übliche Signaturen: `%PDF` (PDF), `PK` (ZIP, XLSX, DOCX), `\x89PNG` (PNG), `GIF8` (GIF).

<Aside type="note">
Ohne `expectMagic` ändert sich nichts -- bestehende Konfigurationen verhalten sich exakt wie bisher.
</Aside>

## 📂 Ordnerstruktur

Die Download-Action organisiert Dateien automatisch in Unterordner basierend auf der **scrapeId** (der Dateiname deiner `.jsonc`-Konfiguration ohne Endung). Die `scrapeId` wird als Unterordner nach dem ersten "echten" Verzeichnissegment in deinem `path` eingefügt.
Expand Down
27 changes: 27 additions & 0 deletions apps/docs/src/content/docs/en/user-guide/actions/download.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,38 @@ The best part: relative URLs are resolved automatically and files are neatly sor
| `url` | `string` | Yes | The file URL. Supports absolute URLs, relative URLs, and `file://` paths. Handlebars templates supported. |
| `path` | `string` | Yes | Target directory for the download (e.g. `./downloads/invoices`). |
| `filename` | `string` | Yes | Filename for the saved file. Handlebars templates supported. |
| `expectMagic` | `string` | No | Signature the file must start with, e.g. `%PDF`. If it does not match, nothing is written and the action returns `null`. |

<Aside type="note">
The `path` is automatically extended with a subfolder using the `scrapeId`. If you specify `./downloads` and your scrape is called `amazon-invoices`, the files end up in `./downloads/amazon-invoices/`.
</Aside>

## 🛡️ Verifying the file really is what you asked for

When a session expires or a bot check kicks in, the other side often answers with **HTTP 200 and an HTML page**. Without a check that page is saved under the name you expected -- an `invoice.pdf` that actually contains `<!DOCTYPE html>`. Whatever processes the file downstream notices late, or not at all.

Set `expectMagic` to the signature the file has to start with:

```jsonc
{
"action": "download",
"params": {
"url": "{{previousData.invoiceUrl}}",
"path": "./downloads",
"filename": "invoice-{{previousData.orderId}}.pdf",
"expectMagic": "%PDF"
}
}
```

If the content does not start with `%PDF`, the file is **not written**, the action logs what it actually received, and returns `null` -- so a `skipIf` can react to it.

Common signatures: `%PDF` (PDF), `PK` (ZIP, XLSX, DOCX), `\x89PNG` (PNG), `GIF8` (GIF).

<Aside type="note">
Without `expectMagic` nothing changes -- existing configs keep behaving exactly as before.
</Aside>

## 📂 Folder Structure

The download action automatically organizes files into subfolders based on the **scrapeId** (the filename of your `.jsonc` config without the extension). The `scrapeId` is inserted as a subfolder after the first "real" directory segment in your `path`.
Expand Down