From 4611051bc185052ed1853aa77d68ff53e76682ef Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:04:58 +0300 Subject: [PATCH 1/2] fix: correct export column labels, redownloadable exports, and webp dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CSV/JSON exports (Promises, AI Extraction Export Rows) showed raw field names as column headers (tenantName, politicalEntityName, ...) instead of the labels already configured on each field. Added an export `before` hook (src/lib/exportColumnLabels.ts) that relabels each row's keys using the field's admin label before the file is written. Saved exports also had no way to be re-downloaded: the plugin's Exports collection denies update access to everyone by design (exports are immutable once generated), so Payload never mounts the edit-view Save/Download button on a saved doc — not merely disabled, entirely absent once you navigate away and back. Added ExportDownloadButton in the unconditionally-rendered `beforeDocumentControls` slot to serve the already-generated file directly, and forked the plugin's create-time button (ExportSaveButton) so it only regenerates when there's no saved file yet. Finally, fixed duplicate Media docs being created for .webp/.gif/.avif images synced from Airtable. Payload always re-encodes those three mimetypes through sharp before persisting them, even with no resize options configured, so the dedup checksum computed from the raw downloaded bytes never matched what was actually stored. computeMediaChecksum now detects those formats by magic bytes and replicates Payload's re-encode step before hashing, so the pre-check and stored checksums are computed over the same bytes. --- src/app/(payload)/admin/importMap.js | 6 +- .../payload/ExportDownloadButton.tsx | 51 +++++++ src/components/payload/ExportSaveButton.tsx | 140 ++++++++++++++++++ src/lib/exportColumnLabels.ts | 49 ++++++ src/plugins/index.ts | 32 ++++ src/tasks/createPoliticalEntity.ts | 4 +- src/tasks/downloadDocuments.ts | 4 +- src/utils/fileSignature.ts | 37 +++++ src/utils/files.ts | 31 +++- tests/int/exportColumnLabels.int.spec.ts | 87 +++++++++++ tests/int/files.int.spec.ts | 77 ++++++++++ 11 files changed, 510 insertions(+), 8 deletions(-) create mode 100644 src/components/payload/ExportDownloadButton.tsx create mode 100644 src/components/payload/ExportSaveButton.tsx create mode 100644 src/lib/exportColumnLabels.ts create mode 100644 tests/int/exportColumnLabels.int.spec.ts diff --git a/src/app/(payload)/admin/importMap.js b/src/app/(payload)/admin/importMap.js index 34a15996..a324683e 100644 --- a/src/app/(payload)/admin/importMap.js +++ b/src/app/(payload)/admin/importMap.js @@ -45,7 +45,8 @@ import { SelectionToUseField as SelectionToUseField_cdf7e044479f899a31f804427d56 import { FieldsToExport as FieldsToExport_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { CollectionField as CollectionField_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { ExportPreview as ExportPreview_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' -import { ExportSaveButton as ExportSaveButton_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' +import { ExportDownloadButton as ExportDownloadButton_e4e9ab26ca06aed2422b0260db830bc4 } from '@/components/payload/ExportDownloadButton' +import { ExportSaveButtonFixed as ExportSaveButtonFixed_fa0881cafe9c287f640a39f3d41a8f6d } from '@/components/payload/ExportSaveButton' import { ImportPreview as ImportPreview_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { ImportSaveButton as ImportSaveButton_cdf7e044479f899a31f804427d568b36 } from '@payloadcms/plugin-import-export/rsc' import { MaskedApiKeyField as MaskedApiKeyField_6e265c33523b378578f5880a6fcfdc0f } from '@/globals/Settings/tabs/MaskedApiKeyField' @@ -107,7 +108,8 @@ export const importMap = { "@payloadcms/plugin-import-export/rsc#FieldsToExport": FieldsToExport_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-import-export/rsc#CollectionField": CollectionField_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-import-export/rsc#ExportPreview": ExportPreview_cdf7e044479f899a31f804427d568b36, - "@payloadcms/plugin-import-export/rsc#ExportSaveButton": ExportSaveButton_cdf7e044479f899a31f804427d568b36, + "@/components/payload/ExportDownloadButton#ExportDownloadButton": ExportDownloadButton_e4e9ab26ca06aed2422b0260db830bc4, + "@/components/payload/ExportSaveButton#ExportSaveButtonFixed": ExportSaveButtonFixed_fa0881cafe9c287f640a39f3d41a8f6d, "@payloadcms/plugin-import-export/rsc#ImportPreview": ImportPreview_cdf7e044479f899a31f804427d568b36, "@payloadcms/plugin-import-export/rsc#ImportSaveButton": ImportSaveButton_cdf7e044479f899a31f804427d568b36, "@/globals/Settings/tabs/MaskedApiKeyField#MaskedApiKeyField": MaskedApiKeyField_6e265c33523b378578f5880a6fcfdc0f, diff --git a/src/components/payload/ExportDownloadButton.tsx b/src/components/payload/ExportDownloadButton.tsx new file mode 100644 index 00000000..3d5508df --- /dev/null +++ b/src/components/payload/ExportDownloadButton.tsx @@ -0,0 +1,51 @@ +"use client"; + +import { Button, useDocumentInfo, useTranslation } from "@payloadcms/ui"; + +/** + * Renders an always-available "Download" control for a saved Export doc. + * + * The exports collection denies update access to everyone by design + * (`access.update: () => false` in @payloadcms/plugin-import-export's + * getExportCollection.js — exports are immutable once generated). Payload's + * Edit view only mounts the `edit.SaveButton` slot when the viewer has + * update permission (see @payloadcms/ui's DocumentControls and + * @payloadcms/next's renderDocumentSlots), so on a saved doc that slot + * never renders at all — not merely disabled. That's why the plugin's own + * Download button (and our ExportSaveButtonFixed fork of it) disappears + * the moment you navigate away and back. + * + * `beforeDocumentControls` is rendered unconditionally, regardless of save + * permission, so it's the right slot for a control that only needs read + * access to the file already attached to the doc. + */ +export const ExportDownloadButton = () => { + const { t } = useTranslation(); + const { id, savedDocumentData } = useDocumentInfo(); + const url = + typeof savedDocumentData?.url === "string" ? savedDocumentData.url : undefined; + const filename = + typeof savedDocumentData?.filename === "string" + ? savedDocumentData.filename + : undefined; + + if (!id || !url) { + return null; + } + + const handleClick = () => { + const a = document.createElement("a"); + a.href = url; + a.download = filename ?? ""; + a.rel = "noopener noreferrer"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + + return ( + + ); +}; diff --git a/src/components/payload/ExportSaveButton.tsx b/src/components/payload/ExportSaveButton.tsx new file mode 100644 index 00000000..ac5e4583 --- /dev/null +++ b/src/components/payload/ExportSaveButton.tsx @@ -0,0 +1,140 @@ +"use client"; + +import { + Button, + SaveButton, + toast, + useConfig, + useField, + useForm, + useFormModified, + useTranslation, +} from "@payloadcms/ui"; +import { formatAdminURL } from "payload/shared"; +import React from "react"; + +/** + * Forks @payloadcms/plugin-import-export's ExportSaveButton to fix two + * issues with its Download button once an export has already been saved: + * + * 1. It's disabled via `disabled: !modified` — as soon as the doc is saved, + * the form is no longer "modified", so the button becomes permanently + * unclickable the next time the doc is opened. + * 2. Even when enabled, it always POSTs to `/exports/download` to + * regenerate a brand-new file from the current form fields rather than + * serving the file already attached to the doc. + * + * Here, whenever the doc already has a saved file, Download just opens that + * file directly. Regeneration is kept as a fallback for brand-new/edited + * exports that have no saved file yet. + */ +export const ExportSaveButtonFixed = () => { + const { t } = useTranslation(); + const { + config: { + routes: { api }, + }, + getEntityConfig, + } = useConfig(); + const { getData, setModified } = useForm(); + const modified = useFormModified(); + const { value: targetCollectionSlug } = useField({ + path: "collectionSlug", + }); + const targetCollectionConfig = getEntityConfig({ + collectionSlug: targetCollectionSlug as string, + }); + const targetPluginConfig = ( + targetCollectionConfig?.admin?.custom as + | { ["plugin-import-export"]?: { disableSave?: boolean; disableDownload?: boolean } } + | undefined + )?.["plugin-import-export"]; + const exportsCollectionConfig = getEntityConfig({ + collectionSlug: "exports", + }); + const exportsAdminCustom = exportsCollectionConfig?.admin?.custom as + | { disableSave?: boolean; disableDownload?: boolean } + | undefined; + const disableSave = + targetPluginConfig?.disableSave ?? exportsAdminCustom?.disableSave === true; + const disableDownload = + targetPluginConfig?.disableDownload ?? + exportsAdminCustom?.disableDownload === true; + const label = t("general:save"); + + const downloadSavedFile = (url: string, filename?: string) => { + const a = document.createElement("a"); + a.href = url; + a.download = filename ?? ""; + a.rel = "noopener noreferrer"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + + const regenerateAndDownload = async (data: Record) => { + let timeoutID: ReturnType | null = null; + let toastID: string | number | null = null; + try { + setModified(false); + timeoutID = setTimeout(() => { + toastID = toast.success("Your export is being processed..."); + }, 200); + const response = await fetch( + formatAdminURL({ apiRoute: api, path: "/exports/download" }), + { + body: JSON.stringify({ data }), + credentials: "include", + headers: { "Content-Type": "application/json" }, + method: "POST", + }, + ); + if (timeoutID) { + clearTimeout(timeoutID); + } + if (toastID) { + toast.dismiss(toastID); + } + if (!response.ok) { + let errorMsg = "Failed to download file"; + try { + const errorJson = await response.json(); + if (errorJson?.errors?.[0]?.message) { + errorMsg = errorJson.errors[0].message; + } + } catch { + // Ignore JSON parse errors, fallback to generic message + } + throw new Error(errorMsg); + } + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + downloadSavedFile(url, `${data.name}-${data.collectionSlug}.${data.format}`); + URL.revokeObjectURL(url); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Error downloading file"); + } + }; + + const handleDownload = async () => { + const data = getData(); + + if (!modified && typeof data?.url === "string" && data.url) { + downloadSavedFile(data.url, typeof data.filename === "string" ? data.filename : undefined); + return; + } + + await regenerateAndDownload(data); + }; + + return ( + + {!disableSave && } + {!disableDownload && ( + + )} + + ); +}; diff --git a/src/lib/exportColumnLabels.ts b/src/lib/exportColumnLabels.ts new file mode 100644 index 00000000..68570f69 --- /dev/null +++ b/src/lib/exportColumnLabels.ts @@ -0,0 +1,49 @@ +import { getTranslation } from "@payloadcms/translations"; +import { CollectionSlug, PayloadRequest } from "payload"; +import { ExportBeforeHook } from "@payloadcms/plugin-import-export/types"; + +const titleCase = (key: string): string => + key + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/^./, (char) => char.toUpperCase()); + +const getFieldLabelMap = ( + req: PayloadRequest, + collectionSlug: CollectionSlug, +): Map => { + const collectionConfig = req.payload.collections[collectionSlug]?.config; + const map = new Map(); + + for (const field of collectionConfig?.flattenedFields ?? []) { + if (!("name" in field) || typeof field.name !== "string") { + continue; + } + + const label = "label" in field ? field.label : undefined; + map.set( + field.name, + label ? String(getTranslation(label, req.i18n)) : titleCase(field.name), + ); + } + + return map; +}; + +/** + * Renames export row keys from raw field names (e.g. "tenantName") to their + * configured admin labels (e.g. "Tenant Name") so downloaded CSV/JSON exports + * are human-readable rather than showing internal field names. + */ +export const createExportHeaderHook = + (collectionSlug: CollectionSlug): ExportBeforeHook => + ({ data, req }) => { + const labelMap = getFieldLabelMap(req, collectionSlug); + + return data.map((row) => { + const relabeledRow: Record = {}; + for (const [key, value] of Object.entries(row)) { + relabeledRow[labelMap.get(key) ?? titleCase(key)] = value; + } + return relabeledRow; + }); + }; diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 4ed11846..c91eb4a4 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -5,6 +5,7 @@ import { multiTenantPlugin } from "@payloadcms/plugin-multi-tenant"; import { importExportPlugin } from "@payloadcms/plugin-import-export"; import { Config } from "@/payload-types"; import { capitalizeFirstLetter, isProd } from "@/utils/utils"; +import { createExportHeaderHook } from "@/lib/exportColumnLabels"; import { s3Storage } from "@payloadcms/storage-s3"; import { seoPlugin } from "@payloadcms/plugin-seo"; import { convertLexicalToPlaintext } from "@payloadcms/richtext-lexical/plaintext"; @@ -21,11 +22,19 @@ export const plugins: Plugin[] = [ collections: [ { slug: "promises", + export: { + hooks: { + before: createExportHeaderHook("promises"), + }, + }, }, { slug: "ai-extraction-export-rows", export: { format: "csv", + hooks: { + before: createExportHeaderHook("ai-extraction-export-rows"), + }, }, import: false, }, @@ -39,6 +48,26 @@ export const plugins: Plugin[] = [ }, admin: { ...collection.admin, + components: { + ...collection.admin?.components, + edit: { + ...collection.admin?.components?.edit, + // The plugin's default Download button is disabled once the doc + // is saved and, even when enabled, regenerates a fresh export + // instead of serving the saved file. This fork keeps it enabled + // and downloads the already-saved file when there is one. + SaveButton: "@/components/payload/ExportSaveButton#ExportSaveButtonFixed", + // The exports collection denies update access to everyone (it's + // immutable once generated), so Payload never mounts the + // edit.SaveButton slot above once a doc is saved — the "Fixed" + // button vanishes entirely on revisit, not just disabled. + // beforeDocumentControls renders unconditionally, so it's the + // only reliable place for an always-available redownload button. + beforeDocumentControls: [ + "@/components/payload/ExportDownloadButton#ExportDownloadButton", + ], + }, + }, group: { en: "Documents", fr: "Documents", @@ -67,6 +96,9 @@ export const plugins: Plugin[] = [ s3Storage({ collections: { media: true, + // Saved exports (Documents > Exports) must survive container + // restarts/redeploys, so they're durable enough to re-download later. + exports: true, }, bucket, config: { diff --git a/src/tasks/createPoliticalEntity.ts b/src/tasks/createPoliticalEntity.ts index f6675b93..8a8b3326 100644 --- a/src/tasks/createPoliticalEntity.ts +++ b/src/tasks/createPoliticalEntity.ts @@ -8,9 +8,9 @@ import { } from "@/lib/airtable"; import { TaskConfig } from "payload"; import { + computeMediaChecksum, downloadFile, removeDownloadedFile, - sha256File, } from "@/utils/files"; import { formatSlug } from "@/fields/slug/formatSlug"; import type { Media } from "@/payload-types"; @@ -254,7 +254,7 @@ export const CreatePoliticalEntity: TaskConfig = { ): Promise => { const filePath = await downloadFile(imageUrl, { fileName: alt }); try { - const checksum = await sha256File(filePath); + const checksum = await computeMediaChecksum(filePath); const existingMedia = await findMediaByChecksum(payload, checksum); if (existingMedia) { diff --git a/src/tasks/downloadDocuments.ts b/src/tasks/downloadDocuments.ts index 436a1bc9..188182a8 100644 --- a/src/tasks/downloadDocuments.ts +++ b/src/tasks/downloadDocuments.ts @@ -8,9 +8,9 @@ import { normalizeMediaSourceUrl, } from "@/lib/mediaUrl"; import { + computeMediaChecksum, downloadFile, removeDownloadedFile, - sha256File, } from "@/utils/files"; import { getTaskLogger, withTaskTracing, type TaskInput } from "./utils"; @@ -410,7 +410,7 @@ export const DownloadDocuments: TaskConfig<"downloadDocuments"> = { filePath = await downloadFile(fileUrl, { fileName: doc.title ?? undefined, }); - const checksum = await sha256File(filePath); + const checksum = await computeMediaChecksum(filePath); const existingMedia = await findMediaByChecksum( payload, checksum, diff --git a/src/utils/fileSignature.ts b/src/utils/fileSignature.ts index 44b92a55..df83f23f 100644 --- a/src/utils/fileSignature.ts +++ b/src/utils/fileSignature.ts @@ -28,6 +28,10 @@ const PNG = startsWith([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const GIF = (header: Buffer) => startsWith([0x47, 0x49, 0x46, 0x38, 0x37, 0x61])(header) || startsWith([0x47, 0x49, 0x46, 0x38, 0x39, 0x61])(header); +const AVIF = (header: Buffer) => + startsWith([0x66, 0x74, 0x79, 0x70], 4)(header) && // "ftyp" box at offset 4 + (startsWith([0x61, 0x76, 0x69, 0x66], 8)(header) || // major brand "avif" + startsWith([0x61, 0x76, 0x69, 0x73], 8)(header)); // major brand "avis" const PDF = startsWith([0x25, 0x50, 0x44, 0x46]); // %PDF const ZIP = (header: Buffer) => startsWith([0x50, 0x4b, 0x03, 0x04])(header) || @@ -144,3 +148,36 @@ export const validateFileSignature = async ( await handle.close(); } }; + +export type PayloadReencodedMimeType = "image/gif" | "image/webp" | "image/avif"; + +/** + * Payload CMS unconditionally re-encodes uploads of these three mimetypes + * through sharp before persisting them (its `fileIsAnimatedType` check in + * `generateFileData.js`), even with no resize/format options configured. + * Detected from magic bytes, independent of any claimed Content-Type or + * file extension, so a pre-upload checksum can be computed against the same + * bytes Payload will actually store. + */ +export const detectPayloadReencodedFormat = async ( + filePath: string, +): Promise => { + const handle = await open(filePath, "r"); + try { + const buffer = Buffer.alloc(HEADER_LENGTH); + const { bytesRead } = await handle.read(buffer, 0, HEADER_LENGTH, 0); + const header = buffer.subarray(0, bytesRead); + if (isRiffWebp(header)) { + return "image/webp"; + } + if (GIF(header)) { + return "image/gif"; + } + if (AVIF(header)) { + return "image/avif"; + } + return null; + } finally { + await handle.close(); + } +}; diff --git a/src/utils/files.ts b/src/utils/files.ts index 5873b2db..c67abdc1 100644 --- a/src/utils/files.ts +++ b/src/utils/files.ts @@ -1,12 +1,16 @@ import { createHash, randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdir, rm, unlink } from "node:fs/promises"; +import { mkdir, readFile, rm, unlink } from "node:fs/promises"; import { dirname, join, sep } from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { fileURLToPath } from "node:url"; +import sharp from "sharp"; -import { validateFileSignature } from "@/utils/fileSignature"; +import { + detectPayloadReencodedFormat, + validateFileSignature, +} from "@/utils/fileSignature"; import { assertResolvesToPublicAddresses, assertSafeRemoteUrl, @@ -339,3 +343,26 @@ export const sha256File = async (filePath: string): Promise => { export const sha256Buffer = (data: Buffer | Uint8Array): string => createHash("sha256").update(data).digest("hex"); + +/** + * Checksum to compare against Payload's stored `checksum` field for + * duplicate detection. For gif/webp/avif, Payload always re-encodes the + * upload through sharp before saving it (see `detectPayloadReencodedFormat`), + * so hashing the raw downloaded bytes would never match what actually gets + * persisted — replicate that re-encode step here before hashing so the two + * checksums are computed over the same bytes. + */ +export const computeMediaChecksum = async ( + filePath: string, +): Promise => { + const reencodedFormat = await detectPayloadReencodedFormat(filePath); + if (!reencodedFormat) { + return sha256File(filePath); + } + + const original = await readFile(filePath); + const reencoded = await sharp(original, { animated: true }) + .rotate() + .toBuffer(); + return sha256Buffer(reencoded); +}; diff --git a/tests/int/exportColumnLabels.int.spec.ts b/tests/int/exportColumnLabels.int.spec.ts new file mode 100644 index 00000000..3099dc87 --- /dev/null +++ b/tests/int/exportColumnLabels.int.spec.ts @@ -0,0 +1,87 @@ +import { createExportHeaderHook } from "@/lib/exportColumnLabels"; +import { describe, expect, it } from "vitest"; + +const fakeReq = (fields: Array<{ name: string; label?: unknown }>) => + ({ + payload: { + collections: { + "ai-extraction-export-rows": { + config: { + flattenedFields: fields, + }, + }, + }, + }, + i18n: { + language: "en", + fallbackLanguage: "en", + t: (key: string) => key, + }, + }) as never; + +describe("createExportHeaderHook", () => { + it("renames raw field keys to their configured admin labels", async () => { + const hook = createExportHeaderHook("ai-extraction-export-rows"); + const req = fakeReq([ + { name: "tenantName", label: "Tenant Name" }, + { name: "politicalEntityName", label: "Political Entity Name" }, + { name: "documentTitle", label: "Document Title" }, + { name: "category", label: "Category" }, + { name: "summary", label: "Summary" }, + { name: "statusLabel", label: "Status Label" }, + { name: "source", label: "Source" }, + { name: "checkMediaURL", label: "CheckMedia URL" }, + ]); + + const result = await hook({ + batchNumber: 1, + data: [ + { + tenantName: "Kenya", + politicalEntityName: "Jane Leader", + documentTitle: "Manifesto PDF", + category: "Health", + summary: "Build new clinics", + statusLabel: "In Progress", + source: "Clinic source quote", + checkMediaURL: "https://check.example.com/media/check-1", + }, + ], + format: "csv", + originalData: [], + req, + totalBatches: 1, + }); + + expect(result).toEqual([ + { + "Tenant Name": "Kenya", + "Political Entity Name": "Jane Leader", + "Document Title": "Manifesto PDF", + Category: "Health", + Summary: "Build new clinics", + "Status Label": "In Progress", + Source: "Clinic source quote", + "CheckMedia URL": "https://check.example.com/media/check-1", + }, + ]); + }); + + it("falls back to a title-cased key when a field has no configured label", async () => { + const hook = createExportHeaderHook("ai-extraction-export-rows"); + const req = fakeReq([{ name: "checkMediaId" }]); + + const result = await hook({ + batchNumber: 1, + data: [{ checkMediaId: "check-1", unknownRawKey: "value" }], + format: "csv", + originalData: [], + req, + totalBatches: 1, + }); + + expect(result).toEqual([ + { "Check Media Id": "check-1", "Unknown Raw Key": "value" }, + ]); + }); +}); diff --git a/tests/int/files.int.spec.ts b/tests/int/files.int.spec.ts index 5a0515e2..34cceada 100644 --- a/tests/int/files.int.spec.ts +++ b/tests/int/files.int.spec.ts @@ -15,12 +15,15 @@ vi.mock("node:dns/promises", () => { return { default: { lookup }, lookup }; }); +import sharp from "sharp"; import { + computeMediaChecksum, downloadFile, removeDownloadedFile, sha256Buffer, sha256File, } from "@/utils/files"; +import { detectPayloadReencodedFormat } from "@/utils/fileSignature"; const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); @@ -49,6 +52,80 @@ describe("sha256File", () => { }); }); +describe("computeMediaChecksum", () => { + const withTempFile = async ( + fileName: string, + data: Buffer, + run: (filePath: string) => Promise, + ): Promise => { + const dir = await mkdtemp(join(tmpdir(), "checksum-spec-")); + const filePath = join(dir, fileName); + await writeFile(filePath, data); + try { + return await run(filePath); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }; + + it("hashes non-reencoded formats (e.g. png) as raw bytes, same as sha256File", async () => { + const png = await sharp({ + create: { width: 4, height: 4, channels: 3, background: "red" }, + }) + .png() + .toBuffer(); + + await withTempFile("image.png", png, async (filePath) => { + expect(await detectPayloadReencodedFormat(filePath)).toBeNull(); + expect(await computeMediaChecksum(filePath)).toBe( + await sha256File(filePath), + ); + }); + }); + + it("hashes webp files against Payload's re-encoded bytes, not the raw download", async () => { + const webp = await sharp({ + create: { width: 4, height: 4, channels: 3, background: "blue" }, + }) + .webp() + .toBuffer(); + + await withTempFile("image.webp", webp, async (filePath) => { + expect(await detectPayloadReencodedFormat(filePath)).toBe( + "image/webp", + ); + + const expectedReencoded = await sharp(webp, { animated: true }) + .rotate() + .toBuffer(); + + const checksum = await computeMediaChecksum(filePath); + expect(checksum).toBe(sha256Buffer(expectedReencoded)); + // Sanity check that this actually differs from a naive raw-byte hash — + // otherwise the test wouldn't be exercising the fix at all. + expect(checksum).not.toBe(await sha256File(filePath)); + }); + }); + + it("produces the same checksum across repeated downloads of the same source image", async () => { + const webp = await sharp({ + create: { width: 6, height: 6, channels: 3, background: "green" }, + }) + .webp() + .toBuffer(); + + const checksums = await Promise.all( + [0, 1].map((run) => + withTempFile(`download-${run}.webp`, webp, (filePath) => + computeMediaChecksum(filePath), + ), + ), + ); + + expect(checksums[0]).toBe(checksums[1]); + }); +}); + describe("downloadFile", () => { const downloadedPaths: string[] = []; From 8fe519317161f12ce85b579d463edafc181c3805 Mon Sep 17 00:00:00 2001 From: kelvin <43873157+kelvinkipruto@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:24:33 +0300 Subject: [PATCH 2/2] docs: clarify AVIF major-brand-only detection matches file-type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR review feedback suggesting the AVIF magic-byte check should also inspect the ISO-BMFF compatible-brands list, not just the major brand. Verified against file-type@21.3.4 (the exact library Payload uses internally to set `file.mimetype`) — it also switches on the major brand alone and never reads compatible brands, so this must match it exactly to correctly predict whether Payload will reencode a given upload. Checking compatible brands would make this detector more accurate than file-type itself and reintroduce the checksum mismatch this change fixes, for a file whose major brand is a generic HEIF brand (e.g. "mif1") with "avif" only listed as a compatible brand. No behavior change — comment only. --- src/utils/fileSignature.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/utils/fileSignature.ts b/src/utils/fileSignature.ts index df83f23f..4ea3b43d 100644 --- a/src/utils/fileSignature.ts +++ b/src/utils/fileSignature.ts @@ -28,6 +28,15 @@ const PNG = startsWith([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const GIF = (header: Buffer) => startsWith([0x47, 0x49, 0x46, 0x38, 0x37, 0x61])(header) || startsWith([0x47, 0x49, 0x46, 0x38, 0x39, 0x61])(header); +// Deliberately checks only the major brand, not the compatible-brands list — +// this must match `file-type`'s own detection exactly (its ISO-BMFF branch +// switches on `brandMajor` alone), since that's what determines the +// `file.mimetype` Payload's `fileIsAnimatedType` check reencodes on. A file +// with a generic major brand (e.g. "mif1") and "avif" only in compatible +// brands is classified by file-type as image/heif, so Payload leaves it +// un-reencoded too — checking compatible brands here would make this more +// accurate than file-type and reintroduce the exact checksum mismatch this +// fixes. const AVIF = (header: Buffer) => startsWith([0x66, 0x74, 0x79, 0x70], 4)(header) && // "ftyp" box at offset 4 (startsWith([0x61, 0x76, 0x69, 0x66], 8)(header) || // major brand "avif"