From 9a9ee4b992f02efb8e707b53265f9fc375f4646d Mon Sep 17 00:00:00 2001 From: caleb Date: Mon, 17 Mar 2025 20:32:41 -0700 Subject: [PATCH 1/7] pdf lib for pdf merging --- archiver/package-lock.json | 43 ++++++++++++++++++++++++++++++++++++++ archiver/package.json | 1 + 2 files changed, 44 insertions(+) diff --git a/archiver/package-lock.json b/archiver/package-lock.json index 46d839b..2acd0ac 100644 --- a/archiver/package-lock.json +++ b/archiver/package-lock.json @@ -24,6 +24,7 @@ "morgan": "^1.10.0", "mssql": "^11.0.1", "node-schedule": "^2.1.1", + "pdf-lib": "^1.17.1", "redis": "^4.7.0", "uuid": "^11.0.2", "winston": "^3.14.1", @@ -1574,6 +1575,24 @@ "node": ">= 8" } }, + "node_modules/@pdf-lib/standard-fonts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", + "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, + "node_modules/@pdf-lib/upng": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz", + "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.10" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -5953,6 +5972,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -6034,6 +6059,24 @@ "node": "*" } }, + "node_modules/pdf-lib": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz", + "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==", + "license": "MIT", + "dependencies": { + "@pdf-lib/standard-fonts": "^1.0.0", + "@pdf-lib/upng": "^1.0.1", + "pako": "^1.0.11", + "tslib": "^1.11.1" + } + }, + "node_modules/pdf-lib/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, "node_modules/pg-connection-string": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", diff --git a/archiver/package.json b/archiver/package.json index b546b05..4de7d7c 100644 --- a/archiver/package.json +++ b/archiver/package.json @@ -32,6 +32,7 @@ "morgan": "^1.10.0", "mssql": "^11.0.1", "node-schedule": "^2.1.1", + "pdf-lib": "^1.17.1", "redis": "^4.7.0", "uuid": "^11.0.2", "winston": "^3.14.1", From 88a4b1aa5130d274a3323ea72a5ac7fe57c5f1a4 Mon Sep 17 00:00:00 2001 From: caleb Date: Mon, 17 Mar 2025 20:32:58 -0700 Subject: [PATCH 2/7] pdf merging class and service --- archiver/src/lib/pdf-merger/index.ts | 1 + archiver/src/lib/pdf-merger/pdf-merger.ts | 75 +++++++++++++++++++++ archiver/src/services/merge-pdfs-service.ts | 19 ++++++ 3 files changed, 95 insertions(+) create mode 100644 archiver/src/lib/pdf-merger/index.ts create mode 100644 archiver/src/lib/pdf-merger/pdf-merger.ts create mode 100644 archiver/src/services/merge-pdfs-service.ts diff --git a/archiver/src/lib/pdf-merger/index.ts b/archiver/src/lib/pdf-merger/index.ts new file mode 100644 index 0000000..7f75207 --- /dev/null +++ b/archiver/src/lib/pdf-merger/index.ts @@ -0,0 +1 @@ +export { PDFMerger } from "./pdf-merger" diff --git a/archiver/src/lib/pdf-merger/pdf-merger.ts b/archiver/src/lib/pdf-merger/pdf-merger.ts new file mode 100644 index 0000000..bbaa528 --- /dev/null +++ b/archiver/src/lib/pdf-merger/pdf-merger.ts @@ -0,0 +1,75 @@ +import { isUndefined } from "lodash" +import { writeFileSync } from "fs" +import { PDFDocument } from "pdf-lib" + +interface PDFMetadata { + producer?: string + author?: string + title?: string + creator?: string +} + +export class PDFMerger { + private doc?: PDFDocument + private options = { + ignoreEncryption: true, + } + + async setMetadata(metadata: PDFMetadata): Promise { + if (isUndefined(this.doc)) { + this.doc = await PDFDocument.create() + this.doc.setProducer("digital-vault") + this.doc.setCreationDate(new Date()) + } + + if (metadata.producer) this.doc.setProducer(metadata.producer) + if (metadata.author) this.doc.setAuthor(metadata.author) + if (metadata.title) this.doc.setTitle(metadata.title) + if (metadata.creator) this.doc.setCreator(metadata.creator) + } + + async add(input: Buffer, pages?: number[]): Promise { + if (isUndefined(this.doc)) { + this.doc = await PDFDocument.create() + this.doc.setProducer("digital-vault") + this.doc.setCreationDate(new Date()) + } + + const srcDoc = await PDFDocument.load(input, this.options) + let indices = [] + if (isUndefined(pages)) { + indices = srcDoc.getPageIndices() + } else { + indices = pages.map((p) => p - 1) + } + + const copiedPages = await this.doc.copyPages(srcDoc, indices) + copiedPages.forEach((page) => { + this.doc?.addPage(page) + }) + } + + async saveAsBuffer(): Promise { + if (isUndefined(this.doc)) { + this.doc = await PDFDocument.create() + this.doc.setProducer("digital-vault") + this.doc.setCreationDate(new Date()) + } + + const uInt8Array = await this.doc.save() + return Buffer.from(uInt8Array) + } + + async save(fileName: string): Promise { + if (isUndefined(this.doc)) { + this.doc = await PDFDocument.create() + this.doc.setProducer("digital-vault") + this.doc.setCreationDate(new Date()) + } + + const pdf = await this.doc.save() + writeFileSync(fileName, pdf) + } +} + +export default PDFMerger diff --git a/archiver/src/services/merge-pdfs-service.ts b/archiver/src/services/merge-pdfs-service.ts new file mode 100644 index 0000000..6d21cd5 --- /dev/null +++ b/archiver/src/services/merge-pdfs-service.ts @@ -0,0 +1,19 @@ +import { PDFMerger } from "@/lib/pdf-merger" + +export class MergePDFsService { + private merger: PDFMerger + private pdfs: Buffer[] + + constructor(pdfs: Buffer[]) { + this.merger = new PDFMerger() + this.pdfs = pdfs + } + + async merge(): Promise { + this.pdfs.forEach(async (pdf) => { + await this.merger.add(pdf) + }) + + return this.merger.saveAsBuffer() + } +} From c04f9ce78bce0aa8f4df805882a90ac0bf9b679b Mon Sep 17 00:00:00 2001 From: caleb Date: Fri, 28 Mar 2025 18:00:22 -0700 Subject: [PATCH 3/7] enhaned pdf signer to take in a buffer --- archiver/src/utils/pdf-signer.ts | 49 ++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/archiver/src/utils/pdf-signer.ts b/archiver/src/utils/pdf-signer.ts index af9dbf0..51f9cd7 100644 --- a/archiver/src/utils/pdf-signer.ts +++ b/archiver/src/utils/pdf-signer.ts @@ -1,5 +1,8 @@ import { exec } from "child_process" import { isEmpty } from "lodash" +import { writeFileSync, readFileSync, unlinkSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" import logger from "@/utils/logger" @@ -10,20 +13,36 @@ if (isEmpty(TIMESTAMP_SERVER)) throw new Error("timestamp server is unset.") if (isEmpty(SSL_FULL_CHAIN_PATH)) throw new Error("ssl fullchain filepath is unset.") if (isEmpty(SSL_CERT_KEY_PATH)) throw new Error("ssl cert key filepath is unset.") -export async function signPDFWithPAdES( - inputPdfFilepath: string, - outputPdfFilepath: string -): Promise { - const cmd: string = `java -jar ${PDF_SIGNER_JAR} --input ${inputPdfFilepath} --output ${outputPdfFilepath} --certificate ${SSL_FULL_CHAIN_PATH} --key ${SSL_CERT_KEY_PATH} --timestamp --tsa ${TIMESTAMP_SERVER} --baseline-lt` - - return new Promise((resolve, reject) => { - exec(cmd, { encoding: "buffer" }, (error, stderr) => { - if (error) { - logger.error("open-pdf-sign error:", stderr.toString()) - reject(new Error(`open-pdf-sign error: ${stderr.toString()}`)) - return - } - resolve() +const getTempFilePath = (prefix: string) => join(tmpdir(), `${prefix}-${Date.now()}.pdf`) + +export async function signPDFWithPAdES(inputPDF: Buffer): Promise { + const tempInputPath = getTempFilePath("input") + const tempOutputPath = getTempFilePath("output") + + try { + writeFileSync(tempInputPath, inputPDF) + + const cmd: string = `java -jar ${PDF_SIGNER_JAR} --input ${tempInputPath} --output ${tempOutputPath} --certificate ${SSL_FULL_CHAIN_PATH} --key ${SSL_CERT_KEY_PATH} --timestamp --tsa ${TIMESTAMP_SERVER} --baseline-lt` + + await new Promise((resolve, reject) => { + exec(cmd, { encoding: "buffer" }, (error, _stdout, stderr) => { + if (error) { + logger.error("open-pdf-sign error:", stderr.toString()) + reject(new Error(`open-pdf-sign error: ${stderr.toString()}`)) + return + } + resolve() + }) }) - }) + + const signedPdfBuffer = readFileSync(tempOutputPath) + return signedPdfBuffer + } finally { + try { + unlinkSync(tempInputPath) + unlinkSync(tempOutputPath) + } catch (cleanupError) { + logger.warn("Failed to clean up temporary files:", cleanupError) + } + } } From 8cb0806fee20a1ed11aa9df708013dbbe502e3cc Mon Sep 17 00:00:00 2001 From: caleb Date: Fri, 28 Mar 2025 18:00:53 -0700 Subject: [PATCH 4/7] pdf converter job changes --- archiver/src/jobs/pdf-converter-job.ts | 34 +++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/archiver/src/jobs/pdf-converter-job.ts b/archiver/src/jobs/pdf-converter-job.ts index c8a0c3b..1545346 100644 --- a/archiver/src/jobs/pdf-converter-job.ts +++ b/archiver/src/jobs/pdf-converter-job.ts @@ -1,5 +1,4 @@ import { isNil } from "lodash" -import { writeFileSync } from "fs" import { FileStorageService } from "@/services" import { ArchiveItemFile } from "@/models" @@ -20,6 +19,8 @@ export class PDFConverterJob { const toConvert = await cache.getKeysByPattern(`CONVERT_`) const fileStore = new FileStorageService() + const archiveItems = new Map() + for (const key of toConvert) { const data = await cache.getValue(key) @@ -35,14 +36,9 @@ export class PDFConverterJob { const fileAsPDF = await bufferToPdf(file) - const pdfPath = "/tmp/input.pdf" - const convertedAndSignedFile = "/tmp/output.pdf" - - writeFileSync(pdfPath, fileAsPDF) - - await signPDFWithPAdES(pdfPath, convertedAndSignedFile) + const signedPdf = await signPDFWithPAdES(fileAsPDF) - const uploadResp = await fileStore.uploadFile(convertedPdfKey, convertedAndSignedFile) + const uploadResp = await fileStore.uploadBuffer(convertedPdfKey, signedPdf) if (uploadResp.errorCode) { throw Error("File upload error") @@ -52,10 +48,30 @@ export class PDFConverterJob { pdfKey: convertedPdfKey, pdfFileName: `${fileRecord.originalFileName}_SIGNED.pdf`, pdfMimeType: "application/pdf", - pdfFileSize: convertedAndSignedFile.length, + pdfFileSize: signedPdf.length, }) cache.deleteValue(key) + + const existingEntry = archiveItems.get(fileRecord.archiveItemId) + if (existingEntry) { + archiveItems.set(fileRecord.archiveItemId, { + archiveItemId: fileRecord.archiveItemId, + fileCount: existingEntry.fileCount + 1, + }) + } else { + archiveItems.set(fileRecord.archiveItemId, { + archiveItemId: fileRecord.archiveItemId, + fileCount: 1, + }) + } } + + archiveItems.forEach((data, archiveItemId) => { + cache.setValueNoExpire( + `PENDING_FILESTORE_UPLOAD_ARCHIVE_ITEM_ID_${archiveItemId}`, + JSON.stringify(data) + ) + }) } } From 115b7ba119caca9c1db9169cde7878da2ec2903e Mon Sep 17 00:00:00 2001 From: caleb Date: Fri, 28 Mar 2025 18:01:06 -0700 Subject: [PATCH 5/7] upload buffer helper function --- archiver/src/services/file-storage-service.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/archiver/src/services/file-storage-service.ts b/archiver/src/services/file-storage-service.ts index 3c9f79b..5200761 100644 --- a/archiver/src/services/file-storage-service.ts +++ b/archiver/src/services/file-storage-service.ts @@ -21,6 +21,12 @@ export class FileStorageService { return uploadBlobResponse } + async uploadBuffer(key: string, file: Buffer) { + const blockBlobClient = this.containerClient.getBlockBlobClient(key) + const uploadBlobResponse = await blockBlobClient.uploadData(file) + return uploadBlobResponse + } + async downloadFile(key: string) { const blockBlobClient = this.containerClient.getBlockBlobClient(key) From ff069e28b2d7fd9a09488491705fbff1b9082029 Mon Sep 17 00:00:00 2001 From: caleb Date: Fri, 28 Mar 2025 18:01:26 -0700 Subject: [PATCH 6/7] archiver item upload job --- archiver/src/jobs/archive-item-upload-job.ts | 67 ++++++++++++++++++++ archiver/src/scheduler.ts | 4 ++ 2 files changed, 71 insertions(+) create mode 100644 archiver/src/jobs/archive-item-upload-job.ts diff --git a/archiver/src/jobs/archive-item-upload-job.ts b/archiver/src/jobs/archive-item-upload-job.ts new file mode 100644 index 0000000..fb111c4 --- /dev/null +++ b/archiver/src/jobs/archive-item-upload-job.ts @@ -0,0 +1,67 @@ +import { isNil } from "lodash" + +import { FileStorageService } from "@/services" +import { ArchiveItemFile } from "@/models" +import cache from "@/db/cache-client" + +import { signPDFWithPAdES } from "@/utils/pdf-signer" +import { PDFMerger } from "@/lib/pdf-merger" +import logger from "@/utils/logger" + +export class ArchiveItemUploadJob { + name = "arhive-item-upload" + schedule = "*/1 * * * *" + + constructor() {} + + async run(statDate: Date) { + logger.info("Running Archive Item Upload Job", statDate) + const toUploads = await cache.getKeysByPattern(`PENDING_FILESTORE_UPLOAD_ARCHIVE_ITEM_ID_`) + const fileStore = new FileStorageService() + + for (const key of toUploads) { + const data = await cache.getValue(key) + + if (isNil(data)) return + + const archiveItemInfo = JSON.parse(data) + + const archiveFiles = await ArchiveItemFile.findAll({ + where: { + archiveItemId: archiveItemInfo.archiveItemId, + }, + }) + + const merger = new PDFMerger() + + for (const archiveFile of archiveFiles) { + if (isNil(archiveFile.pdfKey)) return + + const file = await fileStore.downloadFile(archiveFile.pdfKey) + await merger.add(file) + } + + const mergedPDF = await merger.saveAsBuffer() + const signedMergedPdf = await signPDFWithPAdES(mergedPDF) + + /* FILE STORE UPLOAD */ + // Not sure where it should go + + // const folderKey = archiveItemInfo.originalKey.substring( + // 0, + // archiveItemInfo.originalKey.indexOf("/") + // ) + // const mergedAndSignedPdfKey = `${folderKey}/${fileStore.makeKey()}` + + // const uploadResp = await fileStore.uploadBuffer(mergedAndSignedPdfKey, signedMergedPdf) + + // if (uploadResp.errorCode) { + // throw Error("File upload error") + // } + + // update file record? + + cache.deleteValue(key) + } + } +} diff --git a/archiver/src/scheduler.ts b/archiver/src/scheduler.ts index 18106a3..737fe57 100644 --- a/archiver/src/scheduler.ts +++ b/archiver/src/scheduler.ts @@ -4,6 +4,7 @@ import { APPLICATION_NAME } from "@/config" import { PDFConverterJob } from "@/jobs" import cache from "@/db/cache-client" import logger from "@/utils/logger" +import { ArchiveItemUploadJob } from "./jobs/archive-item-upload-job" export async function startScheduler() { logger.info("Scheduler starting in " + APPLICATION_NAME) @@ -14,5 +15,8 @@ export async function startScheduler() { const converter = new PDFConverterJob() + const archiveItemUploader = new ArchiveItemUploadJob() + scheduleJob(converter.name, converter.schedule, converter.run) + scheduleJob(archiveItemUploader.name, archiveItemUploader.schedule, archiveItemUploader.run) } From 2ff5023965db472970e37599cb10642a8ad218e8 Mon Sep 17 00:00:00 2001 From: caleb Date: Fri, 28 Mar 2025 18:15:21 -0700 Subject: [PATCH 7/7] not using this --- archiver/src/services/merge-pdfs-service.ts | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 archiver/src/services/merge-pdfs-service.ts diff --git a/archiver/src/services/merge-pdfs-service.ts b/archiver/src/services/merge-pdfs-service.ts deleted file mode 100644 index 6d21cd5..0000000 --- a/archiver/src/services/merge-pdfs-service.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { PDFMerger } from "@/lib/pdf-merger" - -export class MergePDFsService { - private merger: PDFMerger - private pdfs: Buffer[] - - constructor(pdfs: Buffer[]) { - this.merger = new PDFMerger() - this.pdfs = pdfs - } - - async merge(): Promise { - this.pdfs.forEach(async (pdf) => { - await this.merger.add(pdf) - }) - - return this.merger.saveAsBuffer() - } -}