Pdf merger

archiver/package.json

@@ -32,6 +32,7 @@
     "morgan": "^1.10.0",
     "mssql": "^11.0.1",
     "node-schedule": "^2.1.1",
+    "pdf-lib": "^1.17.1",
     "redis": "^4.7.0",
     "uuid": "^11.0.2",
     "winston": "^3.14.1",

archiver/src/jobs/archive-item-upload-job.ts

@@ -0,0 +1,67 @@
+import { isNil } from "lodash"
+
+import { FileStorageService } from "@/services"
+import { ArchiveItemFile } from "@/models"
+import cache from "@/db/cache-client"
+
+import { signPDFWithPAdES } from "@/utils/pdf-signer"
+import { PDFMerger } from "@/lib/pdf-merger"
+import logger from "@/utils/logger"
+
+export class ArchiveItemUploadJob {
+  name = "arhive-item-upload"
+  schedule = "*/1 * * * *"
+
+  constructor() {}
+
+  async run(statDate: Date) {
+    logger.info("Running Archive Item Upload Job", statDate)
+    const toUploads = await cache.getKeysByPattern(`PENDING_FILESTORE_UPLOAD_ARCHIVE_ITEM_ID_`)
+    const fileStore = new FileStorageService()
+
+    for (const key of toUploads) {
+      const data = await cache.getValue(key)
+
+      if (isNil(data)) return
+
+      const archiveItemInfo = JSON.parse(data)
+
+      const archiveFiles = await ArchiveItemFile.findAll({
+        where: {
+          archiveItemId: archiveItemInfo.archiveItemId,
+        },
+      })
+
+      const merger = new PDFMerger()
+
+      for (const archiveFile of archiveFiles) {
+        if (isNil(archiveFile.pdfKey)) return
+
+        const file = await fileStore.downloadFile(archiveFile.pdfKey)
+        await merger.add(file)
+      }
+
+      const mergedPDF = await merger.saveAsBuffer()
+      const signedMergedPdf = await signPDFWithPAdES(mergedPDF)
+
+      /* FILE STORE UPLOAD  */
+      // Not sure where it should go
+
+      // const folderKey = archiveItemInfo.originalKey.substring(
+      //   0,
+      //   archiveItemInfo.originalKey.indexOf("/")
+      // )
+      // const mergedAndSignedPdfKey = `${folderKey}/${fileStore.makeKey()}`
+
+      // const uploadResp = await fileStore.uploadBuffer(mergedAndSignedPdfKey, signedMergedPdf)
+
+      // if (uploadResp.errorCode) {
+      //   throw Error("File upload error")
+      // }
+
+      // update file record?
+
+      cache.deleteValue(key)
+    }
+  }
+}

archiver/src/jobs/pdf-converter-job.ts

@@ -1,5 +1,4 @@
 import { isNil } from "lodash"
-import { writeFileSync } from "fs"
 
 import { FileStorageService } from "@/services"
 import { ArchiveItemFile } from "@/models"
@@ -20,6 +19,8 @@ export class PDFConverterJob {
     const toConvert = await cache.getKeysByPattern(`CONVERT_`)
     const fileStore = new FileStorageService()
 
+    const archiveItems = new Map<number, { archiveItemId: number; fileCount: number }>()
+
     for (const key of toConvert) {
       const data = await cache.getValue(key)
 
@@ -35,14 +36,9 @@ export class PDFConverterJob {
 
       const fileAsPDF = await bufferToPdf(file)
 
-      const pdfPath = "/tmp/input.pdf"
-      const convertedAndSignedFile = "/tmp/output.pdf"
-
-      writeFileSync(pdfPath, fileAsPDF)
-
-      await signPDFWithPAdES(pdfPath, convertedAndSignedFile)
+      const signedPdf = await signPDFWithPAdES(fileAsPDF)
 
-      const uploadResp = await fileStore.uploadFile(convertedPdfKey, convertedAndSignedFile)
+      const uploadResp = await fileStore.uploadBuffer(convertedPdfKey, signedPdf)
 
       if (uploadResp.errorCode) {
         throw Error("File upload error")
@@ -52,10 +48,30 @@ export class PDFConverterJob {
         pdfKey: convertedPdfKey,
         pdfFileName: `${fileRecord.originalFileName}_SIGNED.pdf`,
         pdfMimeType: "application/pdf",
-        pdfFileSize: convertedAndSignedFile.length,
+        pdfFileSize: signedPdf.length,
       })
 
       cache.deleteValue(key)
+
+      const existingEntry = archiveItems.get(fileRecord.archiveItemId)
+      if (existingEntry) {
+        archiveItems.set(fileRecord.archiveItemId, {
+          archiveItemId: fileRecord.archiveItemId,
+          fileCount: existingEntry.fileCount + 1,
+        })
+      } else {
+        archiveItems.set(fileRecord.archiveItemId, {
+          archiveItemId: fileRecord.archiveItemId,
+          fileCount: 1,
+        })
+      }
     }
+
+    archiveItems.forEach((data, archiveItemId) => {
+      cache.setValueNoExpire(
+        `PENDING_FILESTORE_UPLOAD_ARCHIVE_ITEM_ID_${archiveItemId}`,
+        JSON.stringify(data)
+      )
+    })
   }
 }

archiver/src/lib/pdf-merger/index.ts

@@ -0,0 +1 @@
+export { PDFMerger } from "./pdf-merger"

archiver/src/lib/pdf-merger/pdf-merger.ts

@@ -0,0 +1,75 @@
+import { isUndefined } from "lodash"
+import { writeFileSync } from "fs"
+import { PDFDocument } from "pdf-lib"
+
+interface PDFMetadata {
+  producer?: string
+  author?: string
+  title?: string
+  creator?: string
+}
+
+export class PDFMerger {
+  private doc?: PDFDocument
+  private options = {
+    ignoreEncryption: true,
+  }
+
+  async setMetadata(metadata: PDFMetadata): Promise<void> {
+    if (isUndefined(this.doc)) {
+      this.doc = await PDFDocument.create()
+      this.doc.setProducer("digital-vault")
+      this.doc.setCreationDate(new Date())
+    }
+
+    if (metadata.producer) this.doc.setProducer(metadata.producer)
+    if (metadata.author) this.doc.setAuthor(metadata.author)
+    if (metadata.title) this.doc.setTitle(metadata.title)
+    if (metadata.creator) this.doc.setCreator(metadata.creator)
+  }
+
+  async add(input: Buffer, pages?: number[]): Promise<void> {
+    if (isUndefined(this.doc)) {
+      this.doc = await PDFDocument.create()
+      this.doc.setProducer("digital-vault")
+      this.doc.setCreationDate(new Date())
+    }
+
+    const srcDoc = await PDFDocument.load(input, this.options)
+    let indices = []
+    if (isUndefined(pages)) {
+      indices = srcDoc.getPageIndices()
+    } else {
+      indices = pages.map((p) => p - 1)
+    }
+
+    const copiedPages = await this.doc.copyPages(srcDoc, indices)
+    copiedPages.forEach((page) => {
+      this.doc?.addPage(page)
+    })
+  }
+
+  async saveAsBuffer(): Promise<Buffer> {
+    if (isUndefined(this.doc)) {
+      this.doc = await PDFDocument.create()
+      this.doc.setProducer("digital-vault")
+      this.doc.setCreationDate(new Date())
+    }
+
+    const uInt8Array = await this.doc.save()
+    return Buffer.from(uInt8Array)
+  }
+
+  async save(fileName: string): Promise<void> {
+    if (isUndefined(this.doc)) {
+      this.doc = await PDFDocument.create()
+      this.doc.setProducer("digital-vault")
+      this.doc.setCreationDate(new Date())
+    }
+
+    const pdf = await this.doc.save()
+    writeFileSync(fileName, pdf)
+  }
+}
+
+export default PDFMerger

archiver/src/scheduler.ts

@@ -4,6 +4,7 @@ import { APPLICATION_NAME } from "@/config"
 import { PDFConverterJob } from "@/jobs"
 import cache from "@/db/cache-client"
 import logger from "@/utils/logger"
+import { ArchiveItemUploadJob } from "./jobs/archive-item-upload-job"
 
 export async function startScheduler() {
   logger.info("Scheduler starting in " + APPLICATION_NAME)
@@ -14,5 +15,8 @@ export async function startScheduler() {
 
   const converter = new PDFConverterJob()
 
+  const archiveItemUploader = new ArchiveItemUploadJob()
+
   scheduleJob(converter.name, converter.schedule, converter.run)
+  scheduleJob(archiveItemUploader.name, archiveItemUploader.schedule, archiveItemUploader.run)
 }

archiver/src/services/file-storage-service.ts

@@ -21,6 +21,12 @@ export class FileStorageService {
     return uploadBlobResponse
   }
 
+  async uploadBuffer(key: string, file: Buffer) {
+    const blockBlobClient = this.containerClient.getBlockBlobClient(key)
+    const uploadBlobResponse = await blockBlobClient.uploadData(file)
+    return uploadBlobResponse
+  }
+
   async downloadFile(key: string) {
     const blockBlobClient = this.containerClient.getBlockBlobClient(key)
 

archiver/src/utils/pdf-signer.ts

@@ -1,5 +1,8 @@
 import { exec } from "child_process"
 import { isEmpty } from "lodash"
+import { writeFileSync, readFileSync, unlinkSync } from "fs"
+import { tmpdir } from "os"
+import { join } from "path"
 
 import logger from "@/utils/logger"
 
@@ -10,20 +13,36 @@ if (isEmpty(TIMESTAMP_SERVER)) throw new Error("timestamp server is unset.")
 if (isEmpty(SSL_FULL_CHAIN_PATH)) throw new Error("ssl fullchain filepath is unset.")
 if (isEmpty(SSL_CERT_KEY_PATH)) throw new Error("ssl cert key filepath is unset.")
 
-export async function signPDFWithPAdES(
-  inputPdfFilepath: string,
-  outputPdfFilepath: string
-): Promise<void> {
-  const cmd: string = `java -jar ${PDF_SIGNER_JAR} --input ${inputPdfFilepath} --output ${outputPdfFilepath} --certificate ${SSL_FULL_CHAIN_PATH} --key ${SSL_CERT_KEY_PATH} --timestamp --tsa ${TIMESTAMP_SERVER} --baseline-lt`
-
-  return new Promise((resolve, reject) => {
-    exec(cmd, { encoding: "buffer" }, (error, stderr) => {
-      if (error) {
-        logger.error("open-pdf-sign error:", stderr.toString())
-        reject(new Error(`open-pdf-sign error: ${stderr.toString()}`))
-        return
-      }
-      resolve()
+const getTempFilePath = (prefix: string) => join(tmpdir(), `${prefix}-${Date.now()}.pdf`)
+
+export async function signPDFWithPAdES(inputPDF: Buffer): Promise<Buffer> {
+  const tempInputPath = getTempFilePath("input")
+  const tempOutputPath = getTempFilePath("output")
+
+  try {
+    writeFileSync(tempInputPath, inputPDF)
+
+    const cmd: string = `java -jar ${PDF_SIGNER_JAR} --input ${tempInputPath} --output ${tempOutputPath} --certificate ${SSL_FULL_CHAIN_PATH} --key ${SSL_CERT_KEY_PATH} --timestamp --tsa ${TIMESTAMP_SERVER} --baseline-lt`
+
+    await new Promise<void>((resolve, reject) => {
+      exec(cmd, { encoding: "buffer" }, (error, _stdout, stderr) => {
+        if (error) {
+          logger.error("open-pdf-sign error:", stderr.toString())
+          reject(new Error(`open-pdf-sign error: ${stderr.toString()}`))
+          return
+        }
+        resolve()
+      })
     })
-  })
+
+    const signedPdfBuffer = readFileSync(tempOutputPath)
+    return signedPdfBuffer
+  } finally {
+    try {
+      unlinkSync(tempInputPath)
+      unlinkSync(tempOutputPath)
+    } catch (cleanupError) {
+      logger.warn("Failed to clean up temporary files:", cleanupError)
+    }
+  }
 }