fireform-core · marcvergees · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026
diff --git a/benchmark/datasets/pdfs/ics_201.pdf b/benchmark/datasets/pdfs/ics_201.pdf
diff --git a/benchmark/datasets/pdfs/ics_202.pdf b/benchmark/datasets/pdfs/ics_202.pdf
diff --git a/benchmark/datasets/pdfs/ics_203.pdf b/benchmark/datasets/pdfs/ics_203.pdf
diff --git a/benchmark/datasets/pdfs/ics_204.pdf b/benchmark/datasets/pdfs/ics_204.pdf
diff --git a/benchmark/datasets/pdfs/ics_205.pdf b/benchmark/datasets/pdfs/ics_205.pdf
diff --git a/benchmark/datasets/pdfs/ics_205a.pdf b/benchmark/datasets/pdfs/ics_205a.pdf
diff --git a/benchmark/datasets/pdfs/ics_206.pdf b/benchmark/datasets/pdfs/ics_206.pdf
diff --git a/benchmark/datasets/pdfs/ics_207.pdf b/benchmark/datasets/pdfs/ics_207.pdf
diff --git a/benchmark/datasets/pdfs/ics_208.pdf b/benchmark/datasets/pdfs/ics_208.pdf
diff --git a/benchmark/datasets/pdfs/ics_213.pdf b/benchmark/datasets/pdfs/ics_213.pdf
diff --git a/benchmark/datasets/reference_candidates/ics201_1.json b/benchmark/datasets/reference_candidates/ics201_1.json
diff --git a/benchmark/datasets/reference_candidates/ics202_1.json b/benchmark/datasets/reference_candidates/ics202_1.json
diff --git a/benchmark/pipelines/base.py b/benchmark/pipelines/base.py
@@ -14,5 +14,5 @@ class BasePipeline(ABC):
     """Base class for all extraction pipelines."""
 
     @abstractmethod
-    def run(self, narrative: str, template_schema: dict) -> PipelineExtractionOutput: 
+    def run(self, narrative: str, template_schema: dict, pdf_path: str) -> PipelineExtractionOutput: 
         pass
diff --git a/benchmark/pipelines/pipeline.py b/benchmark/pipelines/pipeline.py
@@ -2,7 +2,7 @@
 
 
 class Pipeline(BasePipeline):
-    def run(self, narrative: str, template_schema: dict) -> PipelineExtractionOutput:
+    def run(self, narrative: str, template_schema: dict, pdf_path: str) -> PipelineExtractionOutput:
         '''
         You guys should here implement the whole pipeline implementation depending on approach B or C.
 

diff --git a/benchmark/runners/runner.py b/benchmark/runners/runner.py
@@ -15,6 +15,7 @@ def run_benchmark(self) -> dict[str, any]:
         narratives_dir = os.path.join(datasets_dir, "narratives")
         ground_truth_dir = os.path.join(datasets_dir, "ground_truth")
         templates_dir = os.path.join(datasets_dir, "templates")
+        pdfs_dir = os.path.join(templates_dir, "pdfs")
 
         results = []
         total_latency = 0.0
@@ -45,8 +46,9 @@ def run_benchmark(self) -> dict[str, any]:
             narrative_path = os.path.join(narratives_dir, narrative_file)
             gt_path = os.path.join(ground_truth_dir, gt_file) if gt_file else ""
             template_path = os.path.join(templates_dir, template_file)
+            pdf_path = os.path.join(pdfs_dir, template_file.replace(".json", ".pdf"))
 
-            if not os.path.exists(gt_path) or not os.path.exists(template_path):
+            if not os.path.exists(gt_path) or not os.path.exists(template_path) or not os.path.exists(pdf_path):
                 continue
 
             with open(narrative_path, "r") as f:
@@ -62,7 +64,7 @@ def run_benchmark(self) -> dict[str, any]:
                 template_schema = json.load(f)
 
             start_time = time.time()
-            output = self.pipeline.run(narrative_text, template_schema)
+            output = self.pipeline.run(narrative_text, template_schema, pdf_path)
             latency = time.time() - start_time
             total_latency += latency