diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ca8003e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +# Fast, dependency-light checks that run on every PR and push. This is the +# required gate. The full pipeline smoke test lives in smoke-test.yml and is +# run manually because it needs the heavy extraction stack and network access. +on: + pull_request: + branches: + - main + - develop + push: + branches: + - main + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dev dependencies + run: pip install -r requirements-dev.txt + + - name: Run unit tests + run: python -m pytest diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index ea335cf..fb719d2 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -1,28 +1,29 @@ -name: Smoke Test # Name of the workflow, shown in GitHub Actions tab +name: Smoke Test (full pipeline) +# The full DVC pipeline smoke test. It downloads filings and runs the heavy +# extraction stack (torch, docling, layout models), so it is NOT part of the +# required per-PR gate — it is slow and depends on external downloads. Trigger +# it manually from the Actions tab when you want an end-to-end run. on: - pull_request: # Trigger workflow when a pull request is created/updated - branches: - - develop # Run workflow only if PR targets the 'develop' branch - - main # Or if PR targets the 'main' branch + workflow_dispatch: jobs: - smoke-test: - runs-on: ubuntu-latest # Run the job on the latest Ubuntu runner provided by GitHub + smoke-test: + runs-on: ubuntu-latest - steps: - - name: Checkout repository # Fetch the repository code - uses: actions/checkout@v4 # Use the official GitHub Action to check out code + steps: + - name: Checkout repository + uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - - name: Install dependencies # Install project dependencies + - name: Install dependencies run: | - pip install -r requirements.txt # Install packages listed in requirements.txt - pip install dvc[s3] # Install DVC with S3 support (change to dvc[gdrive] or dvc[azure] if needed) + pip install -r requirements.txt + pip install dvc # No remote is configured; plain DVC is enough for `dvc repro`. - - name: Run smoke test # Execute smoke test script - run: bash tests/smoke.sh # Run the smoke.sh + - name: Run smoke test + run: bash tests/smoke.sh diff --git a/data/raw/META/2024_meta_10-k.pdf b/data/raw/META/2024_meta_10-k.pdf deleted file mode 100644 index 38b9202..0000000 Binary files a/data/raw/META/2024_meta_10-k.pdf and /dev/null differ diff --git a/data/raw/META/2024_meta_10-q.pdf b/data/raw/META/2024_meta_10-q.pdf deleted file mode 100644 index 4d8bcbe..0000000 Binary files a/data/raw/META/2024_meta_10-q.pdf and /dev/null differ diff --git a/dvc.yaml b/dvc.yaml index 40b6913..fce9af4 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -8,7 +8,8 @@ stages: - download.filing_types - download.output_dir outs: - - data/raw/META/ + - data/raw/10-K/ + - data/raw/10-Q/ extract_text: cmd: python src/extractors/text_extractor.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..230bdbf --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +# The fast, dependency-light suite. The integration tests under +# tests/integration and evaluation/ require the heavy extraction stack and are +# not collected here. +testpaths = tests/unit +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -q diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..b1db832 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +# Lightweight dependencies for the fast unit-test suite (tests/unit). +# These deliberately exclude the heavy extraction stack (torch, docling, +# detectron2, pdfplumber) so the suite runs quickly and deterministically in CI. +pytest>=8.0.0 +requests>=2.31.0 +PyYAML>=6.0 diff --git a/src/downloaders/sec_downloader.py b/src/downloaders/sec_downloader.py index a9007e7..0227063 100644 --- a/src/downloaders/sec_downloader.py +++ b/src/downloaders/sec_downloader.py @@ -5,6 +5,7 @@ """ import os +import re import sys import yaml import requests @@ -59,33 +60,89 @@ def load_params(self, params_file: str = "params.yaml") -> Dict[str, Any]: logger.error(f"Error parsing {params_file}: {e}") return {} + @staticmethod + def _extract_gdrive_confirm_token(response: "requests.Response") -> Optional[str]: + """Return Google Drive's download-confirmation token, if the response is an interstitial page. + + Large Google Drive files can't be virus-scanned, so the first request returns an + HTML page instead of the file. We must resend the request with the confirm token to + get the real bytes. Without this the downloader silently saves the HTML page as a PDF. + """ + # Older style: token is set as a cookie named ``download_warning*``. + for key, value in response.cookies.items(): + if key.startswith("download_warning"): + return value + + # Newer style: token is embedded in the HTML form of the interstitial page. + content_type = response.headers.get("Content-Type", "") + if "text/html" in content_type: + match = re.search(r'name="confirm"\s+value="([^"]+)"', response.text) + if match: + return match.group(1) + match = re.search(r'confirm=([0-9A-Za-z_\-]+)', response.text) + if match: + return match.group(1) + return None + def download_file(self, url: str, output_path: Path, filename: str) -> bool: - """Download a file from URL to the specified path.""" + """Download a file from URL and verify it is a real PDF before keeping it.""" + file_path = output_path / filename try: logger.info(f"Downloading {filename} from {url}") - + headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' } - - response = requests.get(url, headers=headers, stream=True, timeout=30) + + session = requests.Session() + response = session.get(url, headers=headers, stream=True, timeout=30) response.raise_for_status() - - file_path = output_path / filename + + # Handle Google Drive's large-file confirmation interstitial. + if "drive.google.com" in url or "drive.usercontent.google.com" in url: + token = self._extract_gdrive_confirm_token(response) + if token: + logger.info("Google Drive confirmation required, retrying with token") + response = session.get( + url, headers=headers, params={'confirm': token}, + stream=True, timeout=30 + ) + response.raise_for_status() + with open(file_path, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): if chunk: f.write(chunk) - + + # Guard against silently saving an HTML error/interstitial page as a ".pdf". + if not self._is_pdf(file_path): + logger.error( + f"Downloaded {filename} is not a valid PDF (likely an HTML error page). " + "Discarding the file." + ) + file_path.unlink(missing_ok=True) + return False + file_size = file_path.stat().st_size logger.info(f"Successfully downloaded {filename} ({file_size:,} bytes)") return True - + except requests.RequestException as e: logger.error(f"Failed to download {filename}: {e}") + file_path.unlink(missing_ok=True) return False except Exception as e: logger.error(f"Unexpected error downloading {filename}: {e}") + file_path.unlink(missing_ok=True) + return False + + @staticmethod + def _is_pdf(file_path: Path) -> bool: + """Return True if the file starts with the PDF magic bytes (``%PDF``).""" + try: + with open(file_path, 'rb') as f: + return f.read(5).startswith(b'%PDF') + except OSError: return False def get_filing_url(self, company: str, fiscal_year: int, filing_type: str) -> Optional[str]: @@ -104,32 +161,36 @@ def get_filing_url(self, company: str, fiscal_year: int, filing_type: str) -> Op def download_filings(self, companies: list, fiscal_years: list, filing_types: list) -> Dict[str, bool]: """Download specified filings for given companies and years.""" results = {} - + for company in companies: - company_dir = self.output_dir / company.upper() - company_dir.mkdir(exist_ok=True) - for year in fiscal_years: for filing_type in filing_types: + # Organize downloads by filing type (e.g. data/raw/10-K/) so the + # downstream extractors, which read from data/raw//, + # find their inputs. This also keeps dvc.yaml's `outs` consistent + # with what is actually produced. + filing_dir = self.output_dir / filing_type.upper() + filing_dir.mkdir(parents=True, exist_ok=True) + # Generate filename filename = f"{year}_{company.lower()}_{filing_type.lower()}.pdf" - + # Check if file already exists - file_path = company_dir / filename + file_path = filing_dir / filename if file_path.exists(): logger.info(f"File already exists: {file_path}") results[f"{company}_{year}_{filing_type}"] = True continue - + # Get download URL url = self.get_filing_url(company, year, filing_type) if not url: logger.error(f"No URL found for {company} {year} {filing_type}") results[f"{company}_{year}_{filing_type}"] = False continue - + # Download the file - success = self.download_file(url, company_dir, filename) + success = self.download_file(url, filing_dir, filename) results[f"{company}_{year}_{filing_type}"] = success # Small delay to be respectful to servers diff --git a/src/extractors/layout_detector.py b/src/extractors/layout_detector.py index eb670b8..8807e5f 100644 --- a/src/extractors/layout_detector.py +++ b/src/extractors/layout_detector.py @@ -638,8 +638,8 @@ def _create_comparison_metrics(self, d2_perf: Dict, lmv3_perf: Dict) -> Dict: def process_single_10k_pdf(self, max_pages: int = None) -> Dict: """Process single 10-K PDF for focused analysis""" - # Target specific 10-K file - raw_dir = Path("/Users/HemanthRayudu/Profession/Assignments/DAMG/Docuparse/data/raw") + # Target specific 10-K file (relative to the repo root) + raw_dir = Path("data/raw") target_pdf = raw_dir / "10-K" / "2024_meta_10-k.pdf" if not target_pdf.exists(): @@ -794,7 +794,7 @@ def _print_10k_results(self, summary: Dict): def estimate_processing_time(): """Estimate processing time by testing one PDF""" - raw_dir = Path("/Users/HemanthRayudu/Profession/Assignments/DAMG/Docuparse/data/raw") + raw_dir = Path("data/raw") pdf_files = list(raw_dir.rglob("*.pdf")) if not pdf_files: diff --git a/src/extractors/table_extractor.py b/src/extractors/table_extractor.py index ab0fff9..1456ece 100644 --- a/src/extractors/table_extractor.py +++ b/src/extractors/table_extractor.py @@ -8,7 +8,7 @@ from typing import List, Dict, Optional, Tuple, Any import logging import json -from datetime import datetime +from datetime import datetime, timezone from dataclasses import dataclass, asdict import time @@ -759,7 +759,7 @@ def _save_page_tables(self, doc_id: str, page_num: int, page_result: PageTableEx 'rows': table.rows, 'cols': table.cols, 'extraction_time': table.extraction_time, - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'error': table.error, 'file_path': str(table_path) } @@ -814,7 +814,7 @@ def _save_extraction_log(self, doc_id: str, results: List[PageTableExtraction]): log_data = { 'doc_id': doc_id, - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'total_pages': len(results), 'total_tables': sum(r.total_tables for r in results), 'statistics': { @@ -1006,7 +1006,7 @@ def process_single_pdf(pdf_file: Path) -> Dict: 'quarter': quarter, 'company': company, 'source_file': pdf_file.name, - 'processed_at': datetime.utcnow().isoformat() + 'processed_at': datetime.now(timezone.utc).isoformat() }, 'extraction_stats': results['statistics'], 'file_paths': { @@ -1108,7 +1108,7 @@ def process_single_pdf(pdf_file: Path) -> Dict: if not target_pdf.exists(): logger.warning(f"Target PDF not found: {target_pdf}") return { - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'total_statistics': total_stats, 'file_results': all_results, 'output_directory': str(output_dir) @@ -1143,7 +1143,7 @@ def process_single_pdf(pdf_file: Path) -> Dict: # Save comprehensive processing log processing_log = { - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'total_statistics': total_stats, 'file_results': all_results, 'output_directory': str(output_dir) diff --git a/src/extractors/text_extractor.py b/src/extractors/text_extractor.py index 902bc04..2f81e1c 100644 --- a/src/extractors/text_extractor.py +++ b/src/extractors/text_extractor.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional, Tuple import logging import json -from datetime import datetime +from datetime import datetime, timezone import numpy as np from dataclasses import dataclass, asdict @@ -238,7 +238,7 @@ def _save_page_text(self, doc_id: str, page_num: int, page_result: PageExtractio 'word_count': page_result.word_count, 'char_count': page_result.char_count, 'extraction_time': page_result.extraction_time, - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'error': page_result.error } @@ -260,7 +260,7 @@ def _save_extraction_log(self, doc_id: str, results: List[PageExtraction]): log_data = { 'doc_id': doc_id, - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'total_pages': len(results), 'ocr_pages': [r.page_num for r in results if r.ocr_used], 'failed_pages': [r.page_num for r in results if r.error], @@ -437,7 +437,7 @@ def process_single_pdf(pdf_file: Path, doc_type: str, year: str, quarter: str = 'quarter': quarter, 'company': 'META', 'source_file': pdf_file.name, - 'processed_at': datetime.utcnow().isoformat() + 'processed_at': datetime.now(timezone.utc).isoformat() }, 'extraction_stats': results['statistics'], 'file_paths': { @@ -547,7 +547,7 @@ def process_single_pdf(pdf_file: Path, doc_type: str, year: str, quarter: str = serializable_results[key] = result processing_log = { - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(timezone.utc).isoformat(), 'total_statistics': total_stats, 'file_results': serializable_results, 'output_directory': str(output_dir) diff --git a/src/representations/metadata_storage_formats.py b/src/representations/metadata_storage_formats.py index 0cc0809..59057e8 100644 --- a/src/representations/metadata_storage_formats.py +++ b/src/representations/metadata_storage_formats.py @@ -40,8 +40,9 @@ def convert_metadata(input_file: Path, output_dir: Path): if __name__ == "__main__": - input_dir = Path(r"C:\Users\Pauline\Desktop\PEI\NEU_assignments\DAMG 7245_(2025 Fall)\data\parsed\metadata") - output_dir = Path(r"C:\Users\Pauline\Desktop\PEI\NEU_assignments\DAMG 7245_(2025 Fall)\data\parsed\metadata_representations") + # Paths are relative to the repo root so this runs on any machine. + input_dir = Path("data/parsed/metadata") + output_dir = Path("data/parsed/metadata_representations") output_dir.mkdir(parents=True, exist_ok=True) # find all JSONL files diff --git a/tests/smoke.sh b/tests/smoke.sh index b63bcb0..81525d0 100644 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -1,11 +1,18 @@ -set -e # when error then stop +#!/usr/bin/env bash +set -euo pipefail # stop on error, unset vars, and failed pipes echo "🚀 Running DVC smoke test..." -# 1. run pipeline -dvc repro --pull --force || { echo "❌ DVC pipeline failed"; exit 1; } +# 1. Run the full pipeline. We do NOT pass --pull because no DVC remote is +# configured (.dvc/config is empty); --pull would fail trying to reach a +# remote that does not exist. The pipeline regenerates its outputs locally. +dvc repro --force || { echo "❌ DVC pipeline failed"; exit 1; } -# 2. validate pipeline (at least one *_docling.json exist) -ls data/parsed/docling_or_fallback/*_docling.json >/dev/null 2>&1 || { echo "❌ No parsed outputs found"; exit 1; } +# 2. Validate the pipeline produced Docling output. The docling stage writes to +# data/parsed/docling/_docling.json (see dvc.yaml: extract_docling). +ls data/parsed/docling/*_docling.json >/dev/null 2>&1 || { + echo "❌ No parsed Docling outputs found in data/parsed/docling/" + exit 1 +} echo "✅ Smoke test passed" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_repo_consistency.py b/tests/unit/test_repo_consistency.py new file mode 100644 index 0000000..b3e9b76 --- /dev/null +++ b/tests/unit/test_repo_consistency.py @@ -0,0 +1,73 @@ +"""Static consistency checks that guard against regressions of fixed bugs. + +None of these import the heavy extraction stack, so they run anywhere. They act +as executable documentation of the invariants the pipeline relies on: + + * no machine-specific absolute paths committed to source; + * the smoke test validates the path the pipeline actually writes; + * the DVC download stage declares the directories the extractors read; + * no deprecated ``datetime.utcnow()`` calls. +""" + +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC = REPO_ROOT / "src" + + +def _python_sources(): + return list(SRC.rglob("*.py")) + + +def test_no_hardcoded_absolute_user_paths(): + offenders = [] + for path in _python_sources(): + text = path.read_text(encoding="utf-8") + if "/Users/" in text or "C:\\Users\\" in text: + offenders.append(str(path.relative_to(REPO_ROOT))) + assert not offenders, f"Hardcoded absolute user paths found in: {offenders}" + + +def test_no_deprecated_utcnow(): + offenders = [] + for path in _python_sources(): + if "datetime.utcnow(" in path.read_text(encoding="utf-8"): + offenders.append(str(path.relative_to(REPO_ROOT))) + assert not offenders, f"Deprecated datetime.utcnow() found in: {offenders}" + + +def test_smoke_test_checks_real_docling_output_path(): + smoke = (REPO_ROOT / "tests" / "smoke.sh").read_text(encoding="utf-8") + # The pipeline writes to data/parsed/docling/, not docling_or_fallback/. + assert "data/parsed/docling/" in smoke + assert "docling_or_fallback" not in smoke + + +def test_smoke_test_does_not_pull_from_missing_remote(): + smoke = (REPO_ROOT / "tests" / "smoke.sh").read_text(encoding="utf-8") + # Only inspect actual dvc invocations, not explanatory comments. + dvc_commands = [ + line for line in smoke.splitlines() + if line.strip().startswith("dvc ") + ] + assert dvc_commands, "smoke test should still invoke dvc" + assert all("--pull" not in cmd for cmd in dvc_commands), ( + "No DVC remote is configured; dvc --pull would fail" + ) + + +def test_dvc_download_outs_match_extractor_inputs(): + dvc = yaml.safe_load((REPO_ROOT / "dvc.yaml").read_text(encoding="utf-8")) + outs = dvc["stages"]["download"]["outs"] + # Extractors read from data/raw/10-K/ and data/raw/10-Q/; the download stage + # must declare those as its outputs. + assert "data/raw/10-K/" in outs + assert "data/raw/10-Q/" in outs + + +def test_docling_stage_output_matches_smoke_check(): + dvc = yaml.safe_load((REPO_ROOT / "dvc.yaml").read_text(encoding="utf-8")) + docling_outs = dvc["stages"]["extract_docling"]["outs"] + assert "data/parsed/docling/" in docling_outs diff --git a/tests/unit/test_sec_downloader.py b/tests/unit/test_sec_downloader.py new file mode 100644 index 0000000..aad9f39 --- /dev/null +++ b/tests/unit/test_sec_downloader.py @@ -0,0 +1,152 @@ +"""Unit tests for the SEC/filing downloader. + +These tests cover the concrete bugs that were fixed: + * downloads are organized by filing type (data/raw//) so the extractors + can find their inputs; + * Google Drive's large-file confirmation interstitial is handled; + * an HTML error page is never silently kept as a ".pdf". + +They intentionally avoid any real network access and any heavy dependency +(torch/docling/pdfplumber), so they run fast and deterministically in CI. +""" + +import sys +from pathlib import Path + +import pytest + +# Make ``src`` importable without installing the package. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from downloaders.sec_downloader import SECDownloader # noqa: E402 + + +class FakeResponse: + """Minimal stand-in for a ``requests`` response object.""" + + def __init__(self, body: bytes, *, headers=None, cookies=None, text=""): + self._body = body + self.headers = headers or {} + self.cookies = cookies or {} + self.text = text + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=8192): + for i in range(0, len(self._body), chunk_size): + yield self._body[i:i + chunk_size] + + +class FakeSession: + """Returns queued responses in order for successive ``get`` calls.""" + + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._responses.pop(0) + + +PDF_BYTES = b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\nfake pdf body" +HTML_BYTES = b"Virus scan warning" + + +def test_is_pdf_true_for_pdf_magic(tmp_path): + f = tmp_path / "a.pdf" + f.write_bytes(PDF_BYTES) + assert SECDownloader._is_pdf(f) is True + + +def test_is_pdf_false_for_html(tmp_path): + f = tmp_path / "a.pdf" + f.write_bytes(HTML_BYTES) + assert SECDownloader._is_pdf(f) is False + + +def test_gdrive_token_from_cookie(): + resp = FakeResponse(b"", cookies={"download_warning_abc": "tok123"}) + assert SECDownloader._extract_gdrive_confirm_token(resp) == "tok123" + + +def test_gdrive_token_from_html_form(): + html = '
' + resp = FakeResponse(b"", headers={"Content-Type": "text/html"}, text=html) + assert SECDownloader._extract_gdrive_confirm_token(resp) == "tok999" + + +def test_gdrive_token_absent_for_plain_pdf(): + resp = FakeResponse(PDF_BYTES, headers={"Content-Type": "application/pdf"}) + assert SECDownloader._extract_gdrive_confirm_token(resp) is None + + +def test_download_file_keeps_valid_pdf(tmp_path, monkeypatch): + dl = SECDownloader(output_dir=str(tmp_path)) + session = FakeSession([FakeResponse(PDF_BYTES, + headers={"Content-Type": "application/pdf"})]) + monkeypatch.setattr("downloaders.sec_downloader.requests.Session", + lambda: session) + + ok = dl.download_file("https://example.com/file.pdf", tmp_path, "out.pdf") + + assert ok is True + assert (tmp_path / "out.pdf").read_bytes() == PDF_BYTES + + +def test_download_file_discards_html_error_page(tmp_path, monkeypatch): + dl = SECDownloader(output_dir=str(tmp_path)) + session = FakeSession([FakeResponse(HTML_BYTES, + headers={"Content-Type": "text/html"})]) + monkeypatch.setattr("downloaders.sec_downloader.requests.Session", + lambda: session) + + ok = dl.download_file("https://example.com/file.pdf", tmp_path, "out.pdf") + + assert ok is False + # The bogus HTML must not be left behind masquerading as a PDF. + assert not (tmp_path / "out.pdf").exists() + + +def test_download_file_follows_gdrive_confirmation(tmp_path, monkeypatch): + dl = SECDownloader(output_dir=str(tmp_path)) + interstitial = FakeResponse( + HTML_BYTES, + headers={"Content-Type": "text/html"}, + cookies={"download_warning_x": "tokabc"}, + ) + real_pdf = FakeResponse(PDF_BYTES, headers={"Content-Type": "application/pdf"}) + session = FakeSession([interstitial, real_pdf]) + monkeypatch.setattr("downloaders.sec_downloader.requests.Session", + lambda: session) + + ok = dl.download_file( + "https://drive.google.com/uc?export=download&id=XYZ", tmp_path, "out.pdf" + ) + + assert ok is True + assert (tmp_path / "out.pdf").read_bytes() == PDF_BYTES + # The confirm token must have been sent on the second request. + assert session.calls[1][1]["params"] == {"confirm": "tokabc"} + + +def test_download_filings_organizes_by_filing_type(tmp_path, monkeypatch): + """Downloads must land in data/raw// to match the extractors.""" + dl = SECDownloader(output_dir=str(tmp_path)) + written = [] + + def fake_download_file(url, output_path, filename): + target = Path(output_path) / filename + target.write_bytes(PDF_BYTES) + written.append(target) + return True + + monkeypatch.setattr(dl, "download_file", fake_download_file) + # Both known filings resolve to real URLs. + dl.download_filings(["META"], [2024], ["10-K", "10-Q"]) + + produced = {p.relative_to(tmp_path).as_posix() for p in written} + assert "10-K/2024_meta_10-k.pdf" in produced + assert "10-Q/2024_meta_10-q.pdf" in produced