From c626f495aa62a2e4c1b736730346869ba65d0a73 Mon Sep 17 00:00:00 2001 From: Donald Gray Date: Tue, 4 Aug 2026 09:01:00 +0100 Subject: [PATCH] Support adding headers, scoped by url + customer This is similar to CustomerOriginStrategy in Protagonist. It allows custom headers to be specified per customer and origin URL. Note that setuptools version pining in docker image is temporary, follow up PR will update to django-q2 and this can go --- .env.dist | 4 ++ Dockerfile | 2 +- README.md | 8 +++ src/app/engine/origin_rules.py | 60 ++++++++++++++++++ src/app/engine/origins.py | 15 ++++- src/app/engine/tasks.py | 2 +- src/app/settings.py | 7 ++- src/tests/engine/test_origin_rules.py | 88 +++++++++++++++++++++++++++ 8 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 src/app/engine/origin_rules.py create mode 100644 src/tests/engine/test_origin_rules.py diff --git a/.env.dist b/.env.dist index 323f21f..38b9fb3 100644 --- a/.env.dist +++ b/.env.dist @@ -9,6 +9,10 @@ WEB_SERVER_HOSTNAME=localhost:8000 # Origin Fetching ORIGIN_CHUNK_SIZE=8192 +# Optional JSON rules for injecting headers into origin fetches, keyed by +# customer id then URL regex. Delivered as a secret in real deployments. +# Keep single-quoted and on one line here; see README for escaping notes. +# ORIGIN_HTTP_RULES='{"2":{"^https://customer-url.*":{"addHeader":{"X-ApiKey":"abc12345"}}}}' # Database DATABASE_URL=postgresql://dlcs:password@postgres:5432/compositedb diff --git a/Dockerfile b/Dockerfile index 31b1b02..9c8b088 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get --yes install apt-utils && apt-get --yes upgrade \ && apt-get --yes autoremove && apt-get --yes autoclean && apt-get --yes clean \ && useradd --create-home --home-dir /srv/dlcs --shell /bin/bash --uid 1000 dlcs \ && python -m pip install --upgrade pip \ - && python -m pip install --upgrade setuptools + && python -m pip install --upgrade "setuptools<82" # Copy nginx config and create appropriate folders COPY --chown=dlcs:dlcs ./nginx.conf /etc/nginx/nginx.conf diff --git a/README.md b/README.md index 874284c..3e61925 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ The following list of environment variables are supported: | `WEB_SERVER_SCHEME` | `http` | API | The HTTP scheme used when generating URI's. | | `WEB_SERVER_HOSTNAME` | `localhost:8000` | API | The hostname (and optional port) used when generating URI's. | | `ORIGIN_CHUNK_SIZE` | `8192` | Engine | The chunk size, in bytes, used when retrieving objects from origins. Tailoring this value can theoretically improve download speeds. | +| `ORIGIN_HTTP_RULES` | `{}` | Engine | Optional JSON of request mutations applied when fetching origins, keyed by customer id then by regex matched against the origin URL. The only current operation is `addHeader`. Values are typically secrets and should be injected accordingly. When multiple regexes match, headers are merged with later entries winning. Example: `{"2":{"^https://fraser-staging.*":{"addHeader":{"Auth-Bypass-Key":"..."}}}}`. | | `DATABASE_URL` | None | API, Engine | The URL of the target PostgreSQL database, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `postgresql://dlcs:password@postgres:5432/compositedb`. | | `CACHE_URL` | None | API, Engine | The URL of the target cache, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `dbcache://app_cache`. | | `PDF_RASTERIZER_THREAD_COUNT` | `3` | Engine | The number of concurrent [Poppler](https://poppler.freedesktop.org/) threads spawned when a worker is rasterizing a PDF. Each thread typically consumes 100% of a CPU core. | @@ -94,6 +95,13 @@ The following list of environment variables are supported: | `GUNICORN_WORKERS` | `2` | API | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn | | `SQS_BROKER_QUEUE_NAME` | None | API, Engine | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used | +> [!NOTE] +> `ORIGIN_HTTP_RULES` is safe to inject verbatim as an environment variable (e.g. from a secret) — no escaping is needed at that level. However: +> +> * Backslashes in regexes must be doubled in the JSON, as JSON string escaping consumes one level: write `"^https://customer-url\\.com.*"` to match a literal dot. +> * When set in a `.env` file, wrap the whole value in **single** quotes and keep it on one line — unquoted values are truncated at the first `#`, and double quotes trigger backslash-unescaping that corrupts regex escapes. +> * In a docker-compose `environment:` block, escape any `$` (e.g. a regex end anchor) as `$$` to avoid compose variable interpolation. + Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials. ### Django Q Broker diff --git a/src/app/engine/origin_rules.py b/src/app/engine/origin_rules.py new file mode 100644 index 0000000..89a3db5 --- /dev/null +++ b/src/app/engine/origin_rules.py @@ -0,0 +1,60 @@ +import logging +import re + +logger = logging.getLogger(__name__) + +KNOWN_OPERATIONS = {"addHeader"} + + +def compile_rules(raw_rules): + """Validate ORIGIN_HTTP_RULES config and precompile its regexes. + + Input shape: {customer_id: {url_regex: {operation: {...}}}} + Output: {customer_id (str): [(compiled_regex, {operation: {...}})]} + + Raises ValueError on malformed structure or invalid regex so that a bad + configuration fails at startup rather than silently dropping headers. + """ + compiled = {} + for customer, url_rules in raw_rules.items(): + if not isinstance(url_rules, dict): + raise ValueError( + f"ORIGIN_HTTP_RULES: expected an object of URL regexes " + f"for customer {customer}, got {type(url_rules).__name__}" + ) + entries = [] + for pattern, operations in url_rules.items(): + try: + regex = re.compile(pattern) + except re.error as error: + raise ValueError( + f"ORIGIN_HTTP_RULES: invalid regex {pattern!r} " + f"for customer {customer}: {error}" + ) + if not isinstance(operations, dict): + raise ValueError( + f"ORIGIN_HTTP_RULES: expected an object of operations " + f"for customer {customer}, regex {pattern!r}, " + f"got {type(operations).__name__}" + ) + for operation in operations: + if operation not in KNOWN_OPERATIONS: + logger.warning( + f"ORIGIN_HTTP_RULES: ignoring unknown operation {operation} for customer {customer}" + ) + entries.append((regex, operations)) + compiled[str(customer)] = entries + return compiled + + +def headers_for(rules, customer, url): + """Return the extra headers to apply for this customer/origin URL. + + Headers from all matching rules are merged; later rules override earlier + ones on header-name conflict. Returns {} when nothing matches. + """ + headers = {} + for regex, operations in rules.get(str(customer), []): + if regex.search(url): + headers.update(operations.get("addHeader", {})) + return headers diff --git a/src/app/engine/origins.py b/src/app/engine/origins.py index 527b598..48e5af3 100644 --- a/src/app/engine/origins.py +++ b/src/app/engine/origins.py @@ -1,19 +1,30 @@ +import logging import os from pathlib import Path import requests from django.conf import settings +from app.engine.origin_rules import headers_for + +logger = logging.getLogger(__name__) + class HttpOrigin: def __init__(self): self._scratch_path = settings.SCRATCH_DIRECTORY self._chunk_size = settings.ORIGIN_CONFIG["chunk_size"] + self._http_rules = settings.ORIGIN_CONFIG["http_rules"] - def fetch(self, submission_id, url, file_extension="pdf"): + def fetch(self, submission_id, url, customer, file_extension="pdf"): subfolder_path = self.__generate_subfolder_path(submission_id) file_path = os.path.join(subfolder_path, "source." + file_extension) - with requests.get(url, stream=True) as response: + headers = headers_for(self._http_rules, customer, url) + if headers: + logger.info( + f"Applying custom origin headers {sorted(headers)} for submission {submission_id}", + ) + with requests.get(url, stream=True, headers=headers or None) as response: response.raise_for_status() with open(file_path, "wb") as file: for chunk in response.iter_content(chunk_size=self._chunk_size): diff --git a/src/app/engine/tasks.py b/src/app/engine/tasks.py index 3ae7901..27c40ee 100644 --- a/src/app/engine/tasks.py +++ b/src/app/engine/tasks.py @@ -41,7 +41,7 @@ def process_member(args): def __fetch_origin(member, origin_uri): __update_status(member, "FETCHING_ORIGIN") - return http_origin.fetch(member.id, origin_uri) + return http_origin.fetch(member.id, origin_uri, member.collection.customer) def __rasterize_composite(member, pdf_path): diff --git a/src/app/settings.py b/src/app/settings.py index 1ae1cb8..52799d5 100644 --- a/src/app/settings.py +++ b/src/app/settings.py @@ -14,6 +14,8 @@ import environ +from app.engine.origin_rules import compile_rules + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -180,7 +182,10 @@ "use_cropbox": env("PDF_RASTERIZER_USE_CROPBOX", cast=bool, default=False), } -ORIGIN_CONFIG = {"chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192)} +ORIGIN_CONFIG = { + "chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192), + "http_rules": compile_rules(env.json("ORIGIN_HTTP_RULES", default={})), +} DLCS = { "api_root": env.url("DLCS_API_ROOT", default="https://api.dlcs.digirati.io/"), diff --git a/src/tests/engine/test_origin_rules.py b/src/tests/engine/test_origin_rules.py new file mode 100644 index 0000000..02a4b53 --- /dev/null +++ b/src/tests/engine/test_origin_rules.py @@ -0,0 +1,88 @@ +import pytest + +from app.engine.origin_rules import compile_rules, headers_for + + +def test_empty_rules_returns_no_headers(): + rules = compile_rules({}) + assert headers_for(rules, 2, "https://example.com/file.pdf") == {} + + +def test_matching_customer_and_url_returns_headers(): + rules = compile_rules( + { + "2": { + "^https://fraser-staging.*": { + "addHeader": {"Auth-Bypass-Key": "abc12345"} + } + } + } + ) + assert headers_for(rules, "2", "https://fraser-staging.example.com/a.pdf") == { + "Auth-Bypass-Key": "abc12345" + } + + +def test_int_customer_matches_string_json_key(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == { + "Auth-Bypass-Key": "x" + } + + +def test_non_matching_url_returns_no_headers(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 2, "https://other.example.com/a.pdf") == {} + + +def test_unknown_customer_returns_no_headers(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 3, "https://fraser-staging.example.com/a.pdf") == {} + + +def test_multiple_matches_merge_with_later_rules_winning(): + rules = compile_rules( + { + "2": { + "^https://": {"addHeader": {"X-First": "1", "X-Shared": "first"}}, + "^https://fraser-staging.*": { + "addHeader": {"X-Second": "2", "X-Shared": "second"} + }, + } + } + ) + assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == { + "X-First": "1", + "X-Second": "2", + "X-Shared": "second", + } + + +def test_unknown_operation_is_ignored_with_warning(caplog): + with caplog.at_level("WARNING"): + rules = compile_rules( + {"2": {"^https://.*": {"rewriteUrl": {"target": "somewhere"}}}} + ) + assert headers_for(rules, 2, "https://example.com/a.pdf") == {} + assert "unknown operation rewriteUrl" in caplog.text + + +def test_invalid_regex_raises_value_error(): + with pytest.raises(ValueError, match="invalid regex"): + compile_rules({"2": {"^https://(unclosed": {"addHeader": {"X": "1"}}}}) + + +def test_non_dict_url_rules_raises_value_error(): + with pytest.raises(ValueError, match="expected an object of URL regexes"): + compile_rules({"2": ["not-a-dict"]}) + + +def test_non_dict_operations_raises_value_error(): + with pytest.raises(ValueError, match="expected an object of operations"): + compile_rules({"2": {"^https://.*": "not-a-dict"}})