Support adding headers, scoped by url + customer

.env.dist

@@ -9,6 +9,10 @@ WEB_SERVER_HOSTNAME=localhost:8000
 
 # Origin Fetching
 ORIGIN_CHUNK_SIZE=8192
+# Optional JSON rules for injecting headers into origin fetches, keyed by
+# customer id then URL regex. Delivered as a secret in real deployments.
+# Keep single-quoted and on one line here; see README for escaping notes.
+# ORIGIN_HTTP_RULES='{"2":{"^https://customer-url.*":{"addHeader":{"X-ApiKey":"abc12345"}}}}'
 
 # Database
 DATABASE_URL=postgresql://dlcs:password@postgres:5432/compositedb

Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get update && apt-get --yes install apt-utils && apt-get --yes upgrade \
     && apt-get --yes autoremove && apt-get --yes autoclean && apt-get --yes clean \
     && useradd --create-home --home-dir /srv/dlcs --shell /bin/bash --uid 1000 dlcs \
     && python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools
+    && python -m pip install --upgrade "setuptools<82"
 
 # Copy nginx config and create appropriate folders
 COPY --chown=dlcs:dlcs ./nginx.conf /etc/nginx/nginx.conf

README.md

@@ -73,6 +73,7 @@ The following list of environment variables are supported:
 | `WEB_SERVER_SCHEME`           | `http`                         | API          | The HTTP scheme used when generating URI's.                                                                                                                                                                                                                                  |
 | `WEB_SERVER_HOSTNAME`         | `localhost:8000`               | API          | The hostname (and optional port) used when generating URI's.                                                                                                                                                                                                                 |
 | `ORIGIN_CHUNK_SIZE`           | `8192`                         | Engine       | The chunk size, in bytes, used when retrieving objects from origins. Tailoring this value can theoretically improve download speeds.                                                                                                                                         |
+| `ORIGIN_HTTP_RULES`           | `{}`                           | Engine       | Optional JSON of request mutations applied when fetching origins, keyed by customer id then by regex matched against the origin URL. The only current operation is `addHeader`. Values are typically secrets and should be injected accordingly. When multiple regexes match, headers are merged with later entries winning. Example: `{"2":{"^https://fraser-staging.*":{"addHeader":{"Auth-Bypass-Key":"..."}}}}`.       |
 | `DATABASE_URL`                | None                           | API, Engine  | The URL of the target PostgreSQL database, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `postgresql://dlcs:password@postgres:5432/compositedb`.                                              |
 | `CACHE_URL`                   | None                           | API, Engine  | The URL of the target cache, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `dbcache://app_cache`.                                                                                             |
 | `PDF_RASTERIZER_THREAD_COUNT` | `3`                            | Engine       | The number of concurrent [Poppler](https://poppler.freedesktop.org/) threads spawned when a worker is rasterizing a PDF. Each thread typically consumes 100% of a CPU core.                                                                                                  |
@@ -94,6 +95,13 @@ The following list of environment variables are supported:
 | `GUNICORN_WORKERS`            | `2`                            | API          | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn                                                                                                                                                                           |
 | `SQS_BROKER_QUEUE_NAME`       | None                           | API, Engine  | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used               |
 
+> [!NOTE]
+> `ORIGIN_HTTP_RULES` is safe to inject verbatim as an environment variable (e.g. from a secret) — no escaping is needed at that level. However:
+>
+> * Backslashes in regexes must be doubled in the JSON, as JSON string escaping consumes one level: write `"^https://customer-url\\.com.*"` to match a literal dot.
+> * When set in a `.env` file, wrap the whole value in **single** quotes and keep it on one line — unquoted values are truncated at the first `#`, and double quotes trigger backslash-unescaping that corrupts regex escapes.
+> * In a docker-compose `environment:` block, escape any `$` (e.g. a regex end anchor) as `$$` to avoid compose variable interpolation.
+
 Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials.
 
 ### Django Q Broker

src/app/engine/origin_rules.py

@@ -0,0 +1,60 @@
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+KNOWN_OPERATIONS = {"addHeader"}
+
+
+def compile_rules(raw_rules):
+    """Validate ORIGIN_HTTP_RULES config and precompile its regexes.
+
+    Input shape: {customer_id: {url_regex: {operation: {...}}}}
+    Output: {customer_id (str): [(compiled_regex, {operation: {...}})]}
+
+    Raises ValueError on malformed structure or invalid regex so that a bad
+    configuration fails at startup rather than silently dropping headers.
+    """
+    compiled = {}
+    for customer, url_rules in raw_rules.items():
+        if not isinstance(url_rules, dict):
+            raise ValueError(
+                f"ORIGIN_HTTP_RULES: expected an object of URL regexes "
+                f"for customer {customer}, got {type(url_rules).__name__}"
+            )
+        entries = []
+        for pattern, operations in url_rules.items():
+            try:
+                regex = re.compile(pattern)
+            except re.error as error:
+                raise ValueError(
+                    f"ORIGIN_HTTP_RULES: invalid regex {pattern!r} "
+                    f"for customer {customer}: {error}"
+                )
+            if not isinstance(operations, dict):
+                raise ValueError(
+                    f"ORIGIN_HTTP_RULES: expected an object of operations "
+                    f"for customer {customer}, regex {pattern!r}, "
+                    f"got {type(operations).__name__}"
+                )
+            for operation in operations:
+                if operation not in KNOWN_OPERATIONS:
+                    logger.warning(
+                        f"ORIGIN_HTTP_RULES: ignoring unknown operation {operation} for customer {customer}"
+                    )
+            entries.append((regex, operations))
+        compiled[str(customer)] = entries
+    return compiled
+
+
+def headers_for(rules, customer, url):
+    """Return the extra headers to apply for this customer/origin URL.
+
+    Headers from all matching rules are merged; later rules override earlier
+    ones on header-name conflict. Returns {} when nothing matches.
+    """
+    headers = {}
+    for regex, operations in rules.get(str(customer), []):
+        if regex.search(url):
+            headers.update(operations.get("addHeader", {}))
+    return headers

src/app/engine/origins.py

@@ -1,19 +1,30 @@
+import logging
 import os
 from pathlib import Path
 
 import requests
 from django.conf import settings
 
+from app.engine.origin_rules import headers_for
+
+logger = logging.getLogger(__name__)
+
 
 class HttpOrigin:
     def __init__(self):
         self._scratch_path = settings.SCRATCH_DIRECTORY
         self._chunk_size = settings.ORIGIN_CONFIG["chunk_size"]
+        self._http_rules = settings.ORIGIN_CONFIG["http_rules"]
 
-    def fetch(self, submission_id, url, file_extension="pdf"):
+    def fetch(self, submission_id, url, customer, file_extension="pdf"):
         subfolder_path = self.__generate_subfolder_path(submission_id)
         file_path = os.path.join(subfolder_path, "source." + file_extension)
-        with requests.get(url, stream=True) as response:
+        headers = headers_for(self._http_rules, customer, url)
+        if headers:
+            logger.info(
+                f"Applying custom origin headers {sorted(headers)} for submission {submission_id}",
+            )
+        with requests.get(url, stream=True, headers=headers or None) as response:
             response.raise_for_status()
             with open(file_path, "wb") as file:
                 for chunk in response.iter_content(chunk_size=self._chunk_size):

src/app/engine/tasks.py

@@ -41,7 +41,7 @@ def process_member(args):
 
 def __fetch_origin(member, origin_uri):
     __update_status(member, "FETCHING_ORIGIN")
-    return http_origin.fetch(member.id, origin_uri)
+    return http_origin.fetch(member.id, origin_uri, member.collection.customer)
 
 
 def __rasterize_composite(member, pdf_path):

src/app/settings.py

@@ -14,6 +14,8 @@
 
 import environ
 
+from app.engine.origin_rules import compile_rules
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -180,7 +182,10 @@
     "use_cropbox": env("PDF_RASTERIZER_USE_CROPBOX", cast=bool, default=False),
 }
 
-ORIGIN_CONFIG = {"chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192)}
+ORIGIN_CONFIG = {
+    "chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192),
+    "http_rules": compile_rules(env.json("ORIGIN_HTTP_RULES", default={})),
+}
 
 DLCS = {
     "api_root": env.url("DLCS_API_ROOT", default="https://api.dlcs.digirati.io/"),

src/tests/engine/test_origin_rules.py

@@ -0,0 +1,88 @@
+import pytest
+
+from app.engine.origin_rules import compile_rules, headers_for
+
+
+def test_empty_rules_returns_no_headers():
+    rules = compile_rules({})
+    assert headers_for(rules, 2, "https://example.com/file.pdf") == {}
+
+
+def test_matching_customer_and_url_returns_headers():
+    rules = compile_rules(
+        {
+            "2": {
+                "^https://fraser-staging.*": {
+                    "addHeader": {"Auth-Bypass-Key": "abc12345"}
+                }
+            }
+        }
+    )
+    assert headers_for(rules, "2", "https://fraser-staging.example.com/a.pdf") == {
+        "Auth-Bypass-Key": "abc12345"
+    }
+
+
+def test_int_customer_matches_string_json_key():
+    rules = compile_rules(
+        {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}}
+    )
+    assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == {
+        "Auth-Bypass-Key": "x"
+    }
+
+
+def test_non_matching_url_returns_no_headers():
+    rules = compile_rules(
+        {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}}
+    )
+    assert headers_for(rules, 2, "https://other.example.com/a.pdf") == {}
+
+
+def test_unknown_customer_returns_no_headers():
+    rules = compile_rules(
+        {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}}
+    )
+    assert headers_for(rules, 3, "https://fraser-staging.example.com/a.pdf") == {}
+
+
+def test_multiple_matches_merge_with_later_rules_winning():
+    rules = compile_rules(
+        {
+            "2": {
+                "^https://": {"addHeader": {"X-First": "1", "X-Shared": "first"}},
+                "^https://fraser-staging.*": {
+                    "addHeader": {"X-Second": "2", "X-Shared": "second"}
+                },
+            }
+        }
+    )
+    assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == {
+        "X-First": "1",
+        "X-Second": "2",
+        "X-Shared": "second",
+    }
+
+
+def test_unknown_operation_is_ignored_with_warning(caplog):
+    with caplog.at_level("WARNING"):
+        rules = compile_rules(
+            {"2": {"^https://.*": {"rewriteUrl": {"target": "somewhere"}}}}
+        )
+    assert headers_for(rules, 2, "https://example.com/a.pdf") == {}
+    assert "unknown operation rewriteUrl" in caplog.text
+
+
+def test_invalid_regex_raises_value_error():
+    with pytest.raises(ValueError, match="invalid regex"):
+        compile_rules({"2": {"^https://(unclosed": {"addHeader": {"X": "1"}}}})
+
+
+def test_non_dict_url_rules_raises_value_error():
+    with pytest.raises(ValueError, match="expected an object of URL regexes"):
+        compile_rules({"2": ["not-a-dict"]})
+
+
+def test_non_dict_operations_raises_value_error():
+    with pytest.raises(ValueError, match="expected an object of operations"):
+        compile_rules({"2": {"^https://.*": "not-a-dict"}})