diff --git a/.env.dist b/.env.dist index 323f21f..38b9fb3 100644 --- a/.env.dist +++ b/.env.dist @@ -9,6 +9,10 @@ WEB_SERVER_HOSTNAME=localhost:8000 # Origin Fetching ORIGIN_CHUNK_SIZE=8192 +# Optional JSON rules for injecting headers into origin fetches, keyed by +# customer id then URL regex. Delivered as a secret in real deployments. +# Keep single-quoted and on one line here; see README for escaping notes. +# ORIGIN_HTTP_RULES='{"2":{"^https://customer-url.*":{"addHeader":{"X-ApiKey":"abc12345"}}}}' # Database DATABASE_URL=postgresql://dlcs:password@postgres:5432/compositedb diff --git a/Dockerfile b/Dockerfile index 31b1b02..9c8b088 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get --yes install apt-utils && apt-get --yes upgrade \ && apt-get --yes autoremove && apt-get --yes autoclean && apt-get --yes clean \ && useradd --create-home --home-dir /srv/dlcs --shell /bin/bash --uid 1000 dlcs \ && python -m pip install --upgrade pip \ - && python -m pip install --upgrade setuptools + && python -m pip install --upgrade "setuptools<82" # Copy nginx config and create appropriate folders COPY --chown=dlcs:dlcs ./nginx.conf /etc/nginx/nginx.conf diff --git a/README.md b/README.md index 874284c..3e61925 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ The following list of environment variables are supported: | `WEB_SERVER_SCHEME` | `http` | API | The HTTP scheme used when generating URI's. | | `WEB_SERVER_HOSTNAME` | `localhost:8000` | API | The hostname (and optional port) used when generating URI's. | | `ORIGIN_CHUNK_SIZE` | `8192` | Engine | The chunk size, in bytes, used when retrieving objects from origins. Tailoring this value can theoretically improve download speeds. | +| `ORIGIN_HTTP_RULES` | `{}` | Engine | Optional JSON of request mutations applied when fetching origins, keyed by customer id then by regex matched against the origin URL. The only current operation is `addHeader`. Values are typically secrets and should be injected accordingly. When multiple regexes match, headers are merged with later entries winning. Example: `{"2":{"^https://fraser-staging.*":{"addHeader":{"Auth-Bypass-Key":"..."}}}}`. | | `DATABASE_URL` | None | API, Engine | The URL of the target PostgreSQL database, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `postgresql://dlcs:password@postgres:5432/compositedb`. | | `CACHE_URL` | None | API, Engine | The URL of the target cache, in a format acceptable to [django-environ](https://django-environ.readthedocs.io/en/latest/getting-started.html#usage), e.g. `dbcache://app_cache`. | | `PDF_RASTERIZER_THREAD_COUNT` | `3` | Engine | The number of concurrent [Poppler](https://poppler.freedesktop.org/) threads spawned when a worker is rasterizing a PDF. Each thread typically consumes 100% of a CPU core. | @@ -94,6 +95,13 @@ The following list of environment variables are supported: | `GUNICORN_WORKERS` | `2` | API | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn | | `SQS_BROKER_QUEUE_NAME` | None | API, Engine | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used | +> [!NOTE] +> `ORIGIN_HTTP_RULES` is safe to inject verbatim as an environment variable (e.g. from a secret) — no escaping is needed at that level. However: +> +> * Backslashes in regexes must be doubled in the JSON, as JSON string escaping consumes one level: write `"^https://customer-url\\.com.*"` to match a literal dot. +> * When set in a `.env` file, wrap the whole value in **single** quotes and keep it on one line — unquoted values are truncated at the first `#`, and double quotes trigger backslash-unescaping that corrupts regex escapes. +> * In a docker-compose `environment:` block, escape any `$` (e.g. a regex end anchor) as `$$` to avoid compose variable interpolation. + Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials. ### Django Q Broker diff --git a/src/app/engine/origin_rules.py b/src/app/engine/origin_rules.py new file mode 100644 index 0000000..89a3db5 --- /dev/null +++ b/src/app/engine/origin_rules.py @@ -0,0 +1,60 @@ +import logging +import re + +logger = logging.getLogger(__name__) + +KNOWN_OPERATIONS = {"addHeader"} + + +def compile_rules(raw_rules): + """Validate ORIGIN_HTTP_RULES config and precompile its regexes. + + Input shape: {customer_id: {url_regex: {operation: {...}}}} + Output: {customer_id (str): [(compiled_regex, {operation: {...}})]} + + Raises ValueError on malformed structure or invalid regex so that a bad + configuration fails at startup rather than silently dropping headers. + """ + compiled = {} + for customer, url_rules in raw_rules.items(): + if not isinstance(url_rules, dict): + raise ValueError( + f"ORIGIN_HTTP_RULES: expected an object of URL regexes " + f"for customer {customer}, got {type(url_rules).__name__}" + ) + entries = [] + for pattern, operations in url_rules.items(): + try: + regex = re.compile(pattern) + except re.error as error: + raise ValueError( + f"ORIGIN_HTTP_RULES: invalid regex {pattern!r} " + f"for customer {customer}: {error}" + ) + if not isinstance(operations, dict): + raise ValueError( + f"ORIGIN_HTTP_RULES: expected an object of operations " + f"for customer {customer}, regex {pattern!r}, " + f"got {type(operations).__name__}" + ) + for operation in operations: + if operation not in KNOWN_OPERATIONS: + logger.warning( + f"ORIGIN_HTTP_RULES: ignoring unknown operation {operation} for customer {customer}" + ) + entries.append((regex, operations)) + compiled[str(customer)] = entries + return compiled + + +def headers_for(rules, customer, url): + """Return the extra headers to apply for this customer/origin URL. + + Headers from all matching rules are merged; later rules override earlier + ones on header-name conflict. Returns {} when nothing matches. + """ + headers = {} + for regex, operations in rules.get(str(customer), []): + if regex.search(url): + headers.update(operations.get("addHeader", {})) + return headers diff --git a/src/app/engine/origins.py b/src/app/engine/origins.py index 527b598..48e5af3 100644 --- a/src/app/engine/origins.py +++ b/src/app/engine/origins.py @@ -1,19 +1,30 @@ +import logging import os from pathlib import Path import requests from django.conf import settings +from app.engine.origin_rules import headers_for + +logger = logging.getLogger(__name__) + class HttpOrigin: def __init__(self): self._scratch_path = settings.SCRATCH_DIRECTORY self._chunk_size = settings.ORIGIN_CONFIG["chunk_size"] + self._http_rules = settings.ORIGIN_CONFIG["http_rules"] - def fetch(self, submission_id, url, file_extension="pdf"): + def fetch(self, submission_id, url, customer, file_extension="pdf"): subfolder_path = self.__generate_subfolder_path(submission_id) file_path = os.path.join(subfolder_path, "source." + file_extension) - with requests.get(url, stream=True) as response: + headers = headers_for(self._http_rules, customer, url) + if headers: + logger.info( + f"Applying custom origin headers {sorted(headers)} for submission {submission_id}", + ) + with requests.get(url, stream=True, headers=headers or None) as response: response.raise_for_status() with open(file_path, "wb") as file: for chunk in response.iter_content(chunk_size=self._chunk_size): diff --git a/src/app/engine/tasks.py b/src/app/engine/tasks.py index 3ae7901..27c40ee 100644 --- a/src/app/engine/tasks.py +++ b/src/app/engine/tasks.py @@ -41,7 +41,7 @@ def process_member(args): def __fetch_origin(member, origin_uri): __update_status(member, "FETCHING_ORIGIN") - return http_origin.fetch(member.id, origin_uri) + return http_origin.fetch(member.id, origin_uri, member.collection.customer) def __rasterize_composite(member, pdf_path): diff --git a/src/app/settings.py b/src/app/settings.py index 1ae1cb8..52799d5 100644 --- a/src/app/settings.py +++ b/src/app/settings.py @@ -14,6 +14,8 @@ import environ +from app.engine.origin_rules import compile_rules + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -180,7 +182,10 @@ "use_cropbox": env("PDF_RASTERIZER_USE_CROPBOX", cast=bool, default=False), } -ORIGIN_CONFIG = {"chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192)} +ORIGIN_CONFIG = { + "chunk_size": env("ORIGIN_CHUNK_SIZE", cast=int, default=8192), + "http_rules": compile_rules(env.json("ORIGIN_HTTP_RULES", default={})), +} DLCS = { "api_root": env.url("DLCS_API_ROOT", default="https://api.dlcs.digirati.io/"), diff --git a/src/tests/engine/test_origin_rules.py b/src/tests/engine/test_origin_rules.py new file mode 100644 index 0000000..02a4b53 --- /dev/null +++ b/src/tests/engine/test_origin_rules.py @@ -0,0 +1,88 @@ +import pytest + +from app.engine.origin_rules import compile_rules, headers_for + + +def test_empty_rules_returns_no_headers(): + rules = compile_rules({}) + assert headers_for(rules, 2, "https://example.com/file.pdf") == {} + + +def test_matching_customer_and_url_returns_headers(): + rules = compile_rules( + { + "2": { + "^https://fraser-staging.*": { + "addHeader": {"Auth-Bypass-Key": "abc12345"} + } + } + } + ) + assert headers_for(rules, "2", "https://fraser-staging.example.com/a.pdf") == { + "Auth-Bypass-Key": "abc12345" + } + + +def test_int_customer_matches_string_json_key(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == { + "Auth-Bypass-Key": "x" + } + + +def test_non_matching_url_returns_no_headers(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 2, "https://other.example.com/a.pdf") == {} + + +def test_unknown_customer_returns_no_headers(): + rules = compile_rules( + {"2": {"^https://fraser-staging.*": {"addHeader": {"Auth-Bypass-Key": "x"}}}} + ) + assert headers_for(rules, 3, "https://fraser-staging.example.com/a.pdf") == {} + + +def test_multiple_matches_merge_with_later_rules_winning(): + rules = compile_rules( + { + "2": { + "^https://": {"addHeader": {"X-First": "1", "X-Shared": "first"}}, + "^https://fraser-staging.*": { + "addHeader": {"X-Second": "2", "X-Shared": "second"} + }, + } + } + ) + assert headers_for(rules, 2, "https://fraser-staging.example.com/a.pdf") == { + "X-First": "1", + "X-Second": "2", + "X-Shared": "second", + } + + +def test_unknown_operation_is_ignored_with_warning(caplog): + with caplog.at_level("WARNING"): + rules = compile_rules( + {"2": {"^https://.*": {"rewriteUrl": {"target": "somewhere"}}}} + ) + assert headers_for(rules, 2, "https://example.com/a.pdf") == {} + assert "unknown operation rewriteUrl" in caplog.text + + +def test_invalid_regex_raises_value_error(): + with pytest.raises(ValueError, match="invalid regex"): + compile_rules({"2": {"^https://(unclosed": {"addHeader": {"X": "1"}}}}) + + +def test_non_dict_url_rules_raises_value_error(): + with pytest.raises(ValueError, match="expected an object of URL regexes"): + compile_rules({"2": ["not-a-dict"]}) + + +def test_non_dict_operations_raises_value_error(): + with pytest.raises(ValueError, match="expected an object of operations"): + compile_rules({"2": {"^https://.*": "not-a-dict"}})