From 2073e0a36ffa526a55565bb9ab48123240abed0e Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Wed, 29 Jul 2026 10:17:30 -0500 Subject: [PATCH 1/3] docs: add issue-backed project backlog --- BACKLOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 BACKLOG.md diff --git a/BACKLOG.md b/BACKLOG.md new file mode 100644 index 0000000..95a6363 --- /dev/null +++ b/BACKLOG.md @@ -0,0 +1,20 @@ +# OSApplyTrack Backlog + +This file mirrors the open +[GitHub Issues](https://github.com/CryptoJones/OSApplyTrack/issues) for the project. +Check an item only when its matching issue is closed. Deferred ideas in `README.md` +or `SPRINTS.md` are not committed backlog until they have a corresponding issue. + +## Security and stability + +- [ ] [#51 — Close the Python poller link-check DNS rebinding gap](https://github.com/CryptoJones/OSApplyTrack/issues/51) +- [ ] [#49 — Restrict forwarded-header trust to configured proxies](https://github.com/CryptoJones/OSApplyTrack/issues/49) +- [ ] [#50 — Add global JSON body caps and per-field/cardinality limits](https://github.com/CryptoJones/OSApplyTrack/issues/50) +- [ ] [#52 — Serialize overlapping tenant poll runs](https://github.com/CryptoJones/OSApplyTrack/issues/52) + +## Operations and scalability + +- [ ] [#53 — Add hardened production container defaults](https://github.com/CryptoJones/OSApplyTrack/issues/53) +- [ ] [#54 — Paginate or delta-refresh the applications list](https://github.com/CryptoJones/OSApplyTrack/issues/54) + +Proudly Made in Nebraska. Go Big Red! 🌽 https://xkcd.com/2347/ From c70658749e78edb84715783d83272b2cb202e823 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Wed, 29 Jul 2026 10:17:45 -0500 Subject: [PATCH 2/3] fix(poller): pin link checks to validated DNS addresses --- pyproject.toml | 1 + src/applytrack/linkcheck.py | 138 ++++++++++++++++++++++++++++++------ src/applytrack/poll.py | 4 +- tests/test_linkcheck.py | 88 ++++++++++++++++++++++- uv.lock | 4 +- 5 files changed, 210 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22182e7..ff4bad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license = { text = "Apache-2.0" } authors = [{ name = "Aaron K. Clark" }] dependencies = [ "pyyaml>=6.0", + "httpcore>=1.0,<2.0", "httpx>=0.27", "psycopg[binary]>=3.1", "defusedxml>=0.7", diff --git a/src/applytrack/linkcheck.py b/src/applytrack/linkcheck.py index c971380..becb204 100644 --- a/src/applytrack/linkcheck.py +++ b/src/applytrack/linkcheck.py @@ -11,9 +11,11 @@ import ipaddress import socket +from collections.abc import Iterable from dataclasses import dataclass from urllib.parse import urljoin, urlsplit +import httpcore import httpx # Identify as a current Windows 11 desktop Chrome. Windows 11 still reports @@ -91,30 +93,122 @@ def _ip_is_public(ip: str) -> bool: ) +def _normalize_host(host: str) -> str: + """Canonical key shared by URL parsing, DNS pinning, and httpcore.""" + return (host or "").strip().strip("[]").rstrip(".").casefold() + + +class _PinnedResolver: + """Resolve each hostname once, reject mixed/private answers, and retain one IP.""" + + def __init__(self) -> None: + self._addresses: dict[str, str] = {} + + def pin(self, host: str) -> bool: + key = _normalize_host(host) + if not key: + return False + if key in self._addresses: + return True + + try: + ipaddress.ip_address(key) + addresses = [key] + except ValueError: + try: + infos = socket.getaddrinfo(key, None, type=socket.SOCK_STREAM) + except OSError: + return False + addresses = list(dict.fromkeys(str(info[4][0]) for info in infos)) + + if not addresses or not all(_ip_is_public(address) for address in addresses): + return False + self._addresses[key] = addresses[0] + return True + + def address_for(self, host: str) -> str | None: + """Return only an address previously accepted by :meth:`pin`.""" + return self._addresses.get(_normalize_host(host)) + + +class _PinnedNetworkBackend(httpcore.SyncBackend): + """Connect httpcore's TCP socket to the pinned IP while preserving TLS SNI.""" + + def __init__(self, resolver: _PinnedResolver) -> None: + super().__init__() + self._resolver = resolver + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None, + ) -> httpcore.NetworkStream: + address = self._resolver.address_for(host) + if address is None: + raise httpcore.ConnectError(f"refused unpinned host: {host}") + return super().connect_tcp(address, port, timeout, local_address, socket_options) + + +class _PinnedTransport(httpx.HTTPTransport): + """HTTPX transport whose connection pool cannot perform a second DNS lookup.""" + + def __init__(self, resolver: _PinnedResolver) -> None: + super().__init__(trust_env=False) + self._pool.close() + self._pool = httpcore.ConnectionPool( + ssl_context=httpx.create_ssl_context(trust_env=False), + network_backend=_PinnedNetworkBackend(resolver), + ) + + +class _PinnedClient(httpx.Client): + """HTTP client that connects only to addresses pinned by its resolver.""" + + def __init__(self, *, timeout: float) -> None: + self._resolver = _PinnedResolver() + super().__init__( + timeout=timeout, + follow_redirects=False, + headers=BROWSER_HEADERS, + transport=_PinnedTransport(self._resolver), + trust_env=False, + ) + + def pin(self, host: str) -> bool: + return self._resolver.pin(host) + + +def ssrf_safe_client(*, timeout: float = 12.0) -> httpx.Client: + """Return a reusable client that pins every validated hostname to one public IP.""" + return _PinnedClient(timeout=timeout) + + def _host_is_public(host: str) -> bool: """True only when every address ``host`` resolves to is a public IP. The SSRF gate: a lead URL (or a redirect it chains to) is attacker-influenced, and the poller fetches it server-side, so a link pointing at ``localhost`` or an internal/metadata IP must be refused. An IP literal is checked directly; a - name is resolved and *all* of its A/AAAA records must be public, so a public - hostname that maps to an internal address is still rejected. (A determined - attacker could still race DNS between this check and the connect; closing that - fully needs connect-by-IP, out of scope for v1's self-host threat model.) + name is resolved and *all* of its A/AAAA records must be public. Production + requests additionally use :func:`ssrf_safe_client`, whose transport connects to + the selected validated IP without resolving the hostname again. """ - host = (host or "").strip().strip("[]") # tolerate bracketed IPv6 literals - if not host: - return False + return _PinnedResolver().pin(host) + + +def _pin_for_request(client: httpx.Client, host: str) -> bool: + """Fail closed when a caller supplies an ordinary client for a DNS hostname.""" + if isinstance(client, _PinnedClient): + return client.pin(host) + normalized = _normalize_host(host) try: - ipaddress.ip_address(host) - return _ip_is_public(host) + ipaddress.ip_address(normalized) except ValueError: - pass # not a literal — resolve it below - try: - infos = socket.getaddrinfo(host, None) - except socket.gaierror: return False - return bool(infos) and all(_ip_is_public(str(info[4][0])) for info in infos) + return _ip_is_public(normalized) def _is_home(url: str) -> bool: @@ -160,14 +254,12 @@ def probe( parts = urlsplit(url) if parts.scheme not in ("http", "https"): return LinkStatus(url=url, ok=False, error="not an http(s) URL") - if not _host_is_public(parts.hostname or ""): - return LinkStatus(url=url, ok=False, error="refused non-public address") owns_client = client is None - client = client or httpx.Client( - timeout=timeout, follow_redirects=False, headers=BROWSER_HEADERS - ) + client = client or ssrf_safe_client(timeout=timeout) try: + if not _pin_for_request(client, parts.hostname or ""): + return LinkStatus(url=url, ok=False, error="refused non-public or unpinned address") current = url try: for _ in range(_MAX_REDIRECTS + 1): @@ -181,8 +273,12 @@ def probe( hop = urlsplit(current) if hop.scheme not in ("http", "https"): return LinkStatus(url=url, ok=False, error="redirect to non-http(s) URL") - if not _host_is_public(hop.hostname or ""): - return LinkStatus(url=url, ok=False, error="redirect to non-public address") + if not _pin_for_request(client, hop.hostname or ""): + return LinkStatus( + url=url, + ok=False, + error="redirect to non-public or unpinned address", + ) else: return LinkStatus(url=url, ok=False, error="too many redirects") except httpx.HTTPError as exc: diff --git a/src/applytrack/poll.py b/src/applytrack/poll.py index 16dd079..2bebe93 100644 --- a/src/applytrack/poll.py +++ b/src/applytrack/poll.py @@ -46,7 +46,7 @@ import psycopg from applytrack.criteria import AtsBoard, Criteria -from applytrack.linkcheck import BROWSER_HEADERS, is_reachable +from applytrack.linkcheck import BROWSER_HEADERS, is_reachable, ssrf_safe_client from applytrack.store import AppFields logger = logging.getLogger(__name__) @@ -759,7 +759,7 @@ def score_and_stage( blacklist = Blacklist({_norm_company(c) for c in repo.blacklist_companies()}) verify_client = ( - httpx.Client(timeout=12.0, follow_redirects=True, headers=BROWSER_HEADERS) + ssrf_safe_client(timeout=12.0) if verify_links else None ) diff --git a/tests/test_linkcheck.py b/tests/test_linkcheck.py index e442653..9e74275 100644 --- a/tests/test_linkcheck.py +++ b/tests/test_linkcheck.py @@ -8,11 +8,19 @@ from __future__ import annotations +import socket from collections.abc import Callable +import httpcore import httpx +import pytest -from applytrack.linkcheck import _host_is_public, probe +from applytrack.linkcheck import ( + _host_is_public, + _PinnedNetworkBackend, + _PinnedResolver, + probe, +) Handler = Callable[[httpx.Request], httpx.Response] @@ -81,3 +89,81 @@ def handler(request: httpx.Request) -> httpx.Response: status = probe("http://1.1.1.1/old", client=_client(handler)) assert status.ok is True assert status.final_url == "http://1.0.0.1/jobs/new-67890" + + +def test_dns_answer_is_validated_once_then_tcp_uses_the_pinned_ip( + monkeypatch: pytest.MonkeyPatch, +) -> None: + dns_calls: list[str] = [] + + def fake_getaddrinfo( + host: str, + port: int | None, + *, + type: socket.SocketKind, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + del port, type + dns_calls.append(host) + address = "93.184.216.34" if len(dns_calls) == 1 else "127.0.0.1" + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 0))] + + connected: list[str] = [] + + def fake_connect( + self: httpcore.SyncBackend, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: list[tuple[int, int, int | bytes]] | None = None, + ) -> object: + del self, port, timeout, local_address, socket_options + connected.append(host) + return object() + + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) + monkeypatch.setattr(httpcore.SyncBackend, "connect_tcp", fake_connect) + + resolver = _PinnedResolver() + assert resolver.pin("jobs.example") is True + assert resolver.pin("jobs.example") is True + backend = _PinnedNetworkBackend(resolver) + backend.connect_tcp("jobs.example", 443) + + assert dns_calls == ["jobs.example"] + assert connected == ["93.184.216.34"] + + +def test_resolver_rejects_a_hostname_with_any_private_answer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_getaddrinfo( + host: str, + port: int | None, + *, + type: socket.SocketKind, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + del host, port, type + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.8", 0)), + ] + + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) + assert _PinnedResolver().pin("mixed.example") is False + + +def test_injected_ordinary_client_cannot_bypass_pinning_for_a_hostname() -> None: + touched = False + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal touched + touched = True + return httpx.Response(200, request=request) + + with _client(handler) as client: + status = probe("https://jobs.example/posting/123", client=client) + + assert status.ok is False + assert "unpinned" in status.error + assert touched is False diff --git a/uv.lock b/uv.lock index a9bb558..0a3896c 100644 --- a/uv.lock +++ b/uv.lock @@ -27,6 +27,7 @@ version = "1.11.1" source = { editable = "." } dependencies = [ { name = "defusedxml" }, + { name = "httpcore" }, { name = "httpx" }, { name = "psycopg", extra = ["binary"] }, { name = "pyyaml" }, @@ -45,6 +46,7 @@ dev = [ requires-dist = [ { name = "bandit", marker = "extra == 'dev'", specifier = ">=1.7" }, { name = "defusedxml", specifier = ">=0.7" }, + { name = "httpcore", specifier = ">=1.0,<2.0" }, { name = "httpx", specifier = ">=0.27" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.1" }, @@ -143,7 +145,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ From 5e43e839631eaa0cf7dd2b347949ea085a52cf90 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Wed, 29 Jul 2026 10:24:08 -0500 Subject: [PATCH 3/3] chore: release v1.11.2 --- BACKLOG.md | 2 +- api/ApplyTrack.Api/ApplyTrack.Api.csproj | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 95a6363..ec7b160 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -7,7 +7,7 @@ or `SPRINTS.md` are not committed backlog until they have a corresponding issue. ## Security and stability -- [ ] [#51 — Close the Python poller link-check DNS rebinding gap](https://github.com/CryptoJones/OSApplyTrack/issues/51) +- [x] [#51 — Close the Python poller link-check DNS rebinding gap](https://github.com/CryptoJones/OSApplyTrack/issues/51) - [ ] [#49 — Restrict forwarded-header trust to configured proxies](https://github.com/CryptoJones/OSApplyTrack/issues/49) - [ ] [#50 — Add global JSON body caps and per-field/cardinality limits](https://github.com/CryptoJones/OSApplyTrack/issues/50) - [ ] [#52 — Serialize overlapping tenant poll runs](https://github.com/CryptoJones/OSApplyTrack/issues/52) diff --git a/api/ApplyTrack.Api/ApplyTrack.Api.csproj b/api/ApplyTrack.Api/ApplyTrack.Api.csproj index 63e6e78..2ab5976 100644 --- a/api/ApplyTrack.Api/ApplyTrack.Api.csproj +++ b/api/ApplyTrack.Api/ApplyTrack.Api.csproj @@ -5,7 +5,7 @@ enable enable ApplyTrack.Api - 1.11.1 + 1.11.2 Aaron K. Clark Copyright 2026 Aaron K. Clark Apache-2.0 diff --git a/pyproject.toml b/pyproject.toml index ff4bad2..d0db01b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "applytrack-poller" -version = "1.11.1" +version = "1.11.2" description = "Discovery poller for OSApplyTrack — fetches and scores remote job leads into shared Postgres." requires-python = ">=3.10" license = { text = "Apache-2.0" } diff --git a/uv.lock b/uv.lock index 0a3896c..9e93a55 100644 --- a/uv.lock +++ b/uv.lock @@ -23,7 +23,7 @@ wheels = [ [[package]] name = "applytrack-poller" -version = "1.11.1" +version = "1.11.2" source = { editable = "." } dependencies = [ { name = "defusedxml" },