diff --git a/.github/scripts/build_function_package.py b/.github/scripts/build_function_package.py new file mode 100644 index 00000000..3d776d5d --- /dev/null +++ b/.github/scripts/build_function_package.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Build a build-once Azure Functions package against the HASTE GDAL wheels.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile +from email.parser import BytesParser +from pathlib import Path + +RUNTIME_BINDINGS = { + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + "haste-gdal-runtime", +} +FUNCTION_APP_MODULES = { + "hastefuncapi": "function_app", + "hastefuncqueues": "function_app", + "titilerfuncapi": "app", + "imageryprep-smoke": "hastegeo.workflows.prepare_imagery", +} +HASTEGEO_CONSUMERS = {"hastefuncapi", "hastefuncqueues"} + + +def normalized_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def wheel_metadata(path: Path) -> tuple[str, str]: + with zipfile.ZipFile(path) as wheel: + metadata_files = [ + name + for name in wheel.namelist() + if name.count("/") == 1 and name.endswith(".dist-info/METADATA") + ] + if len(metadata_files) != 1: + raise ValueError(f"{path.name}: expected one wheel METADATA file") + metadata = BytesParser().parsebytes(wheel.read(metadata_files[0])) + name = metadata.get("Name") + version = metadata.get("Version") + if not name or not version: + raise ValueError(f"{path.name}: missing distribution name or version") + return normalized_name(name), version + + +def runtime_wheel_requirements(wheel_dir: Path) -> list[str]: + wheels = sorted(wheel_dir.glob("*.whl")) + if len(wheels) != 6: + raise ValueError(f"expected six runtime wheels, found {len(wheels)}") + requirements: dict[str, str] = {} + for wheel in wheels: + name, version = wheel_metadata(wheel) + if name in requirements: + raise ValueError(f"duplicate runtime wheel for {name}") + requirements[name] = version + if set(requirements) != RUNTIME_BINDINGS: + raise ValueError( + "runtime wheel set mismatch: " + f"missing={sorted(RUNTIME_BINDINGS - set(requirements))}, " + f"unexpected={sorted(set(requirements) - RUNTIME_BINDINGS)}" + ) + return [f"{name}=={requirements[name]}" for name in sorted(requirements)] + + +def _requirement_name(line: str) -> str | None: + content = line.strip() + if not content or content.startswith("#") or content.startswith("-"): + return None + match = re.match(r"([A-Za-z0-9_.-]+)(?=\s|[<=>!~;\[]|$)", content) + return normalized_name(match.group(1)) if match else None + + +def prepare_requirements( + source: Path, + output: Path, + wheel_dir: Path, + hastegeo_wheel: Path | None, +) -> None: + """Replace legacy GDAL bindings and editable hastegeo with the built wheels.""" + lines = source.read_text(encoding="utf-8").splitlines() + retained = [] + for line in lines: + name = _requirement_name(line) + if name in RUNTIME_BINDINGS or ( + name == "hastegeo" or line.strip().startswith("-e ") + ): + continue + retained.append(line) + additions = runtime_wheel_requirements(wheel_dir) + if hastegeo_wheel is not None: + additions.append(f"hastegeo @ {hastegeo_wheel.resolve().as_uri()}") + output.write_text( + "\n".join( + [ + *retained, + "", + "# Exact wheel versions from this build; the lock records hashes.", + *additions, + "", + ] + ), + encoding="utf-8", + ) + + +def _pip_index_arguments() -> list[str]: + index_url = os.environ.get("PIP_INDEX_URL", "https://pypi.org/simple") + return ["--index-url", index_url] + + +def write_lock(wheelhouse: Path, path: Path) -> None: + entries: dict[str, tuple[str, str]] = {} + for wheel in sorted(wheelhouse.glob("*.whl")): + name, version = wheel_metadata(wheel) + if name in entries: + raise ValueError(f"wheelhouse has duplicate distributions: {name}") + entries[name] = (version, sha256(wheel)) + if not entries: + raise ValueError("dependency resolver produced an empty wheelhouse") + path.write_text( + "".join( + f"{name}=={version} --hash=sha256:{digest}\n" + for name, (version, digest) in sorted(entries.items()) + ), + encoding="utf-8", + ) + + +def _copy_function_source(source: Path, target: Path) -> None: + ignored = shutil.ignore_patterns( + "__pycache__", + "*.pyc", + ".pytest_cache", + ".venv", + "venv", + ".python_packages", + ) + shutil.copytree(source, target, ignore=ignored, dirs_exist_ok=True) + + +def _write_zip(root: Path, destination: Path) -> None: + with zipfile.ZipFile( + destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6 + ) as archive: + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + relative = path.relative_to(root).as_posix() + info = zipfile.ZipInfo(relative, date_time=(2020, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + archive.writestr(info, path.read_bytes()) + + +def build_package( + *, + app_name: str, + source_dir: Path, + requirements: Path, + wheel_dir: Path, + hastegeo_wheel: Path | None, + output_dir: Path, +) -> Path: + if app_name not in FUNCTION_APP_MODULES: + raise ValueError(f"unsupported Function App: {app_name}") + if (app_name in HASTEGEO_CONSUMERS) != (hastegeo_wheel is not None): + raise ValueError( + f"{app_name} requires hastegeo wheel input " + f"={app_name in HASTEGEO_CONSUMERS}" + ) + if not source_dir.is_dir() or not requirements.is_file(): + raise FileNotFoundError( + "Function source or requirements file is missing" + ) + output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f"haste-{app_name}-") as temporary: + root = Path(temporary) + wheelhouse = root / "wheelhouse" + wheelhouse.mkdir() + target = root / ".python_packages" / "lib" / "site-packages" + target.mkdir(parents=True) + generated_requirements = root / "requirements.in" + lock_file = root / "requirements.lock" + prepare_requirements( + requirements, generated_requirements, wheel_dir, hastegeo_wheel + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--isolated", + *_pip_index_arguments(), + "--only-binary=:all:", + "--find-links", + str(wheel_dir.resolve()), + "--dest", + str(wheelhouse), + "-r", + str(generated_requirements), + ], + check=True, + ) + write_lock(wheelhouse, lock_file) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--isolated", + "--no-index", + "--find-links", + str(wheelhouse), + "--only-binary=:all:", + "--require-hashes", + "--target", + str(target), + "-r", + str(lock_file), + ], + check=True, + ) + + environment = os.environ.copy() + environment["PYTHONPATH"] = str(target) + environment["HASTE_GDAL_RUNTIME_REQUIRED"] = "true" + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "check", + ], + check=True, + env=environment, + ) + application_environment = environment.copy() + application_environment["PYTHONPATH"] = os.pathsep.join( + [str(target), str(source_dir.resolve())] + ) + application_environment.setdefault("DATA_PATH", str(root / "data")) + subprocess.run( + [ + sys.executable, + "-c", + f"import {FUNCTION_APP_MODULES[app_name]}", + ], + check=True, + cwd=source_dir, + env=application_environment, + ) + subprocess.run( + [ + sys.executable, + "-c", + "import haste_gdal_runtime; " + "policy = haste_gdal_runtime.Policy(" + "expected_gdal='3.13.3', " + "required_bindings=('osgeo.gdal','rasterio','fiona'," + "'pyogrio','pyproj')); " + "haste_gdal_runtime.configure(policy); " + "print(haste_gdal_runtime.secure_startup(policy))", + ], + check=True, + env=environment, + ) + + package_root = root / "package" + package_root.mkdir() + _copy_function_source(source_dir, package_root) + destination_packages = ( + package_root / ".python_packages" / "lib" / "site-packages" + ) + destination_packages.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(target, destination_packages) + shutil.copyfile(lock_file, package_root / "requirements.lock") + shutil.copyfile(lock_file, package_root / "requirements.txt") + shutil.copyfile( + wheel_dir / "manifest.json", + package_root / "gdal-runtime-manifest.json", + ) + archive_path = output_dir / f"{app_name}.zip" + _write_zip(package_root, archive_path) + (output_dir / f"{app_name}.sha256").write_text( + f"{sha256(archive_path)} {archive_path.name}\n", + encoding="utf-8", + ) + return archive_path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--app-name", required=True) + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--requirements", required=True, type=Path) + parser.add_argument("--wheel-dir", required=True, type=Path) + parser.add_argument("--hastegeo-wheel", type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args(argv) + archive = build_package( + app_name=args.app_name, + source_dir=args.source_dir, + requirements=args.requirements, + wheel_dir=args.wheel_dir, + hastegeo_wheel=args.hastegeo_wheel, + output_dir=args.output_dir, + ) + print(f"Built {archive} ({sha256(archive)})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/check_env_drift.py b/.github/scripts/check_env_drift.py index 2bcc575a..82a40898 100644 --- a/.github/scripts/check_env_drift.py +++ b/.github/scripts/check_env_drift.py @@ -99,6 +99,12 @@ "GDAL_SKIP": "read inside the task container", "GDAL_WARP_PARAMS": "read inside the task container", "GDAL_TRANSLATE_PARAMS": "read inside the task container", + # Read by the native GDAL/CPL libraries rather than Python code. + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED": "consumed by GDAL's CPL runtime", + "GDAL_DRIVER_PATH": "consumed by GDAL's driver manager", + "GDAL_DISABLE_READDIR_ON_OPEN": "consumed by GDAL's directory scanner", + "GDAL_VRT_ENABLE_PYTHON": "consumed by GDAL's VRT driver", + "GDAL_VRT_ENABLE_RAWRASTERBAND": "consumed by GDAL's VRT driver", } diff --git a/.github/scripts/check_gdal_hardening.py b/.github/scripts/check_gdal_hardening.py new file mode 100644 index 00000000..a792bb79 --- /dev/null +++ b/.github/scripts/check_gdal_hardening.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Reject configuration that weakens HASTE GDAL runtime protections.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCAN_ROOTS = ("infra", ".github/workflows", ".github/scripts", "api", "docker") +# This checker contains inert GDAL setting names in an environment drift allowlist. +POLICY_ONLY_FILES = { + REPO_ROOT / ".github" / "scripts" / "check_env_drift.py", +} +SCAN_SUFFIXES = { + "", + ".bicep", + ".env", + ".ini", + ".json", + ".py", + ".sh", + ".toml", + ".yml", + ".yaml", +} + +_ASSIGNMENT = r"""(?ix) + (?: + (?:export|ENV)\s+ + | + os\.environ(?:\.get)?\s*\[\s* + )? + ["']?{key}["']? + \s*\]?\s*[:=]\s*["']?{value}\b +""" +_CALL = r"""(?ix) + SetConfigOption\s*\(\s*["']{key}["']\s*,\s*["']?{value}\b +""" +_RECORD = r"""(?ix) + ["']?name["']?\s*:\s*["']{key}["']\s*,\s* + ["']?value["']?\s*:\s*["']?{value}\b +""" +FORBIDDEN_VALUES = { + "GDAL_VRT_ENABLE_RAWRASTERBAND": r"(?:YES|TRUE|ON|1)", + "GDAL_VRT_ENABLE_PYTHON": r"(?:YES|TRUE|ON|1)", + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE": r"ALL", + "GDAL_VRT_PYTHON_TRUSTED_MODULES": r"[^'\"\s][^)]*", + "GDAL_PYTHON_DRIVER_PATH": r"[^'\"\s][^)]*", + "GDAL_DRIVER_PATH": r"(?!disable\b)[^'\"\s][^)]*", + "GDAL_DISABLE_READDIR_ON_OPEN": ( + r"(?!EMPTY_DIR\b)[^'\"\s][^)]*" + ), + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED": ( + r"(?:YES|TRUE|ON|1|ONLY_IN_TEMP)" + ), + "CPL_ENABLE_PATH_TRAVERSAL_DETECTION": r"(?:NO|FALSE|OFF|0)", + "GDAL_HTTP_UNSAFESSL": r"(?:YES|TRUE|ON|1)", +} + + +def _patterns() -> list[tuple[str, re.Pattern[str]]]: + patterns = [] + for key, value in FORBIDDEN_VALUES.items(): + patterns.append( + (key, re.compile(_ASSIGNMENT.format(key=key, value=value))) + ) + patterns.append((key, re.compile(_CALL.format(key=key, value=value)))) + return patterns + + +PATTERNS = _patterns() +# Name/value records (Bicep objects, ARM JSON) may span several lines. +RECORD_PATTERNS = [ + (key, re.compile(_RECORD.format(key=key, value=value))) + for key, value in FORBIDDEN_VALUES.items() +] + + +def _candidate_files(root: Path) -> list[Path]: + files = [] + for relative_root in SCAN_ROOTS: + directory = root / relative_root + if not directory.exists(): + continue + for path in directory.rglob("*"): + if not path.is_file() or ( + path.suffix.lower() not in SCAN_SUFFIXES + and not path.name.startswith("Dockerfile") + ): + continue + if path == Path(__file__).resolve() or path in POLICY_ONLY_FILES: + continue + if ( + any( + part == "tests" or part.startswith("test_") + for part in path.parts + ) + or "__pycache__" in path.parts + ): + continue + files.append(path) + return sorted(set(files)) + + +def find_violations(paths: list[Path]) -> list[str]: + violations = [] + for path in paths: + try: + content = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for line_number, line in enumerate(content.splitlines(), start=1): + for key, pattern in PATTERNS: + if pattern.search(line): + violations.append( + f"{path}:{line_number}: forbidden {key} override" + ) + for key, pattern in RECORD_PATTERNS: + for match in pattern.finditer(content): + line_number = content.count("\n", 0, match.start()) + 1 + violations.append( + f"{path}:{line_number}: forbidden {key} override" + ) + return violations + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help="specific files to scan (default: protected configuration roots)", + ) + args = parser.parse_args(argv) + paths = args.paths or _candidate_files(REPO_ROOT) + violations = find_violations( + [path if path.is_absolute() else REPO_ROOT / path for path in paths] + ) + if violations: + print("\n".join(violations), file=sys.stderr) + return 1 + print( + f"GDAL hardening configuration is valid ({len(paths)} files scanned)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/deploy_apps.sh b/.github/scripts/deploy_apps.sh index 57460cea..b5dddcc9 100644 --- a/.github/scripts/deploy_apps.sh +++ b/.github/scripts/deploy_apps.sh @@ -90,6 +90,7 @@ DEPLOY_RULE_PREFIX="haste-ci-" DEPLOY_RULE_NAME="${DEPLOY_RULE_PREFIX}${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}" DEPLOY_RUNNER_IP="" DEPLOY_ACCESS_APPS=() +GDAL_RUNTIME_PENDING_APPS=() FIXED_TAGS="project=haste created_by=deploy_apps" DYNAMIC_TAGS="env=${ENVIRONMENT} deployed_version=${APP_TAG}" EMAIL_SENDER="DoNotReply@notifications.${STATIC_APP_DOMAIN}" @@ -187,6 +188,18 @@ forget_deploy_access_app() { DEPLOY_ACCESS_APPS=("${KEPT[@]}") } +forget_gdal_runtime_pending_app() { + local APP=$1 + local APP_PENDING + local KEPT=() + for APP_PENDING in "${GDAL_RUNTIME_PENDING_APPS[@]}"; do + if [ "$APP_PENDING" != "$APP" ]; then + KEPT+=("$APP_PENDING") + fi + done + GDAL_RUNTIME_PENDING_APPS=("${KEPT[@]}") +} + # Rule updates can fail transiently (a concurrent change to the site config, # throttling), so a removal is retried before it counts as failed. retry_remove_deploy_access() { @@ -223,6 +236,23 @@ close_deploy_access() { cleanup_deploy_access() { local STATUS=$? local FUNCTION_NAME + for FUNCTION_NAME in "${GDAL_RUNTIME_PENDING_APPS[@]}"; do + if ! az functionapp config appsettings set \ + --name "$FUNCTION_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --settings "HASTE_GDAL_RUNTIME_REQUIRED=true" \ + --output none; then + echo "ERROR: Could not restore fail-closed GDAL runtime enforcement on '$FUNCTION_NAME'." >&2 + STATUS=1 + continue + fi + if ! az functionapp restart \ + --name "$FUNCTION_NAME" \ + --resource-group "$RESOURCE_GROUP"; then + echo "ERROR: Could not restart '$FUNCTION_NAME' after restoring GDAL runtime enforcement." >&2 + STATUS=1 + fi + done for FUNCTION_NAME in "${DEPLOY_ACCESS_APPS[@]}"; do if ! retry_remove_deploy_access "$FUNCTION_NAME"; then echo "ERROR: Could not remove the temporary deploy rules from '$FUNCTION_NAME'. Remove its access rules named ${DEPLOY_RULE_PREFIX}* by hand." >&2 @@ -297,6 +327,21 @@ deploy_function() { # publishing reconciler TimerTrigger's host-scoped Singleton lock does not # collide across apps sharing storage. See data-publishing/rollout.md. HOST_ID=$(printf '%s' "$FUNCTION_NAME" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | cut -c1-32) + local gdal_settings=( + # Keep the currently running worker available while the replacement + # package is built and published. Fail closed after publish succeeds. + "HASTE_GDAL_RUNTIME_REQUIRED=false" + "GDAL_VRT_ENABLE_RAWRASTERBAND=NO" + "GDAL_VRT_ENABLE_PYTHON=NO" + "GDAL_DRIVER_PATH=disable" + "GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR" + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO" + ) + if [ "$FUNCTION_NAME" = "$FUNCTION_TITILER_API" ]; then + gdal_settings+=( + "GDAL_SKIP=HDF4 HDF4Image HDF5 HDF5Image netCDF" + ) + fi if [ "${SET_APPSETTINGS:-true}" = "true" ]; then local size_settings=() @@ -321,8 +366,10 @@ deploy_function() { exit 1 } + GDAL_RUNTIME_PENDING_APPS+=("$FUNCTION_NAME") az functionapp config appsettings set --name "$FUNCTION_NAME" --resource-group "$RESOURCE_GROUP" --settings \ "${size_settings[@]}" \ + "${gdal_settings[@]}" \ "env=${ENVIRONMENT}" \ "IMAGE_QUEUE_NAME=image-layers-queue" \ "INFERENCE_QUEUE_NAME=inference-queue" \ @@ -377,6 +424,11 @@ deploy_function() { "PUBLISH_EXPLORER_RENDER_ENABLED=${PUBLISH_EXPLORER_RENDER_ENABLED}" \ "AzureFunctionsWebHost__hostId=${HOST_ID}" \ --output none + else + GDAL_RUNTIME_PENDING_APPS+=("$FUNCTION_NAME") + az functionapp config appsettings set --name "$FUNCTION_NAME" --resource-group "$RESOURCE_GROUP" --settings \ + "${gdal_settings[@]}" \ + --output none fi # TiTiler reads HASTE imagery only from this storage host (plus its @@ -403,18 +455,58 @@ deploy_function() { echo "Deploying function code..." open_deploy_access "$FUNCTION_NAME" - # A non-zero exit can represent a package/build/deployment failure. Do not - # convert it into a success-shaped warning: set -e propagates the failure. - ( - cd "$FUNCTION_DIR" - func azure functionapp publish "$FUNCTION_NAME" \ - --python \ - --build remote \ - --verbose - ) + local PACKAGE_SHA256="" + if [ -n "${HASTE_FUNCTION_PACKAGE_DIR:-}" ]; then + local PACKAGE_NAME PACKAGE_PATH CHECKSUM_PATH + case "$FUNCTION_NAME" in + "$FUNCTION_API") PACKAGE_NAME="hastefuncapi" ;; + "$FUNCTION_QUEUE_API") PACKAGE_NAME="hastefuncqueues" ;; + "$FUNCTION_TITILER_API") PACKAGE_NAME="titilerfuncapi" ;; + *) + echo "ERROR: No prebuilt package mapping for '$FUNCTION_NAME'." >&2 + return 1 + ;; + esac + PACKAGE_PATH="$HASTE_FUNCTION_PACKAGE_DIR/$PACKAGE_NAME.zip" + CHECKSUM_PATH="$HASTE_FUNCTION_PACKAGE_DIR/$PACKAGE_NAME.sha256" + if [ ! -f "$PACKAGE_PATH" ] || [ ! -f "$CHECKSUM_PATH" ]; then + echo "ERROR: Prebuilt Function App package or checksum is missing for $PACKAGE_NAME." >&2 + return 1 + fi + ( + cd "$HASTE_FUNCTION_PACKAGE_DIR" + sha256sum --check "$PACKAGE_NAME.sha256" + ) + PACKAGE_SHA256=$(awk '{print $1}' "$CHECKSUM_PATH") + echo "Deploying verified prebuilt package $PACKAGE_NAME ($PACKAGE_SHA256)..." + az functionapp deployment source config-zip \ + --name "$FUNCTION_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --src "$PACKAGE_PATH" \ + --build-remote false \ + --output none + else + # A non-zero exit can represent a package/build/deployment failure. Do + # not convert it into a success-shaped warning: set -e propagates it. + ( + cd "$FUNCTION_DIR" + func azure functionapp publish "$FUNCTION_NAME" \ + --python \ + --build remote \ + --verbose + ) + fi close_deploy_access "$FUNCTION_NAME" echo "Function deployment completed successfully." + az functionapp config appsettings set \ + --name "$FUNCTION_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --settings "HASTE_GDAL_RUNTIME_REQUIRED=true" \ + --output none + az functionapp restart --name "$FUNCTION_NAME" --resource-group "$RESOURCE_GROUP" + forget_gdal_runtime_pending_app "$FUNCTION_NAME" + # Record the wheel this app actually installed. Nothing else on the app # says which hastegeo it is running -- the wheel is pinned into # requirements.txt at deploy time and then forgotten -- so a blank @@ -429,6 +521,9 @@ deploy_function() { if [ -n "$PINNED_HASTEGEO" ]; then EXTRA_TAGS="tags.hastegeo_version=${PINNED_HASTEGEO}" fi + if [ -n "$PACKAGE_SHA256" ]; then + EXTRA_TAGS="${EXTRA_TAGS:+$EXTRA_TAGS }tags.function_package_sha256=${PACKAGE_SHA256}" + fi echo "Setting tags for Function App $FUNCTION_NAME ..." az functionapp update --name "$FUNCTION_NAME" --resource-group "$RESOURCE_GROUP" --set $AZ_FUNCTIONAPP_TAGS $EXTRA_TAGS diff --git a/.github/scripts/filter_dependency_requirements.py b/.github/scripts/filter_dependency_requirements.py new file mode 100644 index 00000000..9b14d49d --- /dev/null +++ b/.github/scripts/filter_dependency_requirements.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Filter runtime-owned GDAL wheels from public-index dependency checks. + +The exact HASTE wheels are validated and installed from the build artifact by +the GDAL runtime workflow. This keeps the general dependency resolver from +trying to build GDAL from an sdist before those private build artifacts exist. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +RUNTIME_DISTRIBUTIONS = { + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + "hastegeo", + "haste-gdal-runtime", +} + + +def _name(line: str) -> str | None: + content = line.strip() + if not content or content.startswith("#"): + return None + if content.startswith("-e "): + return "hastegeo" + match = re.match(r"([A-Za-z0-9_.-]+)(?=\s|[<=>!~;\[]|$)", content) + if not match: + return None + return re.sub(r"[-_.]+", "-", match.group(1)).lower() + + +def filter_requirements(source: Path, output: Path) -> None: + retained = [ + line + for line in source.read_text(encoding="utf-8").splitlines() + if _name(line) not in RUNTIME_DISTRIBUTIONS + ] + output.write_text("\n".join(retained) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args(argv) + filter_requirements(args.source, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/prepare_training_runtime_env.py b/.github/scripts/prepare_training_runtime_env.py new file mode 100644 index 00000000..cfa24e8d --- /dev/null +++ b/.github/scripts/prepare_training_runtime_env.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Point the training Conda environment at the current build's GDAL wheels.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path +from urllib.parse import unquote, urlsplit + +RUNTIME_DISTRIBUTIONS = { + "haste-gdal-runtime", + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", +} +WHEEL_URL = re.compile(r"https://[^\s\"']+\.whl", re.IGNORECASE) + + +def wheel_distribution(filename: str) -> str | None: + normalized = filename.casefold() + matches = [ + name + for name in RUNTIME_DISTRIBUTIONS + if normalized.startswith(name.replace("-", "_") + "-") + ] + return matches[0] if len(matches) == 1 else None + + +def prepare_environment( + source: Path, wheel_dir: Path, output: Path +) -> list[Path]: + text = source.read_text(encoding="utf-8") + replacements: dict[str, Path] = {} + for match in WHEEL_URL.findall(text): + filename = unquote(Path(urlsplit(match).path).name) + target = wheel_dir / filename + if not target.is_file(): + raise FileNotFoundError( + f"runtime wheel referenced by env.yml is missing: {filename}" + ) + normalized = wheel_distribution(filename) + if normalized is None: + raise ValueError( + f"unexpected wheel distribution in env.yml: {filename}" + ) + if normalized in replacements: + raise ValueError( + f"duplicate runtime wheel in env.yml: {normalized}" + ) + replacements[normalized] = target + text = text.replace(match, target.resolve().as_uri()) + + if set(replacements) != RUNTIME_DISTRIBUTIONS: + raise ValueError( + "training environment must reference all six runtime wheels; " + f"missing={sorted(RUNTIME_DISTRIBUTIONS - set(replacements))}" + ) + output.write_text(text, encoding="utf-8") + return list(replacements.values()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("wheel_dir", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args(argv) + wheels = prepare_environment(args.source, args.wheel_dir, args.output) + print(f"Prepared {args.output} with {len(wheels)} locked runtime wheels") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/publish_gdal_runtime.py b/.github/scripts/publish_gdal_runtime.py new file mode 100644 index 00000000..68a33623 --- /dev/null +++ b/.github/scripts/publish_gdal_runtime.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Validate and immutably publish one HASTE GDAL runtime bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +import zipfile +from dataclasses import dataclass +from email.message import Message +from email.parser import BytesParser +from pathlib import Path +from typing import Callable, Sequence + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "hastelib")) + +from haste_release import ( # noqa: E402 + RELEASE_TAG, + REPOSITORY, + list_release_assets, +) + +EXPECTED_DISTRIBUTIONS = { + "haste-gdal-runtime", + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", +} +BINDING_MODULES = { + "gdal": "osgeo.gdal", + "rasterio": "rasterio", + "fiona": "fiona", + "pyogrio": "pyogrio", + "pyproj": "pyproj", +} +EVIDENCE_FILES = { + "manifest.json", + "bom.cdx.json", + "THIRD_PARTY_NOTICES.txt", +} +SOURCE_LOCK_PATH = REPO_ROOT / "native" / "gdal-runtime" / "sources.lock" +_CHECKSUM_LINE = re.compile(r"^([0-9a-f]{64}) ([A-Za-z0-9_.+-]+)$") +CommandRunner = Callable[[Sequence[str]], str] + + +@dataclass(frozen=True) +class RuntimeBundle: + """Validated bundle contents ready for immutable publication.""" + + directory: Path + assets: tuple[Path, ...] + gdal_version: str + runtime_version: str + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _wheel_metadata(path: Path) -> tuple[Message, list[str]]: + if not zipfile.is_zipfile(path): + raise ValueError(f"Not a valid wheel ZIP file: {path.name}") + with zipfile.ZipFile(path) as archive: + names = [ + name + for name in archive.namelist() + if name.endswith(".dist-info/METADATA") + ] + wheel_names = [ + name + for name in archive.namelist() + if name.endswith(".dist-info/WHEEL") + ] + if len(names) != 1 or len(wheel_names) != 1: + raise ValueError( + f"{path.name}: expected one METADATA and WHEEL file" + ) + metadata = BytesParser().parsebytes(archive.read(names[0])) + tags = [ + line[5:].strip() + for line in archive.read(wheel_names[0]) + .decode("utf-8") + .splitlines() + if line.startswith("Tag: ") + ] + return metadata, tags + + +def _read_checksums(path: Path) -> dict[str, str]: + checksums = {} + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + match = _CHECKSUM_LINE.fullmatch(line) + if not match: + raise ValueError(f"Invalid SHA256SUMS entry on line {line_number}") + digest, name = match.groups() + if name in checksums: + raise ValueError(f"Duplicate SHA256SUMS entry: {name}") + checksums[name] = digest + return checksums + + +def _locked_native_sources(path: Path | None = None) -> dict[str, str]: + source_lock_path = SOURCE_LOCK_PATH if path is None else path + sources = {} + for line in source_lock_path.read_text(encoding="utf-8").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + fields = line.split() + if fields[0] == "HASTE_GDAL_BUILD": + continue + if len(fields) != 4: + raise ValueError(f"Invalid native source lock row: {line!r}") + name, version, _, _ = fields + if name in sources: + raise ValueError(f"Duplicate native source lock entry: {name}") + sources[name] = version + if not sources: + raise ValueError("Native source lock has no source entries") + return sources + + +def validate_bundle(directory: Path) -> RuntimeBundle: + """Verify exact bundle contents, wheel identities, licenses and hashes.""" + if not directory.is_dir(): + raise FileNotFoundError( + f"Runtime artifact directory is missing: {directory}" + ) + wheels = sorted(directory.glob("*.whl")) + if len(wheels) != len(EXPECTED_DISTRIBUTIONS): + raise ValueError(f"Expected six runtime wheels, found {len(wheels)}") + + manifest = json.loads( + (directory / "manifest.json").read_text(encoding="utf-8") + ) + gdal_version = manifest.get("gdal_release") + if not isinstance(gdal_version, str) or not gdal_version: + raise ValueError("Runtime manifest has no GDAL release") + + wheel_names = set() + wheel_metadata = {} + runtime_version = "" + build_number = "" + for path in wheels: + metadata, tags = _wheel_metadata(path) + name = re.sub(r"[-_.]+", "-", metadata.get("Name", "").lower()) + version = metadata.get("Version", "") + if name not in EXPECTED_DISTRIBUTIONS or name in wheel_names: + raise ValueError( + f"Unexpected or duplicate wheel distribution: {name}" + ) + wheel_names.add(name) + if not metadata.get("License-Expression") and not metadata.get( + "License" + ): + raise ValueError(f"{path.name}: wheel has no license metadata") + if name == "haste-gdal-runtime" and not metadata.get( + "License-Expression" + ): + raise ValueError( + "Runtime wheel must declare an SPDX License-Expression" + ) + normalized_filename_name = re.sub(r"[-_.]+", "_", name).lower() + expected_tag = ( + "py3-none-manylinux_2_28_x86_64" + if name == "haste-gdal-runtime" + else "cp311-cp311-manylinux_2_28_x86_64" + ) + expected_filename = ( + f"{normalized_filename_name}-{version}-{expected_tag}.whl" + ) + if path.name != expected_filename or tags != [expected_tag]: + raise ValueError( + f"{path.name}: wheel filename/tag is not {expected_filename}" + ) + match = re.fullmatch(r"(.+)\+haste\.(\d+)", version) + if not match: + raise ValueError( + f"{path.name}: version must include a HASTE build number" + ) + base_version, number = match.groups() + if build_number and number != build_number: + raise ValueError("Runtime wheels have inconsistent build numbers") + build_number = number + if name in {"gdal", "haste-gdal-runtime"}: + if base_version != gdal_version: + raise ValueError( + f"{path.name}: version does not match GDAL {gdal_version}" + ) + wheel_metadata[name] = metadata + if name == "haste-gdal-runtime": + runtime_version = version + + if wheel_names != EXPECTED_DISTRIBUTIONS or not runtime_version: + raise ValueError("Runtime wheel set is incomplete") + for name, metadata in wheel_metadata.items(): + if name == "haste-gdal-runtime": + continue + required_runtime = ( + re.sub( + r"\s+", + "", + f"haste-gdal-runtime=={runtime_version}", + ) + .replace("_", "-") + .lower() + ) + requirements = [ + re.sub(r"\s+", "", value).replace("_", "-").lower() + for value in metadata.get_all("Requires-Dist", []) + ] + if not any( + required_runtime in requirement for requirement in requirements + ): + raise ValueError( + f"{metadata.get('Name')}: does not pin " + f"haste-gdal-runtime=={runtime_version}; found {requirements}" + ) + + expected_binding_versions = { + module: wheel_metadata[name].get("Version", "").split("+", 1)[0] + for name, module in BINDING_MODULES.items() + } + if manifest.get("binding_versions") != expected_binding_versions: + raise ValueError( + "Runtime manifest binding versions do not match the wheels" + ) + + notices_path = directory / "THIRD_PARTY_NOTICES.txt" + notices = notices_path.read_text(encoding="utf-8") + if "review required" in notices.casefold(): + raise ValueError( + "Third-party notices still contain items marked for review" + ) + bom = json.loads((directory / "bom.cdx.json").read_text(encoding="utf-8")) + if bom.get("bomFormat") != "CycloneDX" or bom.get("specVersion") != "1.6": + raise ValueError("Runtime SBOM must be CycloneDX 1.6") + components = bom.get("components") + if not isinstance(components, list) or not components: + raise ValueError("Runtime SBOM has no components") + unlicensed = sorted( + component.get("name", "") + for component in components + if not component.get("licenses") + ) + if unlicensed: + raise ValueError( + "Runtime SBOM components lack license metadata: " + + ", ".join(unlicensed) + ) + observed_sources = {} + for component in components: + if not component.get("purl", "").startswith("pkg:generic/"): + continue + name = component.get("name") + if not isinstance(name, str) or name in observed_sources: + raise ValueError("Runtime SBOM has an invalid or duplicate source") + observed_sources[name] = component.get("version") + locked_sources = _locked_native_sources() + if observed_sources != locked_sources: + missing_sources = { + name: version + for name, version in locked_sources.items() + if observed_sources.get(name) != version + } + unexpected_sources = sorted( + set(observed_sources) - set(locked_sources) + ) + details = [ + f"{name}=={version}" + for name, version in sorted(missing_sources.items()) + ] + details.extend(f"unexpected {name}" for name in unexpected_sources) + raise ValueError( + "Runtime SBOM does not match sources.lock: " + ", ".join(details) + ) + + checksums_path = directory / "SHA256SUMS" + checksums = _read_checksums(checksums_path) + expected_files = {path.name for path in wheels} | EVIDENCE_FILES + if set(checksums) != expected_files: + raise ValueError("SHA256SUMS does not cover the exact runtime bundle") + for name, digest in checksums.items(): + path = directory / name + if not path.is_file() or _sha256(path) != digest: + raise ValueError(f"SHA-256 mismatch or missing asset: {name}") + + entries = list(directory.iterdir()) + if any(path.is_symlink() or not path.is_file() for path in entries): + raise ValueError("Runtime artifact directory has non-file entries") + actual_files = {path.name for path in entries} + if actual_files != expected_files | {"SHA256SUMS"}: + raise ValueError("Runtime artifact directory has unexpected files") + + return RuntimeBundle( + directory=directory, + assets=tuple( + [ + *wheels, + *(directory / name for name in sorted(EVIDENCE_FILES)), + checksums_path, + ] + ), + gdal_version=gdal_version, + runtime_version=runtime_version, + ) + + +def run_command(command: Sequence[str]) -> str: + result = subprocess.run( + list(command), + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def publish_bundle( + bundle: RuntimeBundle, + *, + runner: CommandRunner = run_command, + asset_lister: Callable[[], list[str]] = list_release_assets, +) -> None: + """Publish assets to haste-binaries without overwriting existing assets.""" + names = {path.name for path in bundle.assets} + collisions = sorted(names & set(asset_lister())) + if collisions: + raise ValueError( + "Release assets already exist and will not be overwritten: " + + ", ".join(collisions) + ) + runner( + [ + "gh", + "release", + "upload", + RELEASE_TAG, + *(str(path) for path in bundle.assets), + "--repo", + REPOSITORY, + ] + ) + missing = sorted(names - set(asset_lister())) + if missing: + raise RuntimeError( + "Upload succeeded but release assets are missing: " + + ", ".join(missing) + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--artifact-dir", required=True, type=Path) + parser.add_argument( + "--validate-only", + action="store_true", + help="validate the bundle without querying or mutating GitHub", + ) + args = parser.parse_args(argv) + + bundle = validate_bundle(args.artifact_dir) + if not args.validate_only: + publish_bundle(bundle) + print( + f"Validated GDAL {bundle.gdal_version} runtime " + f"{bundle.runtime_version}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_build_function_package.py b/.github/scripts/tests/test_build_function_package.py new file mode 100644 index 00000000..3f0dbbc7 --- /dev/null +++ b/.github/scripts/tests/test_build_function_package.py @@ -0,0 +1,96 @@ +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +import build_function_package as package_builder # noqa: E402 + + +class FunctionPackageRequirementTests(unittest.TestCase): + def _wheel(self, directory: Path, name: str, version: str) -> Path: + normalized = package_builder.normalized_name(name).replace("-", "_") + path = directory / f"{normalized}-{version}-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + f"{normalized}-{version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n", + ) + return path + + def test_ignores_metadata_of_vendored_distributions(self): + with tempfile.TemporaryDirectory() as temporary: + path = self._wheel(Path(temporary), "setuptools", "84.0.0") + with zipfile.ZipFile(path, "a") as archive: + archive.writestr( + "setuptools/_vendor/other-1.0.dist-info/METADATA", + "Name: other\nVersion: 1.0\n\n", + ) + self.assertEqual( + package_builder.wheel_metadata(path), ("setuptools", "84.0.0") + ) + + def test_replaces_legacy_bindings_with_this_builds_runtime_wheels(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wheel_dir = root / "runtime" + wheel_dir.mkdir() + versions = { + "GDAL": "3.13.3+haste.1", + "rasterio": "1.4.4+haste.1", + "fiona": "1.10.1+haste.1", + "pyogrio": "0.13.0+haste.1", + "pyproj": "3.7.2+haste.1", + "haste-gdal-runtime": "3.13.3+haste.1", + } + for name, version in versions.items(): + self._wheel(wheel_dir, name, version) + requirements = root / "requirements.txt" + requirements.write_text( + "azure-functions\n" + "GDAL @ https://example.invalid/GDAL-3.9.2.whl\n" + "rasterio==1.3.11\n" + "pyproj==3.6.1\n" + "pyogrio==0.10.0\n" + "-e ../../hastelib\n", + encoding="utf-8", + ) + hastegeo_wheel = root / "hastegeo-1.0.0-py3-none-any.whl" + hastegeo_wheel.touch() + output = root / "resolved.in" + + package_builder.prepare_requirements( + requirements, output, wheel_dir, hastegeo_wheel + ) + generated = output.read_text(encoding="utf-8") + + self.assertIn("azure-functions", generated) + self.assertIn("gdal==3.13.3+haste.1", generated) + self.assertIn("haste-gdal-runtime==3.13.3+haste.1", generated) + self.assertIn(hastegeo_wheel.resolve().as_uri(), generated) + self.assertNotIn("3.9.2", generated) + self.assertNotIn("1.3.11", generated) + self.assertNotIn("-e ../../hastelib", generated) + + def test_generated_lock_covers_each_resolved_wheel_hash(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wheelhouse = root / "wheelhouse" + wheelhouse.mkdir() + wheel = self._wheel(wheelhouse, "attrs", "25.4.0") + output = root / "requirements.lock" + + package_builder.write_lock(wheelhouse, output) + + expected = ( + f"attrs==25.4.0 --hash=sha256:" + f"{package_builder.sha256(wheel)}\n" + ) + self.assertEqual(output.read_text(encoding="utf-8"), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_deploy_network_access.py b/.github/scripts/tests/test_deploy_network_access.py index 8ecd875c..3da433f0 100644 --- a/.github/scripts/tests/test_deploy_network_access.py +++ b/.github/scripts/tests/test_deploy_network_access.py @@ -1,13 +1,14 @@ """Keep deployments from widening a function app's inbound access.""" +import hashlib import json import os -from pathlib import Path import shutil import subprocess import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPTS = ROOT / ".github" / "scripts" @@ -252,6 +253,21 @@ def test_open_titiler_is_contained_before_it_is_published(self) -> None: result, calls, final = self.run_deploy() self.assertEqual(result.returncode, 0, result.stderr) + settings = next( + call + for call in calls + if call.startswith("functionapp config appsettings set") + and "GDAL_VRT_ENABLE_RAWRASTERBAND=NO" in call + ) + self.assertIn("GDAL_VRT_ENABLE_RAWRASTERBAND=NO", settings) + self.assertIn("GDAL_VRT_ENABLE_PYTHON=NO", settings) + self.assertIn("GDAL_DRIVER_PATH=disable", settings) + self.assertIn("GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR", settings) + self.assertIn("HASTE_GDAL_RUNTIME_REQUIRED=false", settings) + self.assertIn( + "GDAL_SKIP=HDF4 HDF4Image HDF5 HDF5Image netCDF", + settings, + ) self.assertEqual(final["defaults"], {"main": "Deny", "scm": "Deny"}) self.assertFalse(final["useMain"]) self.assertEqual( @@ -270,6 +286,21 @@ def test_open_titiler_is_contained_before_it_is_published(self) -> None: if call.startswith("functionapp config access-restriction set") ) self.assertLess(set_defaults, restart) + publish = calls.index(self.publish_line(calls)) + enable_runtime = next( + index + for index, call in enumerate(calls) + if call.startswith("functionapp config appsettings set") + and "HASTE_GDAL_RUNTIME_REQUIRED=true" in call + ) + restarts = [ + index + for index, call in enumerate(calls) + if call.startswith("functionapp restart") + ] + self.assertGreater(len(restarts), 1) + self.assertLess(publish, enable_runtime) + self.assertLess(enable_runtime, restarts[-1]) def test_titiler_storage_host_is_set_before_restart(self) -> None: result, calls, _ = self.run_deploy() @@ -306,6 +337,35 @@ def test_publish_runs_through_a_temporary_runner_rule(self) -> None: self.assertEqual(len(runner_rules), 2) for call in runner_rules: self.assertIn(f"--ip-address {RUNNER_IP}/32", call) + + def test_prebuilt_package_deploys_without_remote_build(self) -> None: + with tempfile.TemporaryDirectory() as package_dir: + package = Path(package_dir) / "titilerfuncapi.zip" + package.write_bytes(b"verified package") + digest = hashlib.sha256(package.read_bytes()).hexdigest() + (Path(package_dir) / "titilerfuncapi.sha256").write_text( + f"{digest} {package.name}\n", + encoding="utf-8", + newline="\n", + ) + result, calls, final = self.run_deploy( + component="titiler", + HASTE_FUNCTION_PACKAGE_DIR=package_dir, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + package_deploy = next( + call + for call in calls + if call.startswith("functionapp deployment source config-zip") + ) + self.assertIn("--build-remote false", package_deploy) + self.assertIn(package.name, package_deploy) + self.assertFalse(any(call.startswith("func ") for call in calls)) + self.assertIn( + f"tags.function_package_sha256={digest}", + "\n".join(calls), + ) self.assertEqual(self.rule_names(final), ["AllowSubnet-1"]) def test_failed_publish_still_removes_the_runner_rule(self) -> None: @@ -314,6 +374,19 @@ def test_failed_publish_still_removes_the_runner_rule(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("scm:haste-ci-77-2", self.publish_line(calls)) self.assertEqual(self.rule_names(final), ["AllowSubnet-1"]) + runtime_settings = [ + index + for index, call in enumerate(calls) + if call.startswith("functionapp config appsettings set") + and "HASTE_GDAL_RUNTIME_REQUIRED=true" in call + ] + self.assertTrue(runtime_settings) + self.assertTrue( + any( + call.startswith("functionapp restart") + for call in calls[runtime_settings[-1] + 1 :] + ) + ) def test_transient_cleanup_failure_is_retried(self) -> None: result, _, final = self.run_deploy(fail_removals=1) diff --git a/.github/scripts/tests/test_filter_dependency_requirements.py b/.github/scripts/tests/test_filter_dependency_requirements.py new file mode 100644 index 00000000..48a0c851 --- /dev/null +++ b/.github/scripts/tests/test_filter_dependency_requirements.py @@ -0,0 +1,58 @@ +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +import filter_dependency_requirements as filter_script # noqa: E402 + + +class FilterDependencyRequirementsTests(unittest.TestCase): + def test_titiler_tests_filter_unpublished_runtime_wheels(self): + workflow = ( + SCRIPT_DIR.parent / "workflows" / "titiler-tests.yml" + ).read_text(encoding="utf-8") + + self.assertIn("filter_dependency_requirements.py", workflow) + self.assertIn("/tmp/titiler-test-requirements.txt", workflow) + self.assertNotIn( + "pip install -r api/titilerfuncapi/requirements.txt", workflow + ) + + def test_removes_runtime_owned_bindings_and_keeps_other_dependencies(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "requirements.txt" + output = root / "filtered.txt" + source.write_text( + "azure-functions\n" + "GDAL @ https://example.test/GDAL.whl\n" + "rasterio==1.4.4\n" + "Fiona==1.10.1\n" + "pyogrio==0.13.0\n" + "pyproj==3.7.2\n" + "-e ../../hastelib\n" + "requests==2.34.2\n", + encoding="utf-8", + ) + + filter_script.filter_requirements(source, output) + filtered = output.read_text(encoding="utf-8") + + self.assertIn("azure-functions", filtered) + self.assertIn("requests==2.34.2", filtered) + for runtime_package in ( + "GDAL", + "rasterio", + "Fiona", + "pyogrio", + "pyproj", + "hastelib", + ): + self.assertNotIn(runtime_package, filtered) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_gdal_hardening_guard.py b/.github/scripts/tests/test_gdal_hardening_guard.py new file mode 100644 index 00000000..f550df0a --- /dev/null +++ b/.github/scripts/tests/test_gdal_hardening_guard.py @@ -0,0 +1,105 @@ +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +from check_gdal_hardening import find_violations # noqa: E402 + + +class GdalHardeningGuardTests(unittest.TestCase): + def test_hardening_settings_are_persisted_in_each_deployment_surface(self): + from check_gdal_hardening import REPO_ROOT + + surfaces = ( + "infra/modules/functions.bicep", + ".github/scripts/deploy_apps.sh", + "api/hastefuncapi/Dockerfile", + "api/hastefuncqueues/Dockerfile", + "docker/docker-compose.yml", + "docker/imageryprep/Dockerfile", + "docker/training/Dockerfile", + "docker/titiler/Dockerfile", + ) + settings = ( + "HASTE_GDAL_RUNTIME_REQUIRED", + "GDAL_VRT_ENABLE_RAWRASTERBAND", + "GDAL_VRT_ENABLE_PYTHON", + "GDAL_DRIVER_PATH", + "GDAL_DISABLE_READDIR_ON_OPEN", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", + ) + for relative_path in surfaces: + with self.subTest(path=relative_path): + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + for setting in settings: + self.assertIn(setting, text) + titiler_docker = ( + REPO_ROOT / "docker" / "titiler" / "Dockerfile" + ).read_text(encoding="utf-8") + self.assertIn('GDAL_SKIP="HDF4 ', titiler_docker) + compose = (REPO_ROOT / "docker" / "docker-compose.yml").read_text( + encoding="utf-8" + ) + self.assertIn('GDAL_SKIP: "HDF4 ', compose) + + def test_rejects_relaxed_settings(self) -> None: + values = ( + "GDAL_VRT_ENABLE_RAWRASTERBAND=YES", + 'os.environ["GDAL_VRT_ENABLE_PYTHON"] = "YES"', + 'GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE = "ALL"', + 'GDAL_PYTHON_DRIVER_PATH = "/tmp/plugins"', + 'GDAL_DRIVER_PATH = "/tmp/plugins"', + "GDAL_DISABLE_READDIR_ON_OPEN=NO", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=ONLY_IN_TEMP", + "CPL_ENABLE_PATH_TRAVERSAL_DETECTION=NO", + 'gdal.SetConfigOption("GDAL_HTTP_UNSAFESSL", "YES")', + ) + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "config.py" + path.write_text("\n".join(values), encoding="utf-8") + violations = find_violations([path]) + + self.assertEqual(len(violations), len(values)) + + def test_rejects_structured_name_value_records(self) -> None: + bicep = "{ name: 'GDAL_VRT_ENABLE_PYTHON', value: 'YES' }\n" + arm = ( + '{\n "name": "GDAL_VRT_ENABLE_RAWRASTERBAND",\n' + ' "value": "YES"\n}\n' + ) + safe = "{ name: 'GDAL_VRT_ENABLE_PYTHON', value: 'NO' }\n" + with tempfile.TemporaryDirectory() as temp_dir: + bicep_path = Path(temp_dir) / "functions.bicep" + arm_path = Path(temp_dir) / "main.json" + safe_path = Path(temp_dir) / "safe.bicep" + bicep_path.write_text(bicep, encoding="utf-8") + arm_path.write_text(arm, encoding="utf-8") + safe_path.write_text(safe, encoding="utf-8") + self.assertEqual(len(find_violations([bicep_path])), 1) + self.assertEqual(len(find_violations([arm_path])), 1) + self.assertEqual(find_violations([safe_path]), []) + + def test_accepts_enforced_values_and_disabled_driver_path(self) -> None: + lines = ( + "GDAL_VRT_ENABLE_RAWRASTERBAND=NO", + "GDAL_VRT_ENABLE_PYTHON=NO", + "GDAL_DRIVER_PATH=disable", + "GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR", + 'GDAL_SKIP="VRT HDF5"', + ) + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "config.env" + path.write_text("\n".join(lines), encoding="utf-8") + self.assertEqual(find_violations([path]), []) + + def test_repository_has_no_relaxed_gdal_configuration(self) -> None: + from check_gdal_hardening import REPO_ROOT, _candidate_files + + self.assertEqual(find_violations(_candidate_files(REPO_ROOT)), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_prepare_training_runtime_env.py b/.github/scripts/tests/test_prepare_training_runtime_env.py new file mode 100644 index 00000000..f3abfcfe --- /dev/null +++ b/.github/scripts/tests/test_prepare_training_runtime_env.py @@ -0,0 +1,67 @@ +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +import prepare_training_runtime_env as prepare_script # noqa: E402 + + +class PrepareTrainingRuntimeEnvironmentTests(unittest.TestCase): + def test_rewrites_every_runtime_wheel_url_to_the_local_bundle(self): + filenames = ( + "haste_gdal_runtime-3.13.3+haste.1-py3-none-manylinux_2_28_x86_64.whl", + "gdal-3.13.3+haste.1-cp311-cp311-manylinux_2_28_x86_64.whl", + "rasterio-1.4.4+haste.1-cp311-cp311-manylinux_2_28_x86_64.whl", + "fiona-1.10.1+haste.1-cp311-cp311-manylinux_2_28_x86_64.whl", + "pyogrio-0.13.0+haste.1-cp311-cp311-manylinux_2_28_x86_64.whl", + "pyproj-3.7.2+haste.1-cp311-cp311-manylinux_2_28_x86_64.whl", + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wheel_dir = root / "wheels" + wheel_dir.mkdir() + urls = [] + for filename in filenames: + (wheel_dir / filename).touch() + urls.append( + "https://example.test/" + filename.replace("+", "%2B") + ) + source = root / "env.yml" + source.write_text( + "dependencies:\n - pip:\n" + + "".join(f" - {url}\n" for url in urls), + encoding="utf-8", + ) + output = root / "prepared.yml" + + resolved = prepare_script.prepare_environment( + source, wheel_dir, output + ) + result = output.read_text(encoding="utf-8") + + self.assertEqual(len(resolved), 6) + self.assertNotIn("https://example.test/", result) + self.assertIn("file:///", result) + self.assertEqual(result.count("file:///"), 6) + + def test_missing_runtime_wheel_fails_instead_of_using_remote_url(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "env.yml" + source.write_text( + "dependencies:\n" + " - pip:\n" + " - https://example.test/gdal-3.13.3+haste.1.whl\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(FileNotFoundError, "gdal-3.13.3"): + prepare_script.prepare_environment( + source, root, root / "out.yml" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_publish_gdal_runtime.py b/.github/scripts/tests/test_publish_gdal_runtime.py new file mode 100644 index 00000000..7951c0a8 --- /dev/null +++ b/.github/scripts/tests/test_publish_gdal_runtime.py @@ -0,0 +1,297 @@ +import json +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +SCRIPT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIR)) + +from publish_gdal_runtime import ( # noqa: E402 + EXPECTED_DISTRIBUTIONS, + _sha256, + publish_bundle, + validate_bundle, +) + + +class PublishGdalRuntimeTests(unittest.TestCase): + def _bundle_directory(self, root: Path) -> Path: + directory = root / "dist" + directory.mkdir() + (directory / "manifest.json").write_text( + json.dumps( + { + "gdal_release": "3.13.3", + "binding_versions": { + "osgeo.gdal": "3.13.3", + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", + }, + } + ), + encoding="utf-8", + ) + source_lock_path = root / "sources.lock" + patcher = mock.patch( + "publish_gdal_runtime.SOURCE_LOCK_PATH", source_lock_path + ) + patcher.start() + self.addCleanup(patcher.stop) + source_lock_path.write_text( + "HASTE_GDAL_BUILD 2\n" + "gdal 3.13.3 https://example.test/gdal.tar.gz " + + "a" * 64 + + "\nproj 9.8.1 https://example.test/proj.tar.gz " + + "b" * 64 + + "\n", + encoding="utf-8", + ) + (directory / "bom.cdx.json").write_text( + json.dumps( + { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "name": name, + "licenses": [{"license": {"id": "MIT"}}], + } + for name in sorted(EXPECTED_DISTRIBUTIONS) + ] + + [ + { + "name": "gdal", + "version": "3.13.3", + "purl": "pkg:generic/gdal@3.13.3", + "licenses": [{"license": {"id": "MIT"}}], + }, + { + "name": "proj", + "version": "9.8.1", + "purl": "pkg:generic/proj@9.8.1", + "licenses": [{"license": {"id": "MIT"}}], + }, + ], + } + ), + encoding="utf-8", + ) + (directory / "THIRD_PARTY_NOTICES.txt").write_text( + "All license texts included.\n", + encoding="utf-8", + ) + + versions = { + "haste-gdal-runtime": "3.13.3+haste.2", + "gdal": "3.13.3+haste.2", + "rasterio": "1.4.4+haste.2", + "fiona": "1.10.1+haste.2", + "pyogrio": "0.13.0+haste.2", + "pyproj": "3.7.2+haste.2", + } + for name in sorted(EXPECTED_DISTRIBUTIONS): + normalized = name.replace("-", "_") + tag = ( + "py3-none-manylinux_2_28_x86_64" + if name == "haste-gdal-runtime" + else "cp311-cp311-manylinux_2_28_x86_64" + ) + path = directory / f"{normalized}-{versions[name]}-{tag}.whl" + requires = ( + "" + if name == "haste-gdal-runtime" + else "Requires-Dist: haste-gdal-runtime==" "3.13.3+haste.2\n" + ) + license_field = ( + "License-Expression: MIT\n" + if name == "haste-gdal-runtime" + else "License: MIT\n" + ) + metadata = ( + "Metadata-Version: 2.1\n" + f"Name: {name}\n" + f"Version: {versions[name]}\n" + f"{license_field}" + f"{requires}\n" + ) + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr(f"{normalized}.dist-info/METADATA", metadata) + wheel.writestr( + f"{normalized}.dist-info/WHEEL", + f"Wheel-Version: 1.0\nTag: {tag}\n", + ) + + assets = [ + path for path in directory.iterdir() if path.name != "SHA256SUMS" + ] + checksums = "\n".join( + f"{_sha256(path)} {path.name}" for path in sorted(assets) + ) + (directory / "SHA256SUMS").write_text( + checksums + "\n", encoding="utf-8" + ) + return directory + + def test_validates_exact_bundle_and_hashes(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + bundle = validate_bundle(self._bundle_directory(Path(temp_dir))) + + self.assertEqual(bundle.gdal_version, "3.13.3") + self.assertEqual(bundle.runtime_version, "3.13.3+haste.2") + self.assertEqual(len(bundle.assets), 10) + + def test_rejects_notice_items_marked_for_review(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + (directory / "THIRD_PARTY_NOTICES.txt").write_text( + "License review required.\n", encoding="utf-8" + ) + with self.assertRaisesRegex(ValueError, "marked for review"): + validate_bundle(directory) + + def test_rejects_unlicensed_sbom_component(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + bom_path = directory / "bom.cdx.json" + bom = json.loads(bom_path.read_text(encoding="utf-8")) + bom["components"][0].pop("licenses") + bom_path.write_text(json.dumps(bom), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "lack license metadata"): + validate_bundle(directory) + + def test_rejects_sbom_missing_locked_source(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + bom_path = directory / "bom.cdx.json" + bom = json.loads(bom_path.read_text(encoding="utf-8")) + bom["components"] = [ + component + for component in bom["components"] + if component.get("name") != "proj" + ] + bom_path.write_text(json.dumps(bom), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "sources.lock"): + validate_bundle(directory) + + def test_rejects_unexpected_native_source(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + bom_path = directory / "bom.cdx.json" + bom = json.loads(bom_path.read_text(encoding="utf-8")) + bom["components"].append( + { + "name": "unlocked-native", + "version": "1.0", + "purl": "pkg:generic/unlocked-native@1.0", + "licenses": [{"license": {"id": "MIT"}}], + } + ) + bom_path.write_text(json.dumps(bom), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "unexpected unlocked-native" + ): + validate_bundle(directory) + + def test_rejects_manifest_binding_version_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + manifest_path = directory / "manifest.json" + manifest = json.loads( + manifest_path.read_text(encoding="utf-8") + ) + manifest["binding_versions"]["fiona"] = "1.10.0" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with self.assertRaisesRegex( + ValueError, "binding versions do not match" + ): + validate_bundle(directory) + + def test_accepts_folded_runtime_dependency_header(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = self._bundle_directory(Path(temp_dir)) + wheel_path = next( + path + for path in directory.glob("*.whl") + if "fiona-" in path.name + ) + with zipfile.ZipFile(wheel_path, "r") as source: + contents = { + name: source.read(name) for name in source.namelist() + } + metadata_path = next( + name + for name in contents + if name.endswith(".dist-info/METADATA") + ) + metadata = contents[metadata_path].decode("utf-8") + metadata = metadata.replace( + "Requires-Dist: haste-gdal-runtime==3.13.3+haste.2", + "Requires-Dist: haste-gdal-runtime==\n 3.13.3+haste.2", + ) + contents[metadata_path] = metadata.encode("utf-8") + with zipfile.ZipFile(wheel_path, "w") as output: + for name, content in contents.items(): + output.writestr(name, content) + checksum_file = directory / "SHA256SUMS" + checksums = [] + for line in checksum_file.read_text(encoding="utf-8").splitlines(): + _, asset_name = line.split(" ", 1) + checksums.append( + f"{_sha256(directory / asset_name)} {asset_name}" + ) + checksum_file.write_text( + "\n".join(checksums) + "\n", encoding="utf-8" + ) + + bundle = validate_bundle(directory) + + self.assertEqual(bundle.runtime_version, "3.13.3+haste.2") + + def test_rejects_overwrite_before_upload(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + bundle = validate_bundle(self._bundle_directory(Path(temp_dir))) + runner = mock.Mock() + with self.assertRaisesRegex(ValueError, "will not be overwritten"): + publish_bundle( + bundle, + runner=runner, + asset_lister=lambda: [bundle.assets[0].name], + ) + + runner.assert_not_called() + + def test_publishes_and_verifies_all_assets(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + bundle = validate_bundle(self._bundle_directory(Path(temp_dir))) + listed = [] + + def list_assets() -> list[str]: + if not listed: + listed.extend(path.name for path in bundle.assets) + return [] + return listed + + runner = mock.Mock(return_value="") + publish_bundle( + bundle, + runner=runner, + asset_lister=list_assets, + ) + + runner.assert_called_once() + command = runner.call_args.args[0] + self.assertIn("haste-binaries", command) + self.assertIn("--repo", command) + self.assertEqual( + len([arg for arg in command if arg.endswith(".whl")]), 6 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_size_limit_workflows.py b/.github/scripts/tests/test_size_limit_workflows.py index ff5f34fd..777b7431 100644 --- a/.github/scripts/tests/test_size_limit_workflows.py +++ b/.github/scripts/tests/test_size_limit_workflows.py @@ -175,6 +175,25 @@ def test_compiled_template_keeps_default_bounds_and_app_ownership(self) -> None: resource for resource in resources if resource["name"] == "functions" )["properties"]["template"]["resources"] + functions_template = next( + resource for resource in resources + if resource["name"] == "functions" + )["properties"]["template"] + hardening_settings = ( + "GDAL_VRT_ENABLE_RAWRASTERBAND", + "GDAL_VRT_ENABLE_PYTHON", + "GDAL_DRIVER_PATH", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", + ) + hardening_config = json.dumps( + functions_template["variables"]["gdalHardeningSettings"] + ) + titiler_config = json.dumps( + functions_template["variables"]["titilerGdalHardeningSettings"] + ) + for key in hardening_settings: + self.assertIn(key, hardening_config) + self.assertIn("GDAL_SKIP", titiler_config) for resource in functions: name = resource["name"] if name not in ("fn-api", "fn-queue", "fn-titiler"): @@ -183,6 +202,12 @@ def test_compiled_template_keeps_default_bounds_and_app_ownership(self) -> None: for key in KEYS: owner = "fn-queue" if key == KEYS[1] else "fn-api" self.assertEqual(key in settings, name == owner, (name, key)) + setting_variable = ( + "titilerGdalHardeningSettings" + if name == "fn-titiler" + else "gdalHardeningSettings" + ) + self.assertIn(setting_variable, settings, name) if __name__ == "__main__": diff --git a/.github/workflows/dependency-validation.yml b/.github/workflows/dependency-validation.yml index e2ce9f12..5690ede2 100644 --- a/.github/workflows/dependency-validation.yml +++ b/.github/workflows/dependency-validation.yml @@ -28,6 +28,8 @@ on: paths: - "api/hastefuncapi/requirements.txt" - "api/hastefuncqueues/requirements.txt" + - "api/titilerfuncapi/requirements.txt" + - "docker/titiler/Dockerfile" - "docker/imageryprep/requirements.txt" - "api/hastefuncapi/Dockerfile" - "api/hastefuncqueues/Dockerfile" @@ -39,6 +41,8 @@ on: - ".github/scripts/check_image_python.py" - ".github/scripts/check_protobuf_pin.py" - ".github/scripts/check_react_pins.py" + - ".github/scripts/filter_dependency_requirements.py" + - ".github/scripts/tests/test_filter_dependency_requirements.py" workflow_dispatch: permissions: {} @@ -57,14 +61,22 @@ jobs: include: - name: hastefuncapi requirements: api/hastefuncapi/requirements.txt + dockerfile: api/hastefuncapi/Dockerfile python: "3.11" functions_worker: true - name: hastefuncqueues requirements: api/hastefuncqueues/requirements.txt + dockerfile: api/hastefuncqueues/Dockerfile python: "3.11" functions_worker: true + - name: titilerfuncapi + requirements: api/titilerfuncapi/requirements.txt + dockerfile: docker/titiler/Dockerfile + python: "3.11" + functions_worker: false - name: imageryprep requirements: docker/imageryprep/requirements.txt + dockerfile: docker/imageryprep/Dockerfile python: "3.11" functions_worker: false steps: @@ -81,25 +93,22 @@ jobs: - name: Confirm the image Python version matches this job run: | python .github/scripts/check_image_python.py \ - "$(dirname "${{ matrix.requirements }}")/Dockerfile" \ + "${{ matrix.dockerfile }}" \ "${{ matrix.python }}" - name: Resolve dependencies run: | set -euo pipefail python -m pip install --upgrade pip - # Run from the requirements file's own directory: the Functions - # images pull hastegeo in with `-e ../../hastelib`, and pip - # resolves relative paths against the working directory rather - # than the file's location. - cd "$(dirname "${{ matrix.requirements }}")" - # --dry-run resolves the full graph and reports conflicts without - # installing, which is all we need and keeps the job quick. Both - # failure modes from #153 surface here: an unsatisfiable pin pair - # raises ResolutionImpossible, and a release that requires a newer - # Python raises "No matching distribution found". + # The HASTE GDAL wheels are not on the public index until the + # approval-gated publisher runs. Their full dependency closure is + # resolved and installed from the built wheel artifact by + # gdal-runtime-build.yml; this job checks the remaining public-index + # dependencies without falling back to a PyPI GDAL sdist. + python .github/scripts/filter_dependency_requirements.py \ + "${{ matrix.requirements }}" /tmp/requirements-resolve.txt pip install --dry-run --report /tmp/report.json \ - -r "$(basename "${{ matrix.requirements }}")" + -r /tmp/requirements-resolve.txt - name: Check protobuf stays compatible with the Functions worker if: matrix.functions_worker diff --git a/.github/workflows/deploy-apps.yml b/.github/workflows/deploy-apps.yml index 785a78c9..53d32358 100644 --- a/.github/workflows/deploy-apps.yml +++ b/.github/workflows/deploy-apps.yml @@ -39,6 +39,11 @@ on: required: false default: "" type: string + gdal_package_run_id: + description: "Successful GDAL runtime build run ID with tested prebuilt Function App packages. Required except for swa-only deployments." + required: false + default: "" + type: string concurrency: group: haste-settings-${{ inputs.environment }} @@ -49,6 +54,8 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.environment }} permissions: + actions: read + attestations: read id-token: write contents: read pull-requests: read @@ -57,6 +64,58 @@ jobs: with: persist-credentials: false + - name: Download tested Function App packages + id: function-packages + if: >- + inputs.gdal_package_run_id != '' && + inputs.component != 'swa' + env: + GH_TOKEN: ${{ github.token }} + PACKAGE_RUN_ID: ${{ inputs.gdal_package_run_id }} + EXPECTED_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [[ ! "$PACKAGE_RUN_ID" =~ ^[0-9]+$ ]]; then + echo "::error::gdal_package_run_id must be a numeric Actions run ID" + exit 1 + fi + read -r status conclusion head_sha < <(gh run view "$PACKAGE_RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --json status,conclusion,headSha \ + --jq '[.status, .conclusion, .headSha] | @tsv') + if [[ "$status" != "completed" || "$conclusion" != "success" ]]; then + echo "::error::GDAL package run must have completed successfully" + exit 1 + fi + if [[ "$head_sha" != "$EXPECTED_SHA" ]]; then + echo "::error::GDAL package run was built from a different source SHA" + exit 1 + fi + package_dir="$RUNNER_TEMP/haste-function-packages" + gh run download "$PACKAGE_RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --name "haste-function-packages-$EXPECTED_SHA" \ + --dir "$package_dir" + for package in hastefuncapi hastefuncqueues titilerfuncapi; do + (cd "$package_dir" && sha256sum --check "$package.sha256") + gh attestation verify "$package_dir/$package.zip" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow \ + "$GITHUB_REPOSITORY/.github/workflows/gdal-runtime-build.yml" \ + --source-ref refs/heads/main + done + echo "directory=$package_dir" >> "$GITHUB_OUTPUT" + + - name: Require prebuilt packages for Function App deployments + if: >- + inputs.component != 'swa' && + inputs.gdal_package_run_id == '' + run: | + echo "::error::Function App deployments require gdal_package_run_id." \ + "Select the successful GDAL runtime build for this exact source" \ + "SHA so deployment uses the tested, attested, build-once zip." + exit 1 + - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: @@ -75,9 +134,10 @@ jobs: - name: Resolve hastegeo wheel URL id: hastegeo if: >- - inputs.component == 'all' || + inputs.gdal_package_run_id == '' && + (inputs.component == 'all' || inputs.component == 'funcapi' || - inputs.component == 'funcqueue' + inputs.component == 'funcqueue') env: GH_TOKEN: ${{ github.token }} run: | @@ -208,6 +268,7 @@ jobs: # reads it back yet -- a blank hastegeo_version still resolves to # latest stable. HASTEGEO_VERSION: ${{ steps.hastegeo.outputs.version }} + HASTE_FUNCTION_PACKAGE_DIR: ${{ steps.function-packages.outputs.directory }} # Batch pool wiring. All optional: unset reproduces the legacy # single-pool behavior. Stored as environment *secrets* (not # variables) so the values are masked in this repo's public diff --git a/.github/workflows/gdal-runtime-build.yml b/.github/workflows/gdal-runtime-build.yml new file mode 100644 index 00000000..716d8da1 --- /dev/null +++ b/.github/workflows/gdal-runtime-build.yml @@ -0,0 +1,614 @@ +name: Build GDAL native runtime + +on: + pull_request: + branches: + - main + paths: + - "api/hastefuncapi/**" + - "api/hastefuncqueues/**" + - "hastelib/src/**" + - "native/gdal-runtime/**" + - ".github/scripts/publish_gdal_runtime.py" + - ".github/scripts/tests/test_publish_gdal_runtime.py" + - ".github/scripts/build_function_package.py" + - ".github/scripts/filter_dependency_requirements.py" + - ".github/scripts/prepare_training_runtime_env.py" + - ".github/scripts/tests/test_build_function_package.py" + - ".github/scripts/tests/test_filter_dependency_requirements.py" + - ".github/scripts/tests/test_prepare_training_runtime_env.py" + - ".github/scripts/check_gdal_hardening.py" + - ".github/scripts/tests/test_gdal_hardening_guard.py" + - ".github/scripts/tests/test_deploy_network_access.py" + - "api/titilerfuncapi/**" + - "hastelib/src/hastegeo/__init__.py" + - "hastelib/src/hastegeo/core/processors/uploader.py" + - "hastelib/src/hastegeo/core/utils/footprints.py" + - "hastelib/src/hastegeo/core/utils/gdal_security.py" + - "hastelib/src/hastegeo/core/utils/imagery.py" + - "hastelib/src/hastegeo/workflows/prepare_imagery.py" + - "hastelib/pyproject.toml" + - "hastelib/tests/build/test_dependency_pins.py" + - "hastelib/tests/core/processors/test_uploader_format.py" + - "hastelib/tests/core/utils/test_footprints.py" + - "hastelib/tests/core/utils/test_gdal_security.py" + - "hastelib/tests/core/utils/test_imagery.py" + - "hastelib/tests/core/utils/test_imagery_runtime.py" + - ".github/scripts/deploy_apps.sh" + - "infra/modules/functions.bicep" + - "infra/main.json" + - "api/hastefuncapi/Dockerfile" + - "api/hastefuncapi/requirements.txt" + - "api/hastefuncqueues/Dockerfile" + - "api/hastefuncqueues/requirements.txt" + - "api/titilerfuncapi/requirements.txt" + - "docker/docker-compose.yml" + - "docker/imageryprep/Dockerfile" + - "docker/imageryprep/requirements.txt" + - "docker/titiler/Dockerfile" + - "docker/training/Dockerfile" + - "docker/training/env/env.yml" + - "docker/training/code/**" + - "docker/training/scripts/entrypoint.sh" + - "docs/security-configuration.md" + - ".github/workflows/dependency-validation.yml" + - ".github/workflows/deploy-apps.yml" + - ".github/workflows/gdal-runtime-build.yml" + push: + branches: + - main + paths: + - "api/hastefuncapi/**" + - "api/hastefuncqueues/**" + - "hastelib/src/**" + - "native/gdal-runtime/**" + - ".github/scripts/publish_gdal_runtime.py" + - ".github/scripts/tests/test_publish_gdal_runtime.py" + - ".github/scripts/build_function_package.py" + - ".github/scripts/filter_dependency_requirements.py" + - ".github/scripts/prepare_training_runtime_env.py" + - ".github/scripts/tests/test_build_function_package.py" + - ".github/scripts/tests/test_filter_dependency_requirements.py" + - ".github/scripts/tests/test_prepare_training_runtime_env.py" + - ".github/scripts/check_gdal_hardening.py" + - ".github/scripts/tests/test_gdal_hardening_guard.py" + - ".github/scripts/tests/test_deploy_network_access.py" + - "api/titilerfuncapi/**" + - "hastelib/src/hastegeo/__init__.py" + - "hastelib/src/hastegeo/core/processors/uploader.py" + - "hastelib/src/hastegeo/core/utils/footprints.py" + - "hastelib/src/hastegeo/core/utils/gdal_security.py" + - "hastelib/src/hastegeo/core/utils/imagery.py" + - "hastelib/src/hastegeo/workflows/prepare_imagery.py" + - "hastelib/pyproject.toml" + - "hastelib/tests/build/test_dependency_pins.py" + - "hastelib/tests/core/processors/test_uploader_format.py" + - "hastelib/tests/core/utils/test_footprints.py" + - "hastelib/tests/core/utils/test_gdal_security.py" + - "hastelib/tests/core/utils/test_imagery.py" + - "hastelib/tests/core/utils/test_imagery_runtime.py" + - ".github/scripts/deploy_apps.sh" + - "infra/modules/functions.bicep" + - "infra/main.json" + - "api/hastefuncapi/Dockerfile" + - "api/hastefuncapi/requirements.txt" + - "api/hastefuncqueues/Dockerfile" + - "api/hastefuncqueues/requirements.txt" + - "api/titilerfuncapi/requirements.txt" + - "docker/docker-compose.yml" + - "docker/imageryprep/Dockerfile" + - "docker/imageryprep/requirements.txt" + - "docker/titiler/Dockerfile" + - "docker/training/Dockerfile" + - "docker/training/env/env.yml" + - "docker/training/code/**" + - "docker/training/scripts/entrypoint.sh" + - "docs/security-configuration.md" + - ".github/workflows/dependency-validation.yml" + - ".github/workflows/deploy-apps.yml" + - ".github/workflows/gdal-runtime-build.yml" + workflow_dispatch: + +permissions: {} + +jobs: + unit: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Run native runtime unit tests + working-directory: native/gdal-runtime + run: python -m unittest discover -s tests -v + + - name: Check repository GDAL hardening configuration + run: python .github/scripts/check_gdal_hardening.py + + build: + needs: unit + runs-on: ubuntu-latest + environment: gdal-runtime-build + env: + PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL || 'https://pypi.org/simple' }} + permissions: + contents: read + container: + image: quay.io/pypa/manylinux_2_28_x86_64@sha256:1e102f2e9ed9b7fd8d64b29f798c76078c3fa8f14d34914d7a4045e39419d8e0 + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Install native build tools + run: dnf install -y nasm openssl perl perl-IPC-Cmd perl-Text-Template sqlite + + - name: Build native runtime and binding wheels + env: + BUILD_ROOT: ${{ runner.temp }}/haste-gdal-runtime + run: bash native/gdal-runtime/build.sh + + - name: Validate the complete runtime release bundle + env: + BUILD_ROOT: ${{ runner.temp }}/haste-gdal-runtime + run: | + python3 .github/scripts/publish_gdal_runtime.py \ + --artifact-dir "$BUILD_ROOT/dist" \ + --validate-only + + - name: Download hash-locked smoke dependencies + env: + BUILD_ROOT: ${{ runner.temp }}/haste-gdal-runtime + run: | + mkdir -p "$BUILD_ROOT/smoke-wheels" + "$BUILD_ROOT/venv/bin/python" -m pip download \ + --isolated \ + --index-url https://pypi.org/simple \ + --only-binary=:all: \ + --no-deps \ + --require-hashes \ + --dest "$BUILD_ROOT/smoke-wheels" \ + -r native/gdal-runtime/requirements-smoke.txt + + - name: Install hash-locked binding smoke dependencies + env: + BUILD_ROOT: ${{ runner.temp }}/haste-gdal-runtime + run: | + "$BUILD_ROOT/venv/bin/python" -m pip install \ + --isolated \ + --no-index \ + --find-links "$BUILD_ROOT/smoke-wheels" \ + --only-binary=:all: \ + --no-deps \ + --require-hashes \ + -r native/gdal-runtime/requirements-smoke.txt + + - name: Smoke-test the native runtime and GDAL binding wheels offline + working-directory: native/gdal-runtime + env: + BUILD_ROOT: ${{ runner.temp }}/haste-gdal-runtime + run: | + build="$(python3 fetch_sources.py --print-build)" + "$BUILD_ROOT/venv/bin/python" smoke_test.py \ + --wheel-dir "$BUILD_ROOT/dist" \ + --build "$build" + + - name: Upload immutable build artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist/* + if-no-files-found: error + retention-days: 7 + + - name: Upload hash-locked smoke dependency wheels + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: haste-gdal-smoke-deps-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/smoke-wheels/*.whl + if-no-files-found: error + retention-days: 7 + + target-image-smoke: + needs: + - build + - function-packages + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - name: functions-python311 + image: mcr.microsoft.com/azure-functions/python:4-python3.11@sha256:2c39d8b50750fbec9a91ef33323fadd1b986358fab4f7823db5b11b6e533f479 + - name: imageryprep-python311 + image: mcr.microsoft.com/azureml/curated/minimal-py311-inference:59@sha256:89a46a46b71b4692731654d0523f53c8d14929e41ea8129293dc4e39fcabb14c + name: smoke-${{ matrix.name }} + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Download build wheels + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist + + - name: Download hash-locked smoke dependencies + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-smoke-deps-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/smoke-wheels + + - name: Smoke-test wheels in ${{ matrix.name }} + env: + SMOKE_IMAGE: ${{ matrix.image }} + WHEEL_DIR: ${{ runner.temp }}/haste-gdal-runtime/dist + SMOKE_WHEEL_DIR: ${{ runner.temp }}/haste-gdal-runtime/smoke-wheels + run: | + docker run --rm --entrypoint sh \ + --mount "type=bind,source=$GITHUB_WORKSPACE/native/gdal-runtime,target=/smoke,readonly" \ + --mount "type=bind,source=$WHEEL_DIR,target=/wheels,readonly" \ + --mount "type=bind,source=$SMOKE_WHEEL_DIR,target=/smoke-wheels,readonly" \ + "$SMOKE_IMAGE" -ec ' + cd /tmp + python -m pip install \ + --isolated \ + --no-index \ + --find-links /smoke-wheels \ + --only-binary=:all: \ + --no-deps \ + --require-hashes \ + -r /smoke/requirements-smoke.txt + build="$(python /smoke/fetch_sources.py --print-build)" + python /smoke/smoke_test.py \ + --wheel-dir /wheels \ + --build "$build" + ' + + - name: Download imageryprep test package + if: matrix.name == 'imageryprep-python311' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-function-packages-${{ github.sha }} + path: ${{ runner.temp }}/haste-function-packages + + - name: Import the packaged imageryprep worker in its target image + if: matrix.name == 'imageryprep-python311' + env: + SMOKE_IMAGE: ${{ matrix.image }} + PACKAGE_DIR: ${{ runner.temp }}/haste-function-packages + run: | + docker run --rm --user root --entrypoint sh \ + --mount "type=bind,source=$PACKAGE_DIR,target=/function-packages,readonly" \ + "$SMOKE_IMAGE" -ec ' + apt-get update + apt-get install -y --no-install-recommends \ + libgl1 libglib2.0-0 libsm6 libxext6 libxrender-dev libgomp1 + python -c "import zipfile; \ + zipfile.ZipFile(\"/function-packages/imageryprep-smoke.zip\").extractall(\"/tmp/haste-imageryprep\")" + PYTHONPATH="/tmp/haste-imageryprep/.python_packages/lib/site-packages:/tmp/haste-imageryprep" \ + HASTE_GDAL_RUNTIME_REQUIRED=true \ + python -c "import hastegeo.workflows.prepare_imagery" + ' + + function-packages: + needs: build + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/azure-functions/python:4-python3.11@sha256:2c39d8b50750fbec9a91ef33323fadd1b986358fab4f7823db5b11b6e533f479 + defaults: + run: + shell: bash + permissions: + actions: read + contents: read + env: + PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL || 'https://pypi.org/simple' }} + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Download native runtime wheels + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist + + - name: Build the matching hastegeo wheel + run: | + python -m pip install --isolated \ + --index-url "$PIP_INDEX_URL" build==1.2.2 + mkdir -p dist + python -m build --wheel --outdir dist + working-directory: hastelib + + - name: Build and smoke-test API and queues packages + run: | + set -euo pipefail + hastegeo_wheel=$(find hastelib/dist -maxdepth 1 \ + -name 'hastegeo-*.whl' -print -quit) + test -n "$hastegeo_wheel" + python .github/scripts/build_function_package.py \ + --app-name hastefuncapi \ + --source-dir api/hastefuncapi \ + --requirements api/hastefuncapi/requirements.txt \ + --wheel-dir "$RUNNER_TEMP/haste-gdal-runtime/dist" \ + --hastegeo-wheel "$hastegeo_wheel" \ + --output-dir "$RUNNER_TEMP/haste-function-packages" + python .github/scripts/build_function_package.py \ + --app-name hastefuncqueues \ + --source-dir api/hastefuncqueues \ + --requirements api/hastefuncqueues/requirements.txt \ + --wheel-dir "$RUNNER_TEMP/haste-gdal-runtime/dist" \ + --hastegeo-wheel "$hastegeo_wheel" \ + --output-dir "$RUNNER_TEMP/haste-function-packages" + python .github/scripts/build_function_package.py \ + --app-name titilerfuncapi \ + --source-dir api/titilerfuncapi \ + --requirements api/titilerfuncapi/requirements.txt \ + --wheel-dir "$RUNNER_TEMP/haste-gdal-runtime/dist" \ + --output-dir "$RUNNER_TEMP/haste-function-packages" + python .github/scripts/build_function_package.py \ + --app-name imageryprep-smoke \ + --source-dir hastelib/src \ + --requirements docker/imageryprep/requirements.txt \ + --wheel-dir "$RUNNER_TEMP/haste-gdal-runtime/dist" \ + --output-dir "$RUNNER_TEMP/haste-function-packages" + + - name: Upload build-once Function App packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: haste-function-packages-${{ github.sha }} + path: ${{ runner.temp }}/haste-function-packages/* + if-no-files-found: error + retention-days: 7 + + vulnerability-scan: + needs: build + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + steps: + - name: Checkout scanner configuration + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + sparse-checkout: .grype.yaml + sparse-checkout-cone-mode: false + + - name: Download build inventory + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist + + - name: Scan the CycloneDX inventory for known vulnerabilities + uses: anchore/scan-action@1638637db639e0ade3258b51db49a9a137574c3e # v6 + with: + sbom: ${{ runner.temp }}/haste-gdal-runtime/dist/bom.cdx.json + fail-build: true + severity-cutoff: medium + add-cpes-if-none: true + output-format: table + + - name: Scan built wheel artifacts for known vulnerabilities + uses: anchore/scan-action@1638637db639e0ade3258b51db49a9a137574c3e # v6 + with: + path: ${{ runner.temp }}/haste-gdal-runtime/dist + fail-build: true + severity-cutoff: medium + output-format: table + + training-base-smoke: + needs: build + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + env: + PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL || 'https://pypi.org/simple' }} + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Download build wheels + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist + + - name: Download hash-locked smoke dependencies + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-smoke-deps-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/smoke-wheels + + - name: Point the full training environment at this build's wheels + env: + WHEEL_DIR: ${{ runner.temp }}/haste-gdal-runtime/dist + TRAINING_ENV: ${{ runner.temp }}/haste-gdal-runtime/training-env.yml + run: | + python .github/scripts/prepare_training_runtime_env.py \ + docker/training/env/env.yml "$WHEEL_DIR" "$TRAINING_ENV" + + - name: Smoke-test the full Python 3.11 training environment + env: + WHEEL_DIR: ${{ runner.temp }}/haste-gdal-runtime/dist + SMOKE_WHEEL_DIR: ${{ runner.temp }}/haste-gdal-runtime/smoke-wheels + TRAINING_ENV: ${{ runner.temp }}/haste-gdal-runtime/training-env.yml + PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL || 'https://pypi.org/simple' }} + run: | + docker run --rm --user root --entrypoint sh \ + --mount "type=bind,source=$GITHUB_WORKSPACE/native/gdal-runtime,target=/smoke,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/docker/training/code,target=/training-code,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/hastelib/src,target=/hastelib-src,readonly" \ + --mount "type=bind,source=$WHEEL_DIR,target=$WHEEL_DIR,readonly" \ + --mount "type=bind,source=$SMOKE_WHEEL_DIR,target=/smoke-wheels,readonly" \ + --mount "type=bind,source=$TRAINING_ENV,target=/training-env.yml,readonly" \ + --env PIP_INDEX_URL \ + --env HASTE_GDAL_RUNTIME_REQUIRED=true \ + --env WHEEL_DIR \ + mcr.microsoft.com/azureml/curated/minimal-py312-cuda12.4-inference@sha256:171ee0f9b43d2295c0266c55b2974d1ab300ca2dbec081bdd0fb2604f75644b3 \ + -ec ' + apt-get update + apt-get install -y --no-install-recommends libgl1-mesa-glx + /opt/miniconda/bin/conda env create --yes \ + --prefix /tmp/gdal-runtime-training \ + --file /training-env.yml + python=/tmp/gdal-runtime-training/bin/python + PYTHONPATH=/training-code:/hastelib-src \ + HASTE_GDAL_RUNTIME_REQUIRED=true \ + "$python" -m unittest discover \ + -s /training-code/tests \ + -p test_gdal_runtime_integration.py -v + "$python" -m pip install \ + --isolated \ + --no-index \ + --find-links /smoke-wheels \ + --only-binary=:all: \ + --no-deps \ + --require-hashes \ + -r /smoke/requirements-smoke.txt + build="$("$python" /smoke/fetch_sources.py --print-build)" + "$python" /smoke/smoke_test.py \ + --wheel-dir "$WHEEL_DIR" \ + --build "$build" + ' + + attest: + needs: + - target-image-smoke + - function-packages + - vulnerability-scan + - training-base-smoke + if: >- + github.repository == 'microsoft/haste' && + github.event_name == 'push' && + github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + attestations: write + contents: read + id-token: write + steps: + - name: Download verified runtime artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/dist + + - name: Attest runtime bundle provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/haste-gdal-runtime/dist/* + + - name: Attest the CycloneDX SBOM + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/haste-gdal-runtime/dist/*.whl + sbom-path: ${{ runner.temp }}/haste-gdal-runtime/dist/bom.cdx.json + + - name: Download build-once Function App packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-function-packages-${{ github.sha }} + path: ${{ runner.temp }}/haste-function-packages + + - name: Attest Function App package artifacts + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: ${{ runner.temp }}/haste-function-packages/*.zip + + publisher-unit: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout source without persisted credentials + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Test immutable publisher validation + run: | + python -m unittest discover \ + -s .github/scripts/tests \ + -p "test_*.py" -v + + publish: + needs: + - attest + - publisher-unit + if: >- + github.repository == 'microsoft/haste' && + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + vars.GDAL_RUNTIME_LICENSE_REVIEWED == 'true' && + vars.GDAL_RUNTIME_PUBLISH_ENABLED == 'true' + runs-on: ubuntu-latest + # Configure this environment with required reviewers before enabling + # either repository variable above. + environment: gdal-runtime-publish + permissions: + actions: read + contents: write + steps: + - name: Checkout trusted publisher source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Download the attested runtime bundle + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: haste-gdal-runtime-${{ github.sha }} + path: ${{ runner.temp }}/haste-gdal-runtime/release + + - name: Verify provenance for each release asset + env: + GH_TOKEN: ${{ github.token }} + run: | + for asset in "${{ runner.temp }}/haste-gdal-runtime/release/"*; do + gh attestation verify "$asset" \ + --repo microsoft/haste \ + --signer-workflow \ + microsoft/haste/.github/workflows/gdal-runtime-build.yml \ + --source-ref refs/heads/main + done + + - name: Validate and publish immutable runtime assets + env: + GH_TOKEN: ${{ github.token }} + run: | + python .github/scripts/publish_gdal_runtime.py \ + --artifact-dir "${{ runner.temp }}/haste-gdal-runtime/release" diff --git a/.github/workflows/titiler-tests.yml b/.github/workflows/titiler-tests.yml index d6fddf35..74b46a27 100644 --- a/.github/workflows/titiler-tests.yml +++ b/.github/workflows/titiler-tests.yml @@ -20,9 +20,17 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" - # Same dependency set the Function App resolves at deploy time. + # The HASTE GDAL wheels are built and validated by gdal-runtime-build.yml + # but are not published until the approval-gated release. This route/API + # test suite uses the public dependencies; runtime-wheel validation stays + # in the GDAL workflow, which installs its exact build artifacts. - name: Install dependencies run: | - python -m pip install -r api/titilerfuncapi/requirements.txt httpx + python .github/scripts/filter_dependency_requirements.py \ + api/titilerfuncapi/requirements.txt \ + /tmp/titiler-test-requirements.txt + python -m pip install -r /tmp/titiler-test-requirements.txt httpx - name: Run tests + env: + PYTHONPATH: api/titilerfuncapi run: python -m unittest discover -s api/titilerfuncapi/tests -t . -v diff --git a/.grype.yaml b/.grype.yaml new file mode 100644 index 00000000..b35949b0 --- /dev/null +++ b/.grype.yaml @@ -0,0 +1,9 @@ +# Grype matches by version, so it still reports zlib 1.3.2 even though the +# runtime build applies the upstream fix (native/gdal-runtime/patches). Remove +# this entry when sources.lock moves to a zlib release that contains the fix. +# Checked 2026-10-21: the latest zlib release is 1.3.2 and does not include +# the fix (madler/zlib df84af25), so the patched build remains in use. +ignore: + - vulnerability: CVE-2026-85091 + package: + name: zlib diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 41a7450c..5fba991a 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -26,6 +26,37 @@ This software includes third-party packages and libraries. The complete list of - **Source**: https://gdal.org/ - **Usage**: Geospatial data processing and format conversion +### HASTE GDAL Runtime (`haste-gdal-runtime`) + +HASTE builds and redistributes a shared native GDAL runtime from the pinned +sources in `native/gdal-runtime/sources.lock`. Each runtime release also +ships the license and NOTICE texts extracted from these sources +(`licenses//`) plus an SBOM; see `native/gdal-runtime/`. + +| Component | Version | License | +|---|---|---| +| GDAL (includes bundled libtiff, libgeotiff, libpng and json-c) | 3.13.3 | MIT/X and bundled third-party licenses (https://gdal.org/en/stable/copyright.html) | +| PROJ | 9.8.1 | MIT | +| SQLite | 3.53.4 | Public Domain (https://www.sqlite.org/copyright.html) | +| curl | 8.22.0 | curl License | +| OpenSSL | 3.5.9 | Apache-2.0 | +| zlib | 1.3.2 | Zlib; HASTE applies the upstream fix for CVE-2026-85091 (`native/gdal-runtime/patches/`) | +| libdeflate | 1.26 | MIT | +| Zstandard | 1.5.7 | BSD-3-Clause (the project is dual-licensed with GPL-2.0; HASTE uses it under BSD) | +| libjpeg-turbo | 3.1.4.1 | BSD-3-Clause, IJG and Zlib | + +The Python bindings built against this runtime are: + +| Package | Version | License | +|---|---|---| +| GDAL Python bindings (`osgeo`) | 3.13.3 | MIT/X | +| rasterio | 1.4.4 | BSD-3-Clause | +| fiona | 1.10.1 | BSD-3-Clause | +| pyogrio | 0.13.0 | MIT | +| pyproj | 3.7.2 | MIT | + +The `haste-gdal-runtime` wheel bundles only the shared libraries of the components in the first table (GDAL, PROJ, SQLite, curl, OpenSSL, zlib, libdeflate, Zstandard and libjpeg-turbo). The binding wheels bundle no native libraries of their own and link to that runtime. Any system library that `auditwheel` adds in a future build keeps its own license and is recorded in the runtime SBOM and license bundle. + ### React - **License**: MIT License - **Source**: https://reactjs.org/ diff --git a/api/hastefuncapi/Dockerfile b/api/hastefuncapi/Dockerfile index 3226c5f0..a6cd76e8 100644 --- a/api/hastefuncapi/Dockerfile +++ b/api/hastefuncapi/Dockerfile @@ -2,6 +2,13 @@ FROM mcr.microsoft.com/azure-functions/python:4-python3.11 WORKDIR /home/site/wwwroot +ENV HASTE_GDAL_RUNTIME_REQUIRED=true \ + GDAL_VRT_ENABLE_RAWRASTERBAND=NO \ + GDAL_VRT_ENABLE_PYTHON=NO \ + GDAL_DRIVER_PATH=disable \ + GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR \ + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO + # Create non-root user first so COPY --chown can reference it; Secrets dir is empty so chown is fast RUN groupadd -r appgroup && useradd -r -g appgroup appuser \ && mkdir -p /azure-functions-host/Secrets \ diff --git a/api/hastefuncapi/requirements.txt b/api/hastefuncapi/requirements.txt index 2380b91b..5c03d9bb 100644 --- a/api/hastefuncapi/requirements.txt +++ b/api/hastefuncapi/requirements.txt @@ -18,9 +18,13 @@ email_validator==2.3.0 psycopg2-binary==2.9.12 requests==2.34.2 requests-auth==8.0.0 -rasterio==1.3.11 +haste-gdal-runtime @ https://github.com/microsoft/haste/releases/download/haste-binaries/haste_gdal_runtime-3.13.3%2Bhaste.1-py3-none-manylinux_2_28_x86_64.whl +GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/gdal-3.13.3%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +rasterio @ https://github.com/microsoft/haste/releases/download/haste-binaries/rasterio-1.4.4%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +fiona @ https://github.com/microsoft/haste/releases/download/haste-binaries/fiona-1.10.1%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyogrio @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyogrio-0.13.0%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyproj @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyproj-3.7.2%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl opencv-python==5.0.0.93 -GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/GDAL-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == 'linux' shapely==2.1.2 pyyaml==6.0.3 # Planetary Computer publishing (vendored REST client) — geo stack used by the @@ -29,8 +33,7 @@ pyyaml==6.0.3 # emitting STAC 1.1.0, but MPC Pro requires STAC 1.0.0 (stac.py STAC_VERSION). pystac[validation]==1.11.0 geopandas==1.1.2 -pyproj==3.6.1 -pyogrio==0.10.0 + boto3==1.43.73 tensorboard==2.19.0 tenacity==9.1.4 diff --git a/api/hastefuncqueues/Dockerfile b/api/hastefuncqueues/Dockerfile index 3ab1f5ea..1f824467 100644 --- a/api/hastefuncqueues/Dockerfile +++ b/api/hastefuncqueues/Dockerfile @@ -2,6 +2,13 @@ FROM mcr.microsoft.com/azure-functions/python:4-python3.11 WORKDIR /home/site/wwwroot +ENV HASTE_GDAL_RUNTIME_REQUIRED=true \ + GDAL_VRT_ENABLE_RAWRASTERBAND=NO \ + GDAL_VRT_ENABLE_PYTHON=NO \ + GDAL_DRIVER_PATH=disable \ + GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR \ + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO + # Create non-root user first so COPY --chown can reference it; Secrets dir is empty so chown is fast RUN groupadd -r appgroup && useradd -r -g appgroup appuser \ && mkdir -p /azure-functions-host/Secrets \ diff --git a/api/hastefuncqueues/requirements.txt b/api/hastefuncqueues/requirements.txt index 53cedaa6..f2860a7c 100644 --- a/api/hastefuncqueues/requirements.txt +++ b/api/hastefuncqueues/requirements.txt @@ -18,9 +18,13 @@ email_validator==2.3.0 psycopg2-binary==2.9.12 requests==2.34.2 requests-auth==8.0.0 -rasterio==1.3.11 +haste-gdal-runtime @ https://github.com/microsoft/haste/releases/download/haste-binaries/haste_gdal_runtime-3.13.3%2Bhaste.1-py3-none-manylinux_2_28_x86_64.whl +GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/gdal-3.13.3%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +rasterio @ https://github.com/microsoft/haste/releases/download/haste-binaries/rasterio-1.4.4%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +fiona @ https://github.com/microsoft/haste/releases/download/haste-binaries/fiona-1.10.1%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyogrio @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyogrio-0.13.0%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyproj @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyproj-3.7.2%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl opencv-python==5.0.0.93 -GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/GDAL-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == 'linux' shapely==2.1.2 pyyaml==6.0.3 # Planetary Computer publishing (vendored REST client) — geo stack used by the @@ -29,8 +33,7 @@ pyyaml==6.0.3 # STAC 1.0.0 (stac.py STAC_VERSION). pystac[validation]==1.11.0 geopandas==1.1.2 -pyproj==3.6.1 -pyogrio==0.10.0 + boto3==1.43.73 tensorboard==2.19.0 tenacity==9.1.4 diff --git a/api/titilerfuncapi/app/__init__.py b/api/titilerfuncapi/app/__init__.py index 804329c7..ff068cc3 100644 --- a/api/titilerfuncapi/app/__init__.py +++ b/api/titilerfuncapi/app/__init__.py @@ -7,31 +7,38 @@ from typing import Annotated import attr -import azure.functions as func -import morecantile -import rasterio -from fastapi import FastAPI, HTTPException, Query -from rio_tiler.io import Reader -from starlette.middleware.cors import CORSMiddleware -from starlette_cramjam.middleware import CompressionMiddleware -from titiler.application import __version__ as titiler_version -from titiler.application.settings import ApiSettings -from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import TilerFactory -from titiler.core.middleware import ( +from gdal_runtime import secure_gdal_runtime + +secure_gdal_runtime() + +import azure.functions as func # noqa: E402 +import morecantile # noqa: E402 +import rasterio # noqa: E402 +from fastapi import FastAPI, HTTPException, Query # noqa: E402 +from rio_tiler.io import Reader # noqa: E402 +from starlette.middleware.cors import CORSMiddleware # noqa: E402 +from starlette_cramjam.middleware import CompressionMiddleware # noqa: E402 +from titiler.application import __version__ as titiler_version # noqa: E402 +from titiler.application.settings import ApiSettings # noqa: E402 +from titiler.core.errors import ( # noqa: E402 + DEFAULT_STATUS_CODES, + add_exception_handlers, +) +from titiler.core.factory import TilerFactory # noqa: E402 +from titiler.core.middleware import ( # noqa: E402 CacheControlMiddleware, LoggerMiddleware, LowerCaseQueryStringMiddleware, TotalTimeMiddleware, ) -from .dataset_url import ( +from .dataset_url import ( # noqa: E402 DatasetUrlError, DatasetUrlPolicy, DestinationChecker, validate_dataset_url, ) -from .gdal_config import apply_gdal_hardening, gdal_environment +from .gdal_config import apply_gdal_hardening, gdal_environment # noqa: E402 logger = logging.getLogger(__name__) diff --git a/api/titilerfuncapi/gdal_runtime.py b/api/titilerfuncapi/gdal_runtime.py new file mode 100644 index 00000000..8b26c071 --- /dev/null +++ b/api/titilerfuncapi/gdal_runtime.py @@ -0,0 +1,56 @@ +"""Shared GDAL runtime initialization for the TiTiler Function App.""" + +from __future__ import annotations + +import importlib +import importlib.util +import logging +import os +from typing import Any + +logger = logging.getLogger(__name__) + +# rio-tiler reprojects through in-memory VRTs, so the driver stays registered. +# Raw bands and Python pixel functions are off, and user-supplied VRT content +# is refused before GDAL opens it. +ALLOWED_DRIVERS = frozenset({"GTiff", "COG", "JPEG", "PNG", "MEM", "VRT"}) + + +def secure_gdal_runtime() -> dict[str, Any] | None: + """Configure and verify the shared runtime before importing TiTiler.""" + required_value = ( + os.environ.get("HASTE_GDAL_RUNTIME_REQUIRED", "").strip().lower() + ) + if required_value in {"1", "true", "yes"}: + required = True + elif required_value in {"", "0", "false", "no"}: + required = False + else: + raise ValueError("HASTE_GDAL_RUNTIME_REQUIRED must be a boolean value") + + if importlib.util.find_spec("haste_gdal_runtime") is None: + if required: + raise RuntimeError( + "HASTE_GDAL_RUNTIME_REQUIRED is set but " + "haste-gdal-runtime is not installed" + ) + logger.warning( + "haste-gdal-runtime is absent; TiTiler is using its current " + "GDAL installation" + ) + return None + + runtime = importlib.import_module("haste_gdal_runtime") + manifest = runtime.load_manifest() + expected_gdal = manifest.get("gdal_release") + if not isinstance(expected_gdal, str) or not expected_gdal: + raise RuntimeError("GDAL runtime manifest has no GDAL release") + policy = runtime.Policy( + expected_gdal=expected_gdal, + allowed_drivers=ALLOWED_DRIVERS, + required_bindings=("rasterio",), + ) + runtime.configure(policy) + report = runtime.secure_startup(policy) + logger.info("verified TiTiler GDAL runtime: %s", report) + return report diff --git a/api/titilerfuncapi/requirements.txt b/api/titilerfuncapi/requirements.txt index 07c7ff89..751abf75 100644 --- a/api/titilerfuncapi/requirements.txt +++ b/api/titilerfuncapi/requirements.txt @@ -4,3 +4,5 @@ azure-functions titiler.application==0.21.1 +haste-gdal-runtime @ https://github.com/microsoft/haste/releases/download/haste-binaries/haste_gdal_runtime-3.13.3%2Bhaste.1-py3-none-manylinux_2_28_x86_64.whl +rasterio @ https://github.com/microsoft/haste/releases/download/haste-binaries/rasterio-1.4.4%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl diff --git a/api/titilerfuncapi/tests/test_gdal_runtime.py b/api/titilerfuncapi/tests/test_gdal_runtime.py new file mode 100644 index 00000000..4120f2d7 --- /dev/null +++ b/api/titilerfuncapi/tests/test_gdal_runtime.py @@ -0,0 +1,65 @@ +import unittest +from unittest import mock + +import gdal_runtime + + +class TestSecureGdalRuntime(unittest.TestCase): + def test_titiler_policy_keeps_vrt_for_reprojection(self) -> None: + self.assertIn("VRT", gdal_runtime.ALLOWED_DRIVERS) + + def test_missing_required_runtime_fails_closed(self): + with ( + mock.patch.dict( + gdal_runtime.os.environ, + {"HASTE_GDAL_RUNTIME_REQUIRED": "true"}, + ), + mock.patch.object( + gdal_runtime.importlib.util, "find_spec", return_value=None + ), + ): + with self.assertRaisesRegex(RuntimeError, "is not installed"): + gdal_runtime.secure_gdal_runtime() + + def test_installed_runtime_is_configured_before_verification(self): + calls = [] + runtime = mock.Mock() + runtime.load_manifest.return_value = {"gdal_release": "3.13.3"} + runtime.Policy.return_value = object() + runtime.configure.side_effect = lambda policy: calls.append( + "configure" + ) + runtime.secure_startup.side_effect = lambda policy: calls.append( + "secure_startup" + ) or {"gdal_release": "3.13.3"} + with ( + mock.patch.object( + gdal_runtime.importlib.util, "find_spec", return_value=object() + ), + mock.patch.object( + gdal_runtime.importlib, + "import_module", + return_value=runtime, + ), + ): + report = gdal_runtime.secure_gdal_runtime() + + self.assertEqual(calls, ["configure", "secure_startup"]) + self.assertEqual(report, {"gdal_release": "3.13.3"}) + runtime.Policy.assert_called_once_with( + expected_gdal="3.13.3", + allowed_drivers=gdal_runtime.ALLOWED_DRIVERS, + required_bindings=("rasterio",), + ) + + def test_invalid_required_setting_is_rejected(self): + with mock.patch.dict( + gdal_runtime.os.environ, + {"HASTE_GDAL_RUNTIME_REQUIRED": "sometimes"}, + ): + with self.assertRaisesRegex(ValueError, "boolean value"): + gdal_runtime.secure_gdal_runtime() + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d738b177..69f2741f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -72,6 +72,15 @@ services: image: haste-titiler ports: - "8000:8000" + environment: + HASTE_GDAL_RUNTIME_REQUIRED: "true" + GDAL_VRT_ENABLE_RAWRASTERBAND: "NO" + GDAL_VRT_ENABLE_PYTHON: "NO" + GDAL_DRIVER_PATH: "disable" + GDAL_DISABLE_READDIR_ON_OPEN: "EMPTY_DIR" + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED: "NO" + GDAL_SKIP: "HDF4 HDF4Image HDF5 HDF5Image netCDF" + TITILER_DEV_ALLOWED_ORIGINS: "http://azurite:10000" depends_on: - azurite @@ -84,9 +93,15 @@ services: context: .. dockerfile: ./api/hastefuncapi/Dockerfile environment: + HASTE_GDAL_RUNTIME_REQUIRED: "true" AzureWebJobsStorage: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;" # pragma: allowlist secret # gitleaks:allow ASPNETCORE_URLS: "http://+:8080" DATA_PATH: "/app/data" + GDAL_VRT_ENABLE_RAWRASTERBAND: "NO" + GDAL_VRT_ENABLE_PYTHON: "NO" + GDAL_DRIVER_PATH: "disable" + GDAL_DISABLE_READDIR_ON_OPEN: "EMPTY_DIR" + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED: "NO" DEVELOPMENT_MODE: "true" FUNCTIONS_WORKER_RUNTIME: "python" AzureWebJobsFeatureFlags: "EnableWorkerIndexing" @@ -132,9 +147,15 @@ services: context: .. dockerfile: ./api/hastefuncqueues/Dockerfile environment: + HASTE_GDAL_RUNTIME_REQUIRED: "true" AzureWebJobsStorage: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;" # pragma: allowlist secret # gitleaks:allow ASPNETCORE_URLS: "http://+:8080" DATA_PATH: "/app/data" + GDAL_VRT_ENABLE_RAWRASTERBAND: "NO" + GDAL_VRT_ENABLE_PYTHON: "NO" + GDAL_DRIVER_PATH: "disable" + GDAL_DISABLE_READDIR_ON_OPEN: "EMPTY_DIR" + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED: "NO" DEVELOPMENT_MODE: "true" FUNCTIONS_WORKER_RUNTIME: "python" AzureWebJobsFeatureFlags: "EnableWorkerIndexing" diff --git a/docker/imageryprep/Dockerfile b/docker/imageryprep/Dockerfile index a6b27966..d336e615 100644 --- a/docker/imageryprep/Dockerfile +++ b/docker/imageryprep/Dockerfile @@ -45,12 +45,22 @@ RUN printf '#!/bin/bash\nexec python -m hastegeo.workflows.prepare_imagery "$@"\ # Set PYTHONPATH so hastegeo modules can be found at runtime ENV PYTHONPATH=/app -# Refuse the GDAL drivers carrying the deferred CVEs (HDF4/HDF-EOS heap -# overflow CVE-2026-8087 et al.) at the GDAL level, so subprocess CLI tools -# (gdalwarp/gdal_translate) and the runtime never register them. In-process -# code additionally enforces a stricter allowlist via hastegeo.core.utils. -# gdal_security.harden_gdal(). See docs/known-vulnerabilities.md Root Cause C. +# Refuse the unused GDAL drivers at the GDAL level. In-process code also +# verifies the shared runtime and enforces its driver/VSI policy. ENV GDAL_SKIP="HDF4 HDF4Image HDF5 HDF5Image netCDF" +ENV HASTE_GDAL_RUNTIME_REQUIRED=true +ENV GDAL_VRT_ENABLE_RAWRASTERBAND=NO +ENV GDAL_VRT_ENABLE_PYTHON=NO +ENV GDAL_DRIVER_PATH=disable +ENV GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR +ENV CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO + +# Fail the build unless every GDAL binding runs on the one shared runtime. +RUN python -c "from hastegeo.core.utils.gdal_security import harden_gdal; harden_gdal(); \ + import fiona, pyogrio, rasterio; from osgeo import gdal; \ + v = gdal.VersionInfo('RELEASE_NAME'); \ + assert v.startswith('3.13.'), v; \ + assert {rasterio.__gdal_version__, fiona.__gdal_version__, pyogrio.__gdal_version_string__} == {v}, v" # Ensure /app itself is writable by appuser (for runtime-created dirs like ./logs) RUN chown appuser:appgroup /app diff --git a/docker/imageryprep/requirements.txt b/docker/imageryprep/requirements.txt index a10cf49f..30288ac8 100644 --- a/docker/imageryprep/requirements.txt +++ b/docker/imageryprep/requirements.txt @@ -18,8 +18,12 @@ email_validator==2.3.0 psycopg2-binary==2.9.12 requests==2.34.2 requests-auth==8.0.0 -GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/GDAL-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -rasterio==1.3.11 +haste-gdal-runtime @ https://github.com/microsoft/haste/releases/download/haste-binaries/haste_gdal_runtime-3.13.3%2Bhaste.1-py3-none-manylinux_2_28_x86_64.whl +GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/gdal-3.13.3%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +rasterio @ https://github.com/microsoft/haste/releases/download/haste-binaries/rasterio-1.4.4%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +fiona @ https://github.com/microsoft/haste/releases/download/haste-binaries/fiona-1.10.1%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyogrio @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyogrio-0.13.0%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl +pyproj @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyproj-3.7.2%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl opencv-python==4.10.0.84 shapely==2.1.2 pyyaml==6.0.3 @@ -28,7 +32,6 @@ tensorboard==2.21.0 tenacity==9.1.4 geopandas==1.1.4 pyarrow==23.0.1 -fiona==1.10.1 fsspec==2024.10.0 adlfs==2024.12.0 # hastegeo source is copied into the image by docker/imageryprep/Dockerfile. diff --git a/docker/titiler/Dockerfile b/docker/titiler/Dockerfile index 371e8028..0ff4989f 100644 --- a/docker/titiler/Dockerfile +++ b/docker/titiler/Dockerfile @@ -1,8 +1,21 @@ -# Use the official TiTiler image from developmentseed -FROM developmentseed/titiler:latest +FROM mcr.microsoft.com/azure-functions/python:4-python3.11@sha256:2c39d8b50750fbec9a91ef33323fadd1b986358fab4f7823db5b11b6e533f479 + +WORKDIR /app + +ENV HASTE_GDAL_RUNTIME_REQUIRED=true \ + GDAL_VRT_ENABLE_RAWRASTERBAND=NO \ + GDAL_VRT_ENABLE_PYTHON=NO \ + GDAL_DRIVER_PATH=disable \ + GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR \ + CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO \ + GDAL_SKIP="HDF4 HDF4Image HDF5 HDF5Image netCDF" + +COPY api/titilerfuncapi/requirements.txt /app/requirements.txt +RUN python -m pip install --no-cache-dir -r /app/requirements.txt \ + "uvicorn>=0.34,<1" + +COPY api/titilerfuncapi /app -# Expose the port that TiTiler runs on EXPOSE 8000 -# Command to run the TiTiler API (module path changed in newer versions) -CMD ["uvicorn", "titiler.application.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/training/Dockerfile b/docker/training/Dockerfile index c06d5593..01f6609c 100644 --- a/docker/training/Dockerfile +++ b/docker/training/Dockerfile @@ -82,18 +82,26 @@ RUN sed -i 's/\r$//' /app/scripts/*.sh && chmod +x /app/scripts/*.sh # Activate the environment by default and expose shared data directories ENV PATH="/opt/conda/envs/bda/bin:${PATH}" -ENV PROJ_LIB=/opt/conda/envs/bda/share/proj -ENV GDAL_DATA=/opt/conda/envs/bda/share/gdal ENV PYTHONPATH=/app ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -# Refuse the GDAL drivers carrying the deferred CVEs (HDF4/HDF-EOS heap -# overflow CVE-2026-8087 et al.) at the GDAL level, so subprocess CLI tools -# (gdalwarp/gdal_translate) and the runtime never register them. In-process -# training step entrypoints additionally call hastegeo.core.utils. -# gdal_security.harden_gdal(). See docs/known-vulnerabilities.md Root Cause C. +# Refuse unused GDAL drivers at the GDAL level. Training invokes GDAL through +# the shared Python runtime rather than shipping separate CLI executables. ENV GDAL_SKIP="HDF4 HDF4Image HDF5 HDF5Image netCDF" +ENV HASTE_GDAL_RUNTIME_REQUIRED=true +ENV GDAL_VRT_ENABLE_RAWRASTERBAND=NO +ENV GDAL_VRT_ENABLE_PYTHON=NO +ENV GDAL_DRIVER_PATH=disable +ENV GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR +ENV CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO + +# Fail the build unless every GDAL binding runs on the one shared runtime. +RUN python -c "from hastegeo.core.utils.gdal_security import harden_gdal; harden_gdal(); \ + import fiona, pyogrio, rasterio; from osgeo import gdal; \ + v = gdal.VersionInfo('RELEASE_NAME'); \ + assert v.startswith('3.13.'), v; \ + assert {rasterio.__gdal_version__, fiona.__gdal_version__, pyogrio.__gdal_version_string__} == {v}, v" # Pre-bake DINOv2 source + weights into an image-level torch.hub cache so diff --git a/docker/training/code/create_masks.py b/docker/training/code/create_masks.py index ca35f47f..5f28eb6b 100644 --- a/docker/training/code/create_masks.py +++ b/docker/training/code/create_masks.py @@ -19,14 +19,15 @@ import glob import os import shutil -import subprocess -from typing import List, Optional +from typing import Dict, List, Optional import cv2 import fiona import fiona.transform +import geopandas as gpd import numpy as np import rasterio +import rasterio.features import rasterio.mask import shapely # shapely>=2 (via geopandas) for STRtree import shapely.geometry @@ -99,26 +100,6 @@ class CropGeometryOutsideRasterError(RuntimeError): """ -def _run(command: List[str], what: str) -> None: - """Runs a subprocess, raising with captured output when it fails. - - Args: - command (List[str]): The command to run. - what (str): Human-readable description used in the error message. - - Raises: - RuntimeError: If the command exits non-zero. - """ - result = subprocess.run(command, capture_output=True, text=True) - if result.returncode != 0: - raise RuntimeError( - f"{what} failed with exit code {result.returncode}.\n" - f"Command: {' '.join(command)}\n" - f"stderr: {result.stderr}\n" - f"stdout: {result.stdout}" - ) - - def write_training_raster( output_fn: str, data: np.ndarray, @@ -141,6 +122,58 @@ def write_training_raster( output.write(data, band) +def rasterize_class_mask( + labels: gpd.GeoDataFrame, + class_names: List[str], + class_name_to_idx_map: Dict[str, int], + profile: dict, + output_fn: str, +) -> None: + """Rasterize projected label polygons without invoking GDAL CLI tools.""" + height = profile["height"] + width = profile["width"] + transform = profile["transform"] + mask = np.zeros((height, width), dtype=np.uint8) + + for class_name in class_names: + class_value = class_name_to_idx_map[class_name] + shapes = [ + (geometry, class_value) + for geometry, label in zip(labels.geometry, labels["class"]) + if label == class_name + and geometry is not None + and not geometry.is_empty + ] + if not shapes: + continue + burned = rasterio.features.rasterize( + shapes, + out_shape=(height, width), + fill=0, + transform=transform, + dtype="uint8", + ) + selected = burned != 0 + mask[selected] = burned[selected] + + mask_profile = profile.copy() + mask_profile.update( + driver="GTiff", + count=1, + dtype="uint8", + nodata=0, + tiled=True, + blockxsize=512, + blockysize=512, + interleave="pixel", + compress="LZW", + predictor=2, + BIGTIFF="YES", + ) + with rasterio.open(output_fn, "w", **mask_profile) as output: + output.write(mask, 1) + + def get_class_names_from_labels(labels_fn: str, key: str = "class") -> set: """Get the class names from a GeoJSON file. @@ -424,8 +457,6 @@ def create_mask_for_labels( output_cropped_image_fn = paths["image"] output_buffered_mask_fn = paths["mask"] - # ogr2ogr refuses to write over an existing GeoJSON, so a leftover from an - # aborted run would fail this cluster on every retry. remove_files([output_warped_label_fn, output_mask_fn]) ########## @@ -436,16 +467,16 @@ def create_mask_for_labels( ########## # Warp the labels to the CRS of the input image - command = [ - "ogr2ogr", - "-f", - "GeoJSON", - "-t_srs", - dst_crs, + warped_labels = gpd.read_file(input_label_fn) + if warped_labels.crs is None: + warped_labels = warped_labels.set_crs("EPSG:4326") + warped_labels = warped_labels.to_crs(dst_crs) + warped_labels.to_file( output_warped_label_fn, - input_label_fn, - ] - _run(command, "ogr2ogr") + driver="GeoJSON", + engine="fiona", + index=False, + ) ########## # Crop the input image to the given extent, or to the extent of the labels @@ -509,80 +540,14 @@ def create_mask_for_labels( # Create mask with rasterio.open(output_cropped_image_fn) as f: profile = f.profile - left, bottom, right, top = f.bounds - width = f.width - height = f.height - dst_crs = f.crs.to_string() - command = [ - "gdal_rasterize", - "-q", # be quiet about it - "-ot", - "Byte", # the output dtype of the raster should be uint8 - "-a_nodata", - "0", # the nodata value should be "0", this value will represent not-labeled in our training process - "-init", - "0", # initialize all values to 0 - "-burn", - str( - class_name_to_idx_map[class_names[0]] - ), # we will burn in the first class value to all polygons in the GeoJSON that match the first class label - "-of", - "GTiff", # the output should be a GeoTIFF - "-co", - "TILED=YES", # the output should be tiled, similar to COGs -- https://www.cogeo.org/ -- this is important for fast windowed reads - "-co", - "BLOCKXSIZE=512", # this is important for fast windowed reads - "-co", - "BLOCKYSIZE=512", # this is important for fast windowed reads - "-co", - "INTERLEAVE=PIXEL", # this is important for fast windowed reads - "-where", - f"class='{class_names[0]}'", # burn in values for polygons where the class label is the first class label - "-te", - str(left), - str(bottom), - str(right), - str( - top - ), # the output GeoTIFF should cover the same bounds as the input image - "-ts", - str(width), - str( - height - ), # the output GeoTIFF should have the same height and width as the input image - "-co", - "COMPRESS=LZW", # compress it - "-co", - "PREDICTOR=2", # compress it good - "-co", - "BIGTIFF=YES", # just incase the image is bigger than 4GB - output_warped_label_fn, + rasterize_class_mask( + warped_labels, + class_names, + class_name_to_idx_map, + profile, output_mask_fn, - ] - _run(command, "gdal_rasterize") - - for i in range(1, len(class_names)): - command = [ - "gdal_rasterize", - "-q", - "-b", - "1", - "-burn", - str(class_name_to_idx_map[class_names[i]]), - "-where", - f"class='{class_names[i]}'", - # The WARPED labels, matching the first class above. Burning from - # the original EPSG:4326 file relies on gdal_rasterize - # reprojecting on the fly; where it does not, every polygon lands - # outside a projected raster's extent and classes 2..N are - # silently dropped, leaving masks of just {0, 1}. Upstream - # measured exactly that. Reading from the file already in the - # target CRS removes the dependency either way. - output_warped_label_fn, - output_mask_fn, - ] - _run(command, f"gdal_rasterize for class '{class_names[i]}'") + ) ########## # Buffer mask around buildings @@ -647,7 +612,7 @@ def main() -> None: # classes, whose names are unconstrained -- a project spelling them # "building"/"background" would otherwise silently lose buffering # entirely. This maps config values onto other config values, so it does - # not affect how label-file class strings are matched by gdal_rasterize. + # not affect how label-file class strings are matched. buffer_value = find_class_value(class_names, class_to_buffer) buffer_by_value = find_class_value(class_names, class_to_buffer_by) @@ -825,13 +790,7 @@ def main() -> None: if __name__ == "__main__": - # GDAL CVE compensating control (docs/known-vulnerabilities.md Root - # Cause C): restrict GDAL drivers in-process. The GDAL_SKIP env in the - # training image also covers this; soft-fail if hastegeo is absent. - try: - from hastegeo.core.utils.gdal_security import harden_gdal - - harden_gdal() - except Exception: - pass + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() main() diff --git a/docker/training/code/fine_tune.py b/docker/training/code/fine_tune.py index a6c56300..1cad9d46 100644 --- a/docker/training/code/fine_tune.py +++ b/docker/training/code/fine_tune.py @@ -270,13 +270,7 @@ def main() -> None: if __name__ == "__main__": - # GDAL CVE compensating control (docs/known-vulnerabilities.md Root - # Cause C): restrict GDAL drivers in-process. The GDAL_SKIP env in the - # training image also covers this; soft-fail if hastegeo is absent. - try: - from hastegeo.core.utils.gdal_security import harden_gdal - - harden_gdal() - except Exception: - pass + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() main() diff --git a/docker/training/code/inference.py b/docker/training/code/inference.py index 004e9cec..78025314 100644 --- a/docker/training/code/inference.py +++ b/docker/training/code/inference.py @@ -243,13 +243,7 @@ def main() -> None: if __name__ == "__main__": - # GDAL CVE compensating control (docs/known-vulnerabilities.md Root - # Cause C): restrict GDAL drivers in-process. The GDAL_SKIP env in the - # training image also covers this; soft-fail if hastegeo is absent. - try: - from hastegeo.core.utils.gdal_security import harden_gdal - - harden_gdal() - except Exception: - pass + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() main() diff --git a/docker/training/code/merge_with_building_footprints.py b/docker/training/code/merge_with_building_footprints.py index e72322af..dcd9636a 100644 --- a/docker/training/code/merge_with_building_footprints.py +++ b/docker/training/code/merge_with_building_footprints.py @@ -248,15 +248,9 @@ def main(args: argparse.Namespace) -> None: if __name__ == "__main__": - # GDAL CVE compensating control (docs/known-vulnerabilities.md Root - # Cause C): restrict GDAL drivers in-process. The GDAL_SKIP env in the - # training image also covers this; soft-fail if hastegeo is absent. - try: - from hastegeo.core.utils.gdal_security import harden_gdal + from hastegeo.core.utils.gdal_security import harden_gdal - harden_gdal() - except Exception: - pass + harden_gdal() parser = set_up_parser() args = parser.parse_args() main(args) diff --git a/docker/training/code/output2visualizer.py b/docker/training/code/output2visualizer.py index 957ce18e..984bba57 100644 --- a/docker/training/code/output2visualizer.py +++ b/docker/training/code/output2visualizer.py @@ -138,15 +138,9 @@ def main(args: argparse.Namespace) -> None: if __name__ == "__main__": - # GDAL CVE compensating control (docs/known-vulnerabilities.md Root - # Cause C): restrict GDAL drivers in-process. The GDAL_SKIP env in the - # training image also covers this; soft-fail if hastegeo is absent. - try: - from hastegeo.core.utils.gdal_security import harden_gdal - - harden_gdal() - except Exception: - pass + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() parser = set_up_parser() args = parser.parse_args() main(args) diff --git a/docker/training/code/run_workflow.py b/docker/training/code/run_workflow.py index 5811b765..acb908d2 100644 --- a/docker/training/code/run_workflow.py +++ b/docker/training/code/run_workflow.py @@ -11,6 +11,7 @@ import glob import os import re +import shlex import subprocess import sys import traceback @@ -71,6 +72,65 @@ def run_subprocess(command, step_name): return result +def gdal_translate_creation_options(parameters: str | None) -> list[str]: + """Parse the supported GDAL_TRANSLATE_PARAMS creation-option syntax.""" + tokens = shlex.split(parameters or "") + if not tokens: + return [] + if not any(token in {"-co", "-of"} for token in tokens): + if any(token.startswith("-") for token in tokens): + raise ValueError( + "GDAL_TRANSLATE_PARAMS accepts creation options, not GDAL " + "command-line switches" + ) + return tokens + + options = [] + index = 0 + while index < len(tokens): + option = tokens[index] + if option not in {"-co", "-of"} or index + 1 >= len(tokens): + raise ValueError( + "GDAL_TRANSLATE_PARAMS accepts creation options and '-of COG'" + ) + value = tokens[index + 1] + if value.startswith("-"): + raise ValueError( + "GDAL_TRANSLATE_PARAMS creation option is missing its value" + ) + if option == "-co": + options.append(value) + elif value.upper() != "COG": + raise ValueError( + "GDAL_TRANSLATE_PARAMS must keep output format COG" + ) + index += 2 + return options + + +def translate_to_cog( + source_path: str, destination_path: str, parameters: str | None +) -> None: + """Convert a raster to COG using the shared in-process GDAL runtime.""" + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() + from osgeo import gdal + + gdal.UseExceptions() + options = gdal.TranslateOptions( + format="COG", + creationOptions=gdal_translate_creation_options(parameters), + ) + output = gdal.Translate(destination_path, source_path, options=options) + if output is None: + raise RuntimeError( + f"GDAL could not translate {source_path} to {destination_path}: " + f"{gdal.GetLastErrorMsg()}" + ) + output = None + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -238,24 +298,10 @@ def main(): log_progress("Converting results to COG") - gdal_translate_params = os.getenv("GDAL_TRANSLATE_PARAMS") - formatted_gdal_params = [] - if gdal_translate_params: - if "-co" in gdal_translate_params: - formatted_gdal_params = gdal_translate_params.split() - else: - for op in gdal_translate_params.split(): - formatted_gdal_params += ["-co", op] - run_subprocess( - [ - "gdal_translate", - "-of", - "COG", - *formatted_gdal_params, - temp_vis_fn, - visualizer_cog_fn, - ], - "gdal_translate", + translate_to_cog( + temp_vis_fn, + visualizer_cog_fn, + os.getenv("GDAL_TRANSLATE_PARAMS"), ) diff --git a/docker/training/code/sitecustomize.py b/docker/training/code/sitecustomize.py new file mode 100644 index 00000000..180589be --- /dev/null +++ b/docker/training/code/sitecustomize.py @@ -0,0 +1,5 @@ +"""Harden GDAL before any training-process module can import a binding.""" + +from hastegeo.core.utils.gdal_security import harden_gdal + +harden_gdal() diff --git a/docker/training/code/tests/test_create_masks_cleanup.py b/docker/training/code/tests/test_create_masks_cleanup.py index e91c1ec0..69ed8e19 100644 --- a/docker/training/code/tests/test_create_masks_cleanup.py +++ b/docker/training/code/tests/test_create_masks_cleanup.py @@ -7,7 +7,7 @@ pair left over from an earlier run silently joins the new training set. Two paths can leave one behind: ``--overwrite`` only replaced the pairs the current run regenerates, and a cluster skipped mid-write left partial files -plus intermediates that would trip the next attempt's ``ogr2ogr``. +plus intermediates from a previous rasterization. """ import os @@ -100,7 +100,7 @@ def test_removes_unclustered_pair(self): self.assertFalse(os.path.exists(path), path) def test_removes_leftover_intermediates(self): - """Aborted runs leave these, and ogr2ogr won't overwrite the GeoJSON.""" + """Aborted runs leave these projected-label and raster intermediates.""" stale = [ self._touch("scene_cluster_0_labels_warped.geojson"), self._touch("scene_cluster_0_mask.tif"), diff --git a/docker/training/code/tests/test_create_masks_writes.py b/docker/training/code/tests/test_create_masks_writes.py index 70fc21dc..a80c484d 100644 --- a/docker/training/code/tests/test_create_masks_writes.py +++ b/docker/training/code/tests/test_create_masks_writes.py @@ -201,28 +201,19 @@ def test_visualizer_output_uses_if_safer(self): def test_raw_mask_rasterizer_uses_bigtiff(self): with open(CREATE_MASKS_PATH) as source_file: - tree = ast.parse(source_file.read()) + source = source_file.read() + tree = ast.parse(source) - rasterize_commands = [ - node.value + rasterize_calls = [ + ast.unparse(node.func) for node in ast.walk(tree) - if isinstance(node, ast.Assign) - and isinstance(node.value, ast.List) - and node.value.elts - and isinstance(node.value.elts[0], ast.Constant) - and node.value.elts[0].value == "gdal_rasterize" + if isinstance(node, ast.Call) ] - self.assertTrue( - any( - any( - isinstance(argument, ast.Constant) - and argument.value == "BIGTIFF=YES" - for argument in command.elts - ) - for command in rasterize_commands - ) - ) + self.assertIn("rasterio.features.rasterize", rasterize_calls) + self.assertIn('BIGTIFF="YES"', source) + self.assertNotIn("gdal_rasterize", source) + self.assertNotIn("ogr2ogr", source) if __name__ == "__main__": diff --git a/docker/training/code/tests/test_gdal_runtime_integration.py b/docker/training/code/tests/test_gdal_runtime_integration.py new file mode 100644 index 00000000..1274364b --- /dev/null +++ b/docker/training/code/tests/test_gdal_runtime_integration.py @@ -0,0 +1,106 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +class TestGdalRuntimeIntegration(unittest.TestCase): + def test_labels_rasterize_in_process_to_tiled_big_tiff_mask(self): + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() + import create_masks + import geopandas as gpd + import numpy as np + import rasterio + from rasterio.transform import from_origin + from shapely.geometry import box + + labels = gpd.GeoDataFrame( + {"class": ["Damaged", "Destroyed"]}, + geometry=[ + box(0, 0, 2, 2), + box(2, 0, 4, 2), + ], + crs="EPSG:3857", + ) + profile = { + "driver": "GTiff", + "height": 4, + "width": 4, + "count": 1, + "dtype": "uint8", + "crs": "EPSG:3857", + "transform": from_origin(0, 4, 1, 1), + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "mask.tif" + create_masks.rasterize_class_mask( + labels, + ["Damaged", "Destroyed"], + {"Damaged": 1, "Destroyed": 2}, + profile, + str(output), + ) + with rasterio.open(output) as mask: + data = mask.read(1) + self.assertEqual(mask.count, 1) + self.assertEqual(mask.nodata, 0) + self.assertEqual(mask.crs.to_epsg(), 3857) + self.assertEqual(mask.transform, from_origin(0, 4, 1, 1)) + self.assertTrue(mask.is_tiled) + self.assertEqual(mask.compression.name.casefold(), "lzw") + + expected = np.zeros((4, 4), dtype=np.uint8) + expected[2:, :2] = 1 + expected[2:, 2:] = 2 + np.testing.assert_array_equal(data, expected) + + @unittest.skipUnless( + importlib.util.find_spec("osgeo"), + "requires GDAL's Python bindings", + ) + def test_visualizer_conversion_uses_in_process_cog_translate(self): + from hastegeo.core.utils.gdal_security import harden_gdal + + harden_gdal() + import numpy as np + import rasterio + import run_workflow + from rasterio.transform import from_origin + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "source.tif" + destination = Path(temporary) / "output.tif" + with rasterio.open( + source, + "w", + driver="GTiff", + height=4, + width=4, + count=1, + dtype="uint8", + crs="EPSG:3857", + transform=from_origin(0, 4, 1, 1), + ) as dataset: + dataset.write(np.ones((4, 4), dtype=np.uint8), 1) + + run_workflow.translate_to_cog( + str(source), + str(destination), + "-co COMPRESS=DEFLATE -of COG", + ) + + with rasterio.open(destination) as translated: + self.assertEqual( + translated.tags(ns="IMAGE_STRUCTURE").get("LAYOUT"), + "COG", + ) + self.assertEqual( + translated.tags(ns="IMAGE_STRUCTURE").get("COMPRESSION"), + "DEFLATE", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/training/code/tests/test_run_workflow_predictions.py b/docker/training/code/tests/test_run_workflow_predictions.py index a9ad568d..4fe8e423 100644 --- a/docker/training/code/tests/test_run_workflow_predictions.py +++ b/docker/training/code/tests/test_run_workflow_predictions.py @@ -73,8 +73,6 @@ def launch(command, step): assert payload["predictionRevision"] == "run-1234" assert payload["classes"] == ["Damaged"] visualizer.main(visualizer.set_up_parser().parse_args(command[2:])) - elif step == "gdal_translate": - rasterio.shutil.copy(command[-2], command[-1], driver="COG") else: pytest.fail(f"Unexpected step: {step}") @@ -87,6 +85,15 @@ def launch(command, step): ) mocker.patch.object(workflow, "run_subprocess", side_effect=launch) + def translate_to_cog(source, destination, parameters): + assert parameters == "" + steps.append("gdal_translate") + rasterio.shutil.copy(source, destination, driver="COG") + + mocker.patch.object( + workflow, "translate_to_cog", side_effect=translate_to_cog + ) + def run(step="inference"): config_path = tmp_path / "config.yml" config_path.write_text(yaml.safe_dump(config)) @@ -99,6 +106,20 @@ def run(step="inference"): return config, run, steps, attrs +def test_translate_creation_options_accepts_plain_and_cli_style_values(): + assert workflow.gdal_translate_creation_options( + "COMPRESS=ZSTD PREDICTOR=2" + ) == ["COMPRESS=ZSTD", "PREDICTOR=2"] + assert workflow.gdal_translate_creation_options( + "-co COMPRESS=ZSTD -co PREDICTOR=2 -of COG" + ) == ["COMPRESS=ZSTD", "PREDICTOR=2"] + + +def test_translate_creation_options_rejects_unsupported_cli_switches(): + with pytest.raises(ValueError, match="format COG"): + workflow.gdal_translate_creation_options("-of GTiff") + + @pytest.mark.parametrize("step", ["inference", None]) def test_inference_and_default_all_write_eager_sidecars(workflow_case, step): config, run, steps, attrs = workflow_case diff --git a/docker/training/env/env.yml b/docker/training/env/env.yml index 31617596..ae20b346 100644 --- a/docker/training/env/env.yml +++ b/docker/training/env/env.yml @@ -2,8 +2,7 @@ name: bda channels: - conda-forge dependencies: - - python=3.10 - - gdal<3.9 + - python=3.11 - numpy - setuptools - pip @@ -25,10 +24,17 @@ dependencies: - jupyter==1.1.1 - mercantile==1.2.1 - osmnx==2.0.2 + # geopandas 1.0.1 breaks on pandas 3.x string dtypes + - pandas==2.2.3 - pigeon-jupyter==0.1.0 - planetary_computer==1.0.0 - pyarrow==23.0.1 - - rasterio==1.4.3 + - haste-gdal-runtime @ https://github.com/microsoft/haste/releases/download/haste-binaries/haste_gdal_runtime-3.13.3%2Bhaste.1-py3-none-manylinux_2_28_x86_64.whl + - GDAL @ https://github.com/microsoft/haste/releases/download/haste-binaries/gdal-3.13.3%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl + - rasterio @ https://github.com/microsoft/haste/releases/download/haste-binaries/rasterio-1.4.4%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl + - fiona @ https://github.com/microsoft/haste/releases/download/haste-binaries/fiona-1.10.1%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl + - pyogrio @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyogrio-0.13.0%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl + - pyproj @ https://github.com/microsoft/haste/releases/download/haste-binaries/pyproj-3.7.2%2Bhaste.1-cp311-cp311-manylinux_2_28_x86_64.whl - s3fs==2025.3.2 - scikit-learn==1.6.1 - tensorboard==2.19.0 diff --git a/docker/training/scripts/entrypoint.sh b/docker/training/scripts/entrypoint.sh index 4afa284a..6c726e16 100644 --- a/docker/training/scripts/entrypoint.sh +++ b/docker/training/scripts/entrypoint.sh @@ -31,7 +31,7 @@ echo "[TRAINING-ENTRYPOINT] Python version:" python --version echo "[TRAINING-ENTRYPOINT] Verifying training workflow import" -python -c "import hastegeo" 2>&1 || echo "[TRAINING-ENTRYPOINT] Failed to import base hastegeo package" +python -c "import hastegeo; print('Successfully imported hastegeo')" 2>&1 # When no command is provided (e.g. compose build helper), drop into bash to avoid exiting with an error if [ $# -eq 0 ]; then diff --git a/docs/security-configuration.md b/docs/security-configuration.md index 2d4d3090..781cc663 100644 --- a/docs/security-configuration.md +++ b/docs/security-configuration.md @@ -365,12 +365,13 @@ The UI's logout flow validates the `redirectPath` parameter via `sanitizeRedirec ### 8.8 GDAL driver allowlist and ingestion size/type limits -GDAL `3.9.2` is pinned (no trusted prebuilt pip wheel exists for the patched 3.13 line under HASTE's runtime), so three memory-safety CVEs — most notably **CVE-2026-8087**, a heap overflow in the HDF4/HDF-EOS driver — are deferred under a documented exception with **compensating controls enforced in code** (see [`known-vulnerabilities.md`](https://github.com/microsoft/haste/blob/main/docs/known-vulnerabilities.md) Root Cause C and [ADR-0004](https://github.com/microsoft/haste/blob/main/spec/architecture/decisions/0004-gdal-driver-allowlist.md)). +The shared-runtime build pins GDAL `3.13.3` and builds the GDAL-linked Python bindings against the same native libraries. Existing deployments continue using their currently deployed packages until each consumer receives the verified runtime and passes its startup check; the legacy runtime exception remains open until rollout is complete (see [`known-vulnerabilities.md`](https://github.com/microsoft/haste/blob/main/docs/known-vulnerabilities.md) Root Cause C and [ADR-0004](https://github.com/microsoft/haste/blob/main/spec/architecture/decisions/0004-gdal-driver-allowlist.md)). -- **Driver allowlist.** At process startup `hastegeo.core.utils.gdal_security.harden_gdal()` restricts GDAL/OGR to the drivers HASTE actually uses (raster `GTiff, COG, VRT, JPEG, PNG, MEM`; vector `GPKG, GeoJSON, Memory`) by deregistering every other driver. Because GDAL dispatches by sniffing file *content* (not the extension), this removes the vulnerable HDF4/HDF-EOS code path from every `gdal.Open`/`rasterio.open`/`pyogrio` read in-process. The imageryprep and training containers also set `GDAL_SKIP="HDF4 HDF4Image HDF5 HDF5Image netCDF"` so subprocess GDAL CLI tools (`gdalwarp`/`gdal_translate`) refuse those drivers too. +- **Shared runtime and driver allowlist.** The immutable runtime bundle contains one `libgdal` and `libproj`, with GDAL `3.13.3` bindings for `osgeo`, Rasterio, Fiona, Pyogrio, and PyProj. At process startup `hastegeo.core.utils.gdal_security.harden_gdal()` verifies those package versions and restricts GDAL/OGR to the drivers HASTE uses (raster `GTiff, COG, VRT, JPEG, PNG, MEM`; vector `GPKG, GeoJSON, Memory`) by deregistering every other driver. The check fails closed when `HASTE_GDAL_RUNTIME_REQUIRED=true` and the runtime is missing or inconsistent. +- **VRT and plugin configuration.** Bicep, `deploy_apps.sh`, Dockerfiles, and local Compose set `GDAL_VRT_ENABLE_RAWRASTERBAND=NO`, `GDAL_VRT_ENABLE_PYTHON=NO`, `GDAL_DRIVER_PATH=disable`, `GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR`, and `CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO`. TiTiler's shared-runtime policy additionally deregisters VRT; internal mosaic workflows retain it. - **Ingestion size/type limits.** Chunked uploads are capped (`HASTE_MAX_UPLOAD_BYTES`, default 5 GiB) and the assembled file's magic bytes must match its declared format before GDAL parses it. Remote imagery fetches are capped (`HASTE_MAX_IMAGERY_DOWNLOAD_BYTES`, default 8 GiB) and refuse cross-host redirects (SSRF guard). Tune the same-named GitHub Environment configuration variables, then run **Update Size Limits**. Both deployment and lightweight updates apply these desired values. Bicep declares the caps explicitly; local `azd` environments must supply matching byte counts before provisioning. The workflow verifies stored settings and samples HTTP worker responses, not queue workers or Batch enforcement. New Batch tasks capture the queues worker's download cap; existing tasks retain their submitted values. See the [operator procedure](configuration.md#ingestion-size-limits) for reset, rollback, resource budgets, and verification. -**Operator action:** keep `GDAL_SKIP` set in any custom GDAL-bearing image, and do not add file formats to the pipeline without extending the allowlist in `gdal_security.py`. If you legitimately need a format outside the allowlist, add its driver explicitly rather than disabling the control. +**Operator action:** keep the GDAL hardening settings in any custom GDAL-bearing image and do not add file formats to the pipeline without extending the allowlist in `gdal_security.py`. If you legitimately need a format outside the allowlist, add its driver explicitly rather than disabling the control. Each HASTE release documents security-relevant changes in [`CHANGELOG.md`](https://github.com/microsoft/haste/blob/main/CHANGELOG.md). The rolling list of acknowledged dependency vulnerabilities — fixed, dismissed with rationale, or pending — is maintained at [`docs/known-vulnerabilities.md`](https://github.com/microsoft/haste/blob/main/docs/known-vulnerabilities.md). diff --git a/hastelib/pyproject.toml b/hastelib/pyproject.toml index 36d8b91a..7bd4cb77 100644 --- a/hastelib/pyproject.toml +++ b/hastelib/pyproject.toml @@ -43,12 +43,14 @@ dependencies = [ "psycopg2-binary", "requests", "requests-auth", - "rasterio", + "rasterio==1.4.4", "opencv-python", - "gdal==3.9.2", + "gdal==3.13.3", "shapely", - "fiona", + "fiona==1.10.1", "geopandas", + "pyogrio==0.13.0", + "pyproj==3.7.2", "pyarrow>=23.0.1", "fsspec", "adlfs", diff --git a/hastelib/src/hastegeo/__init__.py b/hastelib/src/hastegeo/__init__.py index 5b7f7a92..6e8a12d0 100644 --- a/hastelib/src/hastegeo/__init__.py +++ b/hastelib/src/hastegeo/__init__.py @@ -1,2 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + +from .core.utils.gdal_security import harden_gdal + +harden_gdal() diff --git a/hastelib/src/hastegeo/core/processors/uploader.py b/hastelib/src/hastegeo/core/processors/uploader.py index d75f2489..458c537f 100644 --- a/hastelib/src/hastegeo/core/processors/uploader.py +++ b/hastelib/src/hastegeo/core/processors/uploader.py @@ -6,7 +6,11 @@ from ..data_layer.unified import UnifiedDataLayer from ..models.uploader import FileUploadRequest -from ..utils.gdal_security import assert_matches_declared, max_upload_bytes +from ..utils.gdal_security import ( + assert_matches_declared, + assert_matches_declared_bytes, + max_upload_bytes, +) from ..utils.logs import Logger from ..utils.metadata import MetadataUtils @@ -75,6 +79,8 @@ def save_chunk( f"Upload chunk exceeds the max upload size of " f"{max_bytes} bytes." ) + if chunk_number == 0: + assert_matches_declared_bytes(data, data_format) with open(chunk_path, "wb") as chunk_file: chunk_file.write(data) diff --git a/hastelib/src/hastegeo/core/utils/footprints.py b/hastelib/src/hastegeo/core/utils/footprints.py index c1882094..1f5f31f2 100644 --- a/hastelib/src/hastegeo/core/utils/footprints.py +++ b/hastelib/src/hastegeo/core/utils/footprints.py @@ -28,7 +28,7 @@ import pyarrow.fs as fs from geopandas import GeoDataFrame -from .gdal_security import harden_gdal +from .gdal_security import assert_matches_declared, harden_gdal from .validation_config import clamp_validation_sample logger = logging.getLogger(__name__) @@ -364,8 +364,8 @@ def clip_and_normalize_user_footprints( ): """Clip a user-supplied building-footprint GPKG to an AOI polygon. - Reads ``input_path`` (any GDAL-supported vector format readable by - geopandas), reprojects to EPSG:4326 if needed, filters to polygonal + Reads ``input_path`` as a content-verified GeoPackage, reprojects to + EPSG:4326 if needed, filters to polygonal geometries, clips to ``aoi_polygon`` (EPSG:4326), normalizes the schema to ``(id, geometry, subtype, class)`` synthesizing any missing non-geometry column, and writes a GeoPackage to @@ -377,8 +377,7 @@ def clip_and_normalize_user_footprints( interchangeably. Args: - input_path: Path to a local building-footprint GPKG (or any - geopandas-readable vector file). + input_path: Path to a local building-footprint GPKG. aoi_polygon: ``shapely.geometry.Polygon`` (or any geometry) describing the AOI, **in EPSG:4326**. Typically the output of :func:`hastegeo.core.utils.aoi.extract_aoi_polygon`. @@ -405,7 +404,8 @@ def clip_and_normalize_user_footprints( ) os.remove(output_path) - gdf = gpd.read_file(input_path) + assert_matches_declared(input_path, "gpkg") + gdf = gpd.read_file(input_path, engine="fiona", enabled_drivers=["GPKG"]) if gdf.crs is None: raise ValueError( "Input GPKG is missing a CRS; embed one (e.g. EPSG:4326) and retry." diff --git a/hastelib/src/hastegeo/core/utils/gdal_security.py b/hastelib/src/hastegeo/core/utils/gdal_security.py index ffd74997..4eaf0668 100644 --- a/hastelib/src/hastegeo/core/utils/gdal_security.py +++ b/hastelib/src/hastegeo/core/utils/gdal_security.py @@ -21,10 +21,15 @@ (``sniff_file_type``, the size/format helpers) work on any host, and only :func:`harden_gdal` needs ``osgeo``. """ + from __future__ import annotations +import importlib +import importlib.util import logging import os +import re +from typing import Any logger = logging.getLogger(__name__) @@ -40,6 +45,13 @@ # These are the CVE-relevant driver families; an allowlist can't be # expressed via GDAL_SKIP, so we skip the known-dangerous ones explicitly. DISABLED_DRIVERS_ENV = "HDF4 HDF4Image HDF5 HDF5Image netCDF" +LEGACY_REQUIRED_CONFIG = { + "GDAL_VRT_ENABLE_RAWRASTERBAND": "NO", + "GDAL_VRT_ENABLE_PYTHON": "NO", + "GDAL_DRIVER_PATH": "disable", + "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED": "NO", +} # ── Ingestion size caps (env-tunable; generous because satellite COGs are # legitimately large — the goal is to bound pathological/hostile inputs). ─ @@ -57,64 +69,168 @@ _hardened = False -def harden_gdal(*, force: bool = False) -> None: - """Restrict GDAL/OGR to :data:`ALLOWED_RASTER_DRIVERS` + - :data:`ALLOWED_VECTOR_DRIVERS` by deregistering every other driver. - - Idempotent: a no-op after the first successful run unless ``force`` is - set. Safe to call from many import sites. If ``osgeo`` is unavailable - (e.g. a bare host running non-GDAL tests) it logs and returns without - error. - """ - global _hardened - if _hardened and not force: - return +def _harden_legacy_gdal() -> bool: + for key, expected in LEGACY_REQUIRED_CONFIG.items(): + os.environ.setdefault(key, expected) + actual = os.environ[key].strip() + if actual.casefold() != expected.casefold(): + raise RuntimeError( + f"{key}={actual!r} weakens GDAL hardening; " + f"expected {expected!r}" + ) + + forbidden = { + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE": "ALL", + "GDAL_VRT_PYTHON_TRUSTED_MODULES": None, + "GDAL_PYTHON_DRIVER_PATH": None, + "GDAL_HTTP_UNSAFESSL": "YES", + } + for key, forbidden_value in forbidden.items(): + actual = os.environ.get(key) + if actual is None: + continue + if forbidden_value is None and actual.strip(): + raise RuntimeError(f"{key} must be unset for GDAL hardening") + if ( + forbidden_value + and actual.strip().casefold() == forbidden_value.casefold() + ): + raise RuntimeError(f"{key}={actual!r} weakens GDAL hardening") + + existing_skip = [ + name + for name in re.split( + r"[,\s]+", os.environ.get("GDAL_SKIP", "").strip() + ) + if name + ] + required_skip = DISABLED_DRIVERS_ENV.split() + os.environ["GDAL_SKIP"] = " ".join( + dict.fromkeys([*existing_skip, *required_skip]) + ) try: from osgeo import gdal, ogr - except ImportError: - logger.warning("osgeo (GDAL) not importable; skipping harden_gdal()") - return + except ModuleNotFoundError as exc: + if exc.name != "osgeo": + raise + logger.warning( + "osgeo (GDAL) not importable; skipping legacy hardening" + ) + return False gdal.UseExceptions() ogr.UseExceptions() - - # Belt-and-suspenders: make any later (re)registration or subprocess - # honor the denylist too. Respect an operator-provided GDAL_SKIP. - os.environ.setdefault("GDAL_SKIP", DISABLED_DRIVERS_ENV) gdal.SetConfigOption("GDAL_SKIP", os.environ["GDAL_SKIP"]) allowed = ALLOWED_RASTER_DRIVERS | ALLOWED_VECTOR_DRIVERS + removed = _deregister_disallowed_legacy_drivers(gdal, allowed) - # Collect first — deregistering mutates the driver manager and shifts - # indices. In GDAL 3.x the manager is unified (one list for raster + - # vector), so a single pass covers OGR drivers too. - drivers = [ - gdal.GetDriver(i) - for i in range(gdal.GetDriverCount()) - if gdal.GetDriver(i) is not None - ] - - removed: list[str] = [] - for drv in drivers: - name = drv.ShortName - if name not in allowed: - try: - drv.Deregister() - removed.append(name) - except Exception: # pragma: no cover - defensive - logger.warning( - "harden_gdal: failed to deregister driver %s", name - ) - - _hardened = True logger.info( - "harden_gdal: disabled %d GDAL/OGR drivers; kept %s; GDAL_SKIP=%r", + "harden_gdal: disabled %d legacy GDAL/OGR drivers; kept %s; " + "GDAL_SKIP=%r", len(removed), sorted(allowed), os.environ.get("GDAL_SKIP"), ) - logger.debug("harden_gdal: disabled drivers: %s", sorted(removed)) + logger.debug("harden_gdal: disabled legacy drivers: %s", sorted(removed)) + return True + + +def _deregister_disallowed_legacy_drivers( + gdal: Any, allowed: frozenset[str] +) -> list[str]: + def registered_drivers() -> list[Any]: + drivers = [] + for index in range(gdal.GetDriverCount()): + driver = gdal.GetDriver(index) + if driver is not None: + drivers.append(driver) + return drivers + + removed = [] + for driver in registered_drivers(): + name = driver.ShortName + if name not in allowed: + try: + driver.Deregister() + except Exception as exc: + raise RuntimeError( + f"failed to deregister disallowed GDAL driver {name}" + ) from exc + removed.append(name) + + remaining = sorted( + driver.ShortName + for driver in registered_drivers() + if driver.ShortName not in allowed + ) + if remaining: + raise RuntimeError( + "disallowed GDAL drivers remain registered: " + + ", ".join(remaining) + ) + return removed + + +def harden_gdal( + *, force: bool = False, require_runtime: bool | None = None +) -> None: + """Restrict GDAL/OGR to :data:`ALLOWED_RASTER_DRIVERS` + + :data:`ALLOWED_VECTOR_DRIVERS` by deregistering every other driver. + + Idempotent: a no-op after the first successful run unless ``force`` is + set. When the shared runtime is installed, verify it before importing + any GDAL binding. Set ``require_runtime`` or the + HASTE_GDAL_RUNTIME_REQUIRED environment variable to fail closed if the + shared runtime is missing. + """ + global _hardened + if _hardened and not force: + return + + if require_runtime is None: + value = ( + os.environ.get("HASTE_GDAL_RUNTIME_REQUIRED", "").strip().lower() + ) + if value in {"1", "true", "yes"}: + require_runtime = True + elif value in {"", "0", "false", "no"}: + require_runtime = False + else: + raise ValueError( + "HASTE_GDAL_RUNTIME_REQUIRED must be a boolean value" + ) + + if importlib.util.find_spec("haste_gdal_runtime") is None: + if require_runtime: + raise RuntimeError( + "HASTE_GDAL_RUNTIME_REQUIRED is set but " + "haste-gdal-runtime is not installed" + ) + logger.warning( + "haste-gdal-runtime is absent; applying legacy GDAL hardening" + ) + if _harden_legacy_gdal(): + _hardened = True + return + + runtime = importlib.import_module("haste_gdal_runtime") + manifest = runtime.load_manifest() + expected_gdal = manifest.get("gdal_release") + if not isinstance(expected_gdal, str) or not expected_gdal: + raise RuntimeError("GDAL runtime manifest has no GDAL release") + policy = runtime.Policy( + expected_gdal=expected_gdal, + allowed_drivers=ALLOWED_RASTER_DRIVERS | ALLOWED_VECTOR_DRIVERS, + required_bindings=("osgeo.gdal",), + ) + runtime.configure(policy) + report = runtime.secure_startup(policy) + for binding in ("osgeo.gdal", "osgeo.ogr"): + importlib.import_module(binding).UseExceptions() + logger.info("verified HASTE GDAL runtime: %s", report) + _hardened = True def sniff_file_type(path: str) -> str | None: @@ -130,6 +246,10 @@ def sniff_file_type(path: str) -> str | None: except OSError: return None + return _sniff_file_header(head) + + +def _sniff_file_header(head: bytes) -> str | None: if len(head) < 4: return None @@ -154,6 +274,27 @@ def sniff_file_type(path: str) -> str | None: return None +def assert_matches_declared_bytes(data: bytes, declared: str) -> None: + """Validate an upload's first chunk before it is persisted. + + GeoPackage detection needs its application identifier at byte offset 68, + so the first chunk must contain at least 72 bytes. + """ + norm = assert_allowed_upload_format(declared) + expected = _FORMAT_TO_SNIFF[norm] + if expected == "gpkg" and len(data) < 72: + raise ValueError( + "First upload chunk must contain at least 72 bytes to validate " + "GeoPackage content." + ) + actual = _sniff_file_header(data[:72]) + if actual != expected: + raise ValueError( + f"Uploaded file content ({actual!r}) does not match the " + f"declared format {declared!r} (expected {expected!r})." + ) + + def assert_allowed_upload_format(declared: str) -> str: """Normalize and validate a declared chunked-upload format. diff --git a/hastelib/src/hastegeo/core/utils/imagery.py b/hastelib/src/hastegeo/core/utils/imagery.py index f428d1a4..a763f44b 100644 --- a/hastelib/src/hastegeo/core/utils/imagery.py +++ b/hastelib/src/hastegeo/core/utils/imagery.py @@ -3,15 +3,14 @@ import os import traceback from pathlib import Path -from typing import List +from typing import Any, List import cv2 # type: ignore import numpy as np # type: ignore -import rasterio from osgeo import gdal from ..utils.logs import Logger -from .gdal_security import harden_gdal +from .gdal_security import assert_matches_declared, harden_gdal from .source_types import normalize_source_type # Restrict GDAL to an allowlist of drivers (and enable exceptions) before @@ -60,6 +59,19 @@ def adjust_contrast(image): def reduce_noise(image): return cv2.medianBlur(image, 3) + @staticmethod + def open_gtiff(file_path: str) -> Any: + """Open only a content-verified TIFF with the GTiff driver.""" + assert_matches_declared(file_path, "tif") + dataset = gdal.OpenEx( + file_path, + gdal.OF_RASTER | gdal.OF_READONLY, + allowed_drivers=["GTiff"], + ) + if dataset is None: + raise RuntimeError(f"Unable to open GeoTIFF file: {file_path}") + return dataset + @staticmethod def is_gtiff(file_path): """ @@ -75,13 +87,13 @@ def is_gtiff(file_path): bool: True if the file is a GeoTIFF, False otherwise. """ try: - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return False if dataset.GetDriver().ShortName != "GTiff": return False return True - except (OSError, RuntimeError): + except (OSError, RuntimeError, ValueError): return False @staticmethod @@ -98,7 +110,7 @@ def is_cog(file_path): bool: True if the file is a COG, False otherwise. """ try: - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return False metadata = dataset.GetMetadata("IMAGE_STRUCTURE") @@ -113,7 +125,7 @@ def is_cog(file_path): if "LAYOUT" not in metadata or metadata["LAYOUT"] != "COG": return False return True - except (OSError, RuntimeError): + except (OSError, RuntimeError, ValueError): return False @staticmethod @@ -128,7 +140,7 @@ def is_rgb(file_path): bool: True if the file is an RGB image, False otherwise. """ try: - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None or dataset.RasterCount < 3: return False @@ -158,7 +170,7 @@ def is_rgb(file_path): for i in range(1, 4) ] return colors == ["Red", "Green", "Blue"] - except (OSError, RuntimeError): + except (OSError, RuntimeError, ValueError): return False @staticmethod @@ -175,7 +187,7 @@ def fine_tune_gtiff(file_path, output_path, adjust_contrast=False): bool: True if the operation was successful, False otherwise. """ try: - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return False driver = gdal.GetDriverByName("GTiff") @@ -190,7 +202,7 @@ def fine_tune_gtiff(file_path, output_path, adjust_contrast=False): out_band.WriteArray(data) out_dataset.FlushCache() return True - except (OSError, RuntimeError): + except (OSError, RuntimeError, ValueError): return False @staticmethod @@ -219,7 +231,7 @@ def convert_tif_to_jpeg( if output_path is None: output_path = os.path.splitext(tif_path)[0] + ".jpeg" - with gdal.Open(tif_path) as dataset: + with ImageryUtils.open_gtiff(tif_path) as dataset: if not dataset: raise Exception(f"Unable to open TIFF file: {tif_path}") @@ -306,7 +318,7 @@ def get_rgb_band_indexes(tif_file, source_type=None): Fallback: Use GDAL color interpretation. """ - with gdal.Open(tif_file) as dataset: + with ImageryUtils.open_gtiff(tif_file) as dataset: if dataset is None: raise RuntimeError( "Failed to open the dataset for band organization." @@ -473,50 +485,50 @@ def mosaic_imagery( os.path.splitext(tif_files[0])[0] + "_out_mosaic.tif" ) - vrt_input_files = [] + vrt_input_files: list[Any] = [] + source_datasets: list[Any] = [] temp_files = [] + dataset = None + warped = None + vrt = None + result = None # Process each file with gdalwarp using nearest neighbor resampling # This handles rotated geotransforms without changing projection for i, tif_file in enumerate(tif_files): - with rasterio.open(tif_file) as src: - # Check if the file has a rotated geotransform - geotransform = src.transform - has_rotation = geotransform.b != 0 or geotransform.d != 0 - - if has_rotation: - logger.info( - f"Processing rotated geotransform for {tif_file}" - ) - temp_file = ( - os.path.splitext(tif_file)[0] - + f"_temp_warped_{i}.tif" - ) + dataset = ImageryUtils.open_gtiff(tif_file) + source_datasets.append(dataset) + geotransform = dataset.GetGeoTransform() + has_rotation = geotransform[2] != 0 or geotransform[4] != 0 - # Use gdalwarp with nearest neighbor to handle rotation - # while preserving original projection - warp_options = gdal.WarpOptions( - resampleAlg="near", # Nearest neighbor resampling - creationOptions=["COMPRESS=DEFLATE", "TILED=YES"], - ) - - result = gdal.Warp( - temp_file, tif_file, options=warp_options + if has_rotation: + logger.info( + f"Processing rotated geotransform for {tif_file}" + ) + temp_file = ( + os.path.splitext(tif_file)[0] + f"_temp_warped_{i}.tif" + ) + warp_options = gdal.WarpOptions( + resampleAlg="near", + creationOptions=["COMPRESS=DEFLATE", "TILED=YES"], + ) + warped = gdal.Warp( + temp_file, dataset, options=warp_options + ) + if warped is None: + error_msg = gdal.GetLastErrorMsg() + raise RuntimeError( + f"Failed to warp {tif_file}: {error_msg}" ) - if result is None: - error_msg = gdal.GetLastErrorMsg() - raise RuntimeError( - f"Failed to warp {tif_file}: {error_msg}" - ) - - result.FlushCache() - result = None - temp_files.append(temp_file) - vrt_input_files.append(temp_file) - logger.info(f"Warped file created: {temp_file}") - else: - # File doesn't have rotation, use as-is - vrt_input_files.append(tif_file) + warped.FlushCache() + source_datasets.append(warped) + vrt_input_files.append(warped) + warped = None + temp_files.append(temp_file) + logger.info(f"Warped file created: {temp_file}") + else: + vrt_input_files.append(dataset) + dataset = None try: vrt_path = Path(output_file_path).with_suffix(".vrt") @@ -531,7 +543,6 @@ def mosaic_imagery( raise RuntimeError(f"Failed to create VRT: {error_msg}") vrt.FlushCache() - vrt = None logger.info(f"VRT file created at {vrt_path}") # Check if vrt file exists on disk @@ -555,9 +566,7 @@ def mosaic_imagery( ] logger.info(f"Clipping mosaic to AOI bbox {clip_bbox}") warp_options = gdal.WarpOptions(options=warp_option_list) - result = gdal.Warp( - output_file_path, str(vrt_path), options=warp_options - ) + result = gdal.Warp(output_file_path, vrt, options=warp_options) if result is None: error_msg = gdal.GetLastErrorMsg() raise RuntimeError( @@ -566,6 +575,7 @@ def mosaic_imagery( result.FlushCache() result = None + vrt = None logger.info(f"Mosaic COG file created at {output_file_path}") # Clean up VRT @@ -575,6 +585,12 @@ def mosaic_imagery( pass finally: + result = None + vrt = None + warped = None + dataset = None + vrt_input_files.clear() + source_datasets.clear() # Clean up temporary files for temp_file in temp_files: try: @@ -617,7 +633,7 @@ def get_scale_imagery_params(file_path, source_type=None): Exception: If there is an error processing the raster image with GDAL. """ try: - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return None num_bands = dataset.RasterCount @@ -759,7 +775,7 @@ def get_normalization_means(file_path, source_type=None): """ try: normalization_means = [] - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return None num_bands = dataset.RasterCount @@ -794,7 +810,7 @@ def get_normalization_std_devs(file_path, source_type=None): """ try: normalization_stds = [] - with gdal.Open(file_path) as dataset: + with ImageryUtils.open_gtiff(file_path) as dataset: if dataset is None: return None num_bands = dataset.RasterCount diff --git a/hastelib/src/hastegeo/workflows/prepare_imagery.py b/hastelib/src/hastegeo/workflows/prepare_imagery.py index 48b2ddfe..9c39d838 100644 --- a/hastelib/src/hastegeo/workflows/prepare_imagery.py +++ b/hastelib/src/hastegeo/workflows/prepare_imagery.py @@ -158,6 +158,8 @@ def download_pre_event(self): dst_directory=self.dst_directory, ) if preevent_tif_files: + if not _are_all_valid_tif_files(preevent_tif_files): + raise RuntimeError("Invalid pre-event imagery files found.") raw_prefix = self.generate_prefix(PRE_EVENT_RAW_PREFIX) ( self.pre_event_raw_paths, diff --git a/hastelib/tests/build/test_dependency_pins.py b/hastelib/tests/build/test_dependency_pins.py index 17ecc909..7e9dabb7 100644 --- a/hastelib/tests/build/test_dependency_pins.py +++ b/hastelib/tests/build/test_dependency_pins.py @@ -1,21 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -"""Guards against dependency pins that cannot actually be installed. - -PR #153 bumped ``gdal`` to 3.13.3 in ``hastelib/pyproject.toml`` while every -container kept installing the prebuilt GDAL **3.9.2** wheel. Because the -Functions images install hastegeo from the local tree (``-e ../../hastelib``), -pip saw ``gdal==3.13.3`` and ``gdal 3.9.2`` at once and failed with -``ResolutionImpossible``. Nothing caught it: the image build is skipped when a -pull request touches ``hastelib/**``, and the wheel build never installs the -images. - -The same bump also set ``rasterio==1.5.1``, which requires Python >= 3.12 while -all three images run 3.11. That class of break is caught by the dependency -resolution job in ``.github/workflows/dependency-validation.yml``; the checks -here are the fast, offline half that needs no network. -""" +"""Guard that every consumer requests the shared GDAL binding versions.""" import json import re @@ -23,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from urllib.parse import unquote, urlsplit REPO_ROOT = Path(__file__).resolve().parents[3] SCRIPTS_ROOT = REPO_ROOT / ".github" / "scripts" @@ -32,106 +19,161 @@ import check_protobuf_pin # noqa: E402 import check_react_pins # noqa: E402 -# Every requirements file that installs hastegeo from the local tree and so -# must agree with the pin in pyproject.toml. +# Every HASTE API or image dependency file that installs the shared runtime. REQUIREMENTS_FILES = ( "api/hastefuncapi/requirements.txt", "api/hastefuncqueues/requirements.txt", "docker/imageryprep/requirements.txt", + "docker/training/env/env.yml", ) DOCKERFILES = ( "api/hastefuncapi/Dockerfile", "api/hastefuncqueues/Dockerfile", "docker/imageryprep/Dockerfile", + "docker/titiler/Dockerfile", ) PYPROJECT = "hastelib/pyproject.toml" -# GDAL @ https://.../GDAL-3.9.2-cp311-cp311-manylinux...whl -_GDAL_WHEEL_RE = re.compile( - r"GDAL-(?P\d+\.\d+\.\d+)-cp(?P\d+)-", re.IGNORECASE -) -_PYPROJECT_GDAL_RE = re.compile(r'"gdal==(?P\d+\.\d+\.\d+)"') +_GDAL_PINS = { + "gdal": "3.13.3", + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", +} def _read(relative_path): return (REPO_ROOT / relative_path).read_text(encoding="utf-8") -def _pyproject_gdal_version(): - match = _PYPROJECT_GDAL_RE.search(_read(PYPROJECT)) - if match is None: - raise AssertionError(f"No pinned gdal dependency in {PYPROJECT}") - return match.group("version") - - -def _wheel_pins(relative_path): - """Return ``(version, cpython_tag)`` for the GDAL wheel, or ``None``.""" - for line in _read(relative_path).splitlines(): - match = _GDAL_WHEEL_RE.search(line) +def _dependency_pins(text): + pins = {} + for line in text.splitlines(): + content = line.strip() + if content.startswith("- "): + content = content[2:].lstrip() + match = re.match(r"['\"]?([A-Za-z0-9_.-]+)==([^\s;\\'\"]+)", content) + if match: + pins[re.sub(r"[-_.]+", "-", match.group(1)).lower()] = match.group( + 2 + ) + continue + match = re.match( + r"([A-Za-z0-9_.-]+)\s*@\s*https://\S+/([^/\s]+\.whl)", + content, + re.IGNORECASE, + ) if match: - return match.group("version"), match.group("py") - return None + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() + filename = unquote(match.group(2)).lower() + wheel_prefix = re.escape(name.replace("-", "_")) + version = re.match( + rf"{wheel_prefix}-(\d+\.\d+\.\d+)\+haste\.\d+-", filename + ) + if version: + pins[name] = version.group(1) + return pins class GdalPinConsistencyTests(unittest.TestCase): - """The regression PR #153 introduced, expressed as an invariant.""" - - def test_gdal_wheel_matches_the_pyproject_pin(self): - expected = _pyproject_gdal_version() + """All GDAL-linked bindings in HASTE must agree on exact versions.""" - mismatches = [] + def test_every_consumer_pins_the_shared_binding_set(self): + expected = _GDAL_PINS for relative_path in REQUIREMENTS_FILES: - pins = _wheel_pins(relative_path) - if pins is None: - mismatches.append(f"{relative_path}: no GDAL wheel pinned") - continue - version, _ = pins - if version != expected: - mismatches.append( - f"{relative_path}: wheel GDAL {version} != " - f"{PYPROJECT} gdal=={expected}" + actual = _dependency_pins(_read(relative_path)) + with self.subTest(requirements=relative_path): + self.assertEqual( + expected, + {name: actual.get(name) for name in expected}, + f"{relative_path} must pin the same GDAL runtime bindings.", ) + def test_hastegeo_metadata_pins_the_same_binding_versions(self): + actual = _dependency_pins(_read(PYPROJECT)) self.assertEqual( - [], - mismatches, - "The GDAL wheel installed by each image must match the gdal pin " - "hastegeo declares, or pip cannot resolve the two together.", + _GDAL_PINS, + {name: actual.get(name) for name in _GDAL_PINS}, ) - def test_gdal_wheel_targets_the_image_python_version(self): - """A cp311 wheel cannot be installed on a 3.12 base image.""" - image_pythons = {} + def test_training_conda_environment_uses_python_311_and_no_conda_gdal( + self, + ): + text = _read("docker/training/env/env.yml") + self.assertIn("python=3.11", text) + self.assertNotRegex(text, r"(?m)^\s*-\s*gdal\s*[<=>]") + + def test_consumer_wheel_urls_match_the_locked_runtime_build(self): + lock = _read("native/gdal-runtime/sources.lock") + build_match = re.search(r"(?m)^HASTE_GDAL_BUILD\s+(\d+)\s*$", lock) + self.assertIsNotNone(build_match) + expected_build = build_match.group(1) + all_bindings = { + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + "haste-gdal-runtime", + } + consumer_files = ( + *REQUIREMENTS_FILES, + "api/titilerfuncapi/requirements.txt", + ) + for relative_path in consumer_files: + text = _read(relative_path) + names = { + re.sub(r"[-_.]+", "-", name).lower() + for name in re.findall( + r"(?im)^\s*-?\s*([A-Za-z0-9_.-]+)\s*@\s*https://", + text, + ) + } + expected = ( + {"haste-gdal-runtime", "rasterio"} + if relative_path.endswith("titilerfuncapi/requirements.txt") + else all_bindings + ) + with self.subTest(requirements=relative_path): + self.assertEqual(expected, names) + self.assertEqual( + {expected_build}, + set(re.findall(r"\+haste\.(\d+)", unquote(text))), + ) + filenames = [] + for line in text.splitlines(): + match = re.match( + r"^\s*-?\s*([A-Za-z0-9_.-]+)\s*@\s*(https://\S+)", + line, + ) + if not match: + continue + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() + if name in expected: + filenames.append( + Path(unquote(urlsplit(match.group(2)).path)).name + ) + for filename in filenames: + if filename.startswith("haste_gdal_runtime-"): + self.assertIn( + "py3-none-manylinux_2_28_x86_64.whl", filename + ) + else: + self.assertIn( + "cp311-cp311-manylinux_2_28_x86_64.whl", + filename, + ) + + def test_function_and_geospatial_images_use_python_311(self): for relative_path in DOCKERFILES: version = check_image_python.base_python_version( _read(relative_path) ) - if version is not None: - image_pythons[relative_path] = version - - self.assertEqual( - sorted(image_pythons), - sorted(DOCKERFILES), - "Could not determine the Python version of every image.", - ) - - mismatches = [] - for dockerfile, requirements in zip(DOCKERFILES, REQUIREMENTS_FILES): - pins = _wheel_pins(requirements) - if pins is None: - continue - _, wheel_py = pins - image_py = image_pythons[dockerfile].replace(".", "") - if wheel_py != image_py: - mismatches.append( - f"{requirements}: GDAL wheel is cp{wheel_py} but " - f"{dockerfile} runs Python " - f"{image_pythons[dockerfile]}" - ) - - self.assertEqual([], mismatches) + with self.subTest(dockerfile=relative_path): + self.assertEqual("3.11", version) def _write(directory, name, payload): diff --git a/hastelib/tests/core/processors/test_uploader_format.py b/hastelib/tests/core/processors/test_uploader_format.py index 18a4f4e8..1d4d7e52 100644 --- a/hastelib/tests/core/processors/test_uploader_format.py +++ b/hastelib/tests/core/processors/test_uploader_format.py @@ -10,6 +10,7 @@ """ import unittest +from io import BytesIO from unittest.mock import patch @@ -110,6 +111,20 @@ def test_content_check_skipped_when_no_local_chunk(self): # No chunk_0 staged → skip rather than block a legitimate finalize. u._assert_content_matches_declared("absent", "tif") # no raise + def test_first_chunk_is_validated_before_storage(self): + u = self._uploader_with_tmpdir() + chunk = BytesIO(b"") + with self.assertRaises(ValueError): + u.save_chunk( + chunk_number=0, + chunk_data=chunk, + file_id="f5", + total_chunks=2, + data_format="tif", + ) + + u.storage.save_chunk.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/hastelib/tests/core/utils/test_footprints.py b/hastelib/tests/core/utils/test_footprints.py index 8d69f28b..72ee77f0 100644 --- a/hastelib/tests/core/utils/test_footprints.py +++ b/hastelib/tests/core/utils/test_footprints.py @@ -192,8 +192,8 @@ def test_filters_to_polygons_and_writes(self, mock_geodataframe): gdf = MagicMock() # gdf[["id", "geometry", "subtype", "class"]] -> filtered_gdf filtered = MagicMock() - gdf.__getitem__.side_effect = ( - lambda key: filtered if isinstance(key, list) else gdf + gdf.__getitem__.side_effect = lambda key: ( + filtered if isinstance(key, list) else gdf ) # filtered[mask] returns final gdf with .shape / .set_crs / .to_file final = MagicMock() @@ -634,13 +634,17 @@ def test_missing_crs_raises(self): }, crs=None, ) + valid_source = gpd.GeoDataFrame( + { + "id": ["a"], + "geometry": [shapely.geometry.box(1, 1, 2, 2)], + }, + crs="EPSG:4326", + ) with tempfile.TemporaryDirectory() as tmp: input_path = os.path.join(tmp, "in.gpkg") - # Just touch the input path so the helper's existence check - # (if any) doesn't fire; the read is mocked. - with open(input_path, "wb"): - pass + valid_source.to_file(input_path, driver="GPKG") output_path = os.path.join(tmp, "out.gpkg") with patch( "hastegeo.core.utils.footprints.gpd.read_file", @@ -652,6 +656,31 @@ def test_missing_crs_raises(self): ) self.assertIn("CRS", str(cm.exception)) + def test_rejects_vrt_content_before_vector_open(self): + import os + import tempfile + from unittest.mock import patch + + from hastegeo.core.utils.footprints import ( + clip_and_normalize_user_footprints, + ) + + with tempfile.TemporaryDirectory() as tmp: + input_path = os.path.join(tmp, "disguised.gpkg") + with open(input_path, "w", encoding="utf-8") as source: + source.write("") + with patch( + "hastegeo.core.utils.footprints.gpd.read_file" + ) as read_file: + with self.assertRaisesRegex( + ValueError, "does not match the declared format" + ): + clip_and_normalize_user_footprints( + input_path, self._aoi(), os.path.join(tmp, "out.gpkg") + ) + + read_file.assert_not_called() + def test_overwrite_false_rejects_existing_output(self): import os import tempfile diff --git a/hastelib/tests/core/utils/test_gdal_security.py b/hastelib/tests/core/utils/test_gdal_security.py index 2ca7d098..93536f6b 100644 --- a/hastelib/tests/core/utils/test_gdal_security.py +++ b/hastelib/tests/core/utils/test_gdal_security.py @@ -12,6 +12,7 @@ import os import tempfile import unittest +from unittest import mock from hastegeo.core.utils import gdal_security as g @@ -116,6 +117,18 @@ def test_matches_declared_rejects_mismatch(self): os.remove(hdf_like) os.remove(gpkg) + def test_chunk_header_validation_rejects_vrt_and_short_gpkg_chunks(self): + with self.assertRaises(ValueError): + g.assert_matches_declared_bytes(b"", "tif") + with self.assertRaisesRegex(ValueError, "at least 72 bytes"): + g.assert_matches_declared_bytes(b"SQLite format 3\x00", "gpkg") + + def test_chunk_header_validation_accepts_supported_headers(self): + tiff = b"II*\x00" + b"\x00" * 68 + gpkg = b"SQLite format 3\x00" + b"\x00" * (68 - 16) + b"GPKG" + g.assert_matches_declared_bytes(tiff, "tif") + g.assert_matches_declared_bytes(gpkg, "gpkg") + class TestSizeEnvHelpers(unittest.TestCase): def setUp(self): @@ -149,31 +162,159 @@ def test_override_and_invalid(self): self.assertEqual(g.max_upload_bytes(), 5 * 1024**3) +class TestSharedRuntimeActivation(unittest.TestCase): + def setUp(self): + self._was_hardened = g._hardened + g._hardened = False + + def tearDown(self): + g._hardened = self._was_hardened + + def test_shared_runtime_is_configured_before_startup_check(self): + calls = [] + runtime = mock.Mock() + runtime.load_manifest.return_value = {"gdal_release": "3.13.3"} + runtime.Policy.return_value = object() + runtime.configure.side_effect = lambda policy: calls.append( + "configure" + ) + runtime.secure_startup.side_effect = lambda policy: calls.append( + "secure_startup" + ) or {"gdal_release": "3.13.3"} + with mock.patch.object( + g.importlib.util, "find_spec", return_value=object() + ), mock.patch.object( + g.importlib, "import_module", return_value=runtime + ): + g.harden_gdal(force=True, require_runtime=True) + + self.assertEqual(calls, ["configure", "secure_startup"]) + self.assertEqual(runtime.UseExceptions.call_count, 2) + runtime.Policy.assert_called_once_with( + expected_gdal="3.13.3", + allowed_drivers=g.ALLOWED_RASTER_DRIVERS + | g.ALLOWED_VECTOR_DRIVERS, + required_bindings=("osgeo.gdal",), + ) + + def test_required_runtime_missing_fails_closed(self): + with mock.patch.object( + g.importlib.util, "find_spec", return_value=None + ): + with self.assertRaisesRegex(RuntimeError, "is not installed"): + g.harden_gdal(force=True, require_runtime=True) + + +class TestLegacyGdalConfiguration(unittest.TestCase): + def setUp(self): + discovery = mock.patch.object( + g.importlib.util, "find_spec", return_value=None + ) + discovery.start() + self.addCleanup(discovery.stop) + self._saved_settings = { + key: os.environ.get(key) + for key in ( + *g.LEGACY_REQUIRED_CONFIG, + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE", + "GDAL_VRT_PYTHON_TRUSTED_MODULES", + "GDAL_PYTHON_DRIVER_PATH", + "GDAL_HTTP_UNSAFESSL", + "GDAL_SKIP", + ) + } + for key in self._saved_settings: + os.environ.pop(key, None) + self._was_hardened = g._hardened + g._hardened = False + + def tearDown(self): + g._hardened = self._was_hardened + for key, value in self._saved_settings.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def test_legacy_runtime_sets_required_vrt_and_plugin_config(self): + g.harden_gdal(force=True, require_runtime=False) + for key, value in g.LEGACY_REQUIRED_CONFIG.items(): + self.assertEqual(os.environ[key], value) + + def test_legacy_runtime_rejects_relaxed_operator_config(self): + os.environ["GDAL_VRT_ENABLE_PYTHON"] = "YES" + with self.assertRaisesRegex(RuntimeError, "weakens GDAL hardening"): + g.harden_gdal(force=True, require_runtime=False) + + def test_legacy_runtime_merges_required_driver_denylist(self): + os.environ["GDAL_SKIP"] = "JPEG,HDF4" + g.harden_gdal(force=True, require_runtime=False) + skipped = os.environ["GDAL_SKIP"].split() + self.assertIn("JPEG", skipped) + self.assertEqual(skipped.count("HDF4"), 1) + self.assertTrue(set(g.DISABLED_DRIVERS_ENV.split()).issubset(skipped)) + + def test_failed_driver_deregistration_aborts_legacy_hardening(self): + driver = mock.Mock() + driver.ShortName = "HDF4" + driver.Deregister.side_effect = OSError("registration manager failed") + gdal_module = mock.Mock() + gdal_module.GetDriverCount.return_value = 1 + gdal_module.GetDriver.return_value = driver + + with self.assertRaisesRegex( + RuntimeError, "failed to deregister disallowed GDAL driver HDF4" + ): + g._deregister_disallowed_legacy_drivers( + gdal_module, + g.ALLOWED_RASTER_DRIVERS | g.ALLOWED_VECTOR_DRIVERS, + ) + + @unittest.skipUnless(_HAS_GDAL, "requires osgeo/GDAL") class TestHardenGdal(unittest.TestCase): + def setUp(self): + self._saved_settings = { + key: os.environ.get(key) + for key in ( + *g.LEGACY_REQUIRED_CONFIG, + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE", + "GDAL_VRT_PYTHON_TRUSTED_MODULES", + "GDAL_PYTHON_DRIVER_PATH", + "GDAL_HTTP_UNSAFESSL", + "GDAL_SKIP", + ) + } + for key in self._saved_settings: + os.environ.pop(key, None) + + def tearDown(self): + for key, value in self._saved_settings.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + def test_disables_vulnerable_drivers_keeps_allowlist(self): g.harden_gdal(force=True) from osgeo import gdal as _gdal from osgeo import ogr as _ogr - # CVE-bearing families must be gone. for name in ("HDF4", "HDF4Image", "HDF5", "netCDF"): with self.subTest(disabled=name): self.assertIsNone(_gdal.GetDriverByName(name)) - # Allowlisted raster drivers remain. for name in ("GTiff", "COG", "VRT", "JPEG", "PNG"): with self.subTest(kept=name): self.assertIsNotNone(_gdal.GetDriverByName(name)) - # Allowlisted vector drivers remain. for name in ("GPKG", "GeoJSON"): with self.subTest(kept_vector=name): self.assertIsNotNone(_ogr.GetDriverByName(name)) def test_idempotent(self): g.harden_gdal(force=True) - g.harden_gdal() # second call must not raise or remove allowlisted + g.harden_gdal() from osgeo import gdal as _gdal self.assertIsNotNone(_gdal.GetDriverByName("GTiff")) diff --git a/hastelib/tests/core/utils/test_imagery.py b/hastelib/tests/core/utils/test_imagery.py index 373291df..a50012e3 100644 --- a/hastelib/tests/core/utils/test_imagery.py +++ b/hastelib/tests/core/utils/test_imagery.py @@ -3,7 +3,9 @@ """Tests for imagery utility source-specific behavior.""" +import tempfile import unittest +from pathlib import Path from unittest.mock import MagicMock, patch import numpy as np @@ -20,10 +22,7 @@ def test_rgb_no_processing_uses_default_percentiles(self): ) with ( - patch( - "hastegeo.core.utils.imagery.gdal.Open", - return_value=dataset, - ), + patch.object(ImageryUtils, "open_gtiff", return_value=dataset), patch( "hastegeo.core.utils.imagery.np.percentile", side_effect=lambda _, percentile: percentile, @@ -63,10 +62,7 @@ def _band_indexes(source_type, num_bands): dataset.__enter__.return_value = dataset dataset.RasterCount = num_bands - with patch( - "hastegeo.core.utils.imagery.gdal.Open", - return_value=dataset, - ): + with patch.object(ImageryUtils, "open_gtiff", return_value=dataset): return ImageryUtils.get_rgb_band_indexes("image.tif", source_type) def test_vantor_matches_legacy_maxar_for_supported_band_counts(self): @@ -90,5 +86,36 @@ def test_legacy_maxar_still_resolves_to_the_vantor_mapping(self): self.assertEqual(self._band_indexes("maxar", 8), [4, 3, 1]) +class TestOpenGtiff(unittest.TestCase): + def test_vrt_bytes_disguised_as_tiff_are_rejected_before_gdal(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "disguised.tif" + path.write_text("", encoding="utf-8") + with patch("hastegeo.core.utils.imagery.gdal.OpenEx") as open_ex: + with self.assertRaisesRegex( + ValueError, "does not match the declared format" + ): + ImageryUtils.open_gtiff(str(path)) + + open_ex.assert_not_called() + + def test_tiff_open_is_limited_to_gtiff_driver(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "image.tif" + path.write_bytes(b"II*\x00" + b"\x00" * 68) + dataset = MagicMock() + dataset.__enter__.return_value = dataset + with patch( + "hastegeo.core.utils.imagery.gdal.OpenEx", + return_value=dataset, + ) as open_ex: + with ImageryUtils.open_gtiff(str(path)) as actual: + self.assertIs(actual, dataset) + + self.assertEqual( + open_ex.call_args.kwargs["allowed_drivers"], ["GTiff"] + ) + + if __name__ == "__main__": unittest.main() diff --git a/hastelib/tests/core/utils/test_imagery_runtime.py b/hastelib/tests/core/utils/test_imagery_runtime.py index f4e5686c..e2bea12a 100644 --- a/hastelib/tests/core/utils/test_imagery_runtime.py +++ b/hastelib/tests/core/utils/test_imagery_runtime.py @@ -94,6 +94,18 @@ def test_mosaic_combines_adjacent_rasters(self) -> None: dataset.read()[:, :, 64:], right_pixels ) + def test_mosaic_rejects_vrt_bytes_disguised_as_tiff(self) -> None: + disguised = self.directory / "disguised.tif" + disguised.write_text("", encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "does not match the declared format" + ): + ImageryUtils.mosaic_imagery( + [str(disguised)], + str(self.directory / "rejected.tif"), + gdal_warp_params="-of COG", + ) + def test_mosaic_reprojects_using_native_proj_data(self) -> None: output = self.directory / "projected.tif" result = ImageryUtils.mosaic_imagery( diff --git a/infra/main.json b/infra/main.json index 7f380fba..e48534ed 100644 --- a/infra/main.json +++ b/infra/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.47.16.16243", - "templateHash": "9892879045433928477" + "templateHash": "14391226588399507420" } }, "parameters": { @@ -1547,7 +1547,7 @@ "_generator": { "name": "bicep", "version": "0.47.16.16243", - "templateHash": "510276896164916940" + "templateHash": "13225717067480360024" } }, "parameters": { @@ -1851,7 +1851,34 @@ } }, "variables": { - "apimSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('apimSubnetName'))]" + "apimSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('apimSubnetName'))]", + "gdalHardeningSettings": [ + { + "name": "HASTE_GDAL_RUNTIME_REQUIRED", + "value": "true" + }, + { + "name": "GDAL_VRT_ENABLE_RAWRASTERBAND", + "value": "NO" + }, + { + "name": "GDAL_VRT_ENABLE_PYTHON", + "value": "NO" + }, + { + "name": "GDAL_DRIVER_PATH", + "value": "disable" + }, + { + "name": "GDAL_DISABLE_READDIR_ON_OPEN", + "value": "EMPTY_DIR" + }, + { + "name": "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", + "value": "NO" + } + ], + "titilerGdalHardeningSettings": "[concat(variables('gdalHardeningSettings'), createArray(createObject('name', 'GDAL_SKIP', 'value', 'HDF4 HDF4Image HDF5 HDF5Image netCDF')))]" }, "resources": [ { @@ -1901,7 +1928,7 @@ "value": false }, "appSettings": { - "value": "[concat(createArray(createObject('name', 'env', 'value', 'prod'), createObject('name', 'DEVELOPMENT_MODE', 'value', if(parameters('developmentMode'), 'true', 'false')), createObject('name', 'IMAGE_QUEUE_NAME', 'value', 'image-layers-queue'), createObject('name', 'INFERENCE_QUEUE_NAME', 'value', 'inference-queue'), createObject('name', 'STATS_QUEUE_NAME', 'value', 'stats-queue'), createObject('name', 'TRAIN_QUEUE_NAME', 'value', 'train-queue'), createObject('name', 'ZIP_QUEUE_NAME', 'value', 'zip-queue'), createObject('name', 'EMBEDDING_QUEUE_NAME', 'value', 'embedding-queue'), createObject('name', 'PUBLISH_QUEUE_NAME', 'value', 'publish-queue'), createObject('name', 'IMAGERY_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'METADATA_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'ARTIFACT_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'RUNNER_TYPE', 'value', 'azure_batch'), createObject('name', 'TEMP_DATA_PATH', 'value', '/data'), createObject('name', 'DATA_PATH', 'value', '/data'), createObject('name', 'TITILER_ENDPOINT', 'value', '/api/titiler/'), createObject('name', 'BLOB_CONTAINER', 'value', 'data'), createObject('name', 'BLOB_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob), createObject('name', 'QUEUE_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.queue), createObject('name', 'BLOB_CONNECTION_STRING', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};EndpointSuffix={2}', parameters('storageAccountName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').keys[0].value, environment().suffixes.storage)), createObject('name', 'AZURE_BATCH_ACCOUNT_NAME', 'value', parameters('batchAccountName')), createObject('name', 'AZURE_BATCH_URL', 'value', format('https://{0}.{1}.batch.azure.com', parameters('batchAccountName'), parameters('batchAccountLocation'))), createObject('name', 'AZURE_BATCH_ACCOUNT_KEY', 'value', parameters('batchAccountKey')), createObject('name', 'AZURE_BATCH_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_IMAGERYPREP_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('imageryprepImage'))), createObject('name', 'AZURE_BATCH_OUTPUT_CONTAINER_URL', 'value', format('{0}data', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob)), createObject('name', 'AZURE_BATCH_TRAINING_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_REGISTRY_SERVER', 'value', parameters('acrLoginServer')), createObject('name', 'AZURE_BATCH_REGISTRY_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_REGISTRY_IDENTITY_RESOURCE_ID', 'value', parameters('umiResourceId')), createObject('name', 'STATIC_APP_SUBSCRIPTION_ID', 'value', subscription().subscriptionId), createObject('name', 'STATIC_APP_RESOURCE_GROUP', 'value', resourceGroup().name), createObject('name', 'STATIC_APP_NAME', 'value', parameters('staticWebAppName')), createObject('name', 'STATIC_APP_DOMAIN', 'value', parameters('staticAppDomain')), createObject('name', 'EMAIL_CONNECTION_STRING', 'value', parameters('emailConnectionString')), createObject('name', 'EMAIL_SENDER', 'value', format('DoNotReply@{0}', parameters('emailSenderDomain'))), createObject('name', 'AZURE_BATCH_TRAINING_POOL_IDS', 'value', parameters('trainingPoolIds')), createObject('name', 'AZURE_BATCH_INFERENCE_POOL_IDS', 'value', parameters('inferencePoolIds')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_IDS', 'value', parameters('imageryprepPoolIds')), createObject('name', 'AZURE_BATCH_USE_SAS', 'value', if(parameters('useSas'), 'true', 'false')), createObject('name', 'AZURE_BATCH_MANAGE_POOLS', 'value', if(parameters('managePools'), 'true', 'false')), createObject('name', 'PUBLISHING_ENABLED', 'value', if(parameters('publishingEnabled'), 'true', 'false')), createObject('name', 'PC_PROVIDER_ENABLED', 'value', if(parameters('pcProviderEnabled'), 'true', 'false')), createObject('name', 'PC_GEOCATALOG_URL', 'value', parameters('pcGeocatalogUrl')), createObject('name', 'PC_EXPLORER_URL', 'value', parameters('pcExplorerUrl')), createObject('name', 'PC_INGESTION_SOURCE', 'value', parameters('pcIngestionSource')), createObject('name', 'PC_COLLECTION_PREFIX', 'value', parameters('pcCollectionPrefix')), createObject('name', 'PC_PUBLISHING_LICENSE', 'value', parameters('pcPublishingLicense')), createObject('name', 'PUBLISHING_ORGANIZATION_NAME', 'value', parameters('publishingOrganizationName')), createObject('name', 'PUBLISHING_ORGANIZATION_URL', 'value', parameters('publishingOrganizationUrl')), createObject('name', 'PUBLISH_STORAGE_ACCOUNT_URL', 'value', parameters('publishStorageAccountUrl')), createObject('name', 'PUBLISH_BLOB_CONTAINER', 'value', parameters('publishBlobContainer')), createObject('name', 'PUBLISH_EXPLORER_RENDER_ENABLED', 'value', if(parameters('publishExplorerRenderEnabled'), 'true', 'false'))), createArray(createObject('name', 'HASTE_MAX_UPLOAD_BYTES', 'value', string(parameters('maxUploadBytes'))), createObject('name', 'PUBLISH_ASSESSMENT_MAX_TOTAL_BYTES', 'value', string(parameters('publishAssessmentMaxTotalBytes'))), createObject('name', 'AzureFunctionsWebHost__hostId', 'value', toLower(substring(parameters('functionApiName'), 0, min(32, length(parameters('functionApiName'))))))))]" + "value": "[concat(concat(variables('gdalHardeningSettings'), createArray(createObject('name', 'env', 'value', 'prod'), createObject('name', 'DEVELOPMENT_MODE', 'value', if(parameters('developmentMode'), 'true', 'false')), createObject('name', 'IMAGE_QUEUE_NAME', 'value', 'image-layers-queue'), createObject('name', 'INFERENCE_QUEUE_NAME', 'value', 'inference-queue'), createObject('name', 'STATS_QUEUE_NAME', 'value', 'stats-queue'), createObject('name', 'TRAIN_QUEUE_NAME', 'value', 'train-queue'), createObject('name', 'ZIP_QUEUE_NAME', 'value', 'zip-queue'), createObject('name', 'EMBEDDING_QUEUE_NAME', 'value', 'embedding-queue'), createObject('name', 'PUBLISH_QUEUE_NAME', 'value', 'publish-queue'), createObject('name', 'IMAGERY_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'METADATA_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'ARTIFACT_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'RUNNER_TYPE', 'value', 'azure_batch'), createObject('name', 'TEMP_DATA_PATH', 'value', '/data'), createObject('name', 'DATA_PATH', 'value', '/data'), createObject('name', 'TITILER_ENDPOINT', 'value', '/api/titiler/'), createObject('name', 'BLOB_CONTAINER', 'value', 'data'), createObject('name', 'BLOB_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob), createObject('name', 'QUEUE_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.queue), createObject('name', 'BLOB_CONNECTION_STRING', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};EndpointSuffix={2}', parameters('storageAccountName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').keys[0].value, environment().suffixes.storage)), createObject('name', 'AZURE_BATCH_ACCOUNT_NAME', 'value', parameters('batchAccountName')), createObject('name', 'AZURE_BATCH_URL', 'value', format('https://{0}.{1}.batch.azure.com', parameters('batchAccountName'), parameters('batchAccountLocation'))), createObject('name', 'AZURE_BATCH_ACCOUNT_KEY', 'value', parameters('batchAccountKey')), createObject('name', 'AZURE_BATCH_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_IMAGERYPREP_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('imageryprepImage'))), createObject('name', 'AZURE_BATCH_OUTPUT_CONTAINER_URL', 'value', format('{0}data', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob)), createObject('name', 'AZURE_BATCH_TRAINING_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_REGISTRY_SERVER', 'value', parameters('acrLoginServer')), createObject('name', 'AZURE_BATCH_REGISTRY_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_REGISTRY_IDENTITY_RESOURCE_ID', 'value', parameters('umiResourceId')), createObject('name', 'STATIC_APP_SUBSCRIPTION_ID', 'value', subscription().subscriptionId), createObject('name', 'STATIC_APP_RESOURCE_GROUP', 'value', resourceGroup().name), createObject('name', 'STATIC_APP_NAME', 'value', parameters('staticWebAppName')), createObject('name', 'STATIC_APP_DOMAIN', 'value', parameters('staticAppDomain')), createObject('name', 'EMAIL_CONNECTION_STRING', 'value', parameters('emailConnectionString')), createObject('name', 'EMAIL_SENDER', 'value', format('DoNotReply@{0}', parameters('emailSenderDomain'))), createObject('name', 'AZURE_BATCH_TRAINING_POOL_IDS', 'value', parameters('trainingPoolIds')), createObject('name', 'AZURE_BATCH_INFERENCE_POOL_IDS', 'value', parameters('inferencePoolIds')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_IDS', 'value', parameters('imageryprepPoolIds')), createObject('name', 'AZURE_BATCH_USE_SAS', 'value', if(parameters('useSas'), 'true', 'false')), createObject('name', 'AZURE_BATCH_MANAGE_POOLS', 'value', if(parameters('managePools'), 'true', 'false')), createObject('name', 'PUBLISHING_ENABLED', 'value', if(parameters('publishingEnabled'), 'true', 'false')), createObject('name', 'PC_PROVIDER_ENABLED', 'value', if(parameters('pcProviderEnabled'), 'true', 'false')), createObject('name', 'PC_GEOCATALOG_URL', 'value', parameters('pcGeocatalogUrl')), createObject('name', 'PC_EXPLORER_URL', 'value', parameters('pcExplorerUrl')), createObject('name', 'PC_INGESTION_SOURCE', 'value', parameters('pcIngestionSource')), createObject('name', 'PC_COLLECTION_PREFIX', 'value', parameters('pcCollectionPrefix')), createObject('name', 'PC_PUBLISHING_LICENSE', 'value', parameters('pcPublishingLicense')), createObject('name', 'PUBLISHING_ORGANIZATION_NAME', 'value', parameters('publishingOrganizationName')), createObject('name', 'PUBLISHING_ORGANIZATION_URL', 'value', parameters('publishingOrganizationUrl')), createObject('name', 'PUBLISH_STORAGE_ACCOUNT_URL', 'value', parameters('publishStorageAccountUrl')), createObject('name', 'PUBLISH_BLOB_CONTAINER', 'value', parameters('publishBlobContainer')), createObject('name', 'PUBLISH_EXPLORER_RENDER_ENABLED', 'value', if(parameters('publishExplorerRenderEnabled'), 'true', 'false')))), createArray(createObject('name', 'HASTE_MAX_UPLOAD_BYTES', 'value', string(parameters('maxUploadBytes'))), createObject('name', 'PUBLISH_ASSESSMENT_MAX_TOTAL_BYTES', 'value', string(parameters('publishAssessmentMaxTotalBytes'))), createObject('name', 'AzureFunctionsWebHost__hostId', 'value', toLower(substring(parameters('functionApiName'), 0, min(32, length(parameters('functionApiName'))))))))]" }, "tags": { "value": "[parameters('tags')]" @@ -2283,12 +2310,7 @@ "value": true }, "appSettings": { - "value": [ - { - "name": "TITILER_ALLOWED_HOSTS", - "value": "[split(reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob, '/')[2]]" - } - ] + "value": "[concat(variables('titilerGdalHardeningSettings'), createArray(createObject('name', 'TITILER_ALLOWED_HOSTS', 'value', split(reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob, '/')[2])))]" }, "tags": { "value": "[parameters('tags')]" @@ -2656,7 +2678,7 @@ "value": false }, "appSettings": { - "value": "[concat(createArray(createObject('name', 'env', 'value', 'prod'), createObject('name', 'DEVELOPMENT_MODE', 'value', if(parameters('developmentMode'), 'true', 'false')), createObject('name', 'IMAGE_QUEUE_NAME', 'value', 'image-layers-queue'), createObject('name', 'INFERENCE_QUEUE_NAME', 'value', 'inference-queue'), createObject('name', 'STATS_QUEUE_NAME', 'value', 'stats-queue'), createObject('name', 'TRAIN_QUEUE_NAME', 'value', 'train-queue'), createObject('name', 'ZIP_QUEUE_NAME', 'value', 'zip-queue'), createObject('name', 'EMBEDDING_QUEUE_NAME', 'value', 'embedding-queue'), createObject('name', 'PUBLISH_QUEUE_NAME', 'value', 'publish-queue'), createObject('name', 'IMAGERY_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'METADATA_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'ARTIFACT_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'RUNNER_TYPE', 'value', 'azure_batch'), createObject('name', 'TEMP_DATA_PATH', 'value', '/data'), createObject('name', 'DATA_PATH', 'value', '/data'), createObject('name', 'TITILER_ENDPOINT', 'value', '/api/titiler/'), createObject('name', 'BLOB_CONTAINER', 'value', 'data'), createObject('name', 'BLOB_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob), createObject('name', 'QUEUE_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.queue), createObject('name', 'BLOB_CONNECTION_STRING', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};EndpointSuffix={2}', parameters('storageAccountName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').keys[0].value, environment().suffixes.storage)), createObject('name', 'AZURE_BATCH_ACCOUNT_NAME', 'value', parameters('batchAccountName')), createObject('name', 'AZURE_BATCH_URL', 'value', format('https://{0}.{1}.batch.azure.com', parameters('batchAccountName'), parameters('batchAccountLocation'))), createObject('name', 'AZURE_BATCH_ACCOUNT_KEY', 'value', parameters('batchAccountKey')), createObject('name', 'AZURE_BATCH_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_IMAGERYPREP_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('imageryprepImage'))), createObject('name', 'AZURE_BATCH_OUTPUT_CONTAINER_URL', 'value', format('{0}data', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob)), createObject('name', 'AZURE_BATCH_TRAINING_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_REGISTRY_SERVER', 'value', parameters('acrLoginServer')), createObject('name', 'AZURE_BATCH_REGISTRY_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_REGISTRY_IDENTITY_RESOURCE_ID', 'value', parameters('umiResourceId')), createObject('name', 'STATIC_APP_SUBSCRIPTION_ID', 'value', subscription().subscriptionId), createObject('name', 'STATIC_APP_RESOURCE_GROUP', 'value', resourceGroup().name), createObject('name', 'STATIC_APP_NAME', 'value', parameters('staticWebAppName')), createObject('name', 'STATIC_APP_DOMAIN', 'value', parameters('staticAppDomain')), createObject('name', 'EMAIL_CONNECTION_STRING', 'value', parameters('emailConnectionString')), createObject('name', 'EMAIL_SENDER', 'value', format('DoNotReply@{0}', parameters('emailSenderDomain'))), createObject('name', 'AZURE_BATCH_TRAINING_POOL_IDS', 'value', parameters('trainingPoolIds')), createObject('name', 'AZURE_BATCH_INFERENCE_POOL_IDS', 'value', parameters('inferencePoolIds')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_IDS', 'value', parameters('imageryprepPoolIds')), createObject('name', 'AZURE_BATCH_USE_SAS', 'value', if(parameters('useSas'), 'true', 'false')), createObject('name', 'AZURE_BATCH_MANAGE_POOLS', 'value', if(parameters('managePools'), 'true', 'false')), createObject('name', 'PUBLISHING_ENABLED', 'value', if(parameters('publishingEnabled'), 'true', 'false')), createObject('name', 'PC_PROVIDER_ENABLED', 'value', if(parameters('pcProviderEnabled'), 'true', 'false')), createObject('name', 'PC_GEOCATALOG_URL', 'value', parameters('pcGeocatalogUrl')), createObject('name', 'PC_EXPLORER_URL', 'value', parameters('pcExplorerUrl')), createObject('name', 'PC_INGESTION_SOURCE', 'value', parameters('pcIngestionSource')), createObject('name', 'PC_COLLECTION_PREFIX', 'value', parameters('pcCollectionPrefix')), createObject('name', 'PC_PUBLISHING_LICENSE', 'value', parameters('pcPublishingLicense')), createObject('name', 'PUBLISHING_ORGANIZATION_NAME', 'value', parameters('publishingOrganizationName')), createObject('name', 'PUBLISHING_ORGANIZATION_URL', 'value', parameters('publishingOrganizationUrl')), createObject('name', 'PUBLISH_STORAGE_ACCOUNT_URL', 'value', parameters('publishStorageAccountUrl')), createObject('name', 'PUBLISH_BLOB_CONTAINER', 'value', parameters('publishBlobContainer')), createObject('name', 'PUBLISH_EXPLORER_RENDER_ENABLED', 'value', if(parameters('publishExplorerRenderEnabled'), 'true', 'false'))), createArray(createObject('name', 'HASTE_MAX_IMAGERY_DOWNLOAD_BYTES', 'value', string(parameters('maxImageryDownloadBytes'))), createObject('name', 'AzureFunctionsWebHost__hostId', 'value', toLower(substring(parameters('functionQueueName'), 0, min(32, length(parameters('functionQueueName'))))))))]" + "value": "[concat(concat(variables('gdalHardeningSettings'), createArray(createObject('name', 'env', 'value', 'prod'), createObject('name', 'DEVELOPMENT_MODE', 'value', if(parameters('developmentMode'), 'true', 'false')), createObject('name', 'IMAGE_QUEUE_NAME', 'value', 'image-layers-queue'), createObject('name', 'INFERENCE_QUEUE_NAME', 'value', 'inference-queue'), createObject('name', 'STATS_QUEUE_NAME', 'value', 'stats-queue'), createObject('name', 'TRAIN_QUEUE_NAME', 'value', 'train-queue'), createObject('name', 'ZIP_QUEUE_NAME', 'value', 'zip-queue'), createObject('name', 'EMBEDDING_QUEUE_NAME', 'value', 'embedding-queue'), createObject('name', 'PUBLISH_QUEUE_NAME', 'value', 'publish-queue'), createObject('name', 'IMAGERY_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'METADATA_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'ARTIFACT_STORAGE_TYPE', 'value', 'blob'), createObject('name', 'RUNNER_TYPE', 'value', 'azure_batch'), createObject('name', 'TEMP_DATA_PATH', 'value', '/data'), createObject('name', 'DATA_PATH', 'value', '/data'), createObject('name', 'TITILER_ENDPOINT', 'value', '/api/titiler/'), createObject('name', 'BLOB_CONTAINER', 'value', 'data'), createObject('name', 'BLOB_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob), createObject('name', 'QUEUE_ACCOUNT_URL', 'value', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.queue), createObject('name', 'BLOB_CONNECTION_STRING', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};EndpointSuffix={2}', parameters('storageAccountName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').keys[0].value, environment().suffixes.storage)), createObject('name', 'AZURE_BATCH_ACCOUNT_NAME', 'value', parameters('batchAccountName')), createObject('name', 'AZURE_BATCH_URL', 'value', format('https://{0}.{1}.batch.azure.com', parameters('batchAccountName'), parameters('batchAccountLocation'))), createObject('name', 'AZURE_BATCH_ACCOUNT_KEY', 'value', parameters('batchAccountKey')), createObject('name', 'AZURE_BATCH_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_IMAGERYPREP_DOCKER_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('imageryprepImage'))), createObject('name', 'AZURE_BATCH_OUTPUT_CONTAINER_URL', 'value', format('{0}data', reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2023-05-01').primaryEndpoints.blob)), createObject('name', 'AZURE_BATCH_TRAINING_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_ID', 'value', parameters('batchPoolName')), createObject('name', 'AZURE_BATCH_REGISTRY_SERVER', 'value', parameters('acrLoginServer')), createObject('name', 'AZURE_BATCH_REGISTRY_IMAGE', 'value', format('{0}/{1}', parameters('acrLoginServer'), parameters('trainingImage'))), createObject('name', 'AZURE_BATCH_REGISTRY_IDENTITY_RESOURCE_ID', 'value', parameters('umiResourceId')), createObject('name', 'STATIC_APP_SUBSCRIPTION_ID', 'value', subscription().subscriptionId), createObject('name', 'STATIC_APP_RESOURCE_GROUP', 'value', resourceGroup().name), createObject('name', 'STATIC_APP_NAME', 'value', parameters('staticWebAppName')), createObject('name', 'STATIC_APP_DOMAIN', 'value', parameters('staticAppDomain')), createObject('name', 'EMAIL_CONNECTION_STRING', 'value', parameters('emailConnectionString')), createObject('name', 'EMAIL_SENDER', 'value', format('DoNotReply@{0}', parameters('emailSenderDomain'))), createObject('name', 'AZURE_BATCH_TRAINING_POOL_IDS', 'value', parameters('trainingPoolIds')), createObject('name', 'AZURE_BATCH_INFERENCE_POOL_IDS', 'value', parameters('inferencePoolIds')), createObject('name', 'AZURE_BATCH_IMAGERYPREP_POOL_IDS', 'value', parameters('imageryprepPoolIds')), createObject('name', 'AZURE_BATCH_USE_SAS', 'value', if(parameters('useSas'), 'true', 'false')), createObject('name', 'AZURE_BATCH_MANAGE_POOLS', 'value', if(parameters('managePools'), 'true', 'false')), createObject('name', 'PUBLISHING_ENABLED', 'value', if(parameters('publishingEnabled'), 'true', 'false')), createObject('name', 'PC_PROVIDER_ENABLED', 'value', if(parameters('pcProviderEnabled'), 'true', 'false')), createObject('name', 'PC_GEOCATALOG_URL', 'value', parameters('pcGeocatalogUrl')), createObject('name', 'PC_EXPLORER_URL', 'value', parameters('pcExplorerUrl')), createObject('name', 'PC_INGESTION_SOURCE', 'value', parameters('pcIngestionSource')), createObject('name', 'PC_COLLECTION_PREFIX', 'value', parameters('pcCollectionPrefix')), createObject('name', 'PC_PUBLISHING_LICENSE', 'value', parameters('pcPublishingLicense')), createObject('name', 'PUBLISHING_ORGANIZATION_NAME', 'value', parameters('publishingOrganizationName')), createObject('name', 'PUBLISHING_ORGANIZATION_URL', 'value', parameters('publishingOrganizationUrl')), createObject('name', 'PUBLISH_STORAGE_ACCOUNT_URL', 'value', parameters('publishStorageAccountUrl')), createObject('name', 'PUBLISH_BLOB_CONTAINER', 'value', parameters('publishBlobContainer')), createObject('name', 'PUBLISH_EXPLORER_RENDER_ENABLED', 'value', if(parameters('publishExplorerRenderEnabled'), 'true', 'false')))), createArray(createObject('name', 'HASTE_MAX_IMAGERY_DOWNLOAD_BYTES', 'value', string(parameters('maxImageryDownloadBytes'))), createObject('name', 'AzureFunctionsWebHost__hostId', 'value', toLower(substring(parameters('functionQueueName'), 0, min(32, length(parameters('functionQueueName'))))))))]" }, "tags": { "value": "[parameters('tags')]" diff --git a/infra/modules/functions.bicep b/infra/modules/functions.bicep index 35509ebd..142312e8 100644 --- a/infra/modules/functions.bicep +++ b/infra/modules/functions.bicep @@ -164,12 +164,25 @@ var apimSubnetId = resourceId('Microsoft.Network/virtualNetworks/subnets', vnetN // Bare hostname of the blob endpoint HASTE imagery tiles are served from. var titilerStorageHost = split(storageAccount.properties.primaryEndpoints.blob, '/')[2] +// Keep these operator-owned GDAL protections explicit in Azure App Settings. +var gdalHardeningSettings = [ + { name: 'HASTE_GDAL_RUNTIME_REQUIRED', value: 'true' } + { name: 'GDAL_VRT_ENABLE_RAWRASTERBAND', value: 'NO' } + { name: 'GDAL_VRT_ENABLE_PYTHON', value: 'NO' } + { name: 'GDAL_DRIVER_PATH', value: 'disable' } + { name: 'GDAL_DISABLE_READDIR_ON_OPEN', value: 'EMPTY_DIR' } + { name: 'CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED', value: 'NO' } +] +var titilerGdalHardeningSettings = concat(gdalHardeningSettings, [ + { name: 'GDAL_SKIP', value: 'HDF4 HDF4Image HDF5 HDF5Image netCDF' } +]) + // Application settings shared by the api and queues apps. hastegeo's Config() // (instantiated at import time) reads these, so they must be present or the // worker cannot index function_app.py. Storage/queue access is identity-based // (BLOB_ACCOUNT_URL + role grants); only BLOB_CONNECTION_STRING, the Batch key, // and the ACS string are key-based (per ADR: no Key Vault, deploy-time values). -var appConfigSettings = [ +var appConfigSettings = concat(gdalHardeningSettings, [ { name: 'env', value: 'prod' } { name: 'DEVELOPMENT_MODE', value: developmentMode ? 'true' : 'false' } { name: 'IMAGE_QUEUE_NAME', value: 'image-layers-queue' } @@ -237,7 +250,7 @@ var appConfigSettings = [ { name: 'PUBLISH_STORAGE_ACCOUNT_URL', value: publishStorageAccountUrl } { name: 'PUBLISH_BLOB_CONTAINER', value: publishBlobContainer } { name: 'PUBLISH_EXPLORER_RENDER_ENABLED', value: publishExplorerRenderEnabled ? 'true' : 'false' } -] +]) module apiApp 'functionApp.bicep' = { name: 'fn-api' @@ -294,11 +307,11 @@ module titilerApp 'functionApp.bicep' = { ] restrictMainSite: true restrictScmSite: true - // The only storage host TiTiler may read from besides the built-in - // public open-data hosts. Unset means HASTE imagery tiles fail closed. - appSettings: [ + // Keep the GDAL policy and the dataset-host allowlist in one settings + // array so neither control is lost during deployment. + appSettings: concat(titilerGdalHardeningSettings, [ { name: 'TITILER_ALLOWED_HOSTS', value: titilerStorageHost } - ] + ]) tags: tags } // All three apps integrate with the same func-subnet. A subnet's diff --git a/native/gdal-runtime/assemble_wheels.py b/native/gdal-runtime/assemble_wheels.py new file mode 100644 index 00000000..baf7f342 --- /dev/null +++ b/native/gdal-runtime/assemble_wheels.py @@ -0,0 +1,510 @@ +"""Merge native bindings, auditwheel once, then split immutable wheels. + +This script consumes locally built binding wheels and the runtime package. +It performs no package resolution and passes no index URL to pip. +""" + +from __future__ import annotations + +import argparse +import atexit +import base64 +import csv +import hashlib +import io +import json +import os +import re +import subprocess +import tempfile +import zipfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +RUNTIME_NAME = "haste-gdal-runtime" +RUNTIME_WHEEL_TAG = "py3-none-manylinux_2_28_x86_64" +STAGING_TAG = "cp311-cp311-linux_x86_64" +TARGET_TAG = "cp311-cp311-manylinux_2_28_x86_64" +REQUIRED_BINDINGS = {"GDAL", "rasterio", "fiona", "pyogrio", "pyproj"} +BINDING_MODULES = { + "gdal": "osgeo.gdal", + "rasterio": "rasterio", + "fiona": "fiona", + "pyogrio": "pyogrio", + "pyproj": "pyproj", +} + + +def normalized_name(value: str) -> str: + return re.sub(r"[-_.]+", "_", value).lower() + + +def read_metadata_fields(content: bytes) -> tuple[str, str]: + name = "" + version = "" + for line in content.decode("utf-8").splitlines(): + if line.startswith("Name: "): + name = line[6:].strip() + elif line.startswith("Version: "): + version = line[9:].strip() + if name and version: + break + if not name or not version: + raise ValueError("wheel METADATA must contain Name and Version") + return name, version + + +def local_version(version: str, build: str) -> str: + return f"{version.split('+', 1)[0]}+haste.{build}" + + +def replace_header(content: bytes, key: str, value: str) -> bytes: + lines = content.decode("utf-8").splitlines() + prefix = f"{key}: " + found = False + output = [] + for line in lines: + if line.startswith(prefix): + output.append(prefix + value) + found = True + else: + output.append(line) + if not found: + output.insert(0, prefix + value) + return ("\n".join(output) + "\n").encode("utf-8") + + +def update_metadata( + content: bytes, version: str, runtime_version: str | None +) -> bytes: + text = content.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") + header_text, separator, body = text.partition("\n\n") + lines = header_text.splitlines() + output: list[str] = [] + saw_version = False + index = 0 + while index < len(lines): + line = lines[index] + field, delimiter, value = line.partition(":") + field_name = field.casefold() + if delimiter and field_name == "version": + output.append(f"{field}: {version}") + saw_version = True + elif delimiter and field_name == "requires-dist": + field_lines = [line] + index += 1 + while index < len(lines) and lines[index][:1] in {" ", "\t"}: + field_lines.append(lines[index]) + index += 1 + requirement = " ".join( + [value.strip(), *(part.strip() for part in field_lines[1:])] + ) + if not re.match( + r"haste[-_.]gdal[-_.]runtime(?:\s|[<>=!~;\[]|$)", + requirement, + re.IGNORECASE, + ): + output.extend(field_lines) + continue + else: + output.append(line) + index += 1 + if not saw_version: + raise ValueError("wheel METADATA is missing Version") + if runtime_version: + output.append(f"Requires-Dist: {RUNTIME_NAME}=={runtime_version}") + updated_header = "\n".join(output) + updated = ( + f"{updated_header}\n\n{body}" if separator else f"{updated_header}\n" + ) + return updated.encode("utf-8") + + +def rewrite_data_path(path: str, staging_data_prefix: str) -> str: + """Map a wheel's .data tree into the staging wheel's .data tree.""" + match = re.match(r"[^/]+\.data/(.+)$", path) + if match: + return f"{staging_data_prefix}/{match.group(1)}" + return path + + +def restore_data_path(path: str, name: str, version: str) -> str: + """Give a .data member the matching output wheel's name and version.""" + match = re.match(r"[^/]+\.data/(.+)$", path) + if match: + prefix = f"{normalized_name(name)}-{version}.data" + return f"{prefix}/{match.group(1)}" + return path + + +def wheel_record(files: dict[str, bytes], record_path: str) -> bytes: + rows = [] + for path, content in sorted(files.items()): + digest = ( + base64.urlsafe_b64encode(hashlib.sha256(content).digest()) + .rstrip(b"=") + .decode("ascii") + ) + rows.append((path, f"sha256={digest}", str(len(content)))) + rows.append((record_path, "", "")) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerows(rows) + return output.getvalue().encode("utf-8") + + +@dataclass +class WheelInput: + name: str + version: str + archive: Path + dist_info: str + metadata_files: dict[str, bytes] = field(default_factory=dict) + stage_members: dict[str, str] = field(default_factory=dict) + + +def inspect_wheel(path: Path, stage_prefix: str) -> WheelInput: + with zipfile.ZipFile(path) as archive: + infos = { + name.rsplit("/", 1)[0] + for name in archive.namelist() + if name.endswith(".dist-info/METADATA") + } + if len(infos) != 1: + raise ValueError(f"{path.name}: expected one dist-info directory") + dist_info = infos.pop() + metadata = archive.read(f"{dist_info}/METADATA") + name, version = read_metadata_fields(metadata) + result = WheelInput(name, version, path, dist_info) + for member in archive.namelist(): + if member.endswith("/"): + continue + if member.startswith(f"{dist_info}/"): + relative = member[len(dist_info) + 1 :] + if relative not in {"RECORD", "METADATA", "WHEEL"}: + result.metadata_files[relative] = archive.read(member) + continue + stage_member = rewrite_data_path(member, stage_prefix) + result.stage_members[member] = stage_member + return result + + +def read_wheel_payload(wheel: WheelInput) -> dict[str, bytes]: + with zipfile.ZipFile(wheel.archive) as archive: + return { + stage_member: archive.read(source_member) + for source_member, stage_member in wheel.stage_members.items() + } + + +def read_runtime_package(runtime_package: Path) -> dict[str, bytes]: + payload: dict[str, bytes] = {} + for path in sorted(runtime_package.rglob("*")): + if path.is_file(): + relative = path.relative_to(runtime_package).as_posix() + payload[f"haste_gdal_runtime/{relative}"] = path.read_bytes() + return payload + + +def binding_versions(wheels: list[WheelInput]) -> dict[str, str]: + """Return import-module version expectations from the binding wheel set.""" + versions = {} + for wheel in wheels: + module = BINDING_MODULES.get(normalized_name(wheel.name)) + if module is None: + continue + versions[module] = wheel.version.split("+", 1)[0] + if set(versions) != set(BINDING_MODULES.values()): + raise ValueError( + "binding wheel set does not define every binding version" + ) + return versions + + +def runtime_metadata_files(version: str, tag: str) -> dict[str, bytes]: + license_path = Path(__file__).resolve().parents[2] / "LICENSE" + if not license_path.is_file(): + raise FileNotFoundError( + f"runtime package license is missing: {license_path}" + ) + metadata = ( + "Metadata-Version: 2.4\n" + f"Name: {RUNTIME_NAME}\n" + f"Version: {version}\n" + "Summary: HASTE's pinned GDAL native runtime\n" + "Requires-Python: >=3.11\n" + "License-Expression: MIT\n" + "License-File: licenses/LICENSE\n\n" + ).encode("utf-8") + wheel = ( + "Wheel-Version: 1.0\n" + "Generator: haste-gdal-runtime\n" + "Root-Is-Purelib: false\n" + f"Tag: {tag}\n" + ).encode("utf-8") + return { + "METADATA": metadata, + "WHEEL": wheel, + "licenses/LICENSE": license_path.read_bytes(), + } + + +def write_wheel( + destination: Path, + payload: dict[str, bytes], + metadata_files: dict[str, bytes], + *, + name: str, + version: str, + tag: str, + pure: bool, +) -> None: + dist_info = f"{normalized_name(name)}-{version}.dist-info" + files = dict(payload) + for relative, content in metadata_files.items(): + files[f"{dist_info}/{relative}"] = content + files[f"{dist_info}/METADATA"] = metadata_files["METADATA"] + wheel_metadata = replace_header( + metadata_files["WHEEL"], "Root-Is-Purelib", "true" if pure else "false" + ) + wheel_lines = [ + line + for line in wheel_metadata.decode("utf-8").splitlines() + if not line.startswith("Tag: ") + ] + wheel_lines.append(f"Tag: {tag}") + files[f"{dist_info}/WHEEL"] = ("\n".join(wheel_lines) + "\n").encode() + record_path = f"{dist_info}/RECORD" + files[record_path] = wheel_record(files, record_path) + + filename_name = normalized_name(name) + wheel_path = destination / (f"{filename_name}-{version}-{tag}.whl") + with zipfile.ZipFile( + wheel_path, "w", compression=zipfile.ZIP_DEFLATED + ) as out: + for path, content in sorted(files.items()): + out.writestr(path, content) + + +def merge_and_repair( + binding_wheels: list[Path], + runtime_package: Path, + output: Path, + build: str, + gdal_version: str, + prefix: Path, +) -> list[Path]: + output.mkdir(parents=True, exist_ok=True) + runtime_version = local_version(gdal_version, build) + stage_prefix = f"{normalized_name(RUNTIME_NAME)}-{runtime_version}.data" + inputs = [inspect_wheel(path, stage_prefix) for path in binding_wheels] + names = {wheel.name for wheel in inputs} + if names != REQUIRED_BINDINGS: + missing = sorted(REQUIRED_BINDINGS - names) + unexpected = sorted(names - REQUIRED_BINDINGS) + raise ValueError( + f"binding wheel set mismatch: missing={missing}, " + f"unexpected={unexpected}" + ) + + owners: dict[str, dict[str, str]] = {} + payload: dict[str, bytes] = {} + for wheel in inputs: + owners[wheel.name] = wheel.stage_members + for stage_member, content in read_wheel_payload(wheel).items(): + previous = payload.get(stage_member) + if previous is not None and previous != content: + raise ValueError( + "wheel payload collision with different content: " + f"{stage_member}" + ) + payload[stage_member] = content + + runtime_payload = read_runtime_package(runtime_package) + manifest_member = "haste_gdal_runtime/manifest.json" + manifest = json.loads(runtime_payload[manifest_member]) + manifest["binding_versions"] = binding_versions(inputs) + runtime_payload[manifest_member] = ( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + runtime_members = {member: member for member in runtime_payload} + for member, content in runtime_payload.items(): + previous = payload.get(member) + if previous is not None and previous != content: + raise ValueError(f"runtime package payload collision: {member}") + payload[member] = content + owners[RUNTIME_NAME] = runtime_members + + staging_metadata = runtime_metadata_files(runtime_version, STAGING_TAG) + staging_temp = tempfile.TemporaryDirectory(prefix="haste-gdal-assemble-") + atexit.register(staging_temp.cleanup) + staging = Path(staging_temp.name) + stage_payload = dict(payload) + staging_dist_info = ( + f"{normalized_name(RUNTIME_NAME)}-{runtime_version}.dist-info" + ) + staging_record = f"{staging_dist_info}/RECORD" + for relative, content in staging_metadata.items(): + stage_payload[f"{staging_dist_info}/{relative}"] = content + stage_payload[staging_record] = wheel_record(stage_payload, staging_record) + staging_wheel = staging / ( + f"{normalized_name(RUNTIME_NAME)}-{runtime_version}-{STAGING_TAG}.whl" + ) + with zipfile.ZipFile( + staging_wheel, "w", compression=zipfile.ZIP_DEFLATED + ) as archive: + for member, content in sorted(stage_payload.items()): + archive.writestr(member, content) + + repaired_dir = staging / "repaired" + repaired_dir.mkdir() + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = os.pathsep.join( + filter(None, [str(prefix / "lib"), env.get("LD_LIBRARY_PATH", "")]) + ) + subprocess.run( + [ + "auditwheel", + "repair", + "--plat", + "manylinux_2_28_x86_64", + "--wheel-dir", + str(repaired_dir), + str(staging_wheel), + ], + check=True, + env=env, + ) + repaired_wheels = list(repaired_dir.glob("*.whl")) + if len(repaired_wheels) != 1: + raise RuntimeError( + f"auditwheel produced {len(repaired_wheels)} wheels, expected one" + ) + repaired_path = repaired_wheels[0] + with zipfile.ZipFile(repaired_path) as archive: + repaired_names = archive.namelist() + dist_info = next( + name.rsplit("/", 1)[0] + for name in repaired_names + if name.endswith(".dist-info/WHEEL") + ) + wheel_text = archive.read(f"{dist_info}/WHEEL").decode("utf-8") + tags = [ + line[5:].strip() + for line in wheel_text.splitlines() + if line.startswith("Tag: ") + ] + if TARGET_TAG not in tags: + raise RuntimeError( + f"auditwheel did not produce required tag {TARGET_TAG}: {tags}" + ) + libraries = { + name: archive.read(name) + for name in repaired_names + if name.startswith(f"{normalized_name(RUNTIME_NAME)}.libs/") + and not name.endswith("/") + } + if not libraries: + raise RuntimeError("auditwheel did not graft native libraries") + if not any( + PurePosixPath(name).name.startswith("libhastez") + for name in libraries + ): + raise RuntimeError( + "patched zlib was not bundled; the wheel would load the " + "host libz.so.1" + ) + + outputs = [] + for wheel in inputs: + version = local_version(wheel.version, build) + source_metadata = zipfile.ZipFile(wheel.archive) + original_metadata = source_metadata.read( + f"{wheel.dist_info}/METADATA" + ) + original_wheel = source_metadata.read(f"{wheel.dist_info}/WHEEL") + source_metadata.close() + meta_files = dict(wheel.metadata_files) + meta_files["METADATA"] = update_metadata( + original_metadata, version, runtime_version + ) + tags_in_wheel = [f"Tag: {tag}" for tag in tags] + wheel_lines = [ + line + for line in original_wheel.decode("utf-8").splitlines() + if not line.startswith("Tag: ") + and not line.startswith("Root-Is-Purelib: ") + ] + wheel_lines.extend(["Root-Is-Purelib: false", *tags_in_wheel]) + meta_files["WHEEL"] = ("\n".join(wheel_lines) + "\n").encode() + contents = { + restore_data_path( + output_member, wheel.name, version + ): archive.read(stage_member) + for output_member, stage_member in wheel.stage_members.items() + } + wheel_output = ( + output + / f"{normalized_name(wheel.name)}-{version}-{TARGET_TAG}.whl" + ) + write_wheel( + output, + contents, + meta_files, + name=wheel.name, + version=version, + tag=TARGET_TAG, + pure=False, + ) + outputs.append(wheel_output) + + runtime_contents = { + output_member: archive.read(stage_member) + for output_member, stage_member in runtime_members.items() + } + runtime_contents.update(libraries) + runtime_meta = runtime_metadata_files( + runtime_version, RUNTIME_WHEEL_TAG + ) + write_wheel( + output, + runtime_contents, + runtime_meta, + name=RUNTIME_NAME, + version=runtime_version, + tag=RUNTIME_WHEEL_TAG, + pure=False, + ) + outputs.append( + output / f"{normalized_name(RUNTIME_NAME)}-{runtime_version}-" + f"{RUNTIME_WHEEL_TAG}.whl" + ) + return outputs + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel-dir", required=True, type=Path) + parser.add_argument("--runtime-package", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--build", required=True) + parser.add_argument("--gdal-version", required=True) + parser.add_argument("--prefix", required=True, type=Path) + args = parser.parse_args(argv) + + binding_wheels = sorted(args.wheel_dir.glob("*.whl")) + outputs = merge_and_repair( + binding_wheels, + args.runtime_package, + args.output_dir, + args.build, + args.gdal_version, + args.prefix, + ) + for output in outputs: + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/baseline_report.py b/native/gdal-runtime/baseline_report.py new file mode 100644 index 00000000..797bd5c5 --- /dev/null +++ b/native/gdal-runtime/baseline_report.py @@ -0,0 +1,117 @@ +"""Capture GDAL bindings, native libraries, drivers, VSI handlers and config.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import platform +import sys +from pathlib import Path +from typing import Any + +from haste_gdal_runtime._native import NativeGdal, mapped_libraries + +BINDING_MODULES = ( + "osgeo.gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", +) +CONFIG_KEYS = ( + "GDAL_VRT_ENABLE_RAWRASTERBAND", + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE", + "GDAL_VRT_ENABLE_PYTHON", + "GDAL_VRT_PYTHON_TRUSTED_MODULES", + "GDAL_DRIVER_PATH", + "GDAL_PYTHON_DRIVER_PATH", + "GDAL_DISABLE_READDIR_ON_OPEN", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", + "GDAL_DATA", + "PROJ_DATA", + "PROJ_LIB", + "GDAL_SKIP", +) + + +def _binding_info(name: str, module: Any) -> dict[str, str]: + info = {"version": str(getattr(module, "__version__", ""))} + if name == "osgeo.gdal": + info["gdal"] = module.VersionInfo("RELEASE_NAME") + elif name in ("rasterio", "fiona"): + info["gdal"] = str(module.__gdal_version__) + elif name == "pyogrio": + info["gdal"] = str(module.__gdal_version_string__) + elif name == "pyproj": + info["proj"] = str( + getattr(module, "__proj_version__", None) + or module.proj_version_str + ) + return info + + +def collect_report(maps_text: str | None = None) -> dict[str, Any]: + bindings: dict[str, dict[str, str]] = {} + import_errors: dict[str, str] = {} + for name in BINDING_MODULES: + try: + module = importlib.import_module(name) + except (ImportError, OSError, RuntimeError) as error: + import_errors[name] = f"{type(error).__name__}: {error}" + else: + bindings[name] = _binding_info(name, module) + + libraries = { + stem: mapped_libraries(stem, maps_text) + for stem in ("libgdal", "libproj") + } + native_report: dict[str, Any] = { + "library": None, + "version": None, + "drivers": [], + "vsi_prefixes": [], + "config": {}, + "error": None, + } + gdal_libraries = libraries["libgdal"] + if gdal_libraries: + try: + native = NativeGdal(Path(gdal_libraries[0])) + native_report.update( + { + "library": str(native.path), + "version": native.version_info("RELEASE_NAME"), + "drivers": native.drivers(), + "vsi_prefixes": native.vsi_prefixes(), + "config": {key: native.config(key) for key in CONFIG_KEYS}, + } + ) + except (OSError, AttributeError, RuntimeError) as error: + native_report["error"] = f"{type(error).__name__}: {error}" + + return { + "python": sys.version, + "platform": platform.platform(), + "bindings": bindings, + "import_errors": import_errors, + "libraries": libraries, + "native": native_report, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + + output = json.dumps(collect_report(), indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(output, encoding="utf-8") + else: + sys.stdout.write(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/binding_workflow_smoke.py b/native/gdal-runtime/binding_workflow_smoke.py new file mode 100644 index 00000000..74e781ac --- /dev/null +++ b/native/gdal-runtime/binding_workflow_smoke.py @@ -0,0 +1,365 @@ +"""Exercise the GDAL-linked bindings against their shared native runtime.""" + +from __future__ import annotations + +import json +import os +import shutil +import ssl +import subprocess +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +import haste_gdal_runtime as runtime +import numpy as np + +EXPECTED_GDAL = "3.13.3" +EXPECTED_PROJ = "9.8.1" +EXPECTED_BINDINGS = { + "osgeo.gdal": "3.13.3", + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", +} +ALLOWED_DRIVERS = frozenset( + { + "COG", + "ESRIJSON", + "GPKG", + "GTiff", + "GeoJSON", + "GeoJSONSeq", + "JPEG", + "MEM", + "PNG", + "SQLite", + "TopoJSON", + "VRT", + } +) + + +def _create_test_certificate(directory: Path) -> tuple[Path, Path]: + openssl = shutil.which("openssl") + if openssl is None: + raise RuntimeError( + "openssl is required for the HTTPS /vsicurl smoke test" + ) + certificate = directory / "localhost.crt" + private_key = directory / "localhost.key" + subprocess.run( + [ + openssl, + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-keyout", + str(private_key), + "-out", + str(certificate), + "-subj", + "/CN=127.0.0.1", + "-addext", + "subjectAltName=IP:127.0.0.1", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + return certificate, private_key + + +def _https_server( + raster_path: Path, certificate: Path, private_key: Path +) -> tuple[ThreadingHTTPServer, threading.Thread]: + content = raster_path.read_bytes() + + class RangeHandler(BaseHTTPRequestHandler): + def do_HEAD(self) -> None: + self.send_response(200) + self.send_header("Content-Length", str(len(content))) + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + + def do_GET(self) -> None: + range_header = self.headers.get("Range") + if range_header: + unit, _, bounds = range_header.partition("=") + if unit != "bytes": + self.send_error(400) + return + start_text, _, end_text = bounds.partition("-") + start = int(start_text) + end = min( + int(end_text) if end_text else len(content) - 1, + len(content) - 1, + ) + if start < 0 or start > end: + self.send_error(416) + return + payload = content[start : end + 1] + self.send_response(206) + self.send_header( + "Content-Range", + f"bytes {start}-{end}/{len(content)}", + ) + else: + payload = content + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format_string: str, *args: Any) -> None: + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), RangeHandler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(certificate, private_key) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def _smoke_bindings(directory: Path) -> None: + import fiona + import pyogrio + import pyproj + import rasterio + from osgeo import gdal, ogr + from rasterio.shutil import copy as copy_raster + from rasterio.transform import from_origin + from rasterio.vrt import WarpedVRT + + binding_policy = runtime.Policy( + expected_gdal=EXPECTED_GDAL, + allowed_drivers=ALLOWED_DRIVERS, + required_bindings=runtime.BINDINGS, + ) + report = runtime.secure_startup(binding_policy) + for binding, expected_version in EXPECTED_BINDINGS.items(): + actual_version = report["bindings"][binding]["version"].split("+")[0] + if actual_version != expected_version: + raise RuntimeError( + f"{binding} is {actual_version}, expected {expected_version}" + ) + actual_proj = report["bindings"]["pyproj"]["proj"] + if actual_proj != EXPECTED_PROJ: + raise RuntimeError( + f"pyproj reports PROJ {actual_proj}, expected {EXPECTED_PROJ}" + ) + + first = directory / "first.tif" + second = directory / "second.tif" + profile = { + "driver": "GTiff", + "width": 2, + "height": 2, + "count": 1, + "dtype": "uint8", + "crs": "EPSG:4326", + } + first_pixels = np.array([[1, 2], [3, 4]], dtype=np.uint8) + second_pixels = np.array([[5, 6], [7, 8]], dtype=np.uint8) + for path, pixels, x_offset in ( + (first, first_pixels, 0), + (second, second_pixels, 2), + ): + with rasterio.open( + path, + "w", + **profile, + transform=from_origin(x_offset, 2, 1, 1), + ) as dataset: + dataset.write(pixels, 1) + with rasterio.open(path) as dataset: + if not np.array_equal(dataset.read(1), pixels): + raise RuntimeError( + f"rasterio round trip failed for {path.name}" + ) + if dataset.crs.to_epsg() != 4326: + raise RuntimeError(f"unexpected CRS for {path.name}") + + cog_path = directory / "first-cog.tif" + with rasterio.open(first) as source: + copy_raster( + source, + cog_path, + driver="COG", + compress="ZSTD", + ) + with rasterio.open(cog_path) as dataset: + if not np.array_equal(dataset.read(1), first_pixels): + raise RuntimeError("rasterio COG copy changed raster pixels") + cog = gdal.Open(str(cog_path), gdal.GA_ReadOnly) + if ( + cog is None + or cog.GetMetadata("IMAGE_STRUCTURE").get("LAYOUT") != "COG" + ): + raise RuntimeError("GDAL did not recognize the output as a COG") + cog = None + + vrt_path = directory / "mosaic.vrt" + vrt_dataset = gdal.BuildVRT(str(vrt_path), [str(first), str(second)]) + if vrt_dataset is None: + raise RuntimeError("GDAL BuildVRT failed") + vrt_dataset.FlushCache() + vrt_dataset = None + with rasterio.open(vrt_path) as dataset: + expected_pixels = np.concatenate((first_pixels, second_pixels), axis=1) + if dataset.width != 4 or dataset.height != 2: + raise RuntimeError("VRT mosaic has unexpected dimensions") + if not np.array_equal(dataset.read(1), expected_pixels): + raise RuntimeError("VRT mosaic pixels differ from source pixels") + + gdal_source = gdal.Open(str(first), gdal.GA_ReadOnly) + if gdal_source is None: + raise RuntimeError("osgeo.gdal could not open the rasterio GeoTIFF") + warped_path = directory / "warped.tif" + warped = gdal.Warp(str(warped_path), gdal_source, dstSRS="EPSG:3857") + if warped is None: + raise RuntimeError("GDAL Warp reprojection failed") + warped.FlushCache() + warped = None + gdal_source = None + with rasterio.open(warped_path) as dataset: + if dataset.crs.to_epsg() != 3857 or dataset.width == 0: + raise RuntimeError( + "GDAL Warp produced an invalid reprojected raster" + ) + + with rasterio.open(first) as source: + with WarpedVRT(source, crs="EPSG:3857") as warped_vrt: + if warped_vrt.crs.to_epsg() != 3857: + raise RuntimeError("rasterio WarpedVRT reprojection failed") + if warped_vrt.read(1).size == 0: + raise RuntimeError("rasterio WarpedVRT returned no pixels") + + fiona_path = directory / "fiona.gpkg" + schema = {"geometry": "Point", "properties": {"name": "str"}} + feature = { + "geometry": {"type": "Point", "coordinates": (1.0, 2.0)}, + "properties": {"name": "fiona"}, + } + with fiona.open( + fiona_path, + "w", + driver="GPKG", + layer="fiona_features", + schema=schema, + crs="EPSG:4326", + ) as destination: + destination.write(feature) + with fiona.open(fiona_path, layer="fiona_features") as source: + if len(source) != 1 or source.crs.to_epsg() != 4326: + raise RuntimeError("Fiona GeoPackage write/read failed") + + ogr_geometry = ogr.CreateGeometryFromJson(json.dumps(feature["geometry"])) + if ogr_geometry is None: + raise RuntimeError("OGR could not create the test point geometry") + pyogrio_path = directory / "pyogrio.geojson" + geometry_wkb = bytes(ogr_geometry.ExportToWkb()) + pyogrio.raw.write( + pyogrio_path, + np.array([geometry_wkb], dtype=object), + [np.array(["pyogrio"], dtype=object)], + ["name"], + layer="pyogrio_features", + driver="GeoJSON", + geometry_type="Point", + crs=pyproj.CRS.from_epsg(4326).to_wkt(), + ) + metadata, _, geometries, fields = pyogrio.raw.read( + pyogrio_path, + layer="pyogrio_features", + ) + if metadata["geometry_type"] != "Point": + raise RuntimeError("Pyogrio did not report the expected GeoJSON type") + if len(geometries) != 1 or fields[0][0] != "pyogrio": + raise RuntimeError("Pyogrio GeoJSON write/read failed") + with fiona.open(pyogrio_path) as source: + if len(source) != 1 or source[0]["properties"]["name"] != "pyogrio": + raise RuntimeError("Fiona could not read Pyogrio GeoJSON output") + + transformer = pyproj.Transformer.from_crs( + "EPSG:4326", "EPSG:3857", always_xy=True + ) + projected_x, projected_y = transformer.transform(1.0, 2.0) + if not (111000 < projected_x < 112000 and 222000 < projected_y < 223000): + raise RuntimeError( + "pyproj returned an unexpected projected coordinate" + ) + + certificate, private_key = _create_test_certificate(directory) + server, thread = _https_server(first, certificate, private_key) + try: + url = f"/vsicurl/https://127.0.0.1:{server.server_port}/first.tif" + previous_curl_bundle = os.environ.get("CURL_CA_BUNDLE") + previous_ssl_cert = os.environ.get("SSL_CERT_FILE") + previous_no_proxy = os.environ.get("NO_PROXY") + previous_no_proxy_lower = os.environ.get("no_proxy") + previous_gdal_curl_ca = gdal.GetConfigOption("GDAL_CURL_CA_BUNDLE") + os.environ["CURL_CA_BUNDLE"] = str(certificate) + os.environ["SSL_CERT_FILE"] = str(certificate) + os.environ["NO_PROXY"] = "127.0.0.1,localhost" + os.environ["no_proxy"] = "127.0.0.1,localhost" + gdal.SetConfigOption("GDAL_CURL_CA_BUNDLE", str(certificate)) + try: + network_dataset = gdal.Open(url, gdal.GA_ReadOnly) + if network_dataset is None: + detail = gdal.GetLastErrorMsg() + raise RuntimeError( + f"GDAL /vsicurl/ HTTPS read failed: {detail}" + ) + if network_dataset.ReadAsArray().tolist() != first_pixels.tolist(): + raise RuntimeError("GDAL /vsicurl/ pixels differ from source") + network_dataset = None + finally: + if previous_curl_bundle is None: + os.environ.pop("CURL_CA_BUNDLE", None) + else: + os.environ["CURL_CA_BUNDLE"] = previous_curl_bundle + if previous_ssl_cert is None: + os.environ.pop("SSL_CERT_FILE", None) + else: + os.environ["SSL_CERT_FILE"] = previous_ssl_cert + if previous_no_proxy is None: + os.environ.pop("NO_PROXY", None) + else: + os.environ["NO_PROXY"] = previous_no_proxy + if previous_no_proxy_lower is None: + os.environ.pop("no_proxy", None) + else: + os.environ["no_proxy"] = previous_no_proxy_lower + gdal.SetConfigOption("GDAL_CURL_CA_BUNDLE", previous_gdal_curl_ca) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + print( + "Binding workflow smoke passed: rasterio, fiona, pyogrio, pyproj, " + "GDAL Warp/BuildVRT/WarpedVRT and HTTPS /vsicurl/" + ) + + +def main() -> int: + runtime.configure() + with tempfile.TemporaryDirectory(prefix="haste-gdal-smoke-") as temp_dir: + _smoke_bindings(Path(temp_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/build.sh b/native/gdal-runtime/build.sh new file mode 100644 index 00000000..b119b03b --- /dev/null +++ b/native/gdal-runtime/build.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BUILD_ROOT="${BUILD_ROOT:-$ROOT/build}" +PREFIX="${PREFIX:-/opt/haste}" +PYTHON="${PYTHON:-/opt/python/cp311-cp311/bin/python}" +JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN)}" +SOURCES="$BUILD_ROOT/sources" +WORK="$BUILD_ROOT/work" +WHEELS="$BUILD_ROOT/wheels" +TOOL_VENV="$BUILD_ROOT/venv" +FIONA_VENV="$BUILD_ROOT/venv-fiona" + +PIP_INDEX_URL="${PIP_INDEX_URL:-https://pypi.org/simple}" +export PIP_INDEX_URL +if [[ ! -x "$PYTHON" ]]; then + echo "CPython 3.11 interpreter not found: $PYTHON" >&2 + exit 2 +fi + +# Preserve the verified source cache, but rebuild generated state from scratch. +rm -rf "$WORK" "$WHEELS" "$TOOL_VENV" "$FIONA_VENV" \ + "$BUILD_ROOT/dist" "$BUILD_ROOT/package/haste_gdal_runtime" +mkdir -p "$SOURCES" "$WORK" "$WHEELS" +export PYTHON +export PATH="$PREFIX/bin:$PATH" +export LD_LIBRARY_PATH="$PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" +export CMAKE_PREFIX_PATH="$PREFIX${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" +export CFLAGS="-I$PREFIX/include ${CFLAGS:-}" +export CPPFLAGS="-I$PREFIX/include ${CPPFLAGS:-}" +export LDFLAGS="-L$PREFIX/lib -Wl,-rpath,$PREFIX/lib ${LDFLAGS:-}" +export OPENSSL_CONF=/dev/null +export OPENSSL_MODULES="$PREFIX/lib/ossl-modules" +export PYTHON + +python3 "$ROOT/fetch_sources.py" --dest "$SOURCES" +bash "$ROOT/install_build_tools.sh" "$TOOL_VENV" +bash "$ROOT/install_build_tools.sh" "$FIONA_VENV" \ + "$ROOT/requirements-fiona-build.txt" + +extract() { + local name="$1" + local archive + archive="$(find "$SOURCES" -maxdepth 1 -type f -name "$name-*.tar.gz" -print -quit)" + if [[ -z "$archive" ]]; then + echo "locked archive for $name not found" >&2 + return 1 + fi + tar -xf "$archive" -C "$WORK" +} + +extract zlib +( + cd "$WORK"/zlib-*/ + # Upstream fix (madler/zlib df84af25) for CVE-2026-85091. As of 2026-10-21 + # the latest release is 1.3.2 and does not contain it. When a release does, + # bump sources.lock, delete this patch and remove the .grype.yaml ignore. + patch -p1 --batch < "$ROOT/patches/zlib-CVE-2026-85091.patch" + ./configure --prefix="$PREFIX" --shared + # auditwheel never grafts the manylinux-whitelisted libz.so.1, so a stock + # name would make every library load the host zlib instead of this patched + # build. Give it a private name so it is bundled. + sed -i 's/libz\.so/libhastez.so/g' Makefile + make -j"$JOBS" + make install + ln -sf libhastez.so.1 "$PREFIX/lib/libz.so" +) + +extract libdeflate +cmake -S "$WORK"/libdeflate-*/ -B "$WORK/libdeflate-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=ON \ + -DLIBDEFLATE_BUILD_GZIP=OFF \ + -DLIBDEFLATE_BUILD_TESTS=OFF +cmake --build "$WORK/libdeflate-build" --parallel "$JOBS" +cmake --install "$WORK/libdeflate-build" + +extract zstd +cmake -S "$WORK"/zstd-*/build/cmake -B "$WORK/zstd-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DZSTD_BUILD_SHARED=ON \ + -DZSTD_BUILD_STATIC=OFF \ + -DZSTD_BUILD_PROGRAMS=OFF \ + -DZSTD_BUILD_TESTS=OFF +cmake --build "$WORK/zstd-build" --parallel "$JOBS" +cmake --install "$WORK/zstd-build" + +extract libjpeg-turbo +cmake -S "$WORK"/libjpeg-turbo-*/ -B "$WORK/libjpeg-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DENABLE_SHARED=ON \ + -DENABLE_STATIC=OFF \ + -DWITH_TURBOJPEG=OFF \ + -DWITH_TESTS=OFF \ + -DWITH_TOOLS=OFF +cmake --build "$WORK/libjpeg-build" --parallel "$JOBS" +cmake --install "$WORK/libjpeg-build" + +extract openssl +( + cd "$WORK"/openssl-*/ + ./Configure linux-x86_64 shared no-tests no-apps no-docs no-module \ + no-legacy no-engine no-dso no-autoload-config --libdir=lib \ + --prefix="$PREFIX" --openssldir="$PREFIX/ssl" + make -j"$JOBS" + make install_sw +) + +extract curl +( + cd "$WORK"/curl-*/ + ./configure \ + --prefix="$PREFIX" \ + --with-openssl="$PREFIX" \ + --with-zlib="$PREFIX" \ + --with-ca-fallback \ + --without-ca-bundle \ + --without-ca-path \ + --disable-file \ + --disable-ftp \ + --disable-ipfs \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --disable-mqtt \ + --disable-websockets \ + --disable-netrc \ + --disable-docs \ + --disable-manual \ + --disable-unix-sockets \ + --disable-alt-svc \ + --disable-hsts \ + --disable-ech \ + --disable-aws \ + --disable-httpsrr \ + --disable-ares \ + --disable-threaded-resolver \ + --disable-sspi \ + --disable-kerberos-auth \ + --disable-negotiate-auth \ + --disable-ntlm \ + --disable-cookies \ + --without-libpsl \ + --without-libidn2 \ + --without-nghttp2 \ + --without-brotli \ + --without-zstd \ + --without-ngtcp2 \ + --without-nghttp3 + make -j"$JOBS" + make install +) + +extract sqlite +( + cd "$WORK"/sqlite-autoconf-*/ + CFLAGS="-O2 -fPIC -DSQLITE_ENABLE_RTREE=1 \ + -DSQLITE_ENABLE_COLUMN_METADATA=1 -DSQLITE_THREADSAFE=1 \ + -DSQLITE_OMIT_LOAD_EXTENSION=1" \ + ./configure --prefix="$PREFIX" --enable-shared --disable-static + make -j"$JOBS" + make install +) + +PROJ_VERSION="$(python3 "$ROOT/fetch_sources.py" --print-version proj)" +extract proj +cmake -S "$WORK"/proj-*/ -B "$WORK/proj-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + -DSQLite3_INCLUDE_DIR="$PREFIX/include" \ + -DSQLite3_LIBRARY="$PREFIX/lib/libsqlite3.so" \ + -DBUILD_APPS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DENABLE_TIFF=OFF \ + -DENABLE_CURL=OFF \ + -DEMBED_PROJ_DATA_PATH=OFF \ + -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=OFF +cmake --build "$WORK/proj-build" --parallel "$JOBS" +cmake --install "$WORK/proj-build" + +extract gdal +GDAL_SOURCE="$(find "$WORK" -maxdepth 1 -type d -name 'gdal-*' -print -quit)" +cmake -S "$GDAL_SOURCE" -B "$WORK/gdal-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_APPS=OFF \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_TESTING=OFF \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ + -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \ + -DGDAL_ENABLE_DRIVER_GTIFF=ON \ + -DGDAL_ENABLE_DRIVER_VRT=ON \ + -DGDAL_VRT_ENABLE_RAWRASTERBAND=OFF \ + -DGDAL_AUTOLOAD_PLUGINS=OFF \ + -DENABLE_GNM=ON \ + -DGDAL_USE_EXTERNAL_LIBS=OFF \ + -DGDAL_USE_INTERNAL_LIBS=OFF \ + -DGDAL_USE_CURL=ON \ + -DGDAL_USE_SQLITE3=ON \ + -DGDAL_USE_ZLIB=ON \ + -DGDAL_USE_DEFLATE=ON \ + -DGDAL_USE_ZSTD=ON \ + -DGDAL_USE_JPEG=ON \ + -DGDAL_USE_TIFF_INTERNAL=ON \ + -DGDAL_USE_GEOTIFF_INTERNAL=ON \ + -DGDAL_USE_PNG_INTERNAL=ON \ + -DGDAL_USE_JSONC_INTERNAL=ON \ + -DGDAL_USE_GEOS=OFF \ + -DGDAL_USE_LERC=OFF \ + -DGDAL_USE_QHULL=OFF \ + -DGDAL_USE_OPENSSL=OFF \ + -DGDAL_ENABLE_DRIVER_PNG=ON \ + -DGDAL_ENABLE_DRIVER_JPEG=ON \ + -DOGR_ENABLE_DRIVER_GEOJSON=ON \ + -DOGR_ENABLE_DRIVER_GPKG=ON \ + -DOGR_ENABLE_DRIVER_SQLITE=ON \ + -DOGR_ENABLE_DRIVER_SHAPE=OFF \ + -DCMAKE_CXX_FLAGS=-DGDAL_VRT_DISABLE_PYTHON \ + 2>&1 | tee "$BUILD_ROOT/gdal-configure.log" + +# Fail if this source release silently ignores a security-critical CMake +# option. The GDAL configure step also audits every unconsumed cache entry. +cmake -LAH -N "$WORK/gdal-build" > "$BUILD_ROOT/gdal-cmake-cache.txt" +for option in \ + GDAL_VRT_ENABLE_RAWRASTERBAND \ + GDAL_AUTOLOAD_PLUGINS \ + BUILD_PYTHON_BINDINGS; do + grep -q "^${option}:BOOL=OFF$" "$BUILD_ROOT/gdal-cmake-cache.txt" +done +grep -q '^ENABLE_GNM:BOOL=ON$' "$BUILD_ROOT/gdal-cmake-cache.txt" +grep -q '^CMAKE_CXX_FLAGS:STRING=.*GDAL_VRT_DISABLE_PYTHON' \ + "$BUILD_ROOT/gdal-cmake-cache.txt" +if grep -q 'Manually-specified variables were not used' \ + "$BUILD_ROOT/gdal-configure.log"; then + echo "GDAL CMake ignored a manually specified build option" >&2 + exit 1 +fi + +cmake --build "$WORK/gdal-build" --parallel "$JOBS" +cmake --install "$WORK/gdal-build" + +# BUILD_APPS=OFF intentionally omits gdal-config. Generate its build-time +# metadata script with apps enabled, but do not build or install GDAL utilities. +cmake -S "$GDAL_SOURCE" -B "$WORK/gdal-build" -DBUILD_APPS=ON +install -D -m 755 "$WORK/gdal-build/apps/gdal-config" \ + "$PREFIX/bin/gdal-config" +config_version="$("$PREFIX/bin/gdal-config" --version)" +if [[ "$config_version" != "3.13.3" ]]; then + echo "gdal-config reports $config_version, expected 3.13.3" >&2 + exit 1 +fi +ln -sfn "$PREFIX/bin/gdal-config" /usr/local/bin/gdal-config +command -v gdal-config +gdal-config --version + +export GDAL_CONFIG="$PREFIX/bin/gdal-config" +export PROJ_DIR="$PREFIX" +export PROJ_VERSION +export GDAL_DATA="$PREFIX/share/gdal" +export PROJ_DATA="$PREFIX/share/proj" + +# pip inspects GDAL's PEP 517 metadata while downloading its sdist. That +# metadata step runs gdal-config, so resolve binding sdists only after the +# native runtime and its build-only config tool are available. +PYTHON="$TOOL_VENV/bin/python" \ + bash "$ROOT/fetch_binding_sdists.sh" "$SOURCES/bindings" + +mkdir -p "$BUILD_ROOT/package/haste_gdal_runtime/share/gdal" +mkdir -p "$BUILD_ROOT/package/haste_gdal_runtime/share/proj" +cp "$ROOT/haste_gdal_runtime/"*.py "$ROOT/haste_gdal_runtime/gdalrc" \ + "$BUILD_ROOT/package/haste_gdal_runtime/" +cp -a "$PREFIX/share/gdal/." \ + "$BUILD_ROOT/package/haste_gdal_runtime/share/gdal/" +cp -a "$PREFIX/share/proj/." \ + "$BUILD_ROOT/package/haste_gdal_runtime/share/proj/" +PYTHONPATH="$ROOT${PYTHONPATH:+:$PYTHONPATH}" \ + "$PYTHON" -m haste_gdal_runtime manifest \ + --library "$PREFIX/lib/libgdal.so" \ + --output "$BUILD_ROOT/package/haste_gdal_runtime/manifest.json" +python3 "$ROOT/verify_build_manifest.py" \ + "$BUILD_ROOT/package/haste_gdal_runtime/manifest.json" + +python3 "$ROOT/build_bindings.py" \ + --python "$PYTHON" \ + --prefix "$PREFIX" \ + --sdist-dir "$SOURCES/bindings" \ + --wheel-dir "$WHEELS" \ + --common-venv "$TOOL_VENV" \ + --fiona-venv "$FIONA_VENV" + +python3 "$ROOT/assemble_wheels.py" \ + --wheel-dir "$WHEELS" \ + --runtime-package "$BUILD_ROOT/package/haste_gdal_runtime" \ + --output-dir "$BUILD_ROOT/dist" \ + --build "$(python3 "$ROOT/fetch_sources.py" --print-build)" \ + --gdal-version 3.13.3 \ + --prefix "$PREFIX" + +python3 "$ROOT/create_supply_chain.py" \ + --source-lock "$ROOT/sources.lock" \ + --source-dir "$SOURCES" \ + --wheel-dir "$BUILD_ROOT/dist" \ + --manifest "$BUILD_ROOT/package/haste_gdal_runtime/manifest.json" \ + --output-dir "$BUILD_ROOT/dist" + +echo "Native runtime and binding wheels built under $BUILD_ROOT/dist" diff --git a/native/gdal-runtime/build_bindings.py b/native/gdal-runtime/build_bindings.py new file mode 100644 index 00000000..5949f967 --- /dev/null +++ b/native/gdal-runtime/build_bindings.py @@ -0,0 +1,139 @@ +"""Build pinned Python bindings against the local native GDAL runtime. + +The sdists must already be present in ``--sdist-dir`` and the two venvs must +already contain their hash-locked build requirements. This script uses +``--no-index`` and never resolves or installs packages. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import tarfile +from pathlib import Path + + +def find_sdist(directory: Path, distribution: str, version: str) -> Path: + prefix = f"{distribution}-{version}".lower().replace("_", "-") + matches = [ + path + for path in directory.iterdir() + if path.name.lower().replace("_", "-").startswith(prefix) + and path.name.endswith((".tar.gz", ".tgz")) + ] + if len(matches) != 1: + raise FileNotFoundError( + f"expected exactly one sdist for {distribution} {version} " + f"in {directory}, found {len(matches)}" + ) + return matches[0] + + +def check_sdist(path: Path) -> None: + if not tarfile.is_tarfile(path): + raise ValueError(f"not a valid source archive: {path.name}") + + +def build_wheel( + python: Path, sdist: Path, output: Path, env: dict[str, str] +) -> None: + subprocess.run( + [ + str(python), + "-m", + "pip", + "wheel", + "--no-index", + "--no-deps", + "--no-build-isolation", + "--wheel-dir", + str(output), + str(sdist), + ], + check=True, + env=env, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--python", required=True, type=Path) + parser.add_argument("--prefix", required=True, type=Path) + parser.add_argument("--sdist-dir", required=True, type=Path) + parser.add_argument("--wheel-dir", required=True, type=Path) + parser.add_argument("--common-venv", required=True, type=Path) + parser.add_argument("--fiona-venv", required=True, type=Path) + args = parser.parse_args(argv) + + version = subprocess.run( + [ + str(args.python), + "-c", + "import sys; print('.'.join(map(str, sys.version_info[:2])))", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if version != "3.11": + raise RuntimeError( + f"binding builds require CPython 3.11, got {version}" + ) + source_dir = args.sdist_dir.resolve() + wheel_dir = args.wheel_dir.resolve() + prefix = args.prefix.resolve() + wheel_dir.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env["GDAL_CONFIG"] = str(prefix / "bin" / "gdal-config") + env["PROJ_DIR"] = str(prefix) + env["PROJ_VERSION"] = env.get("PROJ_VERSION", "") + env["PATH"] = str(prefix / "bin") + os.pathsep + env.get("PATH", "") + env["LD_LIBRARY_PATH"] = os.pathsep.join( + filter( + None, + [ + str(prefix / "lib"), + env.get("LD_LIBRARY_PATH", ""), + ], + ) + ) + env["CMAKE_PREFIX_PATH"] = os.pathsep.join( + filter( + None, + [ + str(prefix), + env.get("CMAKE_PREFIX_PATH", ""), + ], + ) + ) + + bindings = ( + ("GDAL", "3.13.3", args.common_venv / "bin" / "python"), + ("rasterio", "1.4.4", args.common_venv / "bin" / "python"), + ("fiona", "1.10.1", args.fiona_venv / "bin" / "python"), + ("pyogrio", "0.13.0", args.common_venv / "bin" / "python"), + ("pyproj", "3.7.2", args.common_venv / "bin" / "python"), + ) + for distribution, version, python in bindings: + config_version = subprocess.run( + ["gdal-config", "--version"], + check=True, + capture_output=True, + text=True, + env=env, + ).stdout.strip() + if config_version != "3.13.3": + raise RuntimeError( + f"gdal-config reports {config_version}, expected 3.13.3" + ) + sdist = find_sdist(source_dir, distribution, version) + check_sdist(sdist) + print(f"building {distribution} {version}", flush=True) + build_wheel(python, sdist, wheel_dir, env) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/create_supply_chain.py b/native/gdal-runtime/create_supply_chain.py new file mode 100644 index 00000000..a4d43f6f --- /dev/null +++ b/native/gdal-runtime/create_supply_chain.py @@ -0,0 +1,492 @@ +"""Create an inventory, notices bundle and checksums for runtime artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tarfile +import uuid +import zipfile +from datetime import datetime, timezone +from email.parser import BytesParser +from pathlib import Path, PurePosixPath +from urllib.parse import quote + +from fetch_sources import parse_lock + +LICENSE_NAMES = { + "copying", + "copying.md", + "copying.txt", + "copyright", + "copyright.md", + "copyright.txt", + "license", + "license.md", + "license.rst", + "license.txt", + "notice", + "notice.md", + "notice.txt", +} +MAX_NOTICE_SIZE = 2 * 1024 * 1024 +SOURCE_LICENSES = { + "gdal": { + "name": "MIT and bundled third-party licenses", + "url": "https://gdal.org/en/stable/copyright.html", + }, + "proj": { + "id": "MIT", + "url": "https://proj.org/en/stable/about.html#license", + }, + "sqlite": { + "name": "Public Domain", + "url": "https://www.sqlite.org/copyright.html", + }, + "curl": { + "name": "curl License", + "url": "https://curl.se/docs/copyright.html", + }, + "openssl": { + "id": "Apache-2.0", + "url": "https://www.openssl.org/source/license.html", + }, + "zlib": {"id": "Zlib", "url": "https://zlib.net/zlib_license.html"}, + "libdeflate": { + "id": "MIT", + "url": "https://github.com/ebiggers/libdeflate/blob/v1.26/COPYING", + }, + "zstd": { + "name": "BSD-style or GPL-2.0 licensing", + "url": "https://github.com/facebook/zstd/blob/v1.5.7/LICENSE", + }, + "libjpeg-turbo": { + "name": "BSD-style, IJG, and zlib licenses", + "url": ( + "https://github.com/libjpeg-turbo/libjpeg-turbo/blob/" + "3.1.4.1/LICENSE.md" + ), + }, +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def wheel_license( + expression: str | None, + legacy_name: str | None, + license_texts: list[bytes], +) -> tuple[dict[str, object] | None, str | None]: + if expression and expression.strip(): + value = expression.strip() + return {"expression": value}, value + if not legacy_name or not legacy_name.strip(): + return None, None + + value = legacy_name.strip() + first_line = next( + (line.strip() for line in value.splitlines() if line.strip()), "" + ) + normalized = re.sub(r"\s+", " ", value).casefold() + if normalized.startswith("mit license") or normalized == "mit": + identifier = "MIT" + elif normalized in {"bsd", "bsd 3-clause"} or normalized.startswith( + "bsd 3-clause " + ): + text = b"\n".join(license_texts).decode("utf-8", errors="replace") + identifier = ( + "BSD-3-Clause" + if "neither the name of" in text.casefold() + else None + ) + else: + identifier = None + if identifier: + return {"license": {"id": identifier}}, identifier + return {"license": {"name": first_line[:200]}}, first_line[:200] + + +def _notice_path(name: str, path: str) -> str: + if name in {"", ".", ".."} or "/" in name or "\\" in name or ":" in name: + raise ValueError(f"unsafe notice component: {name!r}") + safe_parts = [] + for part in PurePosixPath(path).parts: + if part in {"", ".", ".."} or ":" in part: + raise ValueError(f"unsafe notice path: {path!r}") + safe_parts.append(part) + return "/".join(["licenses", name, *safe_parts]) + + +def extract_native_notices( + archive_path: Path, component: str +) -> dict[str, bytes]: + notices: dict[str, bytes] = {} + with tarfile.open(archive_path, mode="r:*") as archive: + for member in archive.getmembers(): + if not member.isfile(): + continue + basename = PurePosixPath(member.name).name.casefold() + if basename not in LICENSE_NAMES: + continue + if member.size > MAX_NOTICE_SIZE: + raise ValueError( + f"{archive_path.name}:{member.name} " + "exceeds notice size limit" + ) + stream = archive.extractfile(member) + if stream is None: + raise ValueError( + f"cannot read notice {member.name} " + f"from {archive_path.name}" + ) + relative = PurePosixPath(member.name) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError( + f"unsafe notice path in {archive_path.name}: {member.name}" + ) + notice_path = _notice_path(component, relative.as_posix()) + notices[notice_path] = stream.read() + return notices + + +def wheel_inventory( + wheel_paths: list[Path], +) -> tuple[list[dict[str, object]], dict[str, bytes], dict[str, str]]: + components: list[dict[str, object]] = [] + notices: dict[str, bytes] = {} + licenses: dict[str, str] = {} + for path in sorted(wheel_paths): + with zipfile.ZipFile(path) as wheel: + metadata_names = [ + name + for name in wheel.namelist() + if name.endswith(".dist-info/METADATA") + ] + if len(metadata_names) != 1: + raise ValueError( + f"{path.name}: expected one wheel METADATA file" + ) + metadata = BytesParser().parsebytes(wheel.read(metadata_names[0])) + name = metadata.get("Name") + version = metadata.get("Version") + if not name or not version: + raise ValueError( + f"{path.name}: wheel metadata lacks name or version" + ) + package_name = re.sub(r"[-_.]+", "-", name.lower()) + component: dict[str, object] = { + "type": "library", + "name": name, + "version": version, + "purl": ( + f"pkg:pypi/{quote(package_name, safe='-.')}@" + f"{quote(version, safe='.+-')}" + ), + "hashes": [{"alg": "SHA-256", "content": sha256(path)}], + } + license_expression = metadata.get("License-Expression") + license_name = metadata.get("License") + if package_name == "haste-gdal-runtime" and not license_expression: + raise ValueError( + f"{path.name}: runtime wheel requires an SPDX " + "License-Expression" + ) + license_texts = [] + + for member in wheel.namelist(): + if ".dist-info/licenses/" not in member: + continue + info = wheel.getinfo(member) + if info.file_size > MAX_NOTICE_SIZE: + raise ValueError( + f"{path.name}:{member} exceeds notice size limit" + ) + content = wheel.read(info) + relative = member.split(".dist-info/licenses/", 1)[1] + if relative: + notices[_notice_path(name, relative)] = content + license_texts.append(content) + license_entry, license_value = wheel_license( + license_expression, license_name, license_texts + ) + if package_name == "haste-gdal-runtime" and not license_texts: + raise ValueError( + f"{path.name}: runtime wheel must bundle its license text" + ) + if license_entry: + component["licenses"] = [license_entry] + licenses[name] = license_value or "License metadata present" + else: + licenses[name] = ( + "No license metadata in wheel; review required" + ) + components.append(component) + return components, notices, licenses + + +def runtime_manifest( + wheel_paths: list[Path], build_manifest: dict[str, object] +) -> dict[str, object]: + runtime_wheels = [] + for path in wheel_paths: + with zipfile.ZipFile(path) as wheel: + metadata_names = [ + name + for name in wheel.namelist() + if name.endswith(".dist-info/METADATA") + ] + if len(metadata_names) != 1: + continue + metadata = BytesParser().parsebytes(wheel.read(metadata_names[0])) + name = re.sub(r"[-_.]+", "-", (metadata.get("Name") or "").lower()) + if name == "haste-gdal-runtime": + runtime_wheels.append((path, wheel.namelist())) + + if len(runtime_wheels) != 1: + raise ValueError( + "expected one haste-gdal-runtime wheel when creating the manifest" + ) + path, members = runtime_wheels[0] + manifest_members = [ + member + for member in members + if member.endswith("haste_gdal_runtime/manifest.json") + ] + if len(manifest_members) != 1: + raise ValueError( + f"{path.name}: expected one installed GDAL runtime manifest" + ) + with zipfile.ZipFile(path) as wheel: + installed_manifest = json.loads( + wheel.read(manifest_members[0]).decode("utf-8") + ) + if not isinstance(installed_manifest, dict): + raise ValueError( + f"{path.name}: GDAL runtime manifest is not an object" + ) + mismatched = { + key: (value, installed_manifest.get(key)) + for key, value in build_manifest.items() + if installed_manifest.get(key) != value + } + if mismatched: + raise ValueError( + "installed GDAL runtime manifest differs from build manifest: " + + ", ".join( + f"{key}={expected!r} != {actual!r}" + for key, (expected, actual) in sorted(mismatched.items()) + ) + ) + return installed_manifest + + +def generate( + source_lock: Path, + source_dir: Path, + wheel_dir: Path, + manifest_path: Path, + output_dir: Path, +) -> list[Path]: + _, sources = parse_lock(source_lock.read_text(encoding="utf-8")) + wheel_paths = sorted(wheel_dir.glob("*.whl")) + if not wheel_paths: + raise ValueError(f"no wheels found in {wheel_dir}") + build_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + components: list[dict[str, object]] = [] + notices: dict[str, bytes] = {} + license_status: dict[str, str] = {} + for source in sources: + archive_path = source_dir / source.filename + if not archive_path.is_file(): + raise FileNotFoundError( + f"locked source archive is missing: {source.filename}" + ) + actual_digest = sha256(archive_path) + if actual_digest != source.sha256: + raise ValueError( + f"{source.filename}: sha256 {actual_digest} " + f"does not match sources.lock" + ) + native_notices = extract_native_notices(archive_path, source.name) + if source.name.casefold() == "sqlite": + native_notices[ + "licenses/sqlite/SQLITE_PUBLIC_DOMAIN_NOTICE.txt" + ] = ( + b"SQLite's deliverable code and documentation are dedicated " + b"to the public domain. See " + b"https://www.sqlite.org/copyright.html.\n" + ) + notices.update(native_notices) + license_info = SOURCE_LICENSES.get(source.name.casefold()) + component: dict[str, object] = { + "type": "library", + "name": source.name, + "version": source.version, + "purl": ( + f"pkg:generic/{quote(source.name, safe='-._')}@" + f"{quote(source.version, safe='.+-')}" + ), + "hashes": [{"alg": "SHA-256", "content": source.sha256}], + "externalReferences": [ + { + "type": "distribution", + "url": source.url, + } + ], + } + if license_info: + license_entry = {"url": license_info["url"]} + if "id" in license_info: + license_entry["id"] = license_info["id"] + else: + license_entry["name"] = license_info["name"] + component["licenses"] = [{"license": license_entry}] + component["properties"] = [ + { + "name": "gdal_runtime.license_notice_file", + "value": notice_path, + } + for notice_path in sorted(native_notices) + ] + license_status[source.name] = ( + "Bundled source license file(s): " + + ", ".join(sorted(native_notices)) + if native_notices + else "No license file found in source archive; review required" + ) + else: + license_status[source.name] = ( + "License identification is missing; review required. " + "Bundled notice file(s): " + ", ".join(sorted(native_notices)) + ) + components.append(component) + + python_components, python_notices, python_licenses = wheel_inventory( + wheel_paths + ) + expected_distributions = { + "haste-gdal-runtime", + "gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + } + actual_distributions = { + str(component["name"]).casefold() for component in python_components + } + if actual_distributions != expected_distributions: + raise ValueError( + "runtime wheel set does not match expected distributions: " + f"{sorted(actual_distributions)}" + ) + components.extend(python_components) + notices.update(python_notices) + license_status.update(python_licenses) + manifest = runtime_manifest(wheel_paths, build_manifest) + + runtime_version = next( + component["version"] + for component in python_components + if component["name"].casefold() == "haste-gdal-runtime" + ) + bom = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid4()}", + "version": 1, + "metadata": { + "timestamp": datetime.now(timezone.utc).isoformat(), + "component": { + "type": "application", + "name": "haste-gdal-runtime-bundle", + "version": str(runtime_version), + }, + "properties": [ + { + "name": f"gdal_runtime.{key}", + "value": json.dumps(value, sort_keys=True), + } + for key, value in sorted(manifest.items()) + ], + }, + "components": components, + } + + output_dir.mkdir(parents=True, exist_ok=True) + artifact_manifest = output_dir / "manifest.json" + artifact_manifest.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + bom_path = output_dir / "bom.cdx.json" + bom_path.write_text( + json.dumps(bom, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + notice_path = output_dir / "THIRD_PARTY_NOTICES.txt" + with notice_path.open("w", encoding="utf-8", newline="\n") as output: + output.write( + "HASTE GDAL runtime third-party notices\n" + "=======================================\n\n" + "Native source archives and Python binding wheels are pinned in " + "the build inputs.\n" + "License files discovered in those archives/wheels follow. " + "Any missing license text requires review before publication.\n\n" + ) + output.write("Component/license inventory\n") + output.write("----------------------------\n") + for component, status in sorted(license_status.items()): + output.write(f"- {component}: {status}\n") + output.write("\n") + for member, content in sorted(notices.items()): + output.write(f"\n----- {member} -----\n") + output.write(content.decode("utf-8", errors="replace")) + if not content.endswith(b"\n"): + output.write("\n") + + checksum_paths = [ + *wheel_paths, + artifact_manifest, + bom_path, + notice_path, + ] + checksums_path = output_dir / "SHA256SUMS" + with checksums_path.open("w", encoding="utf-8", newline="\n") as output: + for path in sorted(checksum_paths): + output.write(f"{sha256(path)} {path.name}\n") + return [artifact_manifest, bom_path, notice_path, checksums_path] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-lock", required=True, type=Path) + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--wheel-dir", required=True, type=Path) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args(argv) + + for path in generate( + args.source_lock, + args.source_dir, + args.wheel_dir, + args.manifest, + args.output_dir, + ): + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/fetch_binding_sdists.sh b/native/gdal-runtime/fetch_binding_sdists.sh new file mode 100644 index 00000000..0007b4cc --- /dev/null +++ b/native/gdal-runtime/fetch_binding_sdists.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +DEST="${1:?usage: fetch_binding_sdists.sh DESTINATION}" + +PIP_INDEX_URL="${PIP_INDEX_URL:-https://pypi.org/simple}" + +mkdir -p "$DEST" +"${PYTHON:-python3}" -m pip download \ + --isolated \ + --index-url "$PIP_INDEX_URL" \ + --no-build-isolation \ + --no-deps \ + --no-binary=:all: \ + --require-hashes \ + --dest "$DEST" \ + -r "$ROOT/requirements-sdists.txt" diff --git a/native/gdal-runtime/fetch_sources.py b/native/gdal-runtime/fetch_sources.py new file mode 100644 index 00000000..3eeba2d7 --- /dev/null +++ b/native/gdal-runtime/fetch_sources.py @@ -0,0 +1,142 @@ +"""Download the runtime's native sources and verify them against sources.lock. + +Usage: + python fetch_sources.py --dest DIR download and verify all + python fetch_sources.py --print-build print HASTE_GDAL_BUILD + python fetch_sources.py --print-version NAME print a component version + +A file that is already present is re-verified rather than trusted. Any +mismatch is fatal: the build must never compile a source it cannot verify. +""" + +from __future__ import annotations + +import argparse +import hashlib +import sys +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +LOCK = Path(__file__).with_name("sources.lock") +CHUNK = 1024 * 1024 + + +@dataclass(frozen=True) +class Source: + name: str + version: str + url: str + sha256: str + + @property + def filename(self) -> str: + return self.url.rsplit("/", 1)[-1] + + +def parse_lock(text: str) -> tuple[str, list[Source]]: + build = "" + sources: list[Source] = [] + for number, raw in enumerate(text.splitlines(), start=1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + fields = line.split() + if fields[0] == "HASTE_GDAL_BUILD" and len(fields) == 2: + build = fields[1] + continue + if len(fields) != 4: + raise ValueError(f"sources.lock line {number}: expected 4 fields") + name, version, url, digest = fields + if not url.startswith("https://"): + raise ValueError(f"sources.lock line {number}: URL must be HTTPS") + if len(digest) != 64 or any( + c not in "0123456789abcdef" for c in digest + ): + raise ValueError(f"sources.lock line {number}: bad sha256") + sources.append(Source(name, version, url, digest)) + if not build.isdigit(): + raise ValueError("sources.lock: HASTE_GDAL_BUILD must be a number") + names = [source.name for source in sources] + if len(names) != len(set(names)): + raise ValueError("sources.lock: duplicate component") + return build, sources + + +def sha256_of(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(CHUNK), b""): + digest.update(block) + return digest.hexdigest() + + +def download(source: Source, target: Path, attempts: int = 4) -> None: + partial = target.with_suffix(target.suffix + ".part") + for attempt in range(1, attempts + 1): + try: + request = urllib.request.Request( + source.url, headers={"User-Agent": "haste-gdal-runtime"} + ) + with urllib.request.urlopen(request, timeout=120) as response: + with partial.open("wb") as handle: + for block in iter(lambda: response.read(CHUNK), b""): + handle.write(block) + partial.replace(target) + return + except OSError as error: + if attempt == attempts: + raise + print(f" retrying {source.filename} ({error})", file=sys.stderr) + time.sleep(5 * attempt) + + +def fetch(sources: list[Source], dest: Path) -> list[str]: + dest.mkdir(parents=True, exist_ok=True) + problems = [] + for source in sources: + target = dest / source.filename + if not target.exists(): + print(f"downloading {source.name} {source.version}") + download(source, target) + actual = sha256_of(target) + if actual != source.sha256: + target.unlink() + problems.append( + f"{source.filename}: sha256 {actual} != locked {source.sha256}" + ) + else: + print(f"verified {source.filename}") + return problems + + +def main(argv: list[str]) -> int: + sys.stdout.reconfigure(newline="\n") + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--dest", type=Path) + group.add_argument("--print-build", action="store_true") + group.add_argument("--print-version", metavar="NAME") + args = parser.parse_args(argv) + + build, sources = parse_lock(LOCK.read_text(encoding="utf-8")) + if args.print_build: + print(build) + return 0 + if args.print_version: + for source in sources: + if source.name == args.print_version: + print(source.version) + return 0 + print(f"unknown component {args.print_version!r}", file=sys.stderr) + return 2 + + problems = fetch(sources, args.dest) + for problem in problems: + print(f"ERROR: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/native/gdal-runtime/haste_gdal_runtime/__init__.py b/native/gdal-runtime/haste_gdal_runtime/__init__.py new file mode 100644 index 00000000..9a6e5bcc --- /dev/null +++ b/native/gdal-runtime/haste_gdal_runtime/__init__.py @@ -0,0 +1,450 @@ +"""HASTE GDAL native runtime: configuration, hardening and a fail-closed check. + +The wheel bundles one libgdal (with PROJ and their dependencies) that every +GDAL-linked binding links to. A process uses it in three steps: + + import haste_gdal_runtime as runtime + runtime.configure(POLICY) # before importing any GDAL binding + import rasterio # the binding loads the runtime's libgdal + runtime.secure_startup(POLICY) # restrict, then verify or raise + +``configure`` sets the operator-owned GDAL settings when the environment has +not set them, ``harden`` removes drivers and virtual file systems the policy +does not allow, and ``verify_or_raise`` compares the live process with the +build manifest and the policy. The library location is fixed by the +installed wheel and never read from the environment. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import json +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, MutableMapping + +from ._native import NativeGdal, mapped_libraries + +PACKAGE_DIR = Path(__file__).resolve().parent +LIBS_DIR = PACKAGE_DIR.parent / "haste_gdal_runtime.libs" +GDAL_DATA_DIR = PACKAGE_DIR / "share" / "gdal" +PROJ_DATA_DIR = PACKAGE_DIR / "share" / "proj" +CONFIG_FILE = PACKAGE_DIR / "gdalrc" +MANIFEST_PATH = PACKAGE_DIR / "manifest.json" + +# Operator-owned hardening. configure() applies these when the environment +# leaves them unset; evaluate() fails when the effective value differs. +REQUIRED_CONFIG = { + "GDAL_VRT_ENABLE_RAWRASTERBAND": "NO", + "GDAL_VRT_ENABLE_PYTHON": "NO", + "GDAL_DRIVER_PATH": "disable", + "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", + "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED": "NO", +} + +# Locations GDAL and PROJ read data and settings from. They must point into +# this package so no other file can change the runtime's behaviour. +PINNED_PATHS = { + "GDAL_DATA": GDAL_DATA_DIR, + "PROJ_DATA": PROJ_DATA_DIR, + "GDAL_CONFIG_FILE": CONFIG_FILE, +} + +_TRUE_VALUES = frozenset({"YES", "TRUE", "ON", "1"}) +_FALSE_VALUES = frozenset({"NO", "FALSE", "OFF", "0"}) + +# Settings that relax a protection. Any of these fails the check. +FORBIDDEN_CONFIG = { + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE": lambda v: v.upper() == "ALL", + "GDAL_VRT_PYTHON_TRUSTED_MODULES": lambda v: v != "", + "GDAL_PYTHON_DRIVER_PATH": lambda v: v != "", + "CPL_ENABLE_PATH_TRAVERSAL_DETECTION": lambda v: v.upper() + in _FALSE_VALUES, + "GDAL_HTTP_UNSAFESSL": lambda v: v.upper() in _TRUE_VALUES, +} + +CA_BUNDLE_KEYS = ("CURL_CA_BUNDLE", "SSL_CERT_FILE") + +WATCHED_CONFIG = sorted( + { + *REQUIRED_CONFIG, + *PINNED_PATHS, + *FORBIDDEN_CONFIG, + *CA_BUNDLE_KEYS, + "GDAL_SKIP", + } +) + +# Virtual file systems the HASTE workloads need: in-memory files, HTTPS range +# reads, and the two handlers the GeoTIFF driver uses internally. +DEFAULT_ALLOWED_VSI = frozenset( + {"/vsimem/", "/vsicurl/", "/vsisubfile/", "/vsisparse/"} +) + +BINDINGS = ("osgeo.gdal", "rasterio", "fiona", "pyogrio", "pyproj") + +_CA_BUNDLES = ( + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", +) + + +class RuntimeCheckError(RuntimeError): + """The live GDAL runtime is not the approved one.""" + + def __init__(self, problems: list[str]) -> None: + self.problems = problems + super().__init__( + "GDAL runtime check failed:\n - " + "\n - ".join(problems) + ) + + +@dataclass(frozen=True) +class Policy: + """What one HASTE runtime requires of the GDAL runtime. + + ``allowed_drivers`` of None keeps every driver in the build. + """ + + expected_gdal: str + allowed_drivers: frozenset[str] | None = None + allowed_vsi: frozenset[str] = DEFAULT_ALLOWED_VSI + required_bindings: tuple[str, ...] = () + + +def library_path(stem: str = "libgdal") -> Path: + matches = sorted(LIBS_DIR.glob(f"{stem}-*.so*")) + if len(matches) != 1: + raise RuntimeCheckError( + [f"expected one {stem} in {LIBS_DIR}, found {len(matches)}"] + ) + return matches[0] + + +def load_manifest(path: Path = MANIFEST_PATH) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +_NATIVE: dict[Path, NativeGdal] = {} + + +def _native(library: Path | None) -> NativeGdal: + path = Path(library) if library else library_path() + if path not in _NATIVE: + _NATIVE[path] = NativeGdal(path) + return _NATIVE[path] + + +def skip_list(manifest: dict[str, Any], policy: Policy) -> list[str]: + """Drivers in the build that ``policy`` does not allow.""" + if policy.allowed_drivers is None: + return [] + return sorted(set(manifest.get("drivers", [])) - policy.allowed_drivers) + + +def configure( + policy: Policy | None = None, + environ: MutableMapping[str, str] | None = None, + manifest: dict[str, Any] | None = None, +) -> None: + """Apply data paths and hardening defaults to the process environment. + + Values already set are left alone, so evaluate() still reports any + operator setting that weakens a protection. Drivers the policy does not + allow are added to GDAL_SKIP, which GDAL applies after every driver + registration, including registrations a binding makes later. + """ + env = os.environ if environ is None else environ + for key, path in PINNED_PATHS.items(): + env.setdefault(key, str(path)) + for key, value in REQUIRED_CONFIG.items(): + env.setdefault(key, value) + if not any(env.get(key) for key in CA_BUNDLE_KEYS): + for candidate in _CA_BUNDLES: + if os.path.isfile(candidate): + env["CURL_CA_BUNDLE"] = candidate + break + if policy is not None and policy.allowed_drivers is not None: + skipped = skip_list(manifest or load_manifest(), policy) + existing = [ + name + for name in re.split(r"[,\s]+", env.get("GDAL_SKIP", "")) + if name + ] + merged = list(dict.fromkeys(existing + skipped)) + if merged: + env["GDAL_SKIP"] = ",".join(merged) + + +def harden( + policy: Policy, *, library: Path | None = None +) -> dict[str, list[str]]: + """Remove the drivers and virtual file systems ``policy`` does not allow. + + Run after the bindings are imported. Removed file systems stay removed + for the life of the process; removed drivers also stay out because + configure() lists them in GDAL_SKIP. + """ + native = _native(library) + removed_drivers: list[str] = [] + if policy.allowed_drivers is not None: + removed_drivers = native.deregister_drivers( + set(native.drivers()) - policy.allowed_drivers + ) + removed_vsi = native.remove_vsi_handlers( + set(native.vsi_prefixes()) - policy.allowed_vsi + ) + return {"drivers": removed_drivers, "vsi_prefixes": removed_vsi} + + +def _binding_info(name: str, module: Any) -> dict[str, str]: + info = {"version": str(getattr(module, "__version__", ""))} + if name == "osgeo.gdal": + info["gdal"] = module.VersionInfo("RELEASE_NAME") + elif name in ("rasterio", "fiona"): + info["gdal"] = str(module.__gdal_version__) + elif name == "pyogrio": + info["gdal"] = str(module.__gdal_version_string__) + elif name == "pyproj": + info["proj"] = str( + getattr(module, "__proj_version__", None) + or module.proj_version_str + ) + return info + + +def _import_bindings(names: tuple[str, ...]) -> dict[str, dict[str, str]]: + found = {} + for name in names: + if importlib.util.find_spec(name.split(".")[0]) is None: + continue + found[name] = _binding_info(name, importlib.import_module(name)) + return found + + +def report( + *, bindings: tuple[str, ...] = BINDINGS, library: Path | None = None +) -> dict[str, Any]: + """Collect evidence about the GDAL runtime loaded in this process.""" + imported = _import_bindings(bindings) + native = _native(library) + config = {key: native.config(key) for key in WATCHED_CONFIG} + ca_bundle = next( + (config[key] for key in CA_BUNDLE_KEYS if config.get(key)), None + ) + raw_band_flag = native.driver_metadata( + "VRT", "GDAL_VRT_ENABLE_RAWRASTERBAND" + ) + return { + "library": str(native.path), + "gdal_release": native.version_info("RELEASE_NAME"), + "build_info": native.build_info(), + "drivers": native.drivers(), + "vsi_prefixes": native.vsi_prefixes(), + "vrt_raw_band_compiled": raw_band_flag == "YES", + "config": config, + "ca_bundle": { + "path": ca_bundle, + "exists": bool(ca_bundle) and os.path.isfile(ca_bundle), + }, + "mapped_libgdal": mapped_libraries("libgdal"), + "mapped_libproj": mapped_libraries("libproj"), + "bindings": imported, + "probes": native.probes(), + "probes_permissive": native.probes(permissive=True), + } + + +def _same_path(left: str | None, right: Path) -> bool: + return bool(left) and os.path.realpath(left) == os.path.realpath(right) + + +def _check_versions( + observed: dict[str, Any], manifest: dict[str, Any], policy: Policy +) -> list[str]: + problems = [] + expected = policy.expected_gdal + if observed["gdal_release"] != expected: + problems.append( + f"libgdal is {observed['gdal_release']}, expected {expected}" + ) + if manifest.get("gdal_release") != expected: + problems.append( + f"installed runtime was built for GDAL " + f"{manifest.get('gdal_release')}, expected {expected}" + ) + binding_versions = manifest.get("binding_versions") + if not isinstance(binding_versions, dict): + problems.append("runtime manifest has no binding version inventory") + binding_versions = {} + for name in policy.required_bindings: + if name not in observed["bindings"]: + problems.append(f"required binding {name} is not installed") + for name, info in observed["bindings"].items(): + expected_binding = binding_versions.get(name) + if not isinstance(expected_binding, str): + problems.append( + f"runtime manifest has no expected version for binding {name}" + ) + else: + actual_binding = ( + info.get("gdal", "") + if name == "osgeo.gdal" + else info.get("version", "").split("+", 1)[0] + ) + if actual_binding != expected_binding: + problems.append( + f"{name} reports version {actual_binding!r}, " + f"expected {expected_binding!r}" + ) + if "gdal" in info and info["gdal"] != expected: + problems.append( + f"{name} reports GDAL {info['gdal']}, expected {expected}" + ) + if "proj" in info and info["proj"] != manifest.get("proj_release"): + problems.append( + f"{name} reports PROJ {info['proj']}, runtime has " + f"{manifest.get('proj_release')}" + ) + return problems + + +def _check_libraries(observed: dict[str, Any]) -> list[str]: + problems = [] + library = os.path.realpath(observed["library"]) + mapped = sorted({os.path.realpath(p) for p in observed["mapped_libgdal"]}) + if mapped != [library]: + problems.append( + f"expected only the runtime's libgdal mapped, found {mapped}" + ) + libs_dir = os.path.dirname(library) + proj = sorted({os.path.realpath(p) for p in observed["mapped_libproj"]}) + if len(proj) != 1 or any(os.path.dirname(p) != libs_dir for p in proj): + problems.append( + f"expected exactly one runtime libproj in {libs_dir}, found {proj}" + ) + return problems + + +def _check_inventory( + observed: dict[str, Any], manifest: dict[str, Any], policy: Policy +) -> list[str]: + problems = [] + registered = set(observed["drivers"]) + unknown = sorted(registered - set(manifest.get("drivers", []))) + if unknown: + problems.append(f"drivers not in the build manifest: {unknown}") + expected_drivers = set(manifest.get("drivers", [])) + if policy.allowed_drivers is not None: + expected_drivers &= policy.allowed_drivers + extra = sorted(registered - policy.allowed_drivers) + if extra: + problems.append(f"drivers outside the policy: {extra}") + missing_drivers = sorted(expected_drivers - registered) + if missing_drivers: + problems.append( + f"required drivers are not registered: {missing_drivers}" + ) + + prefixes = set(observed["vsi_prefixes"]) + unknown_vsi = sorted(prefixes - set(manifest.get("vsi_prefixes", []))) + if unknown_vsi: + problems.append(f"file systems not in the manifest: {unknown_vsi}") + expected_vsi = set(manifest.get("vsi_prefixes", [])) & policy.allowed_vsi + missing_vsi = sorted(expected_vsi - prefixes) + if missing_vsi: + problems.append( + f"required file systems are not registered: {missing_vsi}" + ) + extra_vsi = sorted(prefixes - policy.allowed_vsi) + if extra_vsi: + problems.append(f"file systems outside the policy: {extra_vsi}") + if "/vsicurl/" in prefixes and not observed["ca_bundle"]["exists"]: + problems.append( + "HTTPS reads are enabled but no CA bundle file is configured" + ) + return problems + + +def _check_config(observed: dict[str, Any]) -> list[str]: + problems = [] + config = observed["config"] + for key, expected in REQUIRED_CONFIG.items(): + actual = config.get(key) + if (actual or "").upper() != expected.upper(): + problems.append(f"{key} is {actual!r}, expected {expected!r}") + for key, relaxes in FORBIDDEN_CONFIG.items(): + actual = config.get(key) + if actual is not None and relaxes(actual): + problems.append(f"{key}={actual!r} is not allowed") + for key, path in PINNED_PATHS.items(): + if not _same_path(config.get(key), path): + problems.append(f"{key} is {config.get(key)!r}, expected {path}") + return problems + + +def _check_vrt( + observed: dict[str, Any], manifest: dict[str, Any], policy: Policy +) -> list[str]: + problems = [] + if manifest.get("vrt_raw_band_compiled") is not False: + problems.append("the runtime was built with VRT raw bands") + if observed["vrt_raw_band_compiled"]: + problems.append("the loaded libgdal has VRT raw bands compiled in") + for source, probes in ( + ("build", manifest.get("probes_permissive", {})), + ("configured", observed["probes"]), + ("permissive", observed["probes_permissive"]), + ): + for probe in ("raw_band", "python_pixel_function"): + if probes.get(probe) not in ("rejected", "read-failed"): + problems.append( + f"{probe} probe ({source} settings) returned " + f"{probes.get(probe)!r}" + ) + vrt_allowed = ( + policy.allowed_drivers is None or "VRT" in policy.allowed_drivers + ) + if not vrt_allowed and observed["probes"].get("vrt_driver") != "rejected": + problems.append("a VRT document opened although VRT is not allowed") + return problems + + +def evaluate( + observed: dict[str, Any], manifest: dict[str, Any], policy: Policy +) -> list[str]: + """Return every way ``observed`` differs from what is approved.""" + return [ + *_check_versions(observed, manifest, policy), + *_check_libraries(observed), + *_check_inventory(observed, manifest, policy), + *_check_config(observed), + *_check_vrt(observed, manifest, policy), + ] + + +def verify(policy: Policy) -> list[str]: + return evaluate(report(), load_manifest(), policy) + + +def verify_or_raise(policy: Policy) -> dict[str, Any]: + """Raise RuntimeCheckError unless the process matches ``policy``. + + Returns the report so callers can log it as evidence. + """ + observed = report() + problems = evaluate(observed, load_manifest(), policy) + if problems: + raise RuntimeCheckError(problems) + return observed + + +def secure_startup(policy: Policy) -> dict[str, Any]: + """Harden the process, then verify it. Returns the evidence report.""" + removed = harden(policy) + observed = verify_or_raise(policy) + observed["removed"] = removed + return observed diff --git a/native/gdal-runtime/haste_gdal_runtime/__main__.py b/native/gdal-runtime/haste_gdal_runtime/__main__.py new file mode 100644 index 00000000..b926b225 --- /dev/null +++ b/native/gdal-runtime/haste_gdal_runtime/__main__.py @@ -0,0 +1,127 @@ +"""Command line for the HASTE GDAL runtime. + + python -m haste_gdal_runtime report + python -m haste_gdal_runtime verify --expected-gdal 3.13.3 \ + [--allow-driver NAME ...] [--allow-vsi PREFIX ...] + python -m haste_gdal_runtime manifest --library PATH --output FILE + +``manifest`` runs at build time against the freshly built libgdal. The +library is always passed explicitly; nothing is read from the environment. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +from . import ( + BINDINGS, + DEFAULT_ALLOWED_VSI, + Policy, + RuntimeCheckError, + configure, + evaluate, + harden, + load_manifest, + report, +) +from ._native import NativeGdal + + +def _dump(data: Any) -> None: + json.dump(data, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + + +def _policy(args: argparse.Namespace) -> Policy: + return Policy( + expected_gdal=args.expected_gdal, + allowed_drivers=( + frozenset(args.allow_driver) if args.allow_driver else None + ), + allowed_vsi=( + frozenset(args.allow_vsi) + if args.allow_vsi + else DEFAULT_ALLOWED_VSI + ), + required_bindings=tuple(args.require_binding or ()), + ) + + +def _manifest(args: argparse.Namespace) -> int: + native = NativeGdal(Path(args.library)) + build_info = native.build_info() + manifest = { + "schema": 1, + "gdal_release": native.version_info("RELEASE_NAME"), + "proj_release": build_info.get("PROJ_RUNTIME_VERSION") + or os.environ.get("PROJ_VERSION"), + "build_info": build_info, + "drivers": native.drivers(), + "vsi_prefixes": native.vsi_prefixes(), + "vrt_raw_band_compiled": native.driver_metadata( + "VRT", "GDAL_VRT_ENABLE_RAWRASTERBAND" + ) + == "YES", + "probes_permissive": native.probes(permissive=True), + } + Path(args.output).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + _dump(manifest) + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="python -m haste_gdal_runtime") + commands = parser.add_subparsers(dest="command", required=True) + + commands.add_parser("report", help="print the runtime evidence report") + + verify = commands.add_parser( + "verify", help="harden, check against a policy, exit 1 on problems" + ) + verify.add_argument("--expected-gdal", required=True) + verify.add_argument("--allow-driver", action="append") + verify.add_argument("--allow-vsi", action="append") + verify.add_argument("--require-binding", action="append") + verify.add_argument( + "--skip-bindings", + action="store_true", + help="inspect the native runtime without importing bindings", + ) + + manifest = commands.add_parser( + "manifest", help="describe a freshly built libgdal (build time)" + ) + manifest.add_argument("--library", required=True) + manifest.add_argument("--output", required=True) + + args = parser.parse_args(argv) + if args.command == "manifest": + return _manifest(args) + if args.command == "report": + configure() + _dump(report()) + return 0 + + policy = _policy(args) + configure(policy) + try: + removed = harden(policy) + observed = report(bindings=() if args.skip_bindings else BINDINGS) + observed["removed"] = removed + problems = evaluate(observed, load_manifest(), policy) + except RuntimeCheckError as error: + problems, observed = error.problems, {} + _dump({"problems": problems, "report": observed}) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/native/gdal-runtime/haste_gdal_runtime/_native.py b/native/gdal-runtime/haste_gdal_runtime/_native.py new file mode 100644 index 00000000..b7cc7c23 --- /dev/null +++ b/native/gdal-runtime/haste_gdal_runtime/_native.py @@ -0,0 +1,267 @@ +"""ctypes access to the runtime's libgdal. + +The self-check talks to libgdal directly instead of going through a binding. +The same check then works in runtimes that only install rasterio, such as the +tile server, and it inspects the one library every binding links to. +""" + +from __future__ import annotations + +import ctypes +import os +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +GDAL_OF_RASTER = 0x02 +GF_READ = 0 +GDT_BYTE = 1 + +# Settings that would allow each probed feature if it were compiled in. +# Probing under them proves the feature is absent from the binary, not just +# switched off by configuration. +PERMISSIVE_CONFIG = { + "GDAL_VRT_ENABLE_RAWRASTERBAND": "YES", + "GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE": "ALL", + "GDAL_VRT_ENABLE_PYTHON": "YES", +} + +_RAW_BAND_VRT = """ + + raw.bin + 0 + 1 + 1 + +""" + +# Harmless if it ever ran: it only fills the output buffer. Any successful +# read through it fails the check. +_PYTHON_PIXEL_FUNCTION_VRT = """ + + haste_probe + Python + + +""" + +_PLAIN_VRT = """ + +""" + + +class NativeGdal: + """The subset of the GDAL C API the runtime check needs.""" + + def __init__(self, library: Path) -> None: + self.path = Path(library) + self._lib = ctypes.CDLL(str(self.path)) + self._declare() + # Register only when nothing has yet. GDALAllRegister would bring + # back drivers that a hardened process has deregistered. + if self._lib.GDALGetDriverCount() == 0: + self._lib.GDALAllRegister() + + def _declare(self) -> None: + lib = self._lib + c_char_p, c_int, c_void_p = ( + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_void_p, + ) + signatures = { + "GDALAllRegister": ([], None), + "GDALVersionInfo": ([c_char_p], c_char_p), + "GDALGetDriverCount": ([], c_int), + "GDALGetDriver": ([c_int], c_void_p), + "GDALGetDriverByName": ([c_char_p], c_void_p), + "GDALGetDriverShortName": ([c_void_p], c_char_p), + "GDALDeregisterDriver": ([c_void_p], None), + "GDALGetMetadataItem": ([c_void_p, c_char_p, c_char_p], c_char_p), + "VSIGetFileSystemsPrefixes": ([], ctypes.POINTER(c_char_p)), + "VSIRemovePluginHandler": ([c_char_p], c_int), + "CSLDestroy": ([ctypes.POINTER(c_char_p)], None), + "CPLGetConfigOption": ([c_char_p, c_char_p], c_char_p), + "CPLGetThreadLocalConfigOption": ([c_char_p, c_char_p], c_char_p), + "CPLSetThreadLocalConfigOption": ([c_char_p, c_char_p], None), + "CPLPushErrorHandler": ([c_void_p], None), + "CPLPopErrorHandler": ([], None), + "CPLErrorReset": ([], None), + "GDALOpenEx": ( + [c_char_p, ctypes.c_uint, c_void_p, c_void_p, c_void_p], + c_void_p, + ), + "GDALGetRasterBand": ([c_void_p, c_int], c_void_p), + "GDALRasterIO": ( + [c_void_p] + [c_int] * 5 + [c_void_p] + [c_int] * 5, + c_int, + ), + "GDALClose": ([c_void_p], None), + } + for name, (argtypes, restype) in signatures.items(): + function = getattr(lib, name) + function.argtypes = argtypes + function.restype = restype + + @staticmethod + def _text(value: bytes | None) -> str | None: + return None if value is None else value.decode("utf-8", "replace") + + def version_info(self, request: str) -> str: + return self._text(self._lib.GDALVersionInfo(request.encode())) or "" + + def build_info(self) -> dict[str, str]: + info = {} + for line in self.version_info("BUILD_INFO").splitlines(): + key, _, value = line.partition("=") + if key: + info[key.strip()] = value.strip() + return info + + def _driver_handles(self) -> dict[str, int]: + handles = {} + for index in range(self._lib.GDALGetDriverCount()): + driver = self._lib.GDALGetDriver(index) + if driver: + name = self._text(self._lib.GDALGetDriverShortName(driver)) + handles[name or ""] = driver + return handles + + def drivers(self) -> list[str]: + return sorted(self._driver_handles()) + + def driver_metadata(self, driver: str, key: str) -> str | None: + handle = self._lib.GDALGetDriverByName(driver.encode()) + if not handle: + return None + return self._text( + self._lib.GDALGetMetadataItem(handle, key.encode(), None) + ) + + def deregister_drivers(self, names: set[str]) -> list[str]: + removed = [] + for name, handle in self._driver_handles().items(): + if name in names: + self._lib.GDALDeregisterDriver(handle) + removed.append(name) + return sorted(removed) + + def vsi_prefixes(self) -> list[str]: + prefixes = self._lib.VSIGetFileSystemsPrefixes() + names = [] + index = 0 + while prefixes and prefixes[index]: + names.append(prefixes[index].decode("utf-8", "replace")) + index += 1 + self._lib.CSLDestroy(prefixes) + return sorted(names) + + def remove_vsi_handlers(self, prefixes: set[str]) -> list[str]: + removed = [] + for prefix in sorted(prefixes & set(self.vsi_prefixes())): + self._lib.VSIRemovePluginHandler(prefix.encode()) + removed.append(prefix) + return removed + + def config(self, key: str) -> str | None: + return self._text(self._lib.CPLGetConfigOption(key.encode(), None)) + + @contextmanager + def _thread_config(self, values: dict[str, str]) -> Iterator[None]: + previous = { + key: self._lib.CPLGetThreadLocalConfigOption(key.encode(), None) + for key in values + } + for key, value in values.items(): + self._lib.CPLSetThreadLocalConfigOption( + key.encode(), value.encode() + ) + try: + yield + finally: + for key, value in previous.items(): + self._lib.CPLSetThreadLocalConfigOption(key.encode(), value) + + @contextmanager + def _quiet(self) -> Iterator[None]: + handler = ctypes.cast(self._lib.CPLQuietErrorHandler, ctypes.c_void_p) + self._lib.CPLPushErrorHandler(handler) + try: + yield + finally: + self._lib.CPLPopErrorHandler() + self._lib.CPLErrorReset() + + def try_read(self, filename: str) -> str: + """Open ``filename`` as a raster and read one pixel. + + Returns "rejected" when GDAL refuses to open it, "read-failed" when + it opens but refuses the read, and "read" when the read succeeds. + """ + with self._quiet(): + dataset = self._lib.GDALOpenEx( + filename.encode(), GDAL_OF_RASTER, None, None, None + ) + if not dataset: + return "rejected" + try: + band = self._lib.GDALGetRasterBand(dataset, 1) + if not band: + return "read-failed" + buffer = (ctypes.c_ubyte * 1)() + error = self._lib.GDALRasterIO( + band, GF_READ, 0, 0, 1, 1, buffer, 1, 1, GDT_BYTE, 0, 0 + ) + return "read" if error == 0 else "read-failed" + finally: + self._lib.GDALClose(dataset) + + def probes(self, *, permissive: bool = False) -> dict[str, str]: + """Try the VRT features the runtime must never allow. + + With ``permissive`` the probes run under settings that would allow + each feature, so a refusal shows the feature is not in the build. + """ + settings = PERMISSIVE_CONFIG if permissive else {} + with tempfile.TemporaryDirectory(prefix="haste-gdal-probe-") as tmp: + root = Path(tmp) + (root / "raw.bin").write_bytes(b"\x01") + raw_vrt = Path(tmp) / "raw.vrt" + raw_vrt.write_text(_RAW_BAND_VRT, encoding="utf-8") + python_vrt = root / "python.vrt" + python_vrt.write_text(_PYTHON_PIXEL_FUNCTION_VRT, encoding="utf-8") + plain_vrt = root / "plain.vrt" + plain_vrt.write_text(_PLAIN_VRT, encoding="utf-8") + with self._thread_config(settings): + return { + "raw_band": self.try_read(str(raw_vrt)), + "python_pixel_function": self.try_read(str(python_vrt)), + "vrt_driver": self.try_read(str(plain_vrt)), + } + + +def mapped_libraries(stem: str, maps_text: str | None = None) -> list[str]: + """Distinct mapped shared objects whose file name starts with ``stem``. + + Linux only. Every binding links to the runtime's copy, so more than one + path means some package brought its own library. + """ + if maps_text is None: + try: + maps_text = Path("/proc/self/maps").read_text(encoding="utf-8") + except OSError: + return [] + paths = set() + for line in maps_text.splitlines(): + fields = line.split(maxsplit=5) + if len(fields) < 6: + continue + path = fields[5].strip() + name = os.path.basename(path) + if name.startswith(stem) and ".so" in name: + paths.add(path) + return sorted(paths) diff --git a/native/gdal-runtime/haste_gdal_runtime/gdalrc b/native/gdal-runtime/haste_gdal_runtime/gdalrc new file mode 100644 index 00000000..7dabd684 --- /dev/null +++ b/native/gdal-runtime/haste_gdal_runtime/gdalrc @@ -0,0 +1,6 @@ +# GDAL configuration file pinned by the HASTE GDAL runtime. +# +# haste_gdal_runtime.configure() points GDAL_CONFIG_FILE here, so GDAL never +# loads a gdalrc from the user's home or the system configuration directory. +# Keep this file free of settings: operator-owned GDAL settings belong in the +# deployment configuration, where the runtime check can see them. diff --git a/native/gdal-runtime/install_build_tools.sh b/native/gdal-runtime/install_build_tools.sh new file mode 100644 index 00000000..32986d41 --- /dev/null +++ b/native/gdal-runtime/install_build_tools.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +VENV="${1:?usage: install_build_tools.sh VENV [requirements-file]}" +REQUIREMENTS="${2:-$ROOT/requirements-build.txt}" + +PIP_INDEX_URL="${PIP_INDEX_URL:-https://pypi.org/simple}" + +PYTHON="${PYTHON:-python3}" +"$PYTHON" -m venv "$VENV" +"$VENV/bin/python" -m pip install \ + --isolated \ + --index-url "$PIP_INDEX_URL" \ + --only-binary=:all: \ + --require-hashes \ + -r "$REQUIREMENTS" diff --git a/native/gdal-runtime/patches/zlib-CVE-2026-85091.patch b/native/gdal-runtime/patches/zlib-CVE-2026-85091.patch new file mode 100644 index 00000000..f2510190 --- /dev/null +++ b/native/gdal-runtime/patches/zlib-CVE-2026-85091.patch @@ -0,0 +1,17 @@ +diff --git a/gzwrite.c b/gzwrite.c +index b217b85cef..5123292445 100644 +--- a/gzwrite.c ++++ b/gzwrite.c +@@ -242,8 +242,11 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { + n -= state->strm.avail_in; + state->x.pos += n; + len -= n; +- if (ret == -1) ++ if (ret == -1) { ++ state->strm.avail_in = 0; ++ state->strm.next_in = state->in; + return state->again ? put - len : 0; ++ } + } while (len); + } + diff --git a/native/gdal-runtime/requirements-build-base.txt b/native/gdal-runtime/requirements-build-base.txt new file mode 100644 index 00000000..8db4f87e --- /dev/null +++ b/native/gdal-runtime/requirements-build-base.txt @@ -0,0 +1,12 @@ +# Install from PIP_INDEX_URL, or PyPI when no index is configured. Every +# dependency remains hash-locked regardless of the selected package index. +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 +wheel==0.48.0 \ + --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +numpy==2.4.6 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 +versioneer[toml]==0.28 \ + --hash=sha256:11ffc09427ac99db4ce61bdc85785dae819618d0de28153adfce3085956125a8 diff --git a/native/gdal-runtime/requirements-build.txt b/native/gdal-runtime/requirements-build.txt new file mode 100644 index 00000000..b99e8b5e --- /dev/null +++ b/native/gdal-runtime/requirements-build.txt @@ -0,0 +1,3 @@ +-r requirements-build-base.txt +Cython==3.1.8 \ + --hash=sha256:282b3c8e6abc3fea421919e862e898ffdd86fc0796009bdb5ffdf8211413219f diff --git a/native/gdal-runtime/requirements-fiona-build.txt b/native/gdal-runtime/requirements-fiona-build.txt new file mode 100644 index 00000000..eef21638 --- /dev/null +++ b/native/gdal-runtime/requirements-fiona-build.txt @@ -0,0 +1,5 @@ +# Fiona 1.10.1 requires Cython ~=3.0.2; it cannot share the 3.1.x +# toolchain used by rasterio, pyogrio and pyproj. +-r requirements-build-base.txt +Cython==3.0.12 \ + --hash=sha256:563de1728c8e48869d2380a1b76bbc1b1b1d01aba948480d68c1d05e52d20c92 diff --git a/native/gdal-runtime/requirements-sdists.txt b/native/gdal-runtime/requirements-sdists.txt new file mode 100644 index 00000000..6f4208e5 --- /dev/null +++ b/native/gdal-runtime/requirements-sdists.txt @@ -0,0 +1,14 @@ +# Binding sources use PIP_INDEX_URL, or PyPI when no index is configured, +# and are verified by pip against these hashes. +# --no-binary=:all: is supplied by the fetch script; hashes pin the exact +# sdists and are independent of the package feed URL. +GDAL==3.13.3 \ + --hash=sha256:af547af72d2a153d658c1deb2fe595eccb38c18f3d9d298882fbdeee9806a6d2 +rasterio==1.4.4 \ + --hash=sha256:c95424e2c7f009b8f7df1095d645c52895cd332c0c2e1b4c2e073ea28b930320 +fiona==1.10.1 \ + --hash=sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68 +pyogrio==0.13.0 \ + --hash=sha256:9614f27a1891113f80653e0b76b4233ea1fb3beeb1ac46d118ab22e1670f8f13 +pyproj==3.7.2 \ + --hash=sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c diff --git a/native/gdal-runtime/requirements-smoke.txt b/native/gdal-runtime/requirements-smoke.txt new file mode 100644 index 00000000..306f5ce4 --- /dev/null +++ b/native/gdal-runtime/requirements-smoke.txt @@ -0,0 +1,20 @@ +# Minimal runtime dependencies for the five GDAL-linked bindings. +# Install with --only-binary, --no-deps and --require-hashes. +affine==2.4.0 \ + --hash=sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92 +attrs==25.4.0 \ + --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 +certifi==2025.11.12 \ + --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b +click==8.1.8 \ + --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 +cligj==0.7.2 \ + --hash=sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df +numpy==2.4.6 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pyparsing==3.3.2 \ + --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d diff --git a/native/gdal-runtime/smoke_test.py b/native/gdal-runtime/smoke_test.py new file mode 100644 index 00000000..2fda4629 --- /dev/null +++ b/native/gdal-runtime/smoke_test.py @@ -0,0 +1,114 @@ +"""Install and verify the GDAL runtime and binding wheels offline.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +DRIVERS = ( + "GTiff", + "COG", + "VRT", + "MEM", + "PNG", + "JPEG", + "GeoJSON", + "GeoJSONSeq", + "ESRIJSON", + "TopoJSON", + "GPKG", + "SQLite", +) + + +def binding_smoke_command(script: Path, python: str) -> list[str]: + """Run the source smoke driver without shadowing installed packages. + + The driver is loaded by file path, but its directory is appended to + ``sys.path`` so site-packages takes precedence for haste_gdal_runtime. + """ + code = ( + "import runpy, sys\n" + "from pathlib import Path\n" + "script = sys.argv.pop(1)\n" + "sys.path.append(str(Path(script).resolve().parent))\n" + "runpy.run_path(script, run_name='__main__')\n" + ) + return [python, "-c", code, str(script)] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel-dir", required=True, type=Path) + parser.add_argument("--build", required=True) + parser.add_argument("--expected-gdal", default="3.13.3") + args = parser.parse_args(argv) + + distributions = { + "haste-gdal-runtime": args.expected_gdal, + "GDAL": args.expected_gdal, + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", + } + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--isolated", + "--no-index", + "--no-deps", + "--find-links", + str(args.wheel_dir), + *( + f"{name}=={version}+haste.{args.build}" + for name, version in distributions.items() + ), + ], + check=True, + ) + command = [ + sys.executable, + "-m", + "haste_gdal_runtime", + "verify", + "--expected-gdal", + args.expected_gdal, + ] + for binding in ( + "osgeo.gdal", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + ): + command.extend(["--require-binding", binding]) + for driver in DRIVERS: + command.extend(["--allow-driver", driver]) + subprocess.run(command, check=True, cwd=args.wheel_dir.resolve()) + binding_smoke = Path(__file__).with_name("binding_workflow_smoke.py") + subprocess.run( + binding_smoke_command(binding_smoke, sys.executable), + check=True, + cwd=args.wheel_dir.resolve(), + ) + vrt_smoke = Path(__file__).with_name("titiler_vrt_smoke.py") + vrt_environment = os.environ.copy() + vrt_environment["GDAL_SKIP"] = "VRT" + subprocess.run( + binding_smoke_command(vrt_smoke, sys.executable), + check=True, + cwd=args.wheel_dir.resolve(), + env=vrt_environment, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/sources.lock b/native/gdal-runtime/sources.lock new file mode 100644 index 00000000..b940718b --- /dev/null +++ b/native/gdal-runtime/sources.lock @@ -0,0 +1,24 @@ +# Native sources for the HASTE GDAL runtime. +# +# One component per line: name version url sha256 +# Every hash comes from the upstream release itself: +# - GitHub-computed asset digests for GDAL, PROJ, libjpeg-turbo, zlib, +# OpenSSL, curl and libdeflate; +# - the .sha256 file zstd publishes; +# - for SQLite, the published SHA3-256, checked locally and recorded here as +# SHA-256. +# fetch_sources.py refuses any file that doesn't match. +# +# Bump the runtime build number (HASTE_GDAL_BUILD) with any change here. + +HASTE_GDAL_BUILD 1 + +gdal 3.13.3 https://github.com/OSGeo/gdal/releases/download/v3.13.3/gdal-3.13.3.tar.gz 5e0c388d83da2d686cc00a40272882432cdb54edff43d4af173e532844a0a0ea +proj 9.8.1 https://github.com/OSGeo/PROJ/releases/download/9.8.1/proj-9.8.1.tar.gz af5b731c145c1d13c4e3b4eeb7d167e94e845e440f71e3496b4ed8dae0291960 +sqlite 3.53.4 https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz 0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c +curl 8.22.0 https://github.com/curl/curl/releases/download/curl-8_22_0/curl-8.22.0.tar.gz d54dd598bf05927a726deb38df31c6a255ba83ff1de57c5d1464dac3ed8f44a1 +openssl 3.5.9 https://github.com/openssl/openssl/releases/download/openssl-3.5.9/openssl-3.5.9.tar.gz 603f5602e2eef00d77fbd429d34dcd5822bb301757a1bc9cdb24c670f1eb859a +zlib 1.3.2 https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16 +libdeflate 1.26 https://github.com/ebiggers/libdeflate/releases/download/v1.26/libdeflate-1.26.tar.gz 125856d4656e0feab660f94842f835923410c9281fedbcee64598c918da42b5a +zstd 1.5.7 https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 +libjpeg-turbo 3.1.4.1 https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.4.1/libjpeg-turbo-3.1.4.1.tar.gz ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022 diff --git a/native/gdal-runtime/tests/test_runtime.py b/native/gdal-runtime/tests/test_runtime.py new file mode 100644 index 00000000..cc5553e8 --- /dev/null +++ b/native/gdal-runtime/tests/test_runtime.py @@ -0,0 +1,925 @@ +import io +import json +import os +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import assemble_wheels # noqa: E402 +import baseline_report # noqa: E402 +import create_supply_chain # noqa: E402 +import fetch_sources # noqa: E402 +import haste_gdal_runtime as rt # noqa: E402 +import smoke_test # noqa: E402 +from haste_gdal_runtime._native import mapped_libraries # noqa: E402 +from verify_build_manifest import validate as validate_manifest # noqa: E402 + +LIB = Path("/opt/libs/libgdal-abc.so.38") +PROJ_LIB = "/opt/libs/libproj-abc.so.25" + +MANIFEST = { + "gdal_release": "3.13.3", + "proj_release": "9.8.1", + "binding_versions": { + "osgeo.gdal": "3.13.3", + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", + }, + "drivers": [ + "GTiff", + "COG", + "VRT", + "MEM", + "PNG", + "GPKG", + "GNMDatabase", + "GNMFile", + ], + "vsi_prefixes": ["/vsimem/", "/vsicurl/", "/vsisubfile/"], + "vrt_raw_band_compiled": False, + "probes_permissive": { + "raw_band": "rejected", + "python_pixel_function": "rejected", + }, +} + + +def good_report(): + ca = os.path.abspath(__file__) + return { + "library": str(LIB), + "gdal_release": "3.13.3", + "drivers": list(MANIFEST["drivers"]), + "vsi_prefixes": list(MANIFEST["vsi_prefixes"]), + "vrt_raw_band_compiled": False, + "config": { + **rt.REQUIRED_CONFIG, + **{k: str(v) for k, v in rt.PINNED_PATHS.items()}, + "CURL_CA_BUNDLE": ca, + }, + "ca_bundle": {"path": ca, "exists": True}, + "mapped_libgdal": [str(LIB)], + "mapped_libproj": [PROJ_LIB], + "bindings": { + "osgeo.gdal": {"version": "3.13.3", "gdal": "3.13.3"}, + "pyproj": {"version": "3.7.2", "proj": "9.8.1"}, + }, + "probes": { + "raw_band": "rejected", + "python_pixel_function": "read-failed", + "vrt_driver": "opened", + }, + "probes_permissive": { + "raw_band": "rejected", + "python_pixel_function": "rejected", + }, + } + + +class LockTests(unittest.TestCase): + def test_shipped_lock_parses(self): + text = (ROOT / "sources.lock").read_text(encoding="utf-8") + build, sources = fetch_sources.parse_lock(text) + self.assertTrue(build.isdigit()) + self.assertIn("gdal", {s.name for s in sources}) + + def test_every_locked_native_source_has_sbom_license_metadata(self): + text = (ROOT / "sources.lock").read_text(encoding="utf-8") + _, sources = fetch_sources.parse_lock(text) + missing = sorted( + source.name + for source in sources + if source.name.casefold() + not in create_supply_chain.SOURCE_LICENSES + ) + self.assertEqual(missing, []) + + def test_rejects_plain_http(self): + line = "HASTE_GDAL_BUILD 1\nx 1 http://a/x.tgz " + "a" * 64 + with self.assertRaises(ValueError): + fetch_sources.parse_lock(line) + + def test_rejects_bad_digest(self): + line = "HASTE_GDAL_BUILD 1\nx 1 https://a/x.tgz " + "A" * 64 + with self.assertRaises(ValueError): + fetch_sources.parse_lock(line) + + def test_rejects_duplicates_and_missing_build(self): + row = "x 1 https://a/x.tgz " + "a" * 64 + with self.assertRaises(ValueError): + fetch_sources.parse_lock("HASTE_GDAL_BUILD 1\n" + row + "\n" + row) + with self.assertRaises(ValueError): + fetch_sources.parse_lock(row) + + def test_python_sdists_are_not_in_native_fetch_lock(self): + text = (ROOT / "sources.lock").read_text(encoding="utf-8") + self.assertNotIn("pythonhosted.org", text) + + def test_binding_build_helpers_default_to_hash_pinned_public_index(self): + for name in ("install_build_tools.sh", "fetch_binding_sdists.sh"): + text = (ROOT / name).read_text(encoding="utf-8") + self.assertIn("PIP_INDEX_URL", text) + self.assertIn("https://pypi.org/simple", text) + self.assertIn("--index-url", text) + fetch_script = (ROOT / "fetch_binding_sdists.sh").read_text( + encoding="utf-8" + ) + self.assertIn("--no-build-isolation", fetch_script) + build_script = (ROOT / "build.sh").read_text(encoding="utf-8") + public_index = ( + 'PIP_INDEX_URL="${PIP_INDEX_URL:-' 'https://pypi.org/simple}"' + ) + self.assertIn(public_index, build_script) + self.assertIn("export PIP_INDEX_URL", build_script) + build_bindings = (ROOT / "build_bindings.py").read_text( + encoding="utf-8" + ) + self.assertIn('"--no-index"', build_bindings) + + def test_build_workflow_uses_optional_feed_with_public_default(self): + workflow = ( + ROOT.parents[1] + / ".github" + / "workflows" + / "gdal-runtime-build.yml" + ).read_text(encoding="utf-8") + self.assertIn("secrets.PIP_INDEX_URL", workflow) + self.assertIn("https://pypi.org/simple", workflow) + self.assertIn("environment: gdal-runtime-build", workflow) + self.assertIn("requirements-smoke.txt", workflow) + self.assertIn("dnf install -y nasm openssl", workflow) + self.assertIn("Validate the complete runtime release bundle", workflow) + self.assertIn("publish_gdal_runtime.py", workflow) + self.assertIn("--require-hashes", workflow) + self.assertIn("--only-binary=:all:", workflow) + self.assertIn("--index-url https://pypi.org/simple", workflow) + smoke_test = (ROOT / "smoke_test.py").read_text() + self.assertIn("binding_workflow_smoke.py", smoke_test) + self.assertIn("titiler_vrt_smoke.py", smoke_test) + self.assertIn('vrt_environment["GDAL_SKIP"] = "VRT"', smoke_test) + titiler_smoke = (ROOT / "titiler_vrt_smoke.py").read_text() + self.assertLess( + titiler_smoke.index("runtime.configure(policy)"), + titiler_smoke.index("runtime.secure_startup(policy)"), + ) + binding_smoke = (ROOT / "binding_workflow_smoke.py").read_text() + for distribution in ( + "haste-gdal-runtime", + "GDAL", + "rasterio", + "fiona", + "pyogrio", + "pyproj", + ): + self.assertIn(f'"{distribution}"', smoke_test) + for version in ("1.4.4", "1.10.1", "0.13.0", "3.7.2"): + self.assertIn(f'"{version}"', smoke_test) + for binding in ("rasterio", "fiona", "pyogrio", "pyproj"): + self.assertIn(binding, binding_smoke) + self.assertIn("https://127.0.0.1", binding_smoke) + self.assertIn("WarpedVRT", binding_smoke) + self.assertIn("BuildVRT", binding_smoke) + self.assertIn("HTTPS /vsicurl/", binding_smoke) + self.assertIn("bytes(ogr_geometry.ExportToWkb())", binding_smoke) + self.assertIn( + 'gdal.SetConfigOption("GDAL_CURL_CA_BUNDLE"', binding_smoke + ) + self.assertIn("gdal.GetLastErrorMsg()", binding_smoke) + self.assertIn('compress="ZSTD"', binding_smoke) + self.assertIn("EXPECTED_BINDINGS", binding_smoke) + self.assertIn("--no-index", smoke_test) + self.assertIn("distributions.items()", smoke_test) + self.assertIn('"osgeo.gdal"', smoke_test) + self.assertIn("cwd=args.wheel_dir.resolve()", smoke_test) + + def test_build_workflow_smokes_functions_and_imageryprep_base_images(self): + workflow = ( + ROOT.parents[1] + / ".github" + / "workflows" + / "gdal-runtime-build.yml" + ).read_text(encoding="utf-8") + + self.assertIn(".github/scripts/publish_gdal_runtime.py", workflow) + self.assertIn(".github/scripts/check_gdal_hardening.py", workflow) + self.assertIn(".github/scripts/build_function_package.py", workflow) + self.assertIn( + ".github/scripts/prepare_training_runtime_env.py", workflow + ) + self.assertIn( + "Check repository GDAL hardening configuration", workflow + ) + self.assertIn("infra/modules/functions.bicep", workflow) + self.assertIn("infra/main.json", workflow) + self.assertIn("docker/docker-compose.yml", workflow) + self.assertIn("docker/training/Dockerfile", workflow) + self.assertIn("api/titilerfuncapi/**", workflow) + self.assertIn("target-image-smoke:", workflow) + self.assertIn("actions/download-artifact@", workflow) + self.assertIn("functions-python311", workflow) + self.assertIn("imageryprep-python311", workflow) + self.assertIn( + "mcr.microsoft.com/azure-functions/python:4-python3.11@sha256:", + workflow, + ) + self.assertIn( + "mcr.microsoft.com/azureml/curated/" + "minimal-py311-inference:59@sha256:", + workflow, + ) + self.assertIn("docker run --rm --entrypoint sh", workflow) + self.assertIn("haste-gdal-smoke-deps-${{ github.sha }}", workflow) + self.assertIn("--find-links /smoke-wheels", workflow) + self.assertIn("--no-index", workflow) + self.assertIn("dist/*", workflow) + self.assertIn("function-packages:", workflow) + self.assertIn("build_function_package.py", workflow) + self.assertIn("--app-name imageryprep-smoke", workflow) + self.assertIn("haste-function-packages-${{ github.sha }}", workflow) + self.assertIn("Attest Function App package artifacts", workflow) + self.assertIn("vulnerability-scan:", workflow) + self.assertIn("anchore/scan-action@", workflow) + self.assertIn("fail-build: true", workflow) + self.assertIn("severity-cutoff: medium", workflow) + self.assertIn("add-cpes-if-none: true", workflow) + self.assertIn("Scan built wheel artifacts", workflow) + self.assertIn( + "path: ${{ runner.temp }}/haste-gdal-runtime/dist", workflow + ) + self.assertIn("attest:", workflow) + self.assertIn("github.repository == 'microsoft/haste'", workflow) + self.assertIn("actions/attest@", workflow) + self.assertIn("sbom-path:", workflow) + self.assertIn("publisher-unit:", workflow) + self.assertIn( + "publish:\n needs:\n - attest\n - publisher-unit", + workflow, + ) + self.assertIn("publish:", workflow) + self.assertIn("GDAL_RUNTIME_LICENSE_REVIEWED == 'true'", workflow) + self.assertIn("GDAL_RUNTIME_PUBLISH_ENABLED == 'true'", workflow) + self.assertIn("gh attestation verify", workflow) + self.assertIn( + "microsoft/haste/.github/workflows/gdal-runtime-build.yml", + workflow, + ) + self.assertIn("gdal-runtime-publish", workflow) + self.assertIn(".github/scripts/publish_gdal_runtime.py", workflow) + self.assertIn("training-base-smoke:", workflow) + self.assertIn( + "Smoke-test the full Python 3.11 training environment", workflow + ) + self.assertIn("conda env create", workflow) + self.assertIn("test_gdal_runtime_integration.py", workflow) + self.assertIn("Import the packaged imageryprep worker", workflow) + self.assertIn( + "mcr.microsoft.com/azureml/curated/" + "minimal-py312-cuda12.4-inference@sha256:", + workflow, + ) + + def test_build_script_generates_sbom_notices_and_checksums(self): + build_script = (ROOT / "build.sh").read_text(encoding="utf-8") + supply_chain_script = (ROOT / "create_supply_chain.py").read_text( + encoding="utf-8" + ) + self.assertIn('python3 "$ROOT/create_supply_chain.py"', build_script) + self.assertIn("bom.cdx.json", supply_chain_script) + self.assertIn("THIRD_PARTY_NOTICES.txt", supply_chain_script) + self.assertIn("SHA256SUMS", supply_chain_script) + + def test_supply_chain_bundle_includes_source_wheel_and_notice_evidence( + self, + ): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_dir = root / "sources" + wheel_dir = root / "wheels" + output_dir = root / "output" + source_dir.mkdir() + wheel_dir.mkdir() + + source_archive = source_dir / "sample-1.0.tar.gz" + with tarfile.open(source_archive, "w:gz") as archive: + content = b"sample license text\n" + member = tarfile.TarInfo("sample-1.0/LICENSE") + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + source_hash = create_supply_chain.sha256(source_archive) + sqlite_archive = source_dir / "sqlite-1.0.tar.gz" + with tarfile.open(sqlite_archive, "w:gz") as archive: + content = b"autosetup license text\n" + member = tarfile.TarInfo("sqlite-1.0/autosetup/LICENSE") + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + sqlite_hash = create_supply_chain.sha256(sqlite_archive) + source_lock = root / "sources.lock" + source_lock.write_text( + "HASTE_GDAL_BUILD 1\n" + f"sample 1.0 https://example.test/sample-1.0.tar.gz " + f"{source_hash}\n" + f"sqlite 1.0 https://example.test/sqlite-1.0.tar.gz " + f"{sqlite_hash}\n", + encoding="utf-8", + ) + + distributions = ( + ("haste_gdal_runtime", "haste-gdal-runtime"), + ("GDAL", "GDAL"), + ("rasterio", "rasterio"), + ("fiona", "fiona"), + ("pyogrio", "pyogrio"), + ("pyproj", "pyproj"), + ) + for filename, name in distributions: + wheel_path = wheel_dir / f"{filename}-1.0-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w") as wheel: + license_field = ( + "License-Expression: MIT\n" + if name == "haste-gdal-runtime" + else "License: MIT\n" + ) + if name == "pyogrio": + license_field = ( + "License: MIT License\n" + " Copyright (c) Pyogrio contributors\n" + ) + wheel.writestr( + f"{filename}-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\n" + f"Name: {name}\n" + "Version: 1.0\n" + f"{license_field}\n", + ) + if name in {"haste-gdal-runtime", "pyogrio"}: + wheel.writestr( + f"{filename}-1.0.dist-info/licenses/LICENSE", + ( + "runtime license text\n" + if name == "haste-gdal-runtime" + else "MIT License\nPermission is granted\n" + ), + ) + if name == "haste-gdal-runtime": + wheel.writestr( + "haste_gdal_runtime/manifest.json", + json.dumps( + { + "gdal_release": "3.13.3", + "binding_versions": { + "osgeo.gdal": "1.0", + "rasterio": "1.0", + "fiona": "1.0", + "pyogrio": "1.0", + "pyproj": "1.0", + }, + } + ), + ) + manifest = root / "manifest.json" + manifest.write_text('{"gdal_release":"3.13.3"}\n') + + outputs = create_supply_chain.generate( + source_lock, + source_dir, + wheel_dir, + manifest, + output_dir, + ) + + bom = json.loads( + (output_dir / "bom.cdx.json").read_text(encoding="utf-8") + ) + artifact_manifest = json.loads( + (output_dir / "manifest.json").read_text(encoding="utf-8") + ) + notices = (output_dir / "THIRD_PARTY_NOTICES.txt").read_text( + encoding="utf-8" + ) + checksums = (output_dir / "SHA256SUMS").read_text(encoding="utf-8") + + self.assertEqual(len(outputs), 4) + self.assertEqual(bom["bomFormat"], "CycloneDX") + self.assertEqual(len(bom["components"]), 8) + self.assertEqual( + artifact_manifest["binding_versions"]["pyogrio"], "1.0" + ) + pyogrio = next( + component + for component in bom["components"] + if component["name"] == "pyogrio" + ) + self.assertEqual(pyogrio["licenses"], [{"license": {"id": "MIT"}}]) + runtime = next( + component + for component in bom["components"] + if component["name"] == "haste-gdal-runtime" + ) + self.assertEqual(runtime["licenses"], [{"expression": "MIT"}]) + sqlite = next( + component + for component in bom["components"] + if component["name"] == "sqlite" + ) + self.assertEqual( + sqlite["licenses"], + [ + { + "license": { + "name": "Public Domain", + "url": "https://www.sqlite.org/copyright.html", + } + } + ], + ) + self.assertIn("sample license text", notices) + self.assertIn("runtime license text", notices) + self.assertIn("dedicated to the public domain", notices) + self.assertIn("https://www.sqlite.org/copyright.html", notices) + self.assertIn("MIT License", notices) + self.assertIn("haste_gdal_runtime-1.0-py3-none-any.whl", checksums) + self.assertIn("manifest.json", checksums) + + def test_supply_chain_bundle_rejects_changed_locked_source(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_dir = root / "sources" + wheel_dir = root / "wheels" + output_dir = root / "output" + source_dir.mkdir() + wheel_dir.mkdir() + (source_dir / "sample-1.0.tar.gz").write_bytes(b"wrong source") + (wheel_dir / "placeholder.whl").write_bytes(b"") + source_lock = root / "sources.lock" + source_lock.write_text( + "HASTE_GDAL_BUILD 1\n" + f"sample 1.0 https://example.test/sample-1.0.tar.gz " + f'{"0" * 64}\n', + encoding="utf-8", + ) + manifest = root / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match"): + create_supply_chain.generate( + source_lock, + source_dir, + wheel_dir, + manifest, + output_dir, + ) + + def test_binding_smoke_runner_prefers_installed_runtime_over_checkout( + self, + ): + command = smoke_test.binding_smoke_command( + ROOT / "binding_workflow_smoke.py", "python" + ) + + self.assertEqual(command[1], "-c") + self.assertIn("runpy.run_path", command[2]) + self.assertIn("sys.path.append", command[2]) + + def test_smoke_dependencies_are_exact_and_hash_locked(self): + requirements = (ROOT / "requirements-smoke.txt").read_text( + encoding="utf-8" + ) + for name in ( + "affine", + "attrs", + "certifi", + "click", + "click-plugins", + "cligj", + "numpy", + "packaging", + "pyparsing", + ): + self.assertIn(f"{name}==", requirements) + self.assertEqual(requirements.count("--hash=sha256:"), 9) + + def test_build_script_generates_gdal_config_without_gdal_apps(self): + build_script = (ROOT / "build.sh").read_text(encoding="utf-8") + self.assertIn("-DBUILD_APPS=OFF", build_script) + self.assertIn("-DBUILD_APPS=ON", build_script) + self.assertIn("-DBUILD_PYTHON_BINDINGS=OFF", build_script) + self.assertIn("-DENABLE_GNM=ON", build_script) + self.assertIn("--libdir=lib", build_script) + self.assertIn('"$WORK/gdal-build/apps/gdal-config"', build_script) + self.assertIn( + 'ln -sfn "$PREFIX/bin/gdal-config" /usr/local/bin/gdal-config', + build_script, + ) + self.assertIn('"$PREFIX/bin/gdal-config" --version', build_script) + self.assertIn("GDAL_VRT_DISABLE_PYTHON", build_script) + header_file_macro = "CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED" + self.assertNotIn(header_file_macro, build_script) + + def test_patched_zlib_has_a_private_name_so_it_is_bundled(self): + build_script = (ROOT / "build.sh").read_text(encoding="utf-8") + assembler = (ROOT / "assemble_wheels.py").read_text(encoding="utf-8") + self.assertIn("libhastez.so", build_script) + self.assertIn('ln -sf libhastez.so.1 "$PREFIX/lib/libz.so"', build_script) + self.assertIn("libhastez", assembler) + + def test_binding_sdists_are_fetched_after_gdal_config_is_available(self): + build_script = (ROOT / "build.sh").read_text(encoding="utf-8") + config_created = build_script.index( + 'install -D -m 755 "$WORK/gdal-build/apps/gdal-config"' + ) + config_exported = build_script.index( + 'export GDAL_CONFIG="$PREFIX/bin/gdal-config"' + ) + sdists_fetched = build_script.index( + 'bash "$ROOT/fetch_binding_sdists.sh" "$SOURCES/bindings"' + ) + binding_build = 'python3 "$ROOT/build_bindings.py"' + bindings_built = build_script.index(binding_build) + + self.assertLess(config_created, config_exported) + self.assertIn( + 'PYTHON="$TOOL_VENV/bin/python" \\', + build_script[config_exported:sdists_fetched], + ) + self.assertLess(config_exported, sdists_fetched) + self.assertLess(sdists_fetched, bindings_built) + + +class WheelAssemblyTests(unittest.TestCase): + def test_runtime_wheel_declares_and_bundles_the_project_license(self): + metadata = assemble_wheels.runtime_metadata_files( + "3.13.3+haste.1", assemble_wheels.RUNTIME_WHEEL_TAG + ) + self.assertIn(b"License-Expression: MIT", metadata["METADATA"]) + self.assertEqual( + metadata["licenses/LICENSE"], + (ROOT.parents[1] / "LICENSE").read_bytes(), + ) + + def test_runtime_package_payload_is_rooted_in_importable_package(self): + with tempfile.TemporaryDirectory() as temp_dir: + package = Path(temp_dir) + (package / "__init__.py").write_text( + "VERSION = '3.13.3'\n", encoding="utf-8" + ) + (package / "manifest.json").write_text( + '{"gdal_release": "3.13.3"}\n', encoding="utf-8" + ) + + payload = assemble_wheels.read_runtime_package(package) + + self.assertEqual( + set(payload), + { + "haste_gdal_runtime/__init__.py", + "haste_gdal_runtime/manifest.json", + }, + ) + self.assertIn(b"3.13.3", payload["haste_gdal_runtime/manifest.json"]) + + def test_binding_versions_are_read_from_wheel_metadata(self): + wheels = [ + assemble_wheels.WheelInput( + name=name, + version=f"{version}+haste.1", + archive=Path(f"{name}.whl"), + dist_info=f"{name}.dist-info", + ) + for name, version in ( + ("GDAL", "3.13.3"), + ("rasterio", "1.4.4"), + ("fiona", "1.10.1"), + ("pyogrio", "0.13.0"), + ("pyproj", "3.7.2"), + ) + ] + + self.assertEqual( + assemble_wheels.binding_versions(wheels), + { + "osgeo.gdal": "3.13.3", + "rasterio": "1.4.4", + "fiona": "1.10.1", + "pyogrio": "0.13.0", + "pyproj": "3.7.2", + }, + ) + + def test_maps_binding_data_into_the_combined_wheel(self): + member = "rasterio-1.4.4.data/platlib/rasterio/_base.so" + staging = assemble_wheels.rewrite_data_path( + member, "haste_gdal_runtime-3.13.3+haste.1.data" + ) + self.assertEqual( + staging, + "haste_gdal_runtime-3.13.3+haste.1.data/platlib/" + "rasterio/_base.so", + ) + output = assemble_wheels.restore_data_path( + staging, "rasterio", "1.4.4+haste.1" + ) + self.assertEqual( + output, + "rasterio-1.4.4+haste.1.data/platlib/rasterio/_base.so", + ) + + def test_binding_metadata_pins_runtime_and_removes_old_runtime(self): + metadata = ( + b"Metadata-Version: 2.1\n" + b"Name: rasterio\n" + b"Version: 1.4.4\n" + b"Requires-Dist: haste-gdal-runtime>=3\n" + b"Requires-Dist: numpy\n" + ) + updated = assemble_wheels.update_metadata( + metadata, "1.4.4+haste.1", "3.13.3+haste.1" + ) + self.assertIn(b"Version: 1.4.4+haste.1", updated) + self.assertIn( + b"Requires-Dist: haste-gdal-runtime==3.13.3+haste.1", updated + ) + self.assertIn(b"Requires-Dist: numpy", updated) + self.assertNotIn(b"haste-gdal-runtime>=3", updated) + + def test_runtime_pin_stays_in_headers_before_long_description(self): + metadata = ( + b"Metadata-Version: 2.1\n" + b"Name: fiona\n" + b"Version: 1.10.1\n" + b"Requires-Dist: haste-gdal-runtime\n" + b" >=3\n" + b"Requires-Dist: attrs>=19.2\n" + b"\n" + b"Fiona package description.\n" + ) + + updated = assemble_wheels.update_metadata( + metadata, "1.10.1+haste.1", "3.13.3+haste.1" + ) + headers, body = updated.split(b"\n\n", 1) + + self.assertIn( + b"Requires-Dist: haste-gdal-runtime==3.13.3+haste.1", headers + ) + self.assertIn(b"Requires-Dist: attrs>=19.2", headers) + self.assertNotIn(b"haste-gdal-runtime\n >=3", headers) + self.assertEqual(body, b"Fiona package description.\n") + + def test_local_version_discards_any_existing_local_suffix(self): + self.assertEqual( + assemble_wheels.local_version("1.2.3+source", "7"), + "1.2.3+haste.7", + ) + + +class MappedLibrariesTests(unittest.TestCase): + MAPS = ( + "7f00-7f01 r-xp 0 00:00 1 /a/libgdal-1.so.38\n" + "7f02-7f03 r--p 0 00:00 1 /a/libgdal-1.so.38\n" + "7f04-7f05 r-xp 0 00:00 2 /b/libgdal.so.36\n" + "7f06-7f07 r-xp 0 00:00 3 /c/libgdal_extra.txt\n" + "7f08-7f09 r-xp 0 00:00 0\n" + ) + + def test_distinct_paths(self): + self.assertEqual( + mapped_libraries("libgdal", self.MAPS), + ["/a/libgdal-1.so.38", "/b/libgdal.so.36"], + ) + + def test_none_found(self): + self.assertEqual(mapped_libraries("libproj", self.MAPS), []) + + def test_baseline_report_records_distinct_native_libraries(self): + report = baseline_report.collect_report(self.MAPS) + self.assertEqual( + report["libraries"]["libgdal"], + ["/a/libgdal-1.so.38", "/b/libgdal.so.36"], + ) + self.assertEqual(report["libraries"]["libproj"], []) + self.assertIsNone(report["native"]["library"]) + + +class ConfigureTests(unittest.TestCase): + def test_sets_defaults_and_keeps_operator_values(self): + env = {"GDAL_DISABLE_READDIR_ON_OPEN": "TRUE"} + rt.configure(environ=env) + self.assertEqual(env["GDAL_VRT_ENABLE_RAWRASTERBAND"], "NO") + self.assertEqual(env["GDAL_DRIVER_PATH"], "disable") + self.assertEqual(env["CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED"], "NO") + self.assertEqual(env["GDAL_DISABLE_READDIR_ON_OPEN"], "TRUE") + self.assertEqual(env["GDAL_DATA"], str(rt.GDAL_DATA_DIR)) + + def test_skip_list_merges_with_existing(self): + policy = rt.Policy("3.13.3", allowed_drivers=frozenset({"GTiff"})) + env = {"GDAL_SKIP": "PNG, Foo"} + rt.configure(policy, environ=env, manifest=MANIFEST) + self.assertEqual( + sorted(env["GDAL_SKIP"].split(",")), + [ + "COG", + "Foo", + "GNMDatabase", + "GNMFile", + "GPKG", + "MEM", + "PNG", + "VRT", + ], + ) + + def test_skip_list_empty_without_allowlist(self): + self.assertEqual(rt.skip_list(MANIFEST, rt.Policy("3.13.3")), []) + + +class EvaluateTests(unittest.TestCase): + policy = rt.Policy("3.13.3") + + def run_eval(self, observed, manifest=MANIFEST, policy=None): + return rt.evaluate(observed, manifest, policy or self.policy) + + def test_good_report_has_no_problems(self): + self.assertEqual(self.run_eval(good_report()), []) + + def test_wrong_version(self): + observed = good_report() + observed["gdal_release"] = "3.12.0" + self.assertTrue(self.run_eval(observed)) + + def test_binding_on_other_gdal(self): + observed = good_report() + observed["bindings"]["osgeo.gdal"]["gdal"] = "3.9.2" + self.assertTrue(self.run_eval(observed)) + + def test_binding_package_version_mismatch(self): + observed = good_report() + observed["bindings"]["pyproj"]["version"] = "3.7.1" + problems = self.run_eval(observed) + self.assertTrue( + any("pyproj reports version" in item for item in problems) + ) + + def test_binding_version_missing_from_manifest_fails_closed(self): + manifest = dict(MANIFEST) + manifest.pop("binding_versions") + problems = self.run_eval(good_report(), manifest) + self.assertTrue( + any("no binding version inventory" in item for item in problems) + ) + + def test_second_libgdal(self): + observed = good_report() + observed["mapped_libgdal"].append("/x/libgdal.so.36") + self.assertTrue(self.run_eval(observed)) + + def test_missing_libproj(self): + observed = good_report() + observed["mapped_libproj"] = [] + problems = self.run_eval(observed) + self.assertTrue( + any( + "expected exactly one runtime libproj" in item + for item in problems + ) + ) + + def test_unknown_driver_and_vsi(self): + observed = good_report() + observed["drivers"].append("HDF5") + observed["vsi_prefixes"].append("/vsis3/") + self.assertEqual(len(self.run_eval(observed)), 3) + + def test_missing_policy_driver_or_vsi_fails(self): + observed = good_report() + observed["drivers"].remove("GTiff") + observed["vsi_prefixes"].remove("/vsicurl/") + problems = self.run_eval(observed) + self.assertEqual(len(problems), 2) + self.assertTrue(any("GTiff" in problem for problem in problems)) + self.assertTrue(any("/vsicurl/" in problem for problem in problems)) + + def test_relaxed_config(self): + observed = good_report() + observed["config"]["GDAL_VRT_ENABLE_RAWRASTERBAND"] = "YES" + observed["config"]["GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE"] = "ALL" + self.assertEqual(len(self.run_eval(observed)), 2) + + def test_raw_band_compiled_in(self): + observed = good_report() + observed["vrt_raw_band_compiled"] = True + self.assertTrue(self.run_eval(observed)) + + def test_probe_that_succeeds(self): + observed = good_report() + observed["probes_permissive"]["python_pixel_function"] = "opened" + self.assertTrue(self.run_eval(observed)) + + def test_vrt_must_be_rejected_when_not_allowed(self): + policy = rt.Policy("3.13.3", allowed_drivers=frozenset({"GTiff"})) + observed = good_report() + observed["drivers"] = ["GTiff"] + problems = self.run_eval(observed, policy=policy) + self.assertEqual(len(problems), 1) + self.assertIn("VRT", problems[0]) + + def test_missing_ca_bundle(self): + observed = good_report() + observed["ca_bundle"] = {"path": None, "exists": False} + self.assertTrue(self.run_eval(observed)) + + def test_missing_required_binding(self): + policy = rt.Policy("3.13.3", required_bindings=("rasterio",)) + self.assertTrue(self.run_eval(good_report(), policy=policy)) + + +class ManifestTests(unittest.TestCase): + def test_approved_manifest_is_accepted(self): + manifest = { + "gdal_release": "3.13.3", + "proj_release": "9.8.1", + "vrt_raw_band_compiled": False, + "drivers": sorted( + { + "COG", + "GTiff", + "VRT", + "MEM", + "PNG", + "JPEG", + "GeoJSON", + "GeoJSONSeq", + "ESRIJSON", + "TopoJSON", + "GPKG", + "SQLite", + "GNMDatabase", + "GNMFile", + } + ), + "probes_permissive": { + "raw_band": "rejected", + "python_pixel_function": "read-failed", + }, + } + self.assertEqual(validate_manifest(manifest), []) + + def test_manifest_rejects_extra_driver_or_enabled_raw_band(self): + manifest = { + "gdal_release": "3.13.3", + "proj_release": "9.8.1", + "vrt_raw_band_compiled": True, + "drivers": ["GTiff", "HDF5"], + "probes_permissive": { + "raw_band": "read", + "python_pixel_function": "rejected", + }, + } + errors = validate_manifest(manifest) + self.assertGreaterEqual(len(errors), 3) + self.assertTrue(any("inventory" in error for error in errors)) + + def test_manifest_rejects_unapproved_proj_release(self): + manifest = { + "gdal_release": "3.13.3", + "proj_release": "9.7.0", + "vrt_raw_band_compiled": False, + "drivers": sorted( + { + "COG", + "GTiff", + "VRT", + "MEM", + "PNG", + "JPEG", + "GeoJSON", + "GeoJSONSeq", + "ESRIJSON", + "TopoJSON", + "GPKG", + "SQLite", + "GNMDatabase", + "GNMFile", + } + ), + "probes_permissive": { + "raw_band": "rejected", + "python_pixel_function": "read-failed", + }, + } + self.assertTrue( + any( + "PROJ release" in error + for error in validate_manifest(manifest) + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/native/gdal-runtime/titiler_vrt_smoke.py b/native/gdal-runtime/titiler_vrt_smoke.py new file mode 100644 index 00000000..20210f81 --- /dev/null +++ b/native/gdal-runtime/titiler_vrt_smoke.py @@ -0,0 +1,57 @@ +"""Verify rasterio WarpedVRT reprojection with the VRT driver disabled.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import haste_gdal_runtime as runtime + + +def main() -> int: + manifest = runtime.load_manifest() + policy = runtime.Policy( + expected_gdal=manifest["gdal_release"], + allowed_drivers=frozenset({"GTiff", "COG", "JPEG", "PNG", "MEM"}), + required_bindings=("rasterio",), + ) + runtime.configure(policy) + runtime.secure_startup(policy) + + import numpy as np + import rasterio + from osgeo import gdal + from rasterio.transform import from_origin + from rasterio.vrt import WarpedVRT + + if gdal.GetDriverByName("VRT") is not None: + raise RuntimeError( + "The VRT driver is registered in the TiTiler policy" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "source.tif" + with rasterio.open( + path, + "w", + driver="GTiff", + width=16, + height=16, + count=1, + dtype="uint8", + crs="EPSG:4326", + transform=from_origin(-100, 40, 0.01, 0.01), + ) as dataset: + dataset.write(np.ones((1, 16, 16), dtype="uint8")) + + with rasterio.open(path) as source: + with WarpedVRT(source, crs="EPSG:3857") as warped: + if not warped.read(1).any(): + raise RuntimeError("WarpedVRT returned an empty raster") + + print("WarpedVRT reprojection passed with the VRT driver disabled") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/native/gdal-runtime/verify_build_manifest.py b/native/gdal-runtime/verify_build_manifest.py new file mode 100644 index 00000000..ca4c37ae --- /dev/null +++ b/native/gdal-runtime/verify_build_manifest.py @@ -0,0 +1,77 @@ +"""Fail the native build if its GDAL manifest misses the approved baseline.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +EXPECTED_DRIVERS = { + "COG", + "GTiff", + "VRT", + "MEM", + "PNG", + "JPEG", + "GeoJSON", + "GeoJSONSeq", + "ESRIJSON", + "TopoJSON", + "GPKG", + "SQLite", + "GNMDatabase", + "GNMFile", +} + + +def validate(manifest: dict[str, object]) -> list[str]: + errors = [] + if manifest.get("gdal_release") != "3.13.3": + errors.append( + "GDAL release is " + f"{manifest.get('gdal_release')!r}, expected 3.13.3" + ) + if manifest.get("proj_release") != "9.8.1": + errors.append( + "PROJ release is " + f"{manifest.get('proj_release')!r}, expected 9.8.1" + ) + if manifest.get("vrt_raw_band_compiled") is not False: + errors.append("VRTRawRasterBand is compiled into libgdal") + + drivers = set(manifest.get("drivers", [])) + if drivers != EXPECTED_DRIVERS: + errors.append( + "GDAL driver inventory differs: " + f"missing={sorted(EXPECTED_DRIVERS - drivers)}, " + f"unexpected={sorted(drivers - EXPECTED_DRIVERS)}" + ) + + probes = manifest.get("probes_permissive", {}) + if not isinstance(probes, dict): + errors.append("manifest has no permissive VRT probe results") + else: + for name in ("raw_band", "python_pixel_function"): + if probes.get(name) not in ("rejected", "read-failed"): + errors.append( + f"permissive {name} probe returned {probes.get(name)!r}" + ) + return errors + + +def main(argv: list[str]) -> int: + if len(argv) != 1: + print("usage: verify_build_manifest.py MANIFEST.json", file=sys.stderr) + return 2 + manifest = json.loads(Path(argv[0]).read_text(encoding="utf-8")) + errors = validate(manifest) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("GDAL build manifest meets the approved baseline") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/spec/architecture/decisions/0007-gdal-native-runtime.md b/spec/architecture/decisions/0007-gdal-native-runtime.md new file mode 100644 index 00000000..5d9707f2 --- /dev/null +++ b/spec/architecture/decisions/0007-gdal-native-runtime.md @@ -0,0 +1,186 @@ +# ADR-0007: Build a shared GDAL native runtime and deploy immutable Function App packages + +**Status:** accepted +**Date:** 2026-10-02 +**Deciders:** HASTE engineering team + +## Context + +### Too many GDAL builds, and no recipe + +GDAL-bearing code in HASTE runs in several places: + +- the `hastefuncapi`, `hastefuncqueues` and `titilerfuncapi` Function Apps; +- the imageryprep and training Batch images; +- local development and CI. + +Each Python binding brings its own libgdal: + +- the GDAL 3.9.2 wheel on `haste-binaries`, built outside the repository; +- the copies bundled inside the rasterio, fiona and pyogrio wheels; +- conda's GDAL in the training image. + +A single process therefore loads several GDAL versions, each with its own +driver registry. [ADR-0004](0004-gdal-driver-allowlist.md) assumed one +libgdal per process, and that doesn't hold for wheel installs. Its +in-process allowlist only reaches the osgeo copy. + +### Why the patched GDAL hasn't been adopted + +Moving to the patched GDAL 3.13 line was deferred because no trusted wheel +exists for HASTE's Python 3.11 runtimes: + +- Official GDAL Python releases are source-only. +- rasterio wheels that bundle a newer GDAL require Python 3.12 and bundle + GDAL 3.12. +- pyogrio bundles GDAL 3.12. +- fiona's latest release bundles GDAL 3.9. + +### Deployments aren't one artifact + +Function Apps resolve their dependencies on Azure at deploy time, so dev and +prod aren't guaranteed to run the same bytes. + +## Options Considered + +### Option A: Wait for third-party GDAL 3.13 wheels + +- **Pros:** No build infrastructure to own. +- **Cons:** + - No date for such wheels. + - Several libgdal copies per process remain. + - Configuration and driver set stay outside HASTE's control. +- **Impact on HASTE components:** None. The deferral continues. + +### Option B: Move to Python 3.12 and the newest third-party wheels + +- **Pros:** Standard tooling. +- **Cons:** + - Those wheels bundle GDAL 3.12, not 3.13. + - Each binding still vendors its own copy. + - A Python upgrade across every runtime at the same time. +- **Impact on HASTE components:** Every image and Function App changes + Python version. + +### Option C: conda-forge environments + +- **Pros:** One shared libgdal 3.13 per environment, prebuilt. +- **Cons:** + - rasterio 1.5 needs Python 3.12. + - A conda prefix can't safely be shipped inside a Flex Consumption + package, because of C++ runtime conflicts with the Functions worker. + - A second packaging ecosystem. +- **Impact on HASTE components:** The images could adopt it; the Function + Apps couldn't. + +### Option D: Build the native runtime in CI, shared by every binding + +- **Pros:** + - GDAL 3.13.3 on Python 3.11. + - One libgdal and one PROJ per process. + - Build-time driver minimization; raw VRT bands compiled out. + - Reviewable inputs, SBOM and provenance. + - Usable in Function Apps and images alike. +- **Cons:** + - HASTE owns the build. + - HASTE owns rebuilds when a vendored component has a security fix. +- **Impact on HASTE components:** A new build pipeline. Every GDAL consumer + switches to the runtime's wheels. + +## Decision + +Adopt **Option D**. + +### Build the runtime + +CI builds GDAL 3.13.3 and its native dependencies from pinned, +checksum-verified sources in a digest-pinned `manylinux_2_28` container: + +- optional drivers off, then the audited set enabled explicitly; +- raw VRT bands compiled out; +- no GEOS, no CLI tools. + +### Link every binding to it + +GDAL, rasterio, fiona, pyogrio and pyproj are built against the runtime. One +`auditwheel` pass grafts the native libraries into the shared +`haste-gdal-runtime` wheel that every binding links to. + +### Publish with evidence + +The wheels are published to `haste-binaries`, immutable, with: + +- a manifest of versions, drivers and virtual file systems; +- an SBOM; +- a provenance attestation. + +### Check at startup + +Every runtime runs the runtime package's self-check at startup and refuses +to start when the live process differs from the manifest or from HASTE's +policy. The policy covers: + +- the expected version; +- VRT raw bands and Python pixel functions disabled; +- plugin loading disabled; +- no forbidden configuration. + +### Deploy one artifact + +Function App packages are built once, in the digest-pinned Functions Python +image, from hash-locked requirements. That same package is deployed with +`--no-build`. Images are deployed by digest. + +### Amendment to ADR-0004 + +The driver allowlist moves to build time, and runtime deregistration remains +as defense in depth. With one shared libgdal, deregistration now reaches +every binding. Untrusted inputs are opened with an explicit driver allowlist +that excludes VRT. + +TiTiler is the exception to disabling the VRT driver: rio-tiler reprojects +through in-memory VRTs, so `GDAL_SKIP=VRT` makes tile rendering fail. TiTiler +keeps the driver registered, with raw bands and Python pixel functions off, +and still opens only GTiff/COG from untrusted sources. + +### Components Affected + +| Component | Path | Change | +|---|---|---| +| Runtime recipe | `native/gdal-runtime/` | New: sources, build scripts, packaging, self-check | +| CI | `.github/workflows/` | New runtime build and publish workflow; dependency validation for the runtime | +| hastegeo | `hastelib/src/hastegeo/core/utils/` | Self-check policy; untrusted-open helpers | +| Function Apps | `api/*/requirements*`, `.github/scripts/deploy_apps.sh`, `azure.yaml` | Runtime wheels; build-once packages | +| Images | `docker/imageryprep/`, `docker/training/` | Runtime wheels; build-time self-check | + +### Azure Services Affected + +| Service | Change | +|---|---| +| Azure Functions (Flex Consumption) | Deployed from a prebuilt package (`--no-build`); GDAL hardening app settings | +| Azure Batch | Images rebuilt on the runtime | + +## Consequences + +**Easier:** +- Proving which GDAL runs where. +- Hardening one driver registry per process. +- Reviewing exactly what is compiled in. +- Shipping the same bytes to every environment. + +**Harder:** +- HASTE owns rebuilds when GDAL, PROJ, curl, OpenSSL, SQLite or the + compression libraries have a security fix. +- Python-version upgrades need a matching binding build. + +**New constraints:** +- GDAL-linked bindings must come from the runtime set, never from PyPI + binaries. +- Runtime and bindings must share a build number. + +**Local dev:** The Docker Compose services install the same wheels, and the +local TiTiler builds from `api/titilerfuncapi`. + +**CI/CD:** Adds the runtime build workflow and a hash-locked dependency +check. The deploy workflow packages Function Apps instead of building them +remotely. diff --git a/spec/features/gdal-3-13-runtime/README.md b/spec/features/gdal-3-13-runtime/README.md new file mode 100644 index 00000000..eb4c10ef --- /dev/null +++ b/spec/features/gdal-3-13-runtime/README.md @@ -0,0 +1,108 @@ +# Feature: GDAL 3.13 Native Runtime + +**Status:** in-progress +**Author:** HASTE engineering team +**Date:** 2026-10-02 +**Target Release:** next +**Priority:** P0 +**Work Item:** — + +## Summary + +HASTE currently loads several different GDAL builds per process: the +GDAL 3.9.2 bindings wheel, plus the copies bundled inside the rasterio, fiona +and pyogrio wheels. The GDAL 3.9.2 wheel was built outside the repository. + +This feature replaces all of them with **one HASTE-built GDAL 3.13.3 native +runtime**: + +- **Built in CI** from pinned, checksum-verified sources, with a minimal + driver set and raw VRT bands compiled out. +- **Shared by every binding.** Every GDAL-linked Python binding (GDAL/osgeo, + rasterio, fiona, pyogrio, pyproj) is built against it, so each process + loads exactly one libgdal and one PROJ. +- **Published with evidence.** The runtime and bindings are published as + attested, immutable assets alongside a manifest of the drivers and virtual + file systems they contain. + +Every GDAL-bearing runtime then installs exactly those binaries and checks +itself against the manifest at startup. + +## Motivation + +- **Patched GDAL.** GDAL 3.9.2 carries memory-safety CVEs fixed in the 3.13 + line. The upgrade has been deferred because no trusted prebuilt wheel + exists for HASTE's Python 3.11 Linux runtimes (see + [`known-vulnerabilities.md`](../../../docs/known-vulnerabilities.md) Root + Cause C). +- **No such wheel exists today.** Official GDAL Python releases are + source-only. Wheels that bundle a newer GDAL either require Python 3.12 or + bundle GDAL 3.12, not 3.13. +- **One registry to harden.** Several libgdal copies per process mean + several driver registries. Driver restrictions applied through one binding + don't apply to the others. +- **Provenance and immutability.** HASTE needs a GDAL build whose sources, + configuration and resulting driver set are reviewable and reproducible. + Every deployment should run one immutable artifact that can be checked + from inside the running process. + +## Success Criteria + +- [ ] Each deployed runtime reports GDAL 3.13.3, measured from inside the + process through every binding, and loads one libgdal. +- [ ] Each Python binding was built against, and is linked to, the same + runtime. +- [ ] `GDAL_VRT_ENABLE_RAWRASTERBAND=NO` and `GDAL_VRT_ENABLE_PYTHON=NO` are + active. A raw-band VRT and a Python pixel-function VRT both fail to + open. +- [ ] Callers can't change GDAL configuration, driver paths or plugin paths, + and plugin loading is disabled. +- [ ] The driver and virtual-file-system inventory of each runtime is + published, and anything outside the needed set is absent or disabled. +- [ ] Untrusted inputs are only ever opened with an explicit driver + allowlist that excludes VRT. +- [ ] No deployment configuration relaxes raw-VRT source restrictions. A CI + guard and the startup self-check both enforce this. +- [ ] Function App packages are built once and deployed unchanged, and + images are pinned by digest. + +## HASTE Components Affected + +| Component | Impact | +|---|---| +| `native/gdal-runtime/` | New: source lock, build scripts, manifest and smoke tests | +| `.github/workflows/` | New runtime build and publish workflow; dependency validation; deploy packaging | +| `hastelib/src/hastegeo/core/utils/` | Runtime self-check; untrusted-open restrictions; `harden_gdal()` rework | +| `api/hastefuncapi/`, `api/hastefuncqueues/` | Pinned runtime wheels; build-once packages | +| `api/titilerfuncapi/` | Locked dependencies on the HASTE runtime; startup self-check; VRT driver policy | +| `docker/imageryprep/`, `docker/training/` | Runtime wheels; hardening env; build-time self-check | +| `infra/`, `.github/scripts/deploy_apps.sh`, `azure.yaml` | GDAL hardening settings; package deployment | + +## Related Specs + +| Spec | Relationship | +|---|---| +| [gdal-compensating-controls](../gdal-compensating-controls/) | supersedes the deferral this feature closes | +| [function-network-baseline](../function-network-baseline/) | related: limits who can reach TiTiler | +| [ADR-0004](../../architecture/decisions/0004-gdal-driver-allowlist.md) | amended by [ADR-0007](../../architecture/decisions/0007-gdal-native-runtime.md) | + +## Document Index + +| Document | Purpose | Status | +|---|---|---| +| [design.md](design.md) | Technical design | in-progress | +| [impact-analysis.md](impact-analysis.md) | Risk, dependencies, blast radius | in-progress | +| [user-stories.md](user-stories.md) | Stories, acceptance criteria and agent map | in-progress | +| [plan.md](plan.md) | Phases and tasks | in-progress | +| [test-plan.md](test-plan.md) | Test strategy and evidence | in-progress | +| [rollout.md](rollout.md) | Rollout order and rollback | in-progress | + +## Decision Log + +| Date | Decision | Rationale | +|---|---|---| +| 2026-10-02 | Build the GDAL stack from source in CI instead of consuming third-party wheels | No trusted GDAL 3.13 Python 3.11 wheel exists. Owning the build makes the configuration and driver set reviewable. | +| 2026-10-02 | One shared native runtime wheel that every binding links to | One libgdal and one PROJ per process: one driver registry to harden, and version consistency that is provable. | +| 2026-10-02 | Stay on Python 3.11 and build rasterio 1.4.4 | rasterio ≥ 1.5 requires Python 3.12. Moving Python is a separate change; the pipeline is parameterized by Python version. | +| 2026-10-02 | Build GDAL without GEOS | HASTE uses no cutlines or OGR geometry operations, so one fewer library to patch. | +| 2026-10-02 | Function App packages built once and deployed unchanged | A remote build re-resolves dependencies per environment, so a deployment is not one immutable artifact. | diff --git a/spec/features/gdal-3-13-runtime/design.md b/spec/features/gdal-3-13-runtime/design.md new file mode 100644 index 00000000..aac96c27 --- /dev/null +++ b/spec/features/gdal-3-13-runtime/design.md @@ -0,0 +1,280 @@ +# Technical Design: GDAL 3.13 Native Runtime + +## Contents + +- [Overview](#overview) +- [Runtime composition](#runtime-composition) +- [Build](#build) +- [Packaging: one shared native runtime](#packaging-one-shared-native-runtime) +- [Runtime configuration and self-check](#runtime-configuration-and-self-check) +- [Untrusted input](#untrusted-input) +- [VRT and virtual file system policy](#vrt-and-virtual-file-system-policy) +- [Consumers](#consumers) +- [Publishing and supply chain](#publishing-and-supply-chain) +- [Open questions](#open-questions) + +## Overview + +HASTE builds GDAL 3.13.3 and its native dependencies from pinned, +checksum-verified sources in CI, and ships the result as one +`haste-gdal-runtime` wheel. Every GDAL-linked Python binding is compiled +against that runtime and linked to it, so a process loads one libgdal and one +PROJ: + +- GDAL (osgeo) +- rasterio +- fiona +- pyogrio +- pyproj + +The runtime wheel also carries: + +- a **manifest**: component versions, compiled drivers, virtual file system + prefixes and build options; +- a **self-check** that compares a live process against the manifest and a + caller-supplied policy. + +## Runtime composition + +| Component | Version | Source | Notes | +|---|---|---|---| +| GDAL | 3.13.3 | OSGeo release tarball | Minimal driver set; raw VRT bands compiled out | +| PROJ | 9.8.1 | OSGeo release tarball | Built without curl or libtiff, so no network grid downloads | +| SQLite | 3.53.4 | sqlite.org amalgamation | Needed by PROJ (`proj.db`) and GeoPackage | +| curl | 8.22.0 | curl release tarball | HTTPS-only build, for `/vsicurl/` reads | +| OpenSSL | 3.5.9 (LTS) | OpenSSL release tarball | TLS for curl | +| zlib | 1.3.2 | zlib release tarball | DEFLATE | +| libdeflate | 1.26 | release tarball | Fast DEFLATE in GTiff | +| zstd | 1.5.7 | release tarball | ZSTD-compressed TIFFs from providers | +| libjpeg-turbo | 3.1.4.1 | release tarball | JPEG previews; JPEG-in-TIFF | +| libtiff, libgeotiff, libpng, json-c | GDAL internal copies | GDAL tarball | Keeps the external surface small | + +**Python bindings, built from source distributions against the runtime:** + +| Binding | Version | Why this version | +|---|---|---| +| GDAL (osgeo) | 3.13.3 | Matches the runtime; includes `gdal_array` (NumPy) | +| rasterio | 1.4.4 | Last release that supports Python 3.11 | +| fiona | 1.10.1 | Latest release | +| pyogrio | 0.13.0 | Latest release | +| pyproj | 3.7.2 | Last release that supports Python 3.11 (3.8 needs 3.12) | + +**Exact pins live in `native/gdal-runtime/`:** + +- `sources.lock`: URL and SHA-256 for every native tarball. +- `requirements-build.txt`: the hash-locked build toolchain and binding source + distributions. + +The build refuses any file whose hash doesn't match. + +Wheels carry a local version label tied to the runtime build, for example +`rasterio 1.4.4+haste.1`. Binding wheels declare +`Requires-Dist: haste-gdal-runtime==`. pip therefore refuses to +mix bindings from one runtime build with another, or with PyPI binaries. + +## Build + +The build runs in a digest-pinned `manylinux_2_28` container. glibc 2.28 is +the baseline; the Functions Python image and both Batch base images are +newer. + +**GDAL CMake configuration:** + +| Setting | Value | Effect | +|---|---|---| +| `GDAL_BUILD_OPTIONAL_DRIVERS` / `OGR_BUILD_OPTIONAL_DRIVERS` | `OFF` | Only explicitly enabled drivers are built | +| Explicitly enabled | GTiff (includes COG), JPEG, PNG, MEM, VRT, GPKG, GeoJSON, and OGR memory | The audited set HASTE reads and writes | +| Raw VRT bands | compiled out | `VRTRawRasterBand` cannot be created, whatever the configuration | +| `GDAL_USE_CURL` | `ON` | HTTPS range reads for `/vsicurl/` (TiTiler, label tasks) | +| `GDAL_USE_ARCHIVE`, GEOS, HDF4/HDF5/netCDF, JP2, WebP, LERC and every other optional library | `OFF` | Not compiled at all | +| `BUILD_APPS` | `OFF` | No GDAL command-line tools ship | + +The configuration the build actually produced is recorded in the manifest, +including drivers that GDAL always builds. The manifest is the reviewed +driver inventory. + +## Packaging: one shared native runtime + +The goal is that each binding links to one shared libgdal rather than +bundling its own copy. The build does that in four steps: + +1. Build each binding from its source distribution against the installed + runtime. +2. Merge all binding wheels into a single staging wheel. +3. Run `auditwheel repair` once on the staging wheel. auditwheel grafts + libgdal, PROJ and their dependencies into one `haste_gdal_runtime.libs/` + directory. It gives each library a hashed name, so the vendored copies + cannot collide with system libraries of the same name, and points every + extension module at that directory. +4. Split the repaired staging wheel back into per-distribution wheels: + +| Wheel | Contents | +|---|---| +| `haste-gdal-runtime` (`py3-none-manylinux_2_28_x86_64`) | `haste_gdal_runtime.libs/` (shared libraries); `haste_gdal_runtime/` (GDAL and PROJ data, manifest, self-check) | +| `GDAL`, `rasterio`, `fiona`, `pyogrio`, `pyproj` (`cp311-cp311-manylinux_2_28_x86_64`) | The binding's own packages and dist-info, linked to `../haste_gdal_runtime.libs` | + +Because auditwheel names libraries by content hash, bindings built for +another Python version in the same run link to the same runtime file names. + +## Runtime configuration and self-check + +`haste_gdal_runtime` provides a small API: + +- **`configure()`** points `GDAL_DATA` and `PROJ_DATA` at the packaged data. + It also sets the hardening options below as defaults when the environment + doesn't set them. It runs before any binding is imported. +- **`report()`** returns: + - the libgdal paths mapped into the process; + - the version reported by libgdal and by each imported binding; + - registered drivers, VSI prefixes, build info and the relevant + configuration values. + + It queries the single libgdal through `ctypes`, so it doesn't need osgeo + and works in TiTiler. +- **`verify(policy)`** returns every difference between the live process and + the manifest plus the caller's policy, where the policy covers: + - the expected GDAL version; + - whether VRT is allowed; + - the required configuration. + + `verify_or_raise(policy)` turns any difference into a startup failure. + +**Hardening configuration.** The operator sets these, and `configure()` +defaults them: + +| Option | Value | Notes | +|---|---|---| +| `GDAL_VRT_ENABLE_RAWRASTERBAND` | `NO` | Belt and braces; raw bands are already compiled out | +| `GDAL_VRT_ENABLE_PYTHON` | `NO` | No Python pixel functions | +| `GDAL_DRIVER_PATH` | `disable` | No plugin loading | +| `GDAL_DISABLE_READDIR_ON_OPEN` | `EMPTY_DIR` | No sibling-file discovery | +| `CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED` | `NO` | No `header_file=` in `/vsicurl?` URLs | +| `GDAL_SKIP` | HDF and netCDF drivers in TiTiler; `VRT` is never skipped | See [VRT policy](#vrt-and-virtual-file-system-policy) | + +**Settings that are never allowed.** `verify()` fails if any of these is set: + +- `GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE=ALL` +- `GDAL_VRT_PYTHON_TRUSTED_MODULES` +- `GDAL_PYTHON_DRIVER_PATH` +- a `GDAL_DRIVER_PATH` other than `disable` +- `CPL_ENABLE_PATH_TRAVERSAL_DETECTION=NO` + +A CI guard rejects the same settings anywhere in the infrastructure, scripts, +Dockerfiles and app configuration. + +**Probes.** `verify()` also tries to open: + +- an in-memory VRT that uses a raw band; +- an in-memory VRT that uses a Python pixel function; +- in TiTiler, any VRT at all. + +Each attempt must fail. A failure to fail is reported as a violation. + +**Where the check runs:** + +| Runtime | When | Consequence of failure | +|---|---|---| +| TiTiler | At import, before the FastAPI app exists | `/healthz` through APIM can't return 200 | +| API, queues | At hastegeo import | Function indexing fails, and the deploy verification catches it | +| imageryprep, training | As an image build step, and at Batch task start | The image doesn't build, or the task fails before touching input | + +Every run logs the `report()` output. That log is the evidence that the +deployed runtime is the approved one. + +## Untrusted input + +**Untrusted inputs** are anything whose bytes come from outside HASTE: + +- downloaded provider imagery; +- user uploads; +- user-supplied footprints. + +**How they are handled:** + +1. After download, the content is sniffed: TIFF for imagery, GeoPackage or + GeoJSON for vectors. Anything else is rejected before GDAL sees it. +2. They are opened only with an explicit driver allowlist (GTiff/COG, + GPKG/GeoJSON) through `gdal.OpenEx(..., allowed_drivers=...)`, rasterio's + `driver=` and fiona's `enabled_drivers=`. +3. The opened datasets, not file names, are passed on to `gdal.Warp` and + `gdal.BuildVRT`. + +As a result, a VRT document can never be parsed from an untrusted input, even +in runtimes that keep the VRT driver for their own mosaics. + +No request value reaches GDAL configuration, open options, driver paths or +plugin paths. Each GDAL configuration call site uses constant keys and +operator-controlled values. A test pins this. + +## VRT and virtual file system policy + +| Runtime | VRT driver | Why | +|---|---|---| +| TiTiler | Registered | rio-tiler reprojects through in-memory VRTs, and `GDAL_SKIP=VRT` makes tile rendering fail with `Unknown GDAL Error` (verified on a deployed development environment). Raw bands and Python pixel functions stay off, and the reader opens only GTiff/COG, so user-supplied VRT content is still refused by the content check. | +| imageryprep | Registered | `gdal.BuildVRT` mosaics HASTE's own files; untrusted opens exclude VRT | +| training | Registered | Inference accepts HASTE-produced `.vrt` inputs; untrusted opens exclude VRT | +| API, queues | Registered, with untrusted opens restricted | Same library as the workers; no VRT inputs from callers | + +**Virtual file systems.** GDAL 3.13 has no runtime option that allowlists +individual virtual file system handlers. Every network handler comes with +libcurl, and HTTPS range reads are needed. So: + +- the build removes the handlers whose libraries aren't compiled in (for + example the archive handlers that need libarchive); +- the application refuses caller-supplied `/vsi` paths and non-HTTPS URLs, + and only accepts allowlisted hosts. + +The manifest lists the handlers that remain. + +## Consumers + +| Consumer | How it installs the runtime | Immutable artifact | +|---|---|---| +| API, queues, TiTiler (Flex Consumption) | Hash-locked requirements that reference the runtime and binding wheels by URL, installed into `.python_packages` inside the digest-pinned Functions Python image | A deployment zip built once, recorded by SHA-256, deployed with `--no-build` | +| imageryprep | Same wheels, installed into the image virtualenv by hash | Image digest | +| training | Same wheels, installed into the conda environment (Python 3.11); conda no longer provides GDAL | Image digest | +| Local dev and CI tests | Same wheels | — | + +## Publishing and supply chain + +**The build job** has no write credentials. It produces: + +- the wheels; +- the manifest; +- a CycloneDX SBOM; +- a license and NOTICE bundle for every vendored component; +- SHA-256 sums; +- a provenance attestation. + +Grype scans both the CycloneDX SBOM (with CPE inference for source components) +and the built wheel artifact directory. Either scan fails the job on +medium-or-higher findings. The wheel scan supplements SBOM matching; it does +not replace the required review of the bundled native license notices. +The SBOM records license identifiers or descriptive names and notice-file +references for each locked native source and Python wheel; this metadata does +not itself constitute OSS approval. + +**An approval-gated publisher** uploads the assets to `haste-binaries` only +from the upstream main-branch workflow after both license-review and publish +gates are enabled and the protected publishing environment is approved. Before +upload, it validates the six-wheel set, matching runtime pins, notice status, +all locked native source/version entries in the SBOM, provenance attestations, +and SHA-256 manifest. Existing release assets are never overwritten. + +**Rebuilds** go through the same pipeline and bump the `+haste.N` build +number. They happen: + +- when a vendored component or GDAL patch release has a security fix; +- otherwise on a fixed cadence. + +## Open questions + +- [ ] Confirm the CMake option that compiles raw VRT bands out in GDAL 3.13, + and the full list of drivers GDAL always builds. +- [x] Confirm whether rasterio's `WarpedVRT` works with `GDAL_SKIP=VRT`. + It does not: rio-tiler's `calculate_default_transform` needs the VRT + driver, so TiTiler keeps it registered (the documented fallback). +- [ ] Confirm whether values in a GDAL configuration file take precedence + over environment variables. If so, ship a read-only `gdalrc` and pin + `GDAL_CONFIG_FILE` to it. +- [ ] Measure the Function App package size and cold start against today's. diff --git a/spec/features/gdal-3-13-runtime/impact-analysis.md b/spec/features/gdal-3-13-runtime/impact-analysis.md new file mode 100644 index 00000000..862e3bc0 --- /dev/null +++ b/spec/features/gdal-3-13-runtime/impact-analysis.md @@ -0,0 +1,88 @@ +# Impact Analysis: GDAL 3.13 Native Runtime + +## Current state + +| Runtime | Python | GDAL builds in one process | +|---|---|---| +| `hastefuncapi`, `hastefuncqueues` | 3.11 | GDAL 3.9.2 bindings wheel, plus the copies bundled in rasterio, pyogrio and fiona | +| `titilerfuncapi` | 3.11 | rasterio's bundled copy; dependencies besides `titiler.application` are unpinned | +| imageryprep image | 3.11 | GDAL 3.9.2 wheel, plus rasterio, fiona and pyogrio copies | +| training image | 3.10 (conda) | conda GDAL < 3.9, plus rasterio's bundled copy | + +Function Apps resolve their dependencies on Azure at deploy time. + +## Findings that shape the design + +### Several driver registries per process + +Each libgdal copy has its own registry. `harden_gdal()` deregisters drivers +in the osgeo copy only. The rasterio, fiona and pyogrio copies only see the +`GDAL_SKIP` denylist, and only if it is set before they register drivers. + +### Untrusted downloads can be parsed as VRT + +Downloaded imagery is size-checked but not content-checked. It is then opened +with every registered driver, VRT included. A VRT document served from an +allowlisted host would be parsed as VRT, and could reference local files or +internal URLs. GDAL 3.12's raw-band restrictions don't cover ordinary VRT +sources. The fix is a content check plus an explicit driver allowlist at +open. + +### VRT is needed internally + +`gdal.BuildVRT` builds mosaics in imagery preparation, and training inference +reads HASTE-produced `.vrt` files. The tile server never needs VRT as input, +but rasterio's `WarpedVRT` must be tested with the driver skipped. + +### Network handlers can't be selectively disabled at runtime + +GDAL 3.13 has no runtime allowlist for individual virtual file system +handlers. The network handlers come with libcurl, which label tasks and tile +serving need. Controls are therefore build-time removal where possible, plus +HTTPS-only and host allowlisting at the application layer. + +### Private-fork implementation status + +The current private branch validates the first upload chunk before storing +it, checks downloaded imagery for TIFF content before processing, and opens +those rasters with the GTiff driver only. User-supplied footprint GeoPackages +are checked for GeoPackage content and read with Fiona's GPKG driver +allowlist. These controls are not yet an audit of every GDAL open site; the +remaining image, API, and training paths still need review and testing. +`hastegeo` now activates the shared-runtime self-check on package import, and +TiTiler has a startup hook that verifies the runtime when installed. Neither +consumer currently installs the new wheel or sets the required-runtime flag. + +## Blast radius + +**Code paths.** Every GDAL read and write path: + +- imagery preparation (warp, mosaic, COG, JPEG preview); +- label tasks and footprints; +- prediction GeoPackages and assessment reports; +- publishing rasters; +- tile serving. + +**Pipelines:** + +- runtime build and publish; +- `deploy_apps.sh`, `deploy-apps.yml` and the azd hooks; +- dependency validation; +- image builds. + +**Local dev:** setup scripts, Docker Compose and the CI test environment. + +**Deploying older branches.** After this lands, deploying a Function App from +a branch without these changes reinstalls the old GDAL. The rollout notes +that. + +## Risks + +| Risk | Mitigation | +|---|---| +| rasterio 1.4.4 or fiona 1.10.1 fails to build or behave against GDAL 3.13 | Compatibility tests in the build. Fallback: Python 3.12 with rasterio 1.5. | +| rio-tiler reprojection needs the VRT driver | Keep it registered for TiTiler, and restrict the reader to GTiff/COG | +| Two PROJ copies conflict over `proj.db` | pyproj is built against the runtime's PROJ | +| Package size or cold start grows on Flex Consumption | Measured. The minimal driver set should shrink packages compared with today's several copies. | +| HASTE now owns patching of vendored native libraries | Scheduled rebuilds, SBOM vulnerability gate, documented owner | +| The Functions worker's glibc is undocumented | `manylinux_2_28` baseline, proven by the self-check on a dev deployment | diff --git a/spec/features/gdal-3-13-runtime/plan.md b/spec/features/gdal-3-13-runtime/plan.md new file mode 100644 index 00000000..846cdb31 --- /dev/null +++ b/spec/features/gdal-3-13-runtime/plan.md @@ -0,0 +1,207 @@ +# Plan: GDAL 3.13 Native Runtime + +| Phase | Task | Agent | Status | +|---|---|---|---| +| 0 | Spec and ADR-0007 | `backend-dev` | done | +| 0 | Baseline runtime report from every current runtime | `backend-dev` | report script implemented; collection from current runtimes pending | +| 1 | Source lock (`native/gdal-runtime/sources.lock`) and verified fetcher | `backend-dev` | done | +| 1 | Runtime package `haste_gdal_runtime` (configure, harden, report, verify, CLI) | `backend-dev` | implemented; manifest records all binding versions and startup checks compare them with installed wheels; 47 native runtime tests pass locally | +| 1 | Build scripts and shared runtime packaging | `backend-dev` + `gis` | implemented; six-wheel artifact built and downloaded | +| 1 | Build workflow: manifest, SBOM, notices, scan gate, attestation, smoke tests | `backend-dev` | native build and consumer smokes are implemented; the latest private run failed in the TiTiler VRT smoke because it skipped `configure(policy)`; fixed at commit `16d6735`, awaiting upstream CI validation | +| 1 | Approval-gated publish to `haste-binaries`; license review | `backend-dev`, `security` | private publisher verifies provenance, exact source-lock/SBOM match, wheel licenses, checksums and immutable upload; local tests pass; Microsoft OSS approval and upstream confirmation still required | +| 2 | hastegeo self-check policy; `harden_gdal()` rework | `backend-dev` | implemented in the consumer integration; single-libgdal/libproj and per-binding version checks fail closed, and required consumers enable fail-closed startup | +| 2 | Untrusted-input content check and driver allowlist at open; configuration audit | `gis` | uploaded chunk headers, downloaded TIFFs and user GPKG inputs are content-checked; imagery opens are GTiff-only and GPKG reads use Fiona's GPKG allowlist; audit and runtime validation of remaining workflows are still required | +| 2 | GDAL hardening settings in Bicep, `deploy_apps.sh`, Dockerfiles and compose | `backend-dev` | implemented in private fork: runtime-required flag and `GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR` are wired through deployment and container surfaces; Bicep compiles; deploy script keeps old workers running until package publish succeeds, then enables fail-closed startup | +| 3 | TiTiler: hash-locked dependencies on the runtime; startup self-check; VRT driver policy; negative tests | `backend-dev` | upstream route and dataset-URL restrictions are preserved; runtime initialization now runs before GDAL-linked imports; the VRT driver stays registered because tile reprojection needs it (raw bands and Python pixel functions stay off); verified on a deployed development environment | +| 4 | Build-once Function App packages; `--no-build` deploy; post-deploy verification | `backend-dev` | package builder and verified `--no-build` deploy path are implemented; target-image package/import smokes and post-deploy runtime diagnostics must pass before rollout | +| 5 | imageryprep and training images (training to Python 3.11); local dev and CI test environments | `backend-dev` + `gis` | imageryprep and training pin the shared wheels; training uses Python 3.11 and in-process APIs instead of GDAL command-line tools; `pandas` is pinned below 3 because geopandas 1.0.1 fails on its string dtype; both images ran imagery-layer, training and inference jobs on a development environment; local developer setup remains unverified | +| 6 | Dependency pin guards, forbidden-configuration guard, docs, close the GDAL exception | `backend-dev` | in progress: repo guard rejects weakened GDAL settings across infra, workflows, scripts, API and Docker config; release-package guards added; old dependency pins and exception stay until runtime consumer packages are built and approved | +| 7 | Dev rollout, runtime evidence, production after approval | `backend-dev` | first development environment validated with privately built packages and images (tiles, API, queues, imagery layer, training, inference); rollout through the normal attested deploy, further environments and production pending | + +Validation for every phase: `backend-validation`. New native dependencies: +`security` audits them and `security-validation` confirms. + +## Status and next steps (2026-10-19) + +What exists: + +- `native/gdal-runtime/sources.lock`: nine native sources with SHA-256. All + nine downloads were checked against these hashes. +- `native/gdal-runtime/fetch_sources.py`: downloads and verifies the lock; + any mismatch is fatal. +- `native/gdal-runtime/baseline_report.py`: emits versions from every + importable binding, mapped libgdal/libproj paths, driver and VSI inventories, + and a safe allowlist of GDAL/PROJ config values. Reports from the existing + Function Apps and Batch images still need to be collected. +- `native/gdal-runtime/haste_gdal_runtime/`: the runtime package. + - `configure(policy)` sets the operator settings, pins `GDAL_DATA`, + `PROJ_DATA` and `GDAL_CONFIG_FILE` to the package, and puts disallowed + drivers in `GDAL_SKIP`. + - `harden(policy)` deregisters disallowed drivers and removes disallowed + virtual file systems. + - `report()` collects the evidence; `evaluate()` and `verify_or_raise()` + compare it with the manifest and policy. + - `secure_startup(policy)` runs harden, then verify. + - The CLI offers `report`, `verify` and `manifest`. + - Policy logic has synthetic unit coverage; the runtime also passed a + real-libgdal smoke check with GDAL 3.13.3 and PROJ 9.8.1. + +Next, in order: + +1. Update the existing upstream draft PR with the integrated consumer work, + then validate the exact head in upstream CI. The latest private run failed + because the TiTiler VRT smoke called `secure_startup()` without first + applying `configure(policy)`. The fix is committed locally as `16d6735`; + the native runtime unit suite passes (47 tests). This fix has not yet been + validated by a complete CI build. +2. Complete remaining GDAL open-path and compatibility evidence, including + the full training environment, all target-image package imports, and + supported imagery, tile, and training workflows. Baseline reports from + existing runtimes also remain outstanding. +3. Finish the license and supply-chain review before publishing immutable + wheels. No artifact has been published, and no environment has been + deployed. After upstream CI and review pass, keep deployment and release + gated on the required approvals and dev runtime evidence. + +## Verified build facts + +Each fact below was checked in the GDAL 3.13.3, PROJ 9.8.1 and curl 8.22.0 +sources, or in the published package metadata. They replace the matching +open questions in `design.md`. + +GDAL CMake options: + +- `GDAL_VRT_ENABLE_RAWRASTERBAND=OFF` leaves `VRTRawRasterBand` out of the + build. The VRT driver advertises `GDAL_VRT_ENABLE_RAWRASTERBAND=YES` + metadata only when raw bands are compiled in, so its absence is evidence. +- `GDAL_AUTOLOAD_PLUGINS=OFF` defines `GDAL_NO_AUTOLOAD`. Native plugin + loading becomes a no-op, and so does Python driver loading, because the + search path list is empty. Driver and plugin paths are then ignored. +- Adding `-DGDAL_VRT_DISABLE_PYTHON` to the C++ flags forces + `GDAL_VRT_ENABLE_PYTHON` to `NO` whatever the configuration. +- Setting `CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED=NO` rejects `header_file=` + in `/vsicurl?`. The setting also accepts `ONLY_IN_TEMP` (the default) or + `YES`; the runtime package pins it to `NO`. +- Turn optional drivers off, then name the ones to keep: + - `GDAL_BUILD_OPTIONAL_DRIVERS=OFF` and `OGR_BUILD_OPTIONAL_DRIVERS=OFF`. + - GTiff, VRT, GeoJSON and Shape are switched back on automatically unless + they are set explicitly, so set `GDAL_ENABLE_DRIVER_GTIFF=ON`, + `GDAL_ENABLE_DRIVER_VRT=ON`, `OGR_ENABLE_DRIVER_GEOJSON=ON` and + `OGR_ENABLE_DRIVER_SHAPE=OFF`. + - GPKG needs `OGR_ENABLE_DRIVER_SQLITE=ON`. + - MEM is always built. +- The expected native driver inventory is GTiff, COG, VRT, MEM, PNG, JPEG, + GeoJSON, GeoJSONSeq, ESRIJSON, TopoJSON, GPKG, SQLite, GNMDatabase and + GNMFile. The published GDAL Python sdist includes its GNM wrapper based on + the release-time CMake configuration and compiles it unconditionally for + the pinned build, so `ENABLE_GNM` must remain on to supply its headers and + symbols. The runtime manifest records both GNM drivers; HASTE's per-workflow + allowed-driver policy adds them to `GDAL_SKIP` and deregisters them at + startup. In 3.13, "Memory" is a hidden alias of MEM and is not in the driver + list. +- Dependency selection: + - `GDAL_USE_EXTERNAL_LIBS=OFF` and `GDAL_USE_INTERNAL_LIBS=OFF`. + - Then turn on `GDAL_USE_CURL`, `GDAL_USE_SQLITE3`, `GDAL_USE_ZLIB`, + `GDAL_USE_DEFLATE`, `GDAL_USE_ZSTD` and `GDAL_USE_JPEG`. + - Then turn on `GDAL_USE_TIFF_INTERNAL`, `GDAL_USE_GEOTIFF_INTERNAL`, + `GDAL_USE_PNG_INTERNAL` and `GDAL_USE_JSONC_INTERNAL`. json-c is required. +- `BUILD_APPS=OFF` omits `gdal-config`. After installing the native library, + the build reconfigures to generate this build-only metadata script, copies + only `gdal-config` into the build prefix and does not compile or ship GDAL + command-line utilities. CI logs showed that the repeated `gdal-config` + failure happened earlier than the binding-wheel build: pip inspected GDAL's + PEP 517 metadata while fetching binding sdists, before the native runtime + existed. The build now fetches those sdists only after it creates and exposes + `gdal-config`; the latest run passed that stage and reached native + compilation. A proposed compile-time definition for the `/vsicurl?` + header-file restriction collided with a same-named C++ identifier in GDAL + 3.13.3 and has been removed in favor of the supported runtime setting above. + The next run exposed GDAL's in-tree Python binding target using builder + Python 3.15 without setuptools; that target is now disabled because the + pinned CPython 3.11 bindings are built separately. The latest run compiled + native GDAL, generated `gdal-config` and fetched all pinned binding sdists, + then failed when the manifest loader could not find `libssl.so.3` under the + runtime library path. OpenSSL now installs to the same `lib` directory as + the other native dependencies. The latest run exposed a conflict between + the native `ENABLE_GNM=OFF` setting and GDAL's published Python sdist, whose + generated extension list still compiles `gnm_wrap.cpp`. GNM is now retained + so the pinned binding can compile; its two drivers are explicitly captured + in the manifest and excluded by the runtime's allowed-driver policy. The + latest run then failed while pip prepared sdist metadata: isolated build + dependencies caused redundant feed requests and an HTTP 429. Sdist metadata + preparation now uses the preinstalled hash-pinned build-tools venv without + isolation. Upstream run 38918675350 built the six-wheel artifact and passed + runtime plus GDAL-binding offline smoke with the public PyPI fallback. The + private validation branch adds all-binding and functional smoke, using a + separately hash-locked set of runtime dependencies and a local TLS server + for `/vsicurl/`. Run 38946439904 passed imports and version checks for all + five bindings, confirmed one GDAL/PROJ library set, exercised raster/vector + read/write, COG, reprojection, mosaic, WarpedVRT and PyProj, and read a + raster over HTTPS with certificate verification. It uploaded the six-wheel + artifact. Smoke-only dependencies are fetched from public PyPI with exact + hashes to avoid internal-feed throttling. The private fork's CodeQL jobs + completed analysis but could not upload results because code scanning is + disabled for that repository. Private run `38955056455` passed the build, + all-binding smoke, medium-or-higher Grype scan, and smoke tests in Functions, + imageryprep and the training CUDA base. The current unvalidated changes also + scan wheel artifacts alongside the SBOM. Its attestation job was skipped + because publication attestations are restricted to upstream `microsoft/haste` + main. The private build emits a CycloneDX inventory, notice bundle and + checksums. License review, upstream CI validation of the latest fix, durable + publication and dev rollout remain. + +PROJ: + +- Build with `BUILD_APPS=OFF`, `ENABLE_TIFF=OFF`, `ENABLE_CURL=OFF`, + `BUILD_TESTING=OFF` and `EMBED_PROJ_DATA_PATH=OFF`. +- The build needs a `sqlite3` executable (`EXE_SQLITE3`). +- pyproj then needs `PROJ_DIR` and `PROJ_VERSION`, because there is no + `proj` binary to query. + +Virtual file systems: + +- The only build switches are zlib, libarchive and curl. +- `VSIRemovePluginHandler()` (GDAL 3.9+) removes built-in handlers at run + time, and they are not reinstalled. +- The drivers kept here use `/vsimem/`, `/vsisubfile/` and `/vsisparse/` + internally. `/vsisparse/` serves GeoTIFF JPEG overviews. +- `/vsicurl/` and `/vsicurl?` are separate prefixes. Removing `/vsicurl?` + disables caller-supplied curl options and keeps plain HTTPS reads. + +curl and OpenSSL: + +- curl's configure accepts `--disable-file`, `--disable-ftp`, + `--disable-ipfs`, `--disable-ldap`, `--disable-ldaps`, `--disable-rtsp`, + `--disable-dict`, `--disable-telnet`, `--disable-tftp`, `--disable-pop3`, + `--disable-imap`, `--disable-smb`, `--disable-smtp`, `--disable-gopher`, + `--disable-mqtt`, `--disable-websockets`, `--disable-netrc`, + `--with-ca-fallback` and `--without-ca-bundle`. +- OpenSSL 3.5 accepts `no-apps`, `no-docs`, `no-tests`, `no-module`, + `no-legacy`, `no-engine`, `no-dso` and `no-autoload-config`. + - With `no-autoload-config` and a package-private `--openssldir`, the + library never reads the host's `openssl.cnf`. + - CA certificates then come from `CURL_CA_BUNDLE`, which `configure()` + sets from the system bundle. The check fails when HTTPS is enabled but + no bundle exists. + +Build toolchain: + +- fiona 1.10.1 needs Cython 3.0.x (`~=3.0.2`). rasterio 1.4.4 needs Cython + 3.1.x (`~=3.1.0`), and pyogrio and pyproj need 3.1 or later. Build fiona + in its own environment. +- Shared pins: setuptools 84.0.0, wheel 0.48.0, packaging 26.3 (required by + wheel), numpy 2.4.6 (the last line with Python 3.11 wheels) and versioneer + 0.28. +- Wheels built against numpy 2.x also run on numpy 1.x. + +Images and actions: + +- Builder image: + `quay.io/pypa/manylinux_2_28_x86_64@sha256:1e102f2e9ed9b7fd8d64b29f798c76078c3fa8f14d34914d7a4045e39419d8e0` + (tag `2026.10.02-1`). + - It ships CPython 3.11, cmake, auditwheel and patchelf. + - perl modules and nasm must be installed as build-only tools. +- `actions/upload-artifact`: `ea165f8d65b6e75b540449e92b4886f43607fa02` + (v4.6.2). +- `actions/download-artifact`: `d3f86a106a0bac45b974a628896c90dbdf5c8093` + (v4.3.0). diff --git a/spec/features/gdal-3-13-runtime/rollout.md b/spec/features/gdal-3-13-runtime/rollout.md new file mode 100644 index 00000000..bdaf7768 --- /dev/null +++ b/spec/features/gdal-3-13-runtime/rollout.md @@ -0,0 +1,24 @@ +# Rollout: GDAL 3.13 Native Runtime + +## Order + +1. Publish the runtime and binding wheels to `haste-binaries`. They are + additive: nothing references them until a consumer switches. +2. Deploy to dev environments: Function Apps from build-once packages, then + both Batch images. Collect the runtime report from every app and from one + imagery preparation run and one training/inference run. +3. Run the UI smoke tests: tiles, labeling and results views. +4. Deploy to production only after the security reviewer approves the + evidence. + +## Rollback + +- **Function Apps.** Redeploy the previous package by its recorded SHA-256. +- **Batch images.** Point the pools at the previous image digest. +- **Never relax configuration as a rollback.** A runtime that fails its + self-check stays down until a correct artifact is deployed. + +## Notes for contributors + +Deploying a Function App from a branch without this change reinstalls the +old dependencies. Rebase before deploying once this lands. diff --git a/spec/features/gdal-3-13-runtime/test-plan.md b/spec/features/gdal-3-13-runtime/test-plan.md new file mode 100644 index 00000000..c6528c5f --- /dev/null +++ b/spec/features/gdal-3-13-runtime/test-plan.md @@ -0,0 +1,37 @@ +# Test Plan: GDAL 3.13 Native Runtime + +## Build-time tests + +These run in the runtime build workflow, inside each target base image: + +- the Functions Python 3.11 image; +- the imageryprep base; +- the training base. + +| Test | Proves | +|---|---| +| Source hashes | Every input matches `sources.lock` and `requirements-build.txt` | +| Import and report | All five bindings import; the report shows one libgdal and one libproj; versions are 3.13.3 and 9.8.1 | +| Manifest equality | Registered drivers and VSI prefixes equal the manifest | +| Probes | VRTs using a raw band or a Python pixel function don't open; inline VRT doesn't open when VRT is skipped | +| Functional | GeoTIFF/COG read and write; DEFLATE, LZW and ZSTD; `gdal.Warp` reprojection; `gdal.BuildVRT` mosaic from opened datasets; JPEG preview; GPKG and GeoJSON vector read and write through fiona and pyogrio; `pyproj` transforms; rasterio `WarpedVRT` | +| Network handler | `/vsicurl/` range reads against a local HTTPS test server, with no internet | + +## Application tests + +| Area | Test | +|---|---| +| hastegeo | VRT bytes disguised as TIFF/GPKG are rejected before GDAL/Fiona opens; TIFF opens use the GTiff allowlist; GPKG reads use the GPKG allowlist; upload chunk 0 is validated before storage; configuration audit; self-check policy | +| TiTiler | Tiles render with the VRT driver registered (disabling it breaks reprojection); tiles, including reprojected sources, through the app; negative tests for inline VRT, `vrt://`, remote VRT documents (raw band, Python pixel function, local or internal sources) and `/vsi` paths | +| Guards | Dependency pins come only from the runtime set and match everywhere; forbidden GDAL settings are rejected across infrastructure, scripts, Dockerfiles and app settings | + +## Deployment evidence + +| Criterion | Evidence | +|---|---| +| GDAL 3.13.3 in the deployed process | Runtime report logged at startup by each app and each Batch task | +| Bindings match libgdal | Report: one libgdal path; every binding at 3.13.3 | +| Raw-VRT and Python pixel functions disabled | Report configuration plus probe results | +| No caller control over GDAL configuration | Configuration audit test plus TiTiler negative tests | +| Driver and VSI inventory | Published manifest plus the live report | +| No deployment relaxes the restrictions | Guard test plus deployed app-settings scan | diff --git a/spec/features/gdal-3-13-runtime/user-stories.md b/spec/features/gdal-3-13-runtime/user-stories.md new file mode 100644 index 00000000..b6e5c65f --- /dev/null +++ b/spec/features/gdal-3-13-runtime/user-stories.md @@ -0,0 +1,179 @@ +# User Stories: GDAL 3.13 Native Runtime + +## Personas + +| Persona | Description | Key Goals | +|---|---|---| +| Platform Operator | Deploys and runs HASTE environments | Know exactly which GDAL runs where, and deploy the same artifact everywhere | +| Security Reviewer | Verifies HASTE's dependency and configuration posture | Evidence for versions, drivers and configuration, from inside the running service | +| Disaster Analyst | Uses HASTE to assess damage from imagery | Imagery preparation, labeling and tiles keep working | + +--- + +## Stories + +### US-001: Reproducible GDAL runtime + +**As a** Security Reviewer, +**I want** GDAL and its native dependencies built in CI from pinned, verified sources, +**So that** I can review exactly what HASTE ships. + +**Priority:** P0 +**Component(s):** `native/gdal-runtime/`, `.github/workflows/` + +**Acceptance Criteria:** + +```gherkin +Given the runtime source lock +When the build workflow runs +Then every downloaded tarball and source distribution matches its pinned SHA-256 +And the run publishes the wheels, the manifest, an SBOM, license notices and a provenance attestation +``` + +```gherkin +Given a source file whose hash does not match the lock +When the build runs +Then the build fails before compiling anything +``` + +### US-002: One libgdal per process + +**As a** Security Reviewer, +**I want** every GDAL-linked binding to use the same native runtime, +**So that** version and driver checks describe the whole process. + +**Priority:** P0 +**Component(s):** `native/gdal-runtime/` + +**Acceptance Criteria:** + +```gherkin +Given a process that imports GDAL, rasterio, fiona, pyogrio and pyproj +When the runtime report is collected +Then exactly one libgdal and one libproj are mapped +And every binding reports GDAL 3.13.3 +``` + +### US-003: Fail-closed runtime self-check + +**As a** Platform Operator, +**I want** each service to refuse to start when its GDAL runtime differs from the approved one, +**So that** a wrong or weakened runtime never serves traffic. + +**Priority:** P0 +**Component(s):** `native/gdal-runtime/`, `hastelib`, `api/titilerfuncapi/`, Batch images + +**Acceptance Criteria:** + +```gherkin +Given the approved runtime and the default configuration +When the service starts +Then the self-check passes and logs the runtime report +``` + +```gherkin +Given GDAL_VRT_ENABLE_PYTHON=YES, or GDAL_VRT_RAWRASTERBAND_ALLOWED_SOURCE=ALL, or a GDAL_DRIVER_PATH other than "disable" +When the service starts +Then the self-check fails and the service does not start +``` + +```gherkin +Given a VRT document that uses a raw band or a Python pixel function +When the self-check probes it +Then GDAL refuses to open or read it +``` + +### US-004: Untrusted inputs never reach the VRT driver + +**As a** Security Reviewer, +**I want** downloaded and uploaded files opened only with an explicit driver allowlist, +**So that** a VRT document disguised as imagery is rejected. + +**Priority:** P0 +**Component(s):** `hastelib/src/hastegeo/core/utils/` + +**Acceptance Criteria:** + +```gherkin +Given a downloaded "imagery" file whose content is a VRT document +When imagery preparation processes it +Then it is rejected before GDAL parses it, and the layer fails with a clear error +``` + +```gherkin +Given a valid Cloud Optimized GeoTIFF +When imagery preparation processes it +Then the output COG and JPEG preview match those produced by the current release +``` + +### US-005: Tile server on the approved runtime + +**As a** Disaster Analyst, +**I want** imagery tiles to keep rendering, +**So that** labeling and results views work on the hardened tile server. + +**Priority:** P0 +**Component(s):** `api/titilerfuncapi/` + +**Acceptance Criteria:** + +```gherkin +Given a project with pre- and post-event imagery +When the map requests tiles through APIM +Then tiles render, including reprojected sources +``` + +```gherkin +Given a tile request whose source is inline VRT, vrt://, a remote VRT document, or a /vsi path +When TiTiler handles it +Then no VRT is opened and the request fails +``` + +### US-006: One immutable artifact per deployment + +**As a** Platform Operator, +**I want** Function Apps deployed from a package built once, +**So that** dev and prod run the same bytes. + +**Priority:** P0 +**Component(s):** `.github/scripts/deploy_apps.sh`, `.github/workflows/deploy-apps.yml`, `azure.yaml` + +**Acceptance Criteria:** + +```gherkin +Given a deployment run +When a Function App is deployed +Then its package was built once from hash-locked requirements, recorded by SHA-256, and published without a remote build +And the app is tagged with that SHA-256 +``` + +### US-007: No deployment can relax the GDAL hardening + +**As a** Security Reviewer, +**I want** CI to reject any configuration that weakens the GDAL settings, +**So that** the hardening holds across redeployments. + +**Priority:** P1 +**Component(s):** `.github/scripts/tests/`, `hastelib/tests/build/` + +**Acceptance Criteria:** + +```gherkin +Given a change that sets a forbidden GDAL option in infrastructure, scripts, Dockerfiles or app settings +When CI runs +Then the guard test fails and names the file and the option +``` + +--- + +## Agent Assignment Map + +| Story | Implementing Agent | Validating Agent | Notes | +|---|---|---|---| +| US-001 | `backend-dev` + `gis` | `backend-validation` | New native dependencies: `security` audits, `security-validation` confirms | +| US-002 | `gis` | `backend-validation` | | +| US-003 | `backend-dev` | `backend-validation` | | +| US-004 | `gis` | `backend-validation` | | +| US-005 | `backend-dev` | `backend-validation` | Coordinate with the TiTiler route-surface work | +| US-006 | `backend-dev` | `backend-validation` | | +| US-007 | `backend-dev` | `backend-validation` | |