From 4871f8d7a31c2497a4c63f19b50c5c3bb8871168 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Thu, 20 Aug 2026 21:49:05 -0700 Subject: [PATCH 1/7] add RTS to wuhan --- .../configs/centers/wuhan_config.yaml | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/gpm-specs/src/gpm_specs/configs/centers/wuhan_config.yaml b/packages/gpm-specs/src/gpm_specs/configs/centers/wuhan_config.yaml index 9e8fbc0..d2fc91d 100644 --- a/packages/gpm-specs/src/gpm_specs/configs/centers/wuhan_config.yaml +++ b/packages/gpm-specs/src/gpm_specs/configs/centers/wuhan_config.yaml @@ -16,7 +16,12 @@ products: product_name: ORBIT server_id: wuhan_ftp available: true - description: Precise satellite orbits + description: > + Precise satellite orbits. No TTT filter, so this already matches + RTS (Real-Time Service) candidates alongside FIN/RAP; TTT sorting + in pride_pppar.yaml doesn't list RTS, so those candidates just + sort last. See wuhan_clock for why CLOCK/ERP/BIA need an explicit + TTT: RTS entry to get the same behavior. parameters: - {name: AAA, value: WUM} - {name: AAA, value: WMC} @@ -92,16 +97,25 @@ products: product_name: CLOCK server_id: wuhan_ftp available: true - description: Precise satellite and station clocks + description: > + Precise satellite and station clocks. Includes RTS (Real-Time + Service), WUM's own near-real-time product line — no FIN/RAP is + published yet for the most recent 1-3 days, and this is the same + fallback PRIDE-PPPAR's pdp3.sh uses natively for recent days + (USERTS=YES). TTT sorting in pride_pppar.yaml only lists + FIN/RAP/ULT, so RTS candidates naturally sort last without + needing a preference entry. parameters: - {name: AAA, value: WUM} - {name: AAA, value: WMC} - {name: TTT, value: FIN} - {name: TTT, value: RAP} + - {name: TTT, value: RTS} - {name: PPP, value: MGX} - {name: PPP, value: DEM} - {name: SMP, value: 30S} - {name: SMP, value: 05M} + - {name: SMP, value: 05S} directory: {pattern: "pub/whu/phasebias/{YYYY}/clock/"} # ── ERP ───────────────────────────────────────────────────────── @@ -109,11 +123,13 @@ products: product_name: ERP server_id: wuhan_ftp available: true - description: Earth rotation parameters + description: > + Earth rotation parameters. Includes RTS (see wuhan_clock note). parameters: - {name: AAA, value: WUM} - {name: TTT, value: FIN} - {name: TTT, value: RAP} + - {name: TTT, value: RTS} - {name: PPP, value: MGX} directory: {pattern: "pub/whu/phasebias/{YYYY}/orbit/"} @@ -122,12 +138,15 @@ products: product_name: BIA server_id: wuhan_ftp available: true - description: Observable-specific signal biases (OSB) + description: > + Observable-specific signal biases (OSB). Includes RTS (see + wuhan_clock note). parameters: - {name: AAA, value: WUM} - {name: AAA, value: WMC} - {name: TTT, value: FIN} - {name: TTT, value: RAP} + - {name: TTT, value: RTS} - {name: PPP, value: MGX} - {name: PPP, value: DEM} directory: {pattern: "pub/whu/phasebias/{YYYY}/bias/"} From f0972598be2022b2690570e268dd22b3883161d6 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Thu, 27 Aug 2026 12:30:08 -0700 Subject: [PATCH 2/7] changes --- .../client/gnss_client.py | 8 +- .../factories/pipelines/download.py | 10 ++- .../factories/pipelines/resolve.py | 12 ++- .../factories/remote_transport.py | 9 +- .../test/test_download_integrity.py | 19 +++- .../configs/dependencies/pride_pppar.yaml | 12 ++- .../src/pride_ppp/factories/processor.py | 89 +++++++++++++++---- .../src/pride_ppp/specifications/cli.py | 2 +- .../src/pride_ppp/specifications/config.py | 45 +++------- packages/pride-ppp/tests/test_processor.py | 19 ++++ 10 files changed, 165 insertions(+), 60 deletions(-) diff --git a/packages/gnss-product-management/src/gnss_product_management/client/gnss_client.py b/packages/gnss-product-management/src/gnss_product_management/client/gnss_client.py index 3ed9b85..0e425bf 100644 --- a/packages/gnss-product-management/src/gnss_product_management/client/gnss_client.py +++ b/packages/gnss-product-management/src/gnss_product_management/client/gnss_client.py @@ -281,6 +281,7 @@ def resolve_dependencies( date: datetime.datetime, *, sink_id: str, + force_download: bool = False, ) -> tuple[DependencyResolution, AnyPath | None]: """Resolve all dependencies in a spec for the given date. @@ -312,4 +313,9 @@ def resolve_dependencies( workspace=self._workspace, transport=self._transport, ) - return pipeline.run(dep_spec, date, sink_id=sink_id) + return pipeline.run( + dep_spec, + date, + sink_id=sink_id, + force_download=force_download, + ) diff --git a/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/download.py b/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/download.py index 3a94b47..929ae84 100644 --- a/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/download.py +++ b/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/download.py @@ -61,6 +61,7 @@ def run( date: datetime.datetime, *, sink_id: str = "local_config", + force: bool = False, ) -> Path | None | list[Path | None]: """Download found resources to the workspace. @@ -79,7 +80,7 @@ def run( paths: list[Path | None] = [] for r in resources: - path = self._download_one(r, date, sink_id) + path = self._download_one(r, date, sink_id, force=force) paths.append(path) if single: @@ -91,6 +92,8 @@ def _download_one( resource: FoundResource, date: datetime.datetime, sink_id: str, + *, + force: bool = False, ) -> Path | None: """Download a single resource and write its sidecar lockfile. @@ -102,7 +105,7 @@ def _download_one( Returns: Path to the resolved file, or ``None`` on failure. """ - if resource.is_local: + if resource.is_local and not force: local_path = resource.path if local_path and local_path.exists(): logger.debug("Already local: %s", local_path) @@ -121,9 +124,10 @@ def _download_one( local_resource_id=sink_id, local_factory=self._planner._workspace, date=date, + force=force, ) if path is not None: - if get_lock_product(path) is None: + if force or get_lock_product(path) is None: lock = build_lock_product(sink=path, url=resource.uri, name=resource.product) write_lock_product(lock) logger.info("Downloaded %s → %s", resource.product, path) diff --git a/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/resolve.py b/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/resolve.py index db301f6..7e39278 100644 --- a/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/resolve.py +++ b/packages/gnss-product-management/src/gnss_product_management/factories/pipelines/resolve.py @@ -94,6 +94,7 @@ def run( sink_id: str = "local_config", centers: list[str] | None = None, download: bool = True, + force_download: bool = False, ) -> tuple[DependencyResolution, AnyPath | None]: """Resolve all dependencies in *spec* for *date*. @@ -126,7 +127,7 @@ def run( date=date, version=version, ) - if existing is not None: + if existing is not None and not force_download: resolution = self._resolution_from_lockfile(existing, spec) if resolution.all_required_fulfilled: logger.info( @@ -151,6 +152,7 @@ def run( preferences=spec.preferences, centers=centers, download=download, + force_download=force_download, ) with ThreadPoolExecutor(max_workers=15) as executor: resolved = list(executor.map(resolve_one, spec.dependencies)) @@ -176,6 +178,7 @@ def _resolve_one( preferences: list[SearchPreference], centers: list[str] | None, download: bool, + force_download: bool, ) -> ResolvedDependency: """Resolve a single dependency. @@ -201,7 +204,10 @@ def _resolve_one( if centers: q = q.sources(*centers) candidates = q.search() - found: FoundResource | None = candidates[0] if candidates else None + if force_download: + found = next((candidate for candidate in candidates if not candidate.is_local), None) + else: + found = candidates[0] if candidates else None except Exception as exc: logger.debug("No candidates for %s: %s", dep.spec, exc) return ResolvedDependency(spec=dep.spec, required=dep.required, status="missing") @@ -227,7 +233,7 @@ def _resolve_one( remote_url=found.uri, ) - path = self._downloader.run(found, date, sink_id=sink_id) + path = self._downloader.run(found, date, sink_id=sink_id, force=force_download) if path is None: logger.warning("Download failed for dependency %s", dep.spec) return ResolvedDependency(spec=dep.spec, required=dep.required, status="missing") diff --git a/packages/gnss-product-management/src/gnss_product_management/factories/remote_transport.py b/packages/gnss-product-management/src/gnss_product_management/factories/remote_transport.py index 7548eac..5f52ab9 100644 --- a/packages/gnss-product-management/src/gnss_product_management/factories/remote_transport.py +++ b/packages/gnss-product-management/src/gnss_product_management/factories/remote_transport.py @@ -299,6 +299,8 @@ def download_one( local_resource_id: str, local_factory: WorkSpace, date: datetime.datetime, + *, + force: bool = False, ) -> AnyPath | None: """Synchronously download matched files for one search target. @@ -329,10 +331,13 @@ def download_one( destination_dir.mkdir(parents=True, exist_ok=True) destination_path = destination_dir / query.product.filename.value # type: ignore[union-attr] - # Prefer an already-decompressed version on disk + # Prefer an already-decompressed version on disk unless the caller + # explicitly requested a fresh copy from the remote source. if destination_path.suffix == ".gz": decompressed_path = destination_path.with_suffix("") if ( + not force + and decompressed_path.exists() and decompressed_path.stat().st_size > 0 and self._validate_cached_or_evict(decompressed_path) @@ -348,6 +353,8 @@ def download_one( # describes the file as served, so it only applies to the # non-decompressed destination path. if ( + not force + and destination_path.exists() and destination_path.stat().st_size > 0 and self._validate_cached_or_evict(destination_path, query.checksum) diff --git a/packages/gnss-product-management/test/test_download_integrity.py b/packages/gnss-product-management/test/test_download_integrity.py index 63a0cde..27a82fe 100644 --- a/packages/gnss-product-management/test/test_download_integrity.py +++ b/packages/gnss-product-management/test/test_download_integrity.py @@ -96,13 +96,20 @@ def env(tmp_path: Path): } -def _download(env, filename: str = "TEST.SP3", checksum: str | None = None) -> Path | None: +def _download( + env, + filename: str = "TEST.SP3", + checksum: str | None = None, + *, + force: bool = False, +) -> Path | None: query = _make_query(env["remote_root"], filename, checksum) return env["wormhole"].download_one( query=query, local_resource_id="local_config", local_factory=env["workspace"], date=TEST_DATE, + force=force, ) @@ -191,6 +198,16 @@ def test_cache_without_sidecar_is_trusted(self, env, monkeypatch) -> None: assert _download(env) == cached assert calls == [] + def test_force_redownload_replaces_valid_cache(self, env) -> None: + cached = env["sink_dir"] / "TEST.SP3" + cached.parent.mkdir(parents=True) + cached.write_bytes(b"valid but stale product") + + result = _download(env, force=True) + + assert result == cached + assert result.read_bytes() == REMOTE_CONTENT + def test_corrupt_cache_is_evicted_and_redownloaded(self, env) -> None: """A cached file whose hash no longer matches its sidecar must be evicted (with its stale sidecar) and fetched fresh.""" diff --git a/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml b/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml index ede85b7..05808fb 100644 --- a/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml +++ b/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml @@ -44,7 +44,11 @@ dependencies: description: Observable-specific signal biases - spec: ATTOBX - required: true + # pdp3.sh tolerates a missing attitude product outright (sets + # Quaternions = NONE and skips multipath/attitude modeling) — no AC + # publishes ATTOBX faster than FIN/RAP (~1 day lag), so requiring it + # blocks same-day/near-real-time processing pdp3 could otherwise do. + required: false description: Satellite attitude quaternions (OBX) - spec: ATTATX @@ -52,7 +56,11 @@ dependencies: description: Antenna phase center corrections (ANTEX) - spec: RNX3_BRDC - required: true + # pdp3.sh has its own same-day fallback (PrepareRinexNav): when the + # processing date is today, it merges hourly GPS+GLONASS nav from Wuhan + # instead of needing the daily merged multi-GNSS product. That fallback + # never runs if this wrapper aborts the day first for lacking it. + required: false description: Broadcast navigation RINEX 3 - spec: LEAP_SEC diff --git a/packages/pride-ppp/src/pride_ppp/factories/processor.py b/packages/pride-ppp/src/pride_ppp/factories/processor.py index 1ae8c74..db2acf8 100644 --- a/packages/pride-ppp/src/pride_ppp/factories/processor.py +++ b/packages/pride-ppp/src/pride_ppp/factories/processor.py @@ -271,11 +271,19 @@ def _resolution_to_satellite_products( return ( SatelliteProducts( - satellite_orbit=product_fields.get("satellite_orbit"), - satellite_clock=product_fields.get("satellite_clock"), - code_phase_bias=product_fields.get("code_phase_bias"), - quaternions=product_fields.get("quaternions"), - erp=product_fields.get("erp"), + # .get(key, "Default") — not .get(key): a bare None here is passed + # to the pydantic model explicitly, which bypasses the field's own + # "Default" default and gets f-string'd into the config file as the + # literal text "None". pdp3.sh only recognizes "Default" as its + # let-me-resolve-it-myself sentinel, so "None" makes it try to + # download a product file literally named "None" and fail outright + # — worse than the missing-required abort this is meant to replace + # for optional specs like ATTOBX. + satellite_orbit=product_fields.get("satellite_orbit", "Default"), + satellite_clock=product_fields.get("satellite_clock", "Default"), + code_phase_bias=product_fields.get("code_phase_bias", "Default"), + quaternions=product_fields.get("quaternions", "Default"), + erp=product_fields.get("erp", "Default"), product_directory=str(product_dir) if product_dir else "Default", ), product_dir, @@ -380,6 +388,7 @@ def __init__( pride_install_dir: Path | None = None, cli_config: PrideCLIConfig | None = PrideCLIConfig(), mode: ProcessingMode | Literal["FINAL", "DEFAULT"] = ProcessingMode.DEFAULT, + override_products_download: bool = False, ) -> None: """Initialise the processor and all its owned subsystems. @@ -408,6 +417,8 @@ def __init__( Also accepts the string literals ``"DEFAULT"`` or ``"FINAL"`` for convenience. + override_products_download: Ignore product lockfiles and local + cached products, downloading fresh remote copies instead. """ if isinstance(mode, str): mode = ProcessingMode(mode.upper()) @@ -416,6 +427,7 @@ def __init__( self._pride_install_dir = Path(pride_install_dir) if pride_install_dir else None self._cli_config = cli_config if cli_config is not None else PrideCLIConfig() self._mode = mode + self._override_products_download = override_products_download # Load the DependencySpec that matches the requested processing mode. # The dep-spec controls which TTT (timeliness) values the resolver @@ -529,6 +541,7 @@ def _resolve( self._dep_spec, date, sink_id=local_sink_id, + force_download=self._override_products_download, ) return resolution @@ -593,10 +606,28 @@ def _run_pdp3( Returns: ``(kin_path, res_path, returncode, stderr)`` where paths are - ``None`` when the corresponding output was not produced. + ``None`` when the corresponding output was not produced or + didn't pass validation (see below). Raises: FileNotFoundError: If the ``pdp3`` binary is not on ``PATH``. + + Note: + ``pdp3`` can exit 0 and still leave a stale ``kin_*`` file + behind from an earlier, non-fatal stage (e.g. the initial + single-point-positioning seed file written before the real + ambiguity-resolution step) even when a later internal stage + fails outright. Trusting a bare filename-glob match would + silently accept that low-quality leftover as a successful + result, so two independent checks guard against it: (1) + ``pdp3.sh`` prints its own ``error:``-tagged lines to stdout on + a hard stage failure — distinct from its ``warning:`` lines for + recoverable conditions — which is scanned for regardless of + the process's own exit code; (2) any ``kin_*`` match found is + parsed with the same validator used to check cached output + (:func:`kin_to_kin_position_df`) before being trusted, since a + leftover seed file has a different (and much sparser) column + layout that fails to parse. """ if not shutil.which("pdp3"): raise FileNotFoundError("pdp3 binary not found in PATH") @@ -611,9 +642,9 @@ def _run_pdp3( ) # Replay stdout/stderr through the logger for observability - if result.stdout: - for line in result.stdout.strip().splitlines(): - logger.info(line) + stdout_lines = result.stdout.strip().splitlines() if result.stdout else [] + for line in stdout_lines: + logger.info(line) if result.stderr: for line in result.stderr.strip().splitlines(): logger.warning(line) @@ -627,6 +658,15 @@ def _run_pdp3( stderr_tail or "(no stderr)", ) + pdp3_error_lines = [line for line in stdout_lines if "error:" in line.lower()] + if pdp3_error_lines: + logger.error( + "pdp3 reported failure for site %s despite returncode %d: %s", + site, + result.returncode, + " | ".join(pdp3_error_lines), + ) + # pdp3 writes outputs as e.g. "kin_2025254_ncc1" (no extension). # Search recursively in the working dir to find them. kin_files = list(Path(tmpdir).rglob(f"kin_*_{site.lower()}")) @@ -638,20 +678,35 @@ def _run_pdp3( output_dir.mkdir(parents=True, exist_ok=True) # Move outputs to the final output directory with proper extensions - if kin_files: - src = kin_files[0] - dst = output_dir / (src.name + ".kin") - shutil.move(str(src), str(dst)) - kin_out = dst - logger.info("Generated kin file %s", dst) - else: + if not kin_files: logger.error( "pdp3 produced no kin output for site %s (returncode %d)", site, result.returncode, ) + elif pdp3_error_lines: + logger.error( + "Discarding kin output for site %s: pdp3 reported an internal " + "failure, so %s is likely an incomplete/seed-only leftover, " + "not a real result.", + site, + kin_files[0].name, + ) + elif kin_to_kin_position_df(kin_files[0]) is None: + logger.error( + "Discarding kin output for site %s: %s exists but failed to " + "parse as a valid kinematic position file.", + site, + kin_files[0].name, + ) + else: + src = kin_files[0] + dst = output_dir / (src.name + ".kin") + shutil.move(str(src), str(dst)) + kin_out = dst + logger.info("Generated kin file %s", dst) - if res_files: + if kin_out is not None and res_files: src = res_files[0] dst = output_dir / (src.name + ".res") shutil.move(str(src), str(dst)) diff --git a/packages/pride-ppp/src/pride_ppp/specifications/cli.py b/packages/pride-ppp/src/pride_ppp/specifications/cli.py index 1493685..4c7fb2d 100644 --- a/packages/pride-ppp/src/pride_ppp/specifications/cli.py +++ b/packages/pride-ppp/src/pride_ppp/specifications/cli.py @@ -80,7 +80,7 @@ class PrideCLIConfig(BaseModel): sample_frequency: float = 1 system: str = "GREC23J" - frequency: list = ["G12", "R12", "E15", "C26", "J12"] + frequency: list = ["G12", "R12", "E17", "C27", "J12"] loose_edit: bool = True cutoff_elevation: int = 7 interval: float | None = None diff --git a/packages/pride-ppp/src/pride_ppp/specifications/config.py b/packages/pride-ppp/src/pride_ppp/specifications/config.py index 4ef985f..dcda92a 100644 --- a/packages/pride-ppp/src/pride_ppp/specifications/config.py +++ b/packages/pride-ppp/src/pride_ppp/specifications/config.py @@ -222,29 +222,38 @@ class SatelliteProducts(BaseModel): default="Default", description="Directory for satellite products", ) + # Patterns accept the literal "Default" as well as a real filename: pdp3.sh + # compares this field's raw config-file text with `!= Default` to decide + # whether to resolve the product itself, so the sentinel has to survive + # into the file unchanged. A previous version of this pattern only + # accepted a real filename and relied on a validator to rewrite "Default" + # to e.g. "Default.OBX" to satisfy it — but that rewritten value then got + # written to the config file as-is, which pdp3.sh's strict `!= Default` + # check doesn't recognize, so it would try to fetch a file literally named + # "Default.OBX" instead of self-resolving. satellite_orbit: str | None = Field( default="Default", - pattern=r".*\.SP3", + pattern=r"^Default$|.*\.SP3", description="File name of SP3 file", ) satellite_clock: str | None = Field( default="Default", - pattern=r".*\.CLK", + pattern=r"^Default$|.*\.CLK", description="File name of CLK file", ) erp: str | None = Field( default="Default", - pattern=r".*\.ERP", + pattern=r"^Default$|.*\.ERP", description="File name of ERP file", ) quaternions: str | None = Field( default="Default", - pattern=r".*\.OBX", + pattern=r"^Default$|.*\.OBX", description="File name of quaternions file", ) code_phase_bias: str | None = Field( default="Default", - pattern=r".*\.BIA", + pattern=r"^Default$|.*\.BIA", description="File name of code/phase bias file", ) leo_quaternions: str | None = Field( @@ -252,32 +261,6 @@ class SatelliteProducts(BaseModel): description="File name of LEO quaternions file", ) - @field_validator( - "satellite_orbit", - "satellite_clock", - "erp", - "quaternions", - "code_phase_bias", - mode="before", - ) - def override_patternmatch(cls, value: str, field) -> str: - """Set default file extension when value is ``'Default'``.""" - if value != "Default": - return value - match field.field_name: - case "satellite_orbit": - return "Default.SP3" - case "satellite_clock": - return "Default.CLK" - case "erp": - return "Default.ERP" - case "quaternions": - return "Default.OBX" - case "code_phase_bias": - return "Default.BIA" - case _: - return value - class DataProcessingStrategies(BaseModel): """Data processing strategy defaults for the pdp3 config file. diff --git a/packages/pride-ppp/tests/test_processor.py b/packages/pride-ppp/tests/test_processor.py index ce24b33..8b46efe 100644 --- a/packages/pride-ppp/tests/test_processor.py +++ b/packages/pride-ppp/tests/test_processor.py @@ -12,6 +12,7 @@ import os from pathlib import Path from tempfile import TemporaryDirectory +from unittest.mock import MagicMock import pytest @@ -213,6 +214,24 @@ def test_unparseable_kinfile_returns_false( assert processor._validate_kinfile(garbage) is False +def test_resolve_forwards_product_download_override(processor: PrideProcessor) -> None: + processor._client = MagicMock() + processor._dep_spec = MagicMock() + processor._override_products_download = True + expected = MagicMock() + processor._client.resolve_dependencies.return_value = (expected, None) + + result = processor._resolve(datetime.datetime(2026, 8, 27, tzinfo=datetime.timezone.utc)) + + assert result is expected + processor._client.resolve_dependencies.assert_called_once_with( + processor._dep_spec, + datetime.datetime(2026, 8, 27, tzinfo=datetime.timezone.utc), + sink_id="pride", + force_download=True, + ) + + class TestRunPdp3: """Subprocess handling in _run_pdp3, exercised via a fake pdp3 on PATH.""" From 69186f3a96050bc0b9f20012fcf715a09b3ea4f0 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Thu, 27 Aug 2026 13:42:35 -0700 Subject: [PATCH 3/7] Handle non-UTF-8 PRIDE subprocess output --- .../src/pride_ppp/factories/processor.py | 2 ++ packages/pride-ppp/tests/test_processor.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/packages/pride-ppp/src/pride_ppp/factories/processor.py b/packages/pride-ppp/src/pride_ppp/factories/processor.py index db2acf8..6c91284 100644 --- a/packages/pride-ppp/src/pride_ppp/factories/processor.py +++ b/packages/pride-ppp/src/pride_ppp/factories/processor.py @@ -639,6 +639,8 @@ def _run_pdp3( cwd=tmpdir, capture_output=True, text=True, + encoding="utf-8", + errors="replace", ) # Replay stdout/stderr through the logger for observability diff --git a/packages/pride-ppp/tests/test_processor.py b/packages/pride-ppp/tests/test_processor.py index 8b46efe..3272006 100644 --- a/packages/pride-ppp/tests/test_processor.py +++ b/packages/pride-ppp/tests/test_processor.py @@ -276,6 +276,22 @@ def test_nonzero_exit_and_missing_output_are_logged( assert any("pdp3 exited with code 2" in m for m in caplog.messages) assert any("produced no kin output" in m for m in caplog.messages) + def test_non_utf8_process_output_does_not_abort_job( + self, fake_pdp3, tmp_path: Path, caplog + ) -> None: + fake_pdp3("printf '\\200bad output\\n'; exit 2") + out = tmp_path / "out" + + with caplog.at_level(logging.INFO, logger="pride_ppp.factories.processor"): + kin, res, rc, stderr = PrideProcessor._run_pdp3( + command=["pdp3"], site="NCC1", output_dir=out + ) + + assert rc == 2 + assert kin is None and res is None + assert stderr == "" + assert any("bad output" in message for message in caplog.messages) + def _unfulfilled_resolution() -> DependencyResolution: return DependencyResolution( From 67c5176f5afad34afa5e9ace2afcad4af2f09975 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Thu, 27 Aug 2026 14:09:23 -0700 Subject: [PATCH 4/7] Pass PRIDE frequency combinations as separate arguments --- packages/pride-ppp/src/pride_ppp/specifications/cli.py | 4 +++- packages/pride-ppp/tests/test_cli.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/pride-ppp/tests/test_cli.py diff --git a/packages/pride-ppp/src/pride_ppp/specifications/cli.py b/packages/pride-ppp/src/pride_ppp/specifications/cli.py index 4c7fb2d..c1a1dfa 100644 --- a/packages/pride-ppp/src/pride_ppp/specifications/cli.py +++ b/packages/pride-ppp/src/pride_ppp/specifications/cli.py @@ -149,7 +149,9 @@ def generate_pdp_command(self, site: str, local_file_path: str) -> list[str]: command.extend(["--system", self.system]) if self.frequency != ["G12", "R12", "E15", "C26", "J12"]: - command.extend(["--frequency", " ".join(self.frequency)]) + # pdp3.sh consumes each three-character frequency combination as + # a separate argv item until it reaches the next option. + command.extend(["--frequency", *self.frequency]) if self.loose_edit: command.append("--loose-edit") diff --git a/packages/pride-ppp/tests/test_cli.py b/packages/pride-ppp/tests/test_cli.py new file mode 100644 index 0000000..d1c40f4 --- /dev/null +++ b/packages/pride-ppp/tests/test_cli.py @@ -0,0 +1,10 @@ +from pride_ppp.specifications.cli import PrideCLIConfig + + +def test_frequency_combinations_are_separate_arguments() -> None: + config = PrideCLIConfig(frequency=["G12", "R12", "E17", "C27", "J12"]) + + command = config.generate_pdp_command("NTH1", "/tmp/nth1.rnx") + + index = command.index("--frequency") + assert command[index + 1 : index + 6] == ["G12", "R12", "E17", "C27", "J12"] From f7a2683872b0a13db431ba44e7a8cebdce6a0ae5 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Fri, 28 Aug 2026 10:37:53 -0700 Subject: [PATCH 5/7] Handle rounded PRIDE residual timestamps --- .../src/pride_ppp/factories/output.py | 30 ++++++++----------- .../pride-ppp/tests/test_output_parsing.py | 13 ++++++++ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/packages/pride-ppp/src/pride_ppp/factories/output.py b/packages/pride-ppp/src/pride_ppp/factories/output.py index 04aaddf..699272c 100644 --- a/packages/pride-ppp/src/pride_ppp/factories/output.py +++ b/packages/pride-ppp/src/pride_ppp/factories/output.py @@ -7,7 +7,7 @@ import logging import os -from datetime import datetime +from datetime import datetime, timedelta, timezone from pathlib import Path import pandas as pd @@ -59,22 +59,18 @@ def get_wrms_from_res(res_path): sumOfSquares = 0 sumOfWeights = 0 - seconds_str = line_data[6] - if "." in seconds_str: - SS, fractional = seconds_str.split(".") - SS = int(SS) - fractional = fractional.ljust(6, "0")[:6] - else: - SS = int(seconds_str) - fractional = "000000" - - isodate = ( - f"{line_data[1]}-{line_data[2].zfill(2)}-{line_data[3].zfill(2)}" - f"T{line_data[4].zfill(2)}:{line_data[5].zfill(2)}:{str(SS).zfill(2)}" - f".{fractional}+00:00" - ) - - timestamp = datetime.fromisoformat(isodate) + # PRIDE occasionally formats a rounded epoch with seconds + # equal to 60.0000000. Constructing an ISO timestamp with + # second=60 is invalid; adding the seconds as a timedelta + # correctly carries it into the following minute/day. + timestamp = datetime( + int(line_data[1]), + int(line_data[2]), + int(line_data[3]), + int(line_data[4]), + int(line_data[5]), + tzinfo=timezone.utc, + ) + timedelta(seconds=float(line_data[6])) timestamps.append(timestamp) line = res_file.readline() diff --git a/packages/pride-ppp/tests/test_output_parsing.py b/packages/pride-ppp/tests/test_output_parsing.py index d9d0e92..8ae619a 100644 --- a/packages/pride-ppp/tests/test_output_parsing.py +++ b/packages/pride-ppp/tests/test_output_parsing.py @@ -153,3 +153,16 @@ def test_wrms_values_in_mm_range(self, res_file: Path): """WRMS in mm — typical GNSS phase residuals are sub-centimetre.""" df = get_wrms_from_res(res_file) assert (df["wrms"] < 1000).all(), "WRMS suspiciously large (> 1000 mm)" + + def test_seconds_equal_sixty_roll_into_next_minute(self, tmp_path: Path): + res = tmp_path / "res_rollover.res" + res.write_text( + "Residuals COMMENT\n" + "TIM 2026 8 24 19 9 60.0000000 61276 69000.00\n" + "G01 0.0010 0.0000 1.0 1.0 0 45.0 90.0 L1C L2W C1C C2W\n" + ) + + df = get_wrms_from_res(res) + + assert df.loc[0, "date"] == pd.Timestamp("2026-08-24T19:10:00Z") + assert df.loc[0, "wrms"] == pytest.approx(1.0) From 0e905a3f41e6393a357204dee05236b05ee386f2 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Fri, 28 Aug 2026 11:11:41 -0700 Subject: [PATCH 6/7] RTS before RAP --- .../src/pride_ppp/configs/dependencies/pride_pppar.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml b/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml index 05808fb..c67af37 100644 --- a/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml +++ b/packages/pride-ppp/src/pride_ppp/configs/dependencies/pride_pppar.yaml @@ -22,9 +22,11 @@ preferences: description: > Prefer Wuhan, then CODE, GFZ, ESA analysis centres. - parameter: TTT - sorting: [FIN, RAP, ULT] + sorting: [FIN, RTS, RAP, ULT] description: > - Prefer final solutions, then rapid, then ultra-rapid. + Prefer final solutions, then Wuhan real-time streaming products, + then rapid and ultra-rapid products. RTS biases include the Galileo + phase signals needed for E17 processing before RAP products do. dependencies: - spec: ORBIT From 8d12b2e85852b7f87a6ac7c48749a70ba6f96622 Mon Sep 17 00:00:00 2001 From: gottlieb Date: Fri, 28 Aug 2026 11:16:57 -0700 Subject: [PATCH 7/7] Prefer RTS products and support CODE HTTPS archive --- .../client/product_query.py | 13 ++++++- .../factories/connection_pool.py | 19 +++++++++- .../test/test_code_https.py | 37 ++++++++++++++++++ .../gpm_specs/configs/centers/cod_config.yaml | 38 ++++++++++--------- .../src/pride_ppp/defaults/__init__.py | 2 +- .../src/pride_ppp/factories/processor.py | 4 +- packages/pride-ppp/tests/test_processor.py | 10 +++++ 7 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 packages/gnss-product-management/test/test_code_https.py diff --git a/packages/gnss-product-management/src/gnss_product_management/client/product_query.py b/packages/gnss-product-management/src/gnss_product_management/client/product_query.py index 83f0352..1931b3a 100644 --- a/packages/gnss-product-management/src/gnss_product_management/client/product_query.py +++ b/packages/gnss-product-management/src/gnss_product_management/client/product_query.py @@ -24,6 +24,12 @@ logger = logging.getLogger(__name__) +def _remote_uri(protocol: str, hostname: str, directory: str, filename: str) -> str: + """Build a remote URI without duplicating schemes or path separators.""" + base = hostname if "://" in hostname else f"{protocol}://{hostname}" + return f"{base.rstrip('/')}/{directory.strip('/')}/{filename}" + + class ProductQuery: """Fluent builder for constructing and executing a GNSS product search. @@ -352,8 +358,11 @@ def search(self) -> list[FoundResource]: ) else: proto = (rq.server.protocol or "ftp").lower() - uri = ( - f"{proto}://{hostname}/{rq.directory.value or rq.directory.pattern}/{filename}" # type: ignore[union-attr] + uri = _remote_uri( + proto, + hostname, + rq.directory.value or rq.directory.pattern, # type: ignore[union-attr] + filename, ) r = FoundResource( product=rq.product.name, diff --git a/packages/gnss-product-management/src/gnss_product_management/factories/connection_pool.py b/packages/gnss-product-management/src/gnss_product_management/factories/connection_pool.py index 85ee2f3..218f300 100644 --- a/packages/gnss-product-management/src/gnss_product_management/factories/connection_pool.py +++ b/packages/gnss-product-management/src/gnss_product_management/factories/connection_pool.py @@ -2,17 +2,23 @@ import logging import os +import re import threading from contextlib import contextmanager from pathlib import Path from typing import Optional -from urllib.parse import urlparse +from urllib.parse import quote, urlparse import fsspec import fsspec.utils logger = logging.getLogger(__name__) +_AIUB_DOWNLOAD_ROOT = "https://www.aiub.unibe.ch/download" +_AIUB_LISTING_ENDPOINT = ( + "https://code.aiub.unibe.ch/s3_script/aiub_s3_bucket_listing.php?path=" +) + class ConnectionPool: """Thread-safe pool of fsspec filesystem instances for a single host. @@ -249,6 +255,17 @@ def list_directory(self, hostname: str, directory: str) -> list[str]: full_path = pool.full_path(directory) def _ls(conn: "fsspec.AbstractFileSystem") -> list[str]: + if hostname.rstrip("/") == _AIUB_DOWNLOAD_ROOT: + listing_url = _AIUB_LISTING_ENDPOINT + quote(directory.strip("/"), safe="/") + html = conn.cat_file(listing_url) + if isinstance(html, bytes): + html = html.decode("utf-8", errors="replace") + download_prefix = re.escape( + f'{_AIUB_DOWNLOAD_ROOT}/{directory.strip("/")}/' + ) + return sorted( + set(re.findall(rf'href="{download_prefix}([^"/]+)"', html)) + ) raw = conn.ls(full_path, detail=False) return [Path(p).name for p in raw] diff --git a/packages/gnss-product-management/test/test_code_https.py b/packages/gnss-product-management/test/test_code_https.py new file mode 100644 index 0000000..0be1eff --- /dev/null +++ b/packages/gnss-product-management/test/test_code_https.py @@ -0,0 +1,37 @@ +"""Regression tests for CODE's S3-backed HTTPS product archive.""" + +from unittest.mock import MagicMock + +from gnss_product_management.client.product_query import _remote_uri +from gnss_product_management.factories.connection_pool import ConnectionPoolFactory + + +def test_code_https_listing_extracts_download_filenames(monkeypatch) -> None: + html = b""" + + COD0OPSRAP_20262390000_01D_05M_ORB.SP3 + + """ + factory = ConnectionPoolFactory(max_connections=1) + hostname = "https://www.aiub.unibe.ch/download" + factory.add_connection(hostname) + pool = factory._pools[hostname] + connection = MagicMock() + connection.cat_file.return_value = html + monkeypatch.setattr(pool, "_connect", lambda: connection) + + assert factory.list_directory(hostname, "CODE/") == [ + "COD0OPSRAP_20262390000_01D_05M_ORB.SP3" + ] + + +def test_remote_uri_keeps_existing_https_scheme() -> None: + assert _remote_uri( + "https", + "https://www.aiub.unibe.ch/download", + "CODE/", + "COD0OPSRAP_20262390000_01D_05M_ORB.SP3", + ) == ( + "https://www.aiub.unibe.ch/download/CODE/" + "COD0OPSRAP_20262390000_01D_05M_ORB.SP3" + ) diff --git a/packages/gpm-specs/src/gpm_specs/configs/centers/cod_config.yaml b/packages/gpm-specs/src/gpm_specs/configs/centers/cod_config.yaml index efef862..f3d9eb4 100644 --- a/packages/gpm-specs/src/gpm_specs/configs/centers/cod_config.yaml +++ b/packages/gpm-specs/src/gpm_specs/configs/centers/cod_config.yaml @@ -1,20 +1,22 @@ id: COD name: Center for Orbit Determination in Europe (AIUB) -website: http://www.aiub.unibe.ch/ +website: https://www.aiub.unibe.ch/ servers: - - id: code_ftp - name: Primary FTP - hostname: "ftp://ftp.aiub.unibe.ch" - protocol: ftp + - id: code_https + name: AIUB CODE Products HTTPS + hostname: "https://www.aiub.unibe.ch/download" + protocol: https auth_required: false - description: CODE FTP server at University of Bern + description: > + CODE product downloads at the University of Bern. Directory listings + are provided by AIUB's S3-backed product browser; FTP is retired. products: # ── Orbits ────────────────────────────────────────────────────── - id: code_orbit product_name: ORBIT - server_id: code_ftp + server_id: code_https available: true description: CODE precise orbits parameters: @@ -25,12 +27,12 @@ products: - {name: PPP, value: MGX} - {name: SMP, value: 05M} - {name: SMP, value: 15M} - directory: {pattern: "CODE/{YYYY}/"} + directory: {pattern: "CODE/"} # ── Clocks ────────────────────────────────────────────────────── - id: code_clock product_name: CLOCK - server_id: code_ftp + server_id: code_https available: true description: CODE precise clocks parameters: @@ -40,12 +42,12 @@ products: - {name: PPP, value: OPS} - {name: SMP, value: 30S} - {name: SMP, value: 05M} - directory: {pattern: "CODE/{YYYY}/"} + directory: {pattern: "CODE/"} # ── ERP ───────────────────────────────────────────────────────── - id: code_erp product_name: ERP - server_id: code_ftp + server_id: code_https available: true description: Earth rotation parameters parameters: @@ -53,12 +55,12 @@ products: - {name: TTT, value: FIN} - {name: TTT, value: RAP} - {name: PPP, value: OPS} - directory: {pattern: "CODE/{YYYY}/"} + directory: {pattern: "CODE/"} # ── Biases ────────────────────────────────────────────────────── - id: code_bias product_name: BIA - server_id: code_ftp + server_id: code_https available: true description: Differential code biases (DCB/OSB) parameters: @@ -66,12 +68,12 @@ products: - {name: TTT, value: FIN} - {name: TTT, value: RAP} - {name: PPP, value: OPS} - directory: {pattern: "CODE/{YYYY}/"} + directory: {pattern: "CODE/"} # ── Ionosphere ───────────────────────────────────────────────── - id: code_gim product_name: IONEX - server_id: code_ftp + server_id: code_https available: true description: Global Ionosphere Maps (CODE is primary producer) parameters: @@ -82,12 +84,12 @@ products: - {name: PPP, value: OPS} - {name: SMP, value: 01H} - {name: SMP, value: 02H} - directory: {pattern: "CODE/{YYYY}/"} + directory: {pattern: "CODE/"} # ── SINEX weekly solutions ────────────────────────────────────── - id: code_sinex product_name: SINEX - server_id: code_ftp + server_id: code_https available: true description: CODE weekly station coordinate SINEX solutions parameters: @@ -99,7 +101,7 @@ products: # ── Troposphere SINEX ────────────────────────────────────────── - id: code_trop product_name: TROP - server_id: code_ftp + server_id: code_https available: true description: CODE troposphere SINEX (zenith total delay + horizontal gradients) parameters: diff --git a/packages/pride-ppp/src/pride_ppp/defaults/__init__.py b/packages/pride-ppp/src/pride_ppp/defaults/__init__.py index 3dccc59..fbe244a 100644 --- a/packages/pride-ppp/src/pride_ppp/defaults/__init__.py +++ b/packages/pride-ppp/src/pride_ppp/defaults/__init__.py @@ -15,7 +15,7 @@ _CONFIGS_DIR = Path(__file__).resolve().parent.parent / "configs" -# Dependency spec: FIN → RAP → ULT cascade (default processing mode). +# Dependency spec: FIN → RTS → RAP → ULT cascade (default processing mode). PRIDE_PPPAR_SPEC = _CONFIGS_DIR / "dependencies" / "pride_pppar.yaml" # Dependency spec: FINAL-only products (TTT restricted to [FIN]). diff --git a/packages/pride-ppp/src/pride_ppp/factories/processor.py b/packages/pride-ppp/src/pride_ppp/factories/processor.py index 6c91284..d2753a5 100644 --- a/packages/pride-ppp/src/pride_ppp/factories/processor.py +++ b/packages/pride-ppp/src/pride_ppp/factories/processor.py @@ -66,7 +66,7 @@ class ProcessingMode(enum.Enum): Selects which dependency-spec YAML governs which products are accepted. - * ``DEFAULT`` — cascades through FIN → RAP → ULT. Uses the best + * ``DEFAULT`` — cascades through FIN → RTS → RAP → ULT. Uses the best available product at run time. Suitable for near-real-time processing or when the observation date is within the last two weeks. * ``FINAL`` — accepts only IGS final (FIN) products (available ≥13 days @@ -412,7 +412,7 @@ def __init__( mode: Product timeliness mode. Selects which dependency-spec YAML governs product resolution: - * ``ProcessingMode.DEFAULT`` — FIN → RAP → ULT cascade. + * ``ProcessingMode.DEFAULT`` — FIN → RTS → RAP → ULT cascade. * ``ProcessingMode.FINAL`` — only FINAL products. Also accepts the string literals ``"DEFAULT"`` or diff --git a/packages/pride-ppp/tests/test_processor.py b/packages/pride-ppp/tests/test_processor.py index 3272006..f1ee0ec 100644 --- a/packages/pride-ppp/tests/test_processor.py +++ b/packages/pride-ppp/tests/test_processor.py @@ -19,12 +19,14 @@ try: from gnss_product_management.specifications.dependencies.dependencies import ( DependencyResolution, + DependencySpec, ResolvedDependency, ) except ImportError as e: pytest.skip(f"gnss-product-management not installed: {e}", allow_module_level=True) from pride_ppp.factories import processor as processor_module +from pride_ppp.defaults import PRIDE_PPPAR_SPEC from pride_ppp.factories.processor import ( MissingProductsError, PrideProcessor, @@ -33,6 +35,14 @@ ) +def test_default_product_timeliness_prefers_rts_over_rap() -> None: + """RTS should win over RAP so near-real-time Galileo E17 has phase biases.""" + spec = DependencySpec.from_yaml(PRIDE_PPPAR_SPEC) + timeliness = next(p for p in spec.preferences if p.parameter == "TTT") + + assert timeliness.sorting == ["FIN", "RTS", "RAP", "ULT"] + + @pytest.fixture def temp_product_files() -> dict[str, str]: """Create temporary product files and return their paths as strings."""