From fed0799e270c311a476b66f35124bd525da3dc6e Mon Sep 17 00:00:00 2001 From: Jorge Garcia Date: Wed, 23 Sep 2026 10:10:41 -0700 Subject: [PATCH 1/4] fix(observability): authenticate Phoenix OTLP with Bearer token Phoenix authenticates OTLP trace ingestion with an Authorization: Bearer header. Clearwing sent PHOENIX_API_KEY as an "api_key" header, which Phoenix rejects with 401. The BatchSpanProcessor swallows the export failure, so traces were dropped silently with no error surfaced. Send "Authorization: Bearer " instead. Standard OTEL_EXPORTER_OTLP header variables still take precedence when set. --- clearwing/observability/otel.py | 5 ++++- tests/test_otel.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/clearwing/observability/otel.py b/clearwing/observability/otel.py index 25895423..402fc58b 100644 --- a/clearwing/observability/otel.py +++ b/clearwing/observability/otel.py @@ -101,7 +101,10 @@ def _otlp_exporter() -> SpanExporter: ) if api_key := os.environ.get("PHOENIX_API_KEY"): if not standard_headers_configured: - kwargs["headers"] = {"api_key": api_key} + # Phoenix authenticates OTLP ingestion via a Bearer token; an + # ``api_key`` header is rejected with 401 and the batch span + # processor drops the spans silently. + kwargs["headers"] = {"authorization": f"Bearer {api_key}"} return OTLPSpanExporter(**kwargs) return OTLPSpanExporter() diff --git a/tests/test_otel.py b/tests/test_otel.py index 212771f2..86a6c36a 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -108,7 +108,7 @@ def test_standard_and_phoenix_environment_detection(monkeypatch): assert otel.telemetry_configured() is False -def test_phoenix_api_key_becomes_otlp_header(monkeypatch): +def test_phoenix_api_key_becomes_bearer_header(monkeypatch): monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", raising=False) monkeypatch.delenv("OTEL_EXPORTER_OTLP_HEADERS", raising=False) @@ -123,7 +123,7 @@ def test_phoenix_api_key_becomes_otlp_header(monkeypatch): exporter.assert_called_once_with( endpoint="https://phoenix.example/v1/traces", - headers={"api_key": "secret-key"}, + headers={"authorization": "Bearer secret-key"}, ) From be1388dedba24201830256a382145943b64fc50f Mon Sep 17 00:00:00 2001 From: Jorge Garcia Date: Wed, 23 Sep 2026 10:31:41 -0700 Subject: [PATCH 2/4] fix(observability): bootstrap OTLP tracing for programmatic sourcehunt runs SourceHuntRunner.run()/arun() are reached directly by the eval harness, the sourcehunt agent tool, notebooks and per-repo campaign runs, all of which bypass the CLI and web entrypoints that call ObservabilityIntegration.bootstrap_from_env(). Without a bootstrap the process-wide tracer provider stays the no-op proxy, so instrumented LLM spans are silently dropped and never reach Phoenix. Bootstrap from env at the top of arun() -- the single async entry every run funnels through -- and force_flush() at the run() boundary so each run's spans are exported before the caller exits or advances to the next repo. Both are no-ops when OTLP export is not configured and bootstrap is idempotent, so CLI/web behavior is unchanged. --- clearwing/sourcehunt/runner.py | 38 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/clearwing/sourcehunt/runner.py b/clearwing/sourcehunt/runner.py index 38327844..360b5028 100644 --- a/clearwing/sourcehunt/runner.py +++ b/clearwing/sourcehunt/runner.py @@ -1233,19 +1233,26 @@ def _finalize_spend_ledger(self, status: str | None = None) -> dict[str, Any]: return summary def run(self) -> SourceHuntResult: + from clearwing.observability.otel import force_flush from clearwing.ui.llm_activity import llm_activity_panel self._ensure_spend_ledger() - with llm_activity_panel( - live=self._live, - budget_usd=self.budget_usd or None, - spend_ledger=self._spend_ledger, - trace_context=lambda: ( - getattr(self, "_otel_trace_id", None), - getattr(self, "_otel_span_id", None), - ), - ): - return asyncio.run(self.arun()) + try: + with llm_activity_panel( + live=self._live, + budget_usd=self.budget_usd or None, + spend_ledger=self._spend_ledger, + trace_context=lambda: ( + getattr(self, "_otel_trace_id", None), + getattr(self, "_otel_span_id", None), + ), + ): + return asyncio.run(self.arun()) + finally: + # Flush this run's spans before the caller exits or advances to the + # next repo. We deliberately do not disconnect: the shared provider + # must persist across repeated run() calls within one process. + force_flush() async def _arun_proof_flow(self) -> SourceHuntResult: """Run the proof-carrying engine and adapt its typed output.""" @@ -1953,6 +1960,17 @@ def _finalize_proof_manifest( @tracer.chain(name="SourceHunt") async def arun(self) -> SourceHuntResult: + # Programmatic callers (the eval harness, the sourcehunt agent tool, + # notebooks, campaign per-repo runs) reach the runner directly, + # bypassing the CLI/web entrypoints that normally wire up OTLP tracing. + # Bootstrap here — the single async entry all runs pass through — so + # their spans are exported instead of dropping into the no-op proxy + # provider. It is a no-op unless OTLP export is configured and is + # idempotent (a process-wide singleton), so it stays safe under the CLI. + from clearwing.observability.integration import ObservabilityIntegration + + ObservabilityIntegration.bootstrap_from_env() + self._run_started_at = datetime.now(timezone.utc).isoformat() self._run_started_monotonic = time.monotonic() span_context = otel_trace.get_current_span().get_span_context() From 61b5510d76f410a778c7e1e48c1a488e060f9fd9 Mon Sep 17 00:00:00 2001 From: Jorge Garcia Date: Wed, 23 Sep 2026 11:55:27 -0700 Subject: [PATCH 3/4] style(observability): trim inline commentary on tracing bootstrap --- clearwing/observability/otel.py | 3 --- clearwing/sourcehunt/runner.py | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/clearwing/observability/otel.py b/clearwing/observability/otel.py index 402fc58b..65e21cb6 100644 --- a/clearwing/observability/otel.py +++ b/clearwing/observability/otel.py @@ -101,9 +101,6 @@ def _otlp_exporter() -> SpanExporter: ) if api_key := os.environ.get("PHOENIX_API_KEY"): if not standard_headers_configured: - # Phoenix authenticates OTLP ingestion via a Bearer token; an - # ``api_key`` header is rejected with 401 and the batch span - # processor drops the spans silently. kwargs["headers"] = {"authorization": f"Bearer {api_key}"} return OTLPSpanExporter(**kwargs) return OTLPSpanExporter() diff --git a/clearwing/sourcehunt/runner.py b/clearwing/sourcehunt/runner.py index 360b5028..04555883 100644 --- a/clearwing/sourcehunt/runner.py +++ b/clearwing/sourcehunt/runner.py @@ -1960,8 +1960,7 @@ def _finalize_proof_manifest( @tracer.chain(name="SourceHunt") async def arun(self) -> SourceHuntResult: - # Programmatic callers (the eval harness, the sourcehunt agent tool, - # notebooks, campaign per-repo runs) reach the runner directly, + # Programmatic callers reach the runner directly, # bypassing the CLI/web entrypoints that normally wire up OTLP tracing. # Bootstrap here — the single async entry all runs pass through — so # their spans are exported instead of dropping into the no-op proxy From b98af9ac3deca3342e1167e29f16f90185f8529d Mon Sep 17 00:00:00 2001 From: Jorge Garcia Date: Wed, 23 Sep 2026 13:34:54 -0700 Subject: [PATCH 4/4] =?UTF-8?q?refactor(observability):=20address=20review?= =?UTF-8?q?=20=E2=80=94=20drop=20bootstrap=20in=20arun=20and=20inline=20co?= =?UTF-8?q?mments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clearwing/sourcehunt/runner.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/clearwing/sourcehunt/runner.py b/clearwing/sourcehunt/runner.py index 04555883..a45361e3 100644 --- a/clearwing/sourcehunt/runner.py +++ b/clearwing/sourcehunt/runner.py @@ -1249,9 +1249,6 @@ def run(self) -> SourceHuntResult: ): return asyncio.run(self.arun()) finally: - # Flush this run's spans before the caller exits or advances to the - # next repo. We deliberately do not disconnect: the shared provider - # must persist across repeated run() calls within one process. force_flush() async def _arun_proof_flow(self) -> SourceHuntResult: @@ -1960,16 +1957,6 @@ def _finalize_proof_manifest( @tracer.chain(name="SourceHunt") async def arun(self) -> SourceHuntResult: - # Programmatic callers reach the runner directly, - # bypassing the CLI/web entrypoints that normally wire up OTLP tracing. - # Bootstrap here — the single async entry all runs pass through — so - # their spans are exported instead of dropping into the no-op proxy - # provider. It is a no-op unless OTLP export is configured and is - # idempotent (a process-wide singleton), so it stays safe under the CLI. - from clearwing.observability.integration import ObservabilityIntegration - - ObservabilityIntegration.bootstrap_from_env() - self._run_started_at = datetime.now(timezone.utc).isoformat() self._run_started_monotonic = time.monotonic() span_context = otel_trace.get_current_span().get_span_context()