From 669f54c99b2a5e6c6242e37d101d6b9bca31a0a7 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:42:45 +0530 Subject: [PATCH 01/35] add verdict dataclasses and canonical sentence renderer --- test/test_sentence.py | 184 +++++++++++++++++++++++++++++++++++++ vitals/verdict/__init__.py | 19 ++++ vitals/verdict/types.py | 170 ++++++++++++++++++++++++++++++++++ 3 files changed, 373 insertions(+) create mode 100644 test/test_sentence.py create mode 100644 vitals/verdict/__init__.py create mode 100644 vitals/verdict/types.py diff --git a/test/test_sentence.py b/test/test_sentence.py new file mode 100644 index 0000000..d76481d --- /dev/null +++ b/test/test_sentence.py @@ -0,0 +1,184 @@ +"""Contract test — Verdict.sentence snapshot for each of the five §7 examples.""" + +from vitals.verdict.types import ( + Cause, + InconclusiveReason, + Subject, + Verdict, + VerdictState, +) + + +def test_sentence_steady(): + v = Verdict( + verdict_id="1234567890abcdef", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=VerdictState.STEADY, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=0.3, + cost_sigma=0.1, + cost_usd_per_req=0.001, + baseline_cost_usd_per_req=0.001, + velocity_ratio=1.0, + samples=412, + baseline_samples=30, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=None, + caveats=(), + falsifier="would flip to CHANGED at behavior >=3σ (currently 0.3σ)", + warming_progress=None, + exemplars=(), + ) + assert v.sentence == "STEADY · ragapp v1 · behavior +0.3σ · cost +0.1σ · n=412" + + +def test_sentence_changed_behavior(): + v = Verdict( + verdict_id="1234567890abcdef", + ts_unix=1774276327.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v2", + baseline_version="v1", + state=VerdictState.CHANGED, + subject=Subject.RELEASE, + cause=Cause.RELEASE, + flag_cost=False, + flag_behavior=True, + runaway=False, + behavior_sigma=4.2, + cost_sigma=0.3, + cost_usd_per_req=0.001, + baseline_cost_usd_per_req=0.001, + velocity_ratio=1.0, + samples=1240, + baseline_samples=30, + onset_ts_unix=1774276327.0, # 14:32:07 UTC + seconds_after_deploy=90.0, + inconclusive_reason=None, + caveats=("output_length_-31%",), + falsifier="would flip to STEADY if input drift >=3σ (currently 0.4σ)", + warming_progress=None, + exemplars=(), + ) + assert ( + v.sentence + == "CHANGED · behavior · v2 vs v1 · +4.2σ (normal ±1σ) · cost flat +0.3σ · onset 14:32:07, 90s after v2 deployed · n=1240" + ) + + +def test_sentence_changed_cost_runaway(): + v = Verdict( + verdict_id="1234567890abcdef", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=VerdictState.CHANGED, + subject=Subject.TIME, + cause=Cause.UNATTRIBUTED, + flag_cost=True, + flag_behavior=False, + runaway=True, + behavior_sigma=0.4, + cost_sigma=15.0, + cost_usd_per_req=0.05, + baseline_cost_usd_per_req=0.001, + velocity_ratio=51.0, + samples=88, + baseline_samples=30, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=None, + caveats=(), + falsifier="would flip to STEADY if velocity returns within 3σ for 120s", + warming_progress=None, + exemplars=(), + ) + assert ( + v.sentence + == "CHANGED · cost · runaway: 51× baseline burn rate · behavior flat +0.4σ · cause unattributed — no release in the last 5m · n=88" + ) + + +def test_sentence_inconclusive_input_shift(): + v = Verdict( + verdict_id="1234567890abcdef", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=VerdictState.INCONCLUSIVE, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=3.8, + cost_sigma=0.2, + cost_usd_per_req=0.001, + baseline_cost_usd_per_req=0.001, + velocity_ratio=1.0, + samples=205, + baseline_samples=30, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=InconclusiveReason.INPUT_SHIFT, + caveats=(), + falsifier="would resolve if input drift drops below 3σ", + warming_progress=None, + exemplars=(), + input_sigma=4.1, + ) + assert ( + v.sentence + == "INCONCLUSIVE · input_shift · behavior +3.8σ but input +4.1σ — your traffic changed, not your model · n=205" + ) + + +def test_sentence_warming(): + v = Verdict( + verdict_id="1234567890abcdef", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=VerdictState.WARMING, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=None, + cost_sigma=None, + cost_usd_per_req=None, + baseline_cost_usd_per_req=None, + velocity_ratio=None, + samples=340, + baseline_samples=30, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=InconclusiveReason.WARMING, + caveats=(), + falsifier="", + warming_progress=(340, 1000), + exemplars=(), + ) + assert v.sentence == "WARMING · ragapp v1 · collecting reference 340/1000 · est. 22m" diff --git a/vitals/verdict/__init__.py b/vitals/verdict/__init__.py new file mode 100644 index 0000000..fe1718b --- /dev/null +++ b/vitals/verdict/__init__.py @@ -0,0 +1,19 @@ +"""Verdict package — evaluation models, signals, scope state, attribution, and evaluator.""" + +from vitals.verdict.types import ( + Cause, + Exemplar, + InconclusiveReason, + Subject, + Verdict, + VerdictState, +) + +__all__ = [ + "VerdictState", + "Subject", + "Cause", + "InconclusiveReason", + "Exemplar", + "Verdict", +] diff --git a/vitals/verdict/types.py b/vitals/verdict/types.py new file mode 100644 index 0000000..e63dfcc --- /dev/null +++ b/vitals/verdict/types.py @@ -0,0 +1,170 @@ +"""Verdict data models, enums, and canonical sentence renderer (spec §3, §7).""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum + + +class VerdictState(str, Enum): + WARMING = "warming" + STEADY = "steady" + CHANGED = "changed" + INCONCLUSIVE = "inconclusive" + + +class Subject(str, Enum): + TIME = "time" # this window vs frozen reference + RELEASE = "release" # version B vs last STEADY version A + + +class Cause(str, Enum): + RELEASE = "release" + UNATTRIBUTED = "unattributed" + NONE = "none" + + +class InconclusiveReason(str, Enum): + LOW_SAMPLE = "low_sample" + INPUT_SHIFT = "input_shift" + WARMING = "warming" + + +@dataclass(frozen=True, slots=True) +class Exemplar: + kind: str # "worst" | "median" + trace_id: str + span_id: str + output_excerpt: str # <= 240 chars, whitespace-collapsed + behavior_sigma: float + + +def format_sigma(z: float | None, with_flat: bool = False) -> str: + if z is None: + return "N/A" + sign = "+" if z >= 0 else "" + val_str = f"{sign}{z:.1f}σ" + if with_flat and abs(z) < 1.0: + return f"flat {val_str}" + return val_str + + +@dataclass(frozen=True, slots=True) +class Verdict: + verdict_id: str # uuid4 hex, 16 chars + ts_unix: float + # scope + service_name: str + gen_ai_system: str + model: str + version: str # the version under evaluation + baseline_version: str | None + # judgment + state: VerdictState + subject: Subject + cause: Cause + flag_cost: bool + flag_behavior: bool + runaway: bool # cost velocity ratio breach + # evidence (sigma units — D4) + behavior_sigma: float | None + cost_sigma: float | None + cost_usd_per_req: float | None + baseline_cost_usd_per_req: float | None + velocity_ratio: float | None + samples: int + baseline_samples: int + # onset & attribution + onset_ts_unix: float | None + seconds_after_deploy: float | None + # honesty surface + inconclusive_reason: InconclusiveReason | None + caveats: tuple[str, ...] # e.g. ("output_length_-31%",) + falsifier: str # "would flip to STEADY if ..." + warming_progress: tuple[int, int] | None # (have, need) + exemplars: tuple[Exemplar, ...] + input_sigma: float | None = None + + @property + def sentence(self) -> str: + """Render the canonical one-liner (§7).""" + if self.state == VerdictState.WARMING: + have, need = self.warming_progress if self.warming_progress else (0, 30) + rem = max(0, need - have) + est_m = int(round(rem * 2 / 60)) if rem > 0 else 0 + return ( + f"WARMING · {self.service_name} {self.version} · " + f"collecting reference {have}/{need} · est. {est_m}m" + ) + + if self.state == VerdictState.STEADY: + b_str = format_sigma(self.behavior_sigma) + c_str = format_sigma(self.cost_sigma) + return ( + f"STEADY · {self.service_name} {self.version} · " + f"behavior {b_str} · cost {c_str} · n={self.samples}" + ) + + if self.state == VerdictState.INCONCLUSIVE: + reason = self.inconclusive_reason.value if self.inconclusive_reason else "unknown" + if self.inconclusive_reason == InconclusiveReason.INPUT_SHIFT: + b_str = format_sigma(self.behavior_sigma) + i_str = format_sigma(self.input_sigma) + return ( + f"INCONCLUSIVE · {reason} · behavior {b_str} but input {i_str} — " + f"your traffic changed, not your model · n={self.samples}" + ) + elif self.inconclusive_reason == InconclusiveReason.LOW_SAMPLE: + return ( + f"INCONCLUSIVE · {reason} · sample count {self.samples} below minimum · " + f"n={self.samples}" + ) + else: + return f"INCONCLUSIVE · {reason} · n={self.samples}" + + if self.state == VerdictState.CHANGED: + flags_list = [] + if self.flag_behavior: + flags_list.append("behavior") + if self.flag_cost: + flags_list.append("cost") + flags_str = " + ".join(flags_list) if flags_list else "change" + + parts = [f"CHANGED · {flags_str}"] + + if self.subject == Subject.RELEASE and self.baseline_version: + parts.append(f"{self.version} vs {self.baseline_version}") + + if self.runaway and self.velocity_ratio is not None: + parts.append(f"runaway: {int(round(self.velocity_ratio))}× baseline burn rate") + if self.behavior_sigma is not None: + parts.append(f"behavior {format_sigma(self.behavior_sigma, with_flat=True)}") + else: + if self.flag_behavior and self.behavior_sigma is not None: + parts.append(f"{format_sigma(self.behavior_sigma)} (normal ±1σ)") + elif self.behavior_sigma is not None: + parts.append(f"behavior {format_sigma(self.behavior_sigma, with_flat=True)}") + + if self.flag_cost and self.cost_sigma is not None: + parts.append(f"cost {format_sigma(self.cost_sigma)}") + elif self.cost_sigma is not None: + parts.append(f"cost {format_sigma(self.cost_sigma, with_flat=True)}") + + if self.cause == Cause.RELEASE and self.onset_ts_unix is not None: + dt_str = datetime.fromtimestamp(self.onset_ts_unix, tz=timezone.utc).strftime( + "%H:%M:%S" + ) + sec_str = ( + f"{int(round(self.seconds_after_deploy))}s" + if self.seconds_after_deploy is not None + else "0s" + ) + parts.append(f"onset {dt_str}, {sec_str} after {self.version} deployed") + elif self.cause == Cause.UNATTRIBUTED: + parts.append("cause unattributed — no release in the last 5m") + + parts.append(f"n={self.samples}") + return " · ".join(parts) + + return f"{self.state.value.upper()} · {self.service_name} {self.version} · n={self.samples}" From 4af3c5faa5f698db3c17c30843a9a62fff87ecca Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:42:59 +0530 Subject: [PATCH 02/35] implement calibrated signal --- test/test_signal.py | 51 ++++++++++++++++++++++++++++++++++++++ vitals/verdict/__init__.py | 2 ++ vitals/verdict/signal.py | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 test/test_signal.py create mode 100644 vitals/verdict/signal.py diff --git a/test/test_signal.py b/test/test_signal.py new file mode 100644 index 0000000..5906e3a --- /dev/null +++ b/test/test_signal.py @@ -0,0 +1,51 @@ +"""Unit tests for CalibratedSignal primitive (spec §17).""" + +import math +import pytest +from vitals.verdict.signal import CalibratedSignal + + +def test_signal_calibration_not_ready(): + sig = CalibratedSignal(calib_n=30) + assert not sig.calibrated() + for _ in range(29): + sig.observe_calibration(1.0) + assert not sig.calibrated() + + with pytest.raises(RuntimeError, match="Signal not calibrated"): + sig.z(1.0) + + +def test_signal_calibration_ready(): + sig = CalibratedSignal(calib_n=30) + for i in range(30): + sig.observe_calibration(10.0 + (i % 2)) # alternating 10 and 11 + assert sig.calibrated() + assert math.isclose(sig.mu, 10.5, abs_tol=1e-5) + # Z-score check + z_val = sig.z(10.5) + assert math.isclose(z_val, 0.0, abs_tol=1e-5) + + +def test_signal_sigma_floor(): + sig = CalibratedSignal(calib_n=5, sigma_floor=1e-4) + for _ in range(5): + sig.observe_calibration(5.0) # zero variance + assert sig.calibrated() + assert sig.sigma == 1e-4 + # z should use sigma_floor + assert sig.z(5.0) == 0.0 + assert math.isclose(sig.z(5.001), 10.0, abs_tol=1e-5) + + +def test_signal_z_score_calculation(): + sig = CalibratedSignal(calib_n=10) + values = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0] + for v in values: + sig.observe_calibration(v) + assert sig.calibrated() + assert math.isclose(sig.mu, 5.5, abs_tol=1e-5) + # sample std dev of 1..10 is ~3.02765 + expected_sigma = math.sqrt(sum((x - 5.5) ** 2 for x in values) / 9) + assert math.isclose(sig.sigma, expected_sigma, abs_tol=1e-5) + assert math.isclose(sig.z(8.52765), 1.0, abs_tol=1e-2) diff --git a/vitals/verdict/__init__.py b/vitals/verdict/__init__.py index fe1718b..52961f1 100644 --- a/vitals/verdict/__init__.py +++ b/vitals/verdict/__init__.py @@ -1,5 +1,6 @@ """Verdict package — evaluation models, signals, scope state, attribution, and evaluator.""" +from vitals.verdict.signal import CalibratedSignal from vitals.verdict.types import ( Cause, Exemplar, @@ -16,4 +17,5 @@ "InconclusiveReason", "Exemplar", "Verdict", + "CalibratedSignal", ] diff --git a/vitals/verdict/signal.py b/vitals/verdict/signal.py new file mode 100644 index 0000000..a7d7ec4 --- /dev/null +++ b/vitals/verdict/signal.py @@ -0,0 +1,47 @@ +"""CalibratedSignal — statistical primitive wrapping every measured quantity (spec §4.2).""" + +from __future__ import annotations + +import math + + +class CalibratedSignal: + + def __init__(self, calib_n: int, sigma_floor: float = 1e-6) -> None: + if calib_n < 2: + raise ValueError("calib_n must be >= 2") + self.calib_n: int = calib_n + self.sigma_floor: float = sigma_floor + self.n: int = 0 + self.mu: float = 0.0 + self._m2: float = 0.0 + self._sigma: float = 0.0 + + def observe_calibration(self, x: float) -> None: + """Observe a data point during the calibration phase.""" + if self.calibrated(): + return + self.n += 1 + delta = x - self.mu + self.mu += delta / self.n + delta2 = x - self.mu + self._m2 += delta * delta2 + + if self.n >= 2: + var = self._m2 / (self.n - 1) + self._sigma = math.sqrt(max(0.0, var)) + + def calibrated(self) -> bool: + """Return True if calibration window has reached required sample count.""" + return self.n >= self.calib_n + + @property + def sigma(self) -> float: + """Standard deviation with sigma_floor enforced.""" + return max(self._sigma, self.sigma_floor) + + def z(self, x: float) -> float: + """Compute (x - mu) / sigma. Raises RuntimeError if not calibrated.""" + if not self.calibrated(): + raise RuntimeError("Signal not calibrated") + return (x - self.mu) / self.sigma From c348316e5ef4f98210f6c9367fd61ca9e6715663 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:43:19 +0530 Subject: [PATCH 03/35] add verdict config and contract additions --- test/test_config.py | 52 ++++++++++++++++++++++++++++++++ vitals.yaml | 26 ++++++++++++++++ vitals/config/settings.py | 62 +++++++++++++++++++++++++++++++++++++++ vitals/contract.py | 28 ++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 test/test_config.py diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 0000000..1feb0d2 --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,52 @@ +"""Unit tests for VerdictConfig, ConsoleConfig, StoreConfig and validation rules.""" + +import pytest +from vitals.config.settings import VitalsConfig, load_config + + +def test_config_defaults(tmp_path): + cfg_file = tmp_path / "vitals.yaml" + cfg_file.write_text("") + cfg = load_config(str(cfg_file)) + + assert cfg.verdict.enabled is True + assert cfg.verdict.evaluate_interval_s == 10 + assert cfg.verdict.sigma_threshold == 3.0 + assert cfg.console.port == 8787 + assert cfg.store.path == "vitals.db" + + +def test_config_env_overrides(monkeypatch, tmp_path): + cfg_file = tmp_path / "vitals.yaml" + cfg_file.write_text("") + + monkeypatch.setenv("VITALS_CONSOLE_PORT", "9999") + monkeypatch.setenv("VITALS_STORE_PATH", "custom.db") + monkeypatch.setenv("VITALS_VERDICT_ENABLED", "false") + + cfg = load_config(str(cfg_file)) + assert cfg.console.port == 9999 + assert cfg.store.path == "custom.db" + assert cfg.verdict.enabled is False + + +def test_config_validation(): + cfg = VitalsConfig() + cfg.verdict.sigma_threshold = -1.0 + with pytest.raises(ValueError, match="verdict.sigma_threshold must be > 0"): + cfg.validate() + + cfg = VitalsConfig() + cfg.verdict.consecutive_ticks = 0 + with pytest.raises(ValueError, match="verdict.consecutive_ticks must be >= 1"): + cfg.validate() + + cfg = VitalsConfig() + cfg.verdict.min_samples = 2 + with pytest.raises(ValueError, match="verdict.min_samples must be >= 5"): + cfg.validate() + + cfg = VitalsConfig() + cfg.verdict.runaway_ratio = 0.5 + with pytest.raises(ValueError, match="verdict.runaway_ratio must be > 1"): + cfg.validate() diff --git a/vitals.yaml b/vitals.yaml index a3c6858..1cdf696 100644 --- a/vitals.yaml +++ b/vitals.yaml @@ -26,3 +26,29 @@ quality: weight_drift: 0.6 weight_consistency: 0.2 weight_stability: 0.2 + +verdict: + enabled: true + evaluate_interval_s: 10 # tick period + window_s: 300 # current-window lookback + window_max: 500 # per-version rolling record cap + min_samples: 30 # G1 floor + sigma_threshold: 3.0 # CHANGED entry + consecutive_ticks: 2 # hysteresis (bypassed by runaway) + min_hold_s: 120 # CHANGED -> STEADY dwell + heartbeat_s: 60 # re-emit unchanged verdict + runaway_ratio: 20.0 # velocity multiple -> immediate CHANGED + attribution_window_s: 300 # deploy->onset window for RELEASE cause + length_caveat_pct: 0.25 # G3 caveat trigger + calibration_samples: 30 # per-signal calibration window + exemplars_worst: 2 + exemplars_median: 1 + +console: + enabled: true + host: 127.0.0.1 + port: 8787 + +store: + path: vitals.db + retain_verdicts: 1000 diff --git a/vitals/config/settings.py b/vitals/config/settings.py index 55b136f..04bd63a 100644 --- a/vitals/config/settings.py +++ b/vitals/config/settings.py @@ -49,12 +49,47 @@ class QualityConfig: weight_stability: float = 0.2 +@dataclass +class VerdictConfig: + enabled: bool = True + evaluate_interval_s: int = 10 # tick period + window_s: int = 300 # current-window lookback + window_max: int = 500 # per-version rolling record cap + min_samples: int = 30 # G1 floor + sigma_threshold: float = 3.0 # CHANGED entry + consecutive_ticks: int = 2 # hysteresis (bypassed by runaway) + min_hold_s: int = 120 # CHANGED -> STEADY dwell + heartbeat_s: int = 60 # re-emit unchanged verdict + runaway_ratio: float = 20.0 # velocity multiple -> immediate CHANGED + attribution_window_s: int = 300 # deploy->onset window for RELEASE cause + length_caveat_pct: float = 0.25 # G3 caveat trigger + calibration_samples: int = 30 # per-signal calibration window + exemplars_worst: int = 2 + exemplars_median: int = 1 + + +@dataclass +class ConsoleConfig: + enabled: bool = True + host: str = "127.0.0.1" + port: int = 8787 + + +@dataclass +class StoreConfig: + path: str = "vitals.db" + retain_verdicts: int = 1000 + + @dataclass class VitalsConfig: receiver: ReceiverConfig = field(default_factory=ReceiverConfig) emit: EmitConfig = field(default_factory=EmitConfig) cost: CostConfig = field(default_factory=CostConfig) quality: QualityConfig = field(default_factory=QualityConfig) + verdict: VerdictConfig = field(default_factory=VerdictConfig) + console: ConsoleConfig = field(default_factory=ConsoleConfig) + store: StoreConfig = field(default_factory=StoreConfig) def validate(self) -> None: q = self.quality @@ -66,6 +101,22 @@ def validate(self) -> None: if self.emit.queue_max < 1: raise ValueError("emit.queue_max must be >= 1") + v = self.verdict + if v.sigma_threshold <= 0: + raise ValueError("verdict.sigma_threshold must be > 0") + if v.consecutive_ticks < 1: + raise ValueError("verdict.consecutive_ticks must be >= 1") + if v.min_samples < 5: + raise ValueError("verdict.min_samples must be >= 5") + if v.runaway_ratio <= 1: + raise ValueError("verdict.runaway_ratio must be > 1") + if v.evaluate_interval_s < 1: + raise ValueError("verdict.evaluate_interval_s must be >= 1") + if not (0 < v.length_caveat_pct < 1): + raise ValueError("verdict.length_caveat_pct must be between 0 and 1") + if v.exemplars_median < 1: + raise ValueError("verdict.exemplars_median must be >= 1") + def _apply_env_overrides(cfg: VitalsConfig) -> None: if v := os.getenv("VITALS_OTLP_GRPC_PORT"): @@ -74,6 +125,14 @@ def _apply_env_overrides(cfg: VitalsConfig) -> None: cfg.receiver.http_port = int(v) if v := os.getenv("SIGNOZ_OTLP_ENDPOINT"): cfg.emit.endpoint = v + if v := os.getenv("VITALS_CONSOLE_PORT"): + cfg.console.port = int(v) + if v := os.getenv("VITALS_CONSOLE_ENABLED"): + cfg.console.enabled = v.lower() == "true" + if v := os.getenv("VITALS_STORE_PATH"): + cfg.store.path = v + if v := os.getenv("VITALS_VERDICT_ENABLED"): + cfg.verdict.enabled = v.lower() == "true" def load_config(path: str | os.PathLike | None = "vitals.yaml") -> VitalsConfig: @@ -87,6 +146,9 @@ def load_config(path: str | os.PathLike | None = "vitals.yaml") -> VitalsConfig: ("emit", cfg.emit), ("cost", cfg.cost), ("quality", cfg.quality), + ("verdict", cfg.verdict), + ("console", cfg.console), + ("store", cfg.store), ): for key, value in (raw.get(section) or {}).items(): if hasattr(dc, key): diff --git a/vitals/contract.py b/vitals/contract.py index dac37c4..9d620ee 100644 --- a/vitals/contract.py +++ b/vitals/contract.py @@ -32,6 +32,25 @@ METRIC_HEALTH_SPANS_SKIPPED = "vitals.health.spans_skipped" # malformed / non-gen_ai METRIC_HEALTH_EMIT_ERRORS = "vitals.health.emit_errors" METRIC_HEALTH_BASELINE_STATE = "vitals.health.baseline_state" # 0=warming, 1=ready +METRIC_HEALTH_SCOPES = "vitals.health.scopes" # live scope count +METRIC_HEALTH_VERDICTS_EMITTED = "vitals.health.verdicts_emitted" + +# --- Verdict metrics (vitals.verdict.*) (V2 additive) --- +METRIC_VERDICT_STATE = "vitals.verdict.state" # 0=warming, 1=steady, 2=changed, 3=inconclusive +METRIC_VERDICT_BEHAVIOR_SIGMA = "vitals.verdict.behavior_sigma" # signed +METRIC_VERDICT_COST_SIGMA = "vitals.verdict.cost_sigma" # signed +METRIC_VERDICT_VELOCITY_RATIO = "vitals.verdict.velocity_ratio" # current / baseline burn rate +METRIC_VERDICT_SAMPLES = "vitals.verdict.samples" # window n + +# --- Verdict attribute names --- +ATTR_SUBJECT = "vitals.subject" +ATTR_CAUSE = "vitals.cause" +ATTR_FLAG_COST = "vitals.flag_cost" +ATTR_FLAG_BEHAVIOR = "vitals.flag_behavior" +ATTR_RUNAWAY = "vitals.runaway" +ATTR_FALSIFIER = "vitals.falsifier" +ATTR_CAVEATS = "vitals.caveats" +ATTR_EXEMPLARS = "vitals.exemplars" # --- Eval log record schema (one per scored response, trace_id-linked) --- # SigNoz native logs<->traces correlation keys off trace_id / span_id. @@ -49,5 +68,14 @@ STATE_WARMING = "warming" # baseline window not yet full — never emit a score STATE_SCORED = "scored" +# Verdict state enum numeric mapping for metrics: +VERDICT_STATE_NUM = { + "warming": 0, + "steady": 1, + "changed": 2, + "inconclusive": 3, +} + # The instrumentation scope name stamped on all Vitals-emitted telemetry. SCOPE_NAME = "vitals" + From 1f05359a9ca37ae2cc52cc5c5e529fd41f87c463 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:43:43 +0530 Subject: [PATCH 04/35] implement sqlite verdict store --- test/test_store.py | 113 ++++++++++++++++++++++++++++++ vitals/store/__init__.py | 5 ++ vitals/store/db.py | 147 +++++++++++++++++++++++++++++++++++++++ vitals/verdict/types.py | 113 ++++++++++++++++++++++++++++++ 4 files changed, 378 insertions(+) create mode 100644 test/test_store.py create mode 100644 vitals/store/__init__.py create mode 100644 vitals/store/db.py diff --git a/test/test_store.py b/test/test_store.py new file mode 100644 index 0000000..b48fbc5 --- /dev/null +++ b/test/test_store.py @@ -0,0 +1,113 @@ +"""Unit tests for SQLite VerdictStore (spec §17).""" + +import pytest +from vitals.store.db import VerdictStore +from vitals.verdict.types import ( + Cause, + Exemplar, + InconclusiveReason, + Subject, + Verdict, + VerdictState, +) + + +def _make_verdict(vid: str, ts: float) -> Verdict: + return Verdict( + verdict_id=vid, + ts_unix=ts, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=VerdictState.STEADY, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=0.2, + cost_sigma=0.1, + cost_usd_per_req=0.001, + baseline_cost_usd_per_req=0.001, + velocity_ratio=1.0, + samples=50, + baseline_samples=30, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=None, + caveats=(), + falsifier="falsifier string", + warming_progress=None, + exemplars=( + Exemplar( + kind="worst", + trace_id="tr1", + span_id="sp1", + output_excerpt="excerpt", + behavior_sigma=1.2, + ), + ), + ) + + +def test_store_insert_get_list(tmp_path): + db_path = tmp_path / "test_verdicts.db" + store = VerdictStore(path=str(db_path)) + + v1 = _make_verdict("v100000000000001", 100.0) + v2 = _make_verdict("v100000000000002", 200.0) + + store.insert(v1) + store.insert(v2) + + fetched_v1 = store.get("v100000000000001") + assert fetched_v1 is not None + assert fetched_v1.verdict_id == "v100000000000001" + assert fetched_v1.ts_unix == 100.0 + assert fetched_v1.state == VerdictState.STEADY + assert len(fetched_v1.exemplars) == 1 + assert fetched_v1.exemplars[0].trace_id == "tr1" + + verdicts = store.list(limit=50) + assert len(verdicts) == 2 + # newest first + assert verdicts[0].verdict_id == "v100000000000002" + assert verdicts[1].verdict_id == "v100000000000001" + + store.close() + + +def test_store_retention_pruning(tmp_path): + db_path = tmp_path / "test_retention.db" + store = VerdictStore(path=str(db_path), retain_verdicts=3) + + for i in range(5): + v = _make_verdict(f"v10000000000000{i}", 100.0 + i) + store.insert(v) + + verdicts = store.list(limit=10) + assert len(verdicts) == 3 + # should keep the 3 newest (timestamps 104.0, 103.0, 102.0) + retained_ids = [v.verdict_id for v in verdicts] + assert retained_ids == ["v100000000000004", "v100000000000003", "v100000000000002"] + assert store.get("v100000000000000") is None + + store.close() + + +def test_store_error_swallowing(tmp_path, caplog): + db_path = tmp_path / "test_err.db" + store = VerdictStore(path=str(db_path)) + + # Force connection to fail by closing it prematurely + store.close() + + v = _make_verdict("v100000000000009", 500.0) + # Should not raise exception + store.insert(v) + res = store.get("v100000000000009") + assert res is None + res_list = store.list() + assert res_list == [] diff --git a/vitals/store/__init__.py b/vitals/store/__init__.py new file mode 100644 index 0000000..ab4cd92 --- /dev/null +++ b/vitals/store/__init__.py @@ -0,0 +1,5 @@ +"""SQLite Verdict Store package.""" + +from vitals.store.db import VerdictStore + +__all__ = ["VerdictStore"] diff --git a/vitals/store/db.py b/vitals/store/db.py new file mode 100644 index 0000000..24c9772 --- /dev/null +++ b/vitals/store/db.py @@ -0,0 +1,147 @@ +"""SQLite Verdict Store implementation (spec §12, §15).""" + +from __future__ import annotations + +import json +import logging +import os +import sqlite3 +import threading +from typing import Any + +from vitals.verdict.types import Verdict + +logger = logging.getLogger("vitals.store") + + +class VerdictStore: + """Single SQLite verdict store, WAL mode, locked single-connection.""" + + def __init__(self, path: str | os.PathLike = "vitals.db", retain_verdicts: int = 1000) -> None: + self.path = str(path) + self.retain_verdicts = retain_verdicts + self._lock = threading.Lock() + self._conn: sqlite3.Connection | None = None + self._init_db() + + def _init_db(self) -> None: + try: + with self._lock: + self._conn = sqlite3.connect(self.path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL;") + self._conn.execute( + """ + CREATE TABLE IF NOT EXISTS verdicts ( + verdict_id TEXT PRIMARY KEY, + ts_unix REAL NOT NULL, + service_name TEXT NOT NULL, + version TEXT NOT NULL, + state TEXT NOT NULL, + subject TEXT NOT NULL, + cause TEXT NOT NULL, + behavior_sigma REAL, + cost_sigma REAL, + samples INTEGER NOT NULL, + sentence TEXT NOT NULL, + payload_json TEXT NOT NULL + ); + """ + ) + self._conn.execute( + "CREATE INDEX IF NOT EXISTS idx_verdicts_ts ON verdicts(ts_unix DESC);" + ) + self._conn.commit() + except Exception as exc: + logger.warning("Failed to initialize VerdictStore database at %s: %s", self.path, exc) + + def insert(self, verdict: Verdict, retain_count: int | None = None) -> None: + """Insert a verdict and prune beyond retain_count. Swallows exceptions (§15).""" + limit = retain_count if retain_count is not None else self.retain_verdicts + try: + payload = json.dumps(verdict.to_dict()) + with self._lock: + if self._conn is None: + return + self._conn.execute( + """ + INSERT INTO verdicts ( + verdict_id, ts_unix, service_name, version, state, subject, + cause, behavior_sigma, cost_sigma, samples, sentence, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + verdict.verdict_id, + verdict.ts_unix, + verdict.service_name, + verdict.version, + verdict.state.value, + verdict.subject.value, + verdict.cause.value, + verdict.behavior_sigma, + verdict.cost_sigma, + verdict.samples, + verdict.sentence, + payload, + ), + ) + # Retention pruning + self._conn.execute( + """ + DELETE FROM verdicts + WHERE verdict_id NOT IN ( + SELECT verdict_id FROM verdicts ORDER BY ts_unix DESC LIMIT ? + ) + """, + (limit,), + ) + self._conn.commit() + except Exception as exc: + logger.warning("Failed to insert verdict %s: %s", verdict.verdict_id, exc) + + def get(self, verdict_id: str) -> Verdict | None: + """Fetch a single verdict by ID. Swallows exceptions (§15).""" + try: + with self._lock: + if self._conn is None: + return None + cursor = self._conn.execute( + "SELECT payload_json FROM verdicts WHERE verdict_id = ?", (verdict_id,) + ) + row = cursor.fetchone() + if not row: + return None + payload_dict = json.loads(row["payload_json"]) + return Verdict.from_dict(payload_dict) + except Exception as exc: + logger.warning("Failed to get verdict %s: %s", verdict_id, exc) + return None + + def list(self, limit: int = 50) -> list[Verdict]: + """Fetch newest verdicts up to limit. Swallows exceptions (§15).""" + try: + with self._lock: + if self._conn is None: + return [] + cursor = self._conn.execute( + "SELECT payload_json FROM verdicts ORDER BY ts_unix DESC LIMIT ?", (limit,) + ) + rows = cursor.fetchall() + results = [] + for row in rows: + payload_dict = json.loads(row["payload_json"]) + results.append(Verdict.from_dict(payload_dict)) + return results + except Exception as exc: + logger.warning("Failed to list verdicts: %s", exc) + return [] + + def close(self) -> None: + """Close database connection cleanly.""" + try: + with self._lock: + if self._conn is not None: + self._conn.close() + self._conn = None + except Exception as exc: + logger.warning("Failed to close VerdictStore connection: %s", exc) diff --git a/vitals/verdict/types.py b/vitals/verdict/types.py index e63dfcc..bc6147e 100644 --- a/vitals/verdict/types.py +++ b/vitals/verdict/types.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum +from typing import Any class VerdictState(str, Enum): @@ -168,3 +169,115 @@ def sentence(self) -> str: return " · ".join(parts) return f"{self.state.value.upper()} · {self.service_name} {self.version} · n={self.samples}" + + def to_dict(self) -> dict[str, Any]: + """Serialize Verdict to a JSON-compatible dictionary.""" + return { + "verdict_id": self.verdict_id, + "ts_unix": self.ts_unix, + "service_name": self.service_name, + "gen_ai_system": self.gen_ai_system, + "model": self.model, + "version": self.version, + "baseline_version": self.baseline_version, + "state": self.state.value, + "subject": self.subject.value, + "cause": self.cause.value, + "flag_cost": self.flag_cost, + "flag_behavior": self.flag_behavior, + "runaway": self.runaway, + "behavior_sigma": self.behavior_sigma, + "cost_sigma": self.cost_sigma, + "cost_usd_per_req": self.cost_usd_per_req, + "baseline_cost_usd_per_req": self.baseline_cost_usd_per_req, + "velocity_ratio": self.velocity_ratio, + "samples": self.samples, + "baseline_samples": self.baseline_samples, + "onset_ts_unix": self.onset_ts_unix, + "seconds_after_deploy": self.seconds_after_deploy, + "inconclusive_reason": ( + self.inconclusive_reason.value if self.inconclusive_reason else None + ), + "caveats": list(self.caveats), + "falsifier": self.falsifier, + "warming_progress": list(self.warming_progress) if self.warming_progress else None, + "exemplars": [ + { + "kind": ex.kind, + "trace_id": ex.trace_id, + "span_id": ex.span_id, + "output_excerpt": ex.output_excerpt, + "behavior_sigma": ex.behavior_sigma, + } + for ex in self.exemplars + ], + "input_sigma": self.input_sigma, + "sentence": self.sentence, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> Verdict: + """Deserialize Verdict from a dictionary.""" + exemplars = tuple( + Exemplar( + kind=ex["kind"], + trace_id=ex["trace_id"], + span_id=ex["span_id"], + output_excerpt=ex["output_excerpt"], + behavior_sigma=float(ex["behavior_sigma"]), + ) + for ex in d.get("exemplars", []) + ) + inc_reason = ( + InconclusiveReason(d["inconclusive_reason"]) + if d.get("inconclusive_reason") + else None + ) + wp = tuple(d["warming_progress"]) if d.get("warming_progress") is not None else None + return cls( + verdict_id=d["verdict_id"], + ts_unix=float(d["ts_unix"]), + service_name=d["service_name"], + gen_ai_system=d["gen_ai_system"], + model=d["model"], + version=d["version"], + baseline_version=d.get("baseline_version"), + state=VerdictState(d["state"]), + subject=Subject(d["subject"]), + cause=Cause(d["cause"]), + flag_cost=bool(d["flag_cost"]), + flag_behavior=bool(d["flag_behavior"]), + runaway=bool(d["runaway"]), + behavior_sigma=( + float(d["behavior_sigma"]) if d.get("behavior_sigma") is not None else None + ), + cost_sigma=float(d["cost_sigma"]) if d.get("cost_sigma") is not None else None, + cost_usd_per_req=( + float(d["cost_usd_per_req"]) if d.get("cost_usd_per_req") is not None else None + ), + baseline_cost_usd_per_req=( + float(d["baseline_cost_usd_per_req"]) + if d.get("baseline_cost_usd_per_req") is not None + else None + ), + velocity_ratio=( + float(d["velocity_ratio"]) if d.get("velocity_ratio") is not None else None + ), + samples=int(d["samples"]), + baseline_samples=int(d["baseline_samples"]), + onset_ts_unix=( + float(d["onset_ts_unix"]) if d.get("onset_ts_unix") is not None else None + ), + seconds_after_deploy=( + float(d["seconds_after_deploy"]) + if d.get("seconds_after_deploy") is not None + else None + ), + inconclusive_reason=inc_reason, + caveats=tuple(d.get("caveats", ())), + falsifier=d.get("falsifier", ""), + warming_progress=wp, + exemplars=exemplars, + input_sigma=float(d["input_sigma"]) if d.get("input_sigma") is not None else None, + ) + From 2771d2f3b2123dd685e9c020c1b463a52c86f5d0 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:46:02 +0530 Subject: [PATCH 05/35] plumb input PSI and output length through quality engine and add cost velocity_for --- vitals/cost/engine.py | 7 +++++++ vitals/quality/baseline.py | 7 +++++++ vitals/quality/engine.py | 22 ++++++++++++++++++++-- vitals/quality/types.py | 2 ++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/vitals/cost/engine.py b/vitals/cost/engine.py index a94a4f4..5d5530a 100644 --- a/vitals/cost/engine.py +++ b/vitals/cost/engine.py @@ -91,3 +91,10 @@ def sample(self, now: float | None = None) -> list[CostSample]: ) ) return out + + def velocity_for(self, dims: dict[str, str], now: float | None = None) -> float: + """USD/min velocity for specific dimensions.""" + now = time.monotonic() if now is None else now + key = _dim_key(dims) + with self._lock: + return self._velocity(key, now) diff --git a/vitals/quality/baseline.py b/vitals/quality/baseline.py index fef4e49..65d856a 100644 --- a/vitals/quality/baseline.py +++ b/vitals/quality/baseline.py @@ -31,6 +31,7 @@ def __init__(self, window: int, calib_n: int, k_sigma: float, h_sigma: float): self._h_sigma = h_sigma self.outputs: list[str] = [] + self.reference_inputs: list[str] = [] self.ready = False # False while warming self._calib_values: list[float] = [] @@ -49,6 +50,12 @@ def add_warming_output(self, output: str) -> None: if len(self.outputs) >= self._window: self.ready = True + def add_warming_input(self, text: str) -> None: + """Collect an input into reference inputs; freeze when full.""" + if len(self.reference_inputs) >= self._window: + return + self.reference_inputs.append(text) + def observe_drift(self, drift: float) -> bool: """Feed one post-warming drift value into CUSUM. Returns True on the first onset only (the transition), False otherwise.""" diff --git a/vitals/quality/engine.py b/vitals/quality/engine.py index 5071486..6f84ffc 100644 --- a/vitals/quality/engine.py +++ b/vitals/quality/engine.py @@ -79,17 +79,20 @@ def score(self, span: GenAISpan) -> EvalLogRecord: if not baseline.ready: baseline.add_warming_output(span.output_text) + baseline.add_warming_input(span.input_text or " ") self._latest[tuple(sorted(dims.items()))] = QualityMetricSample( dims=dims, drift=None, consistency=None, stability=None, score=None, drift_onset=0, baseline_ready=False, ) return EvalLogRecord( trace_id=span.trace_id, span_id=span.span_id, dims=dims, - state="warming", reason="baseline warming " + state="warming", input_drift=None, output_len=len(span.output_text or ""), + reason="baseline warming " f"({len(baseline.outputs)}/{self._cfg.baseline_window})", ) drift = self._measure_drift(span, baseline) + input_drift = self._measure_input_drift(span, baseline) consistency, stability = self._measure_optional(span) baseline.observe_drift(drift) onset = 1 if baseline.onset else 0 @@ -106,7 +109,8 @@ def score(self, span: GenAISpan) -> EvalLogRecord: ) return EvalLogRecord( trace_id=span.trace_id, span_id=span.span_id, dims=dims, state="scored", - drift=drift, consistency=consistency, stability=stability, score=score, + drift=drift, input_drift=input_drift, output_len=len(span.output_text or ""), + consistency=consistency, stability=stability, score=score, drift_onset=onset, reason=reason, ) @@ -122,6 +126,20 @@ def _measure_drift(self, span: GenAISpan, baseline: Baseline) -> float: log.exception("quality: drift measure failed") return 0.0 + def _measure_input_drift(self, span: GenAISpan, baseline: Baseline) -> float | None: + if not baseline.reference_inputs: + return None + try: + tc = LLMTestCase( + input=" ", + actual_output=span.input_text or " ", + baseline_outputs=baseline.reference_inputs, + ) + return float(self._drift_metric.measure(tc)) + except Exception: # noqa: BLE001 — never crash the scorer + log.exception("quality: input drift measure failed") + return 0.0 + def _measure_optional(self, span: GenAISpan) -> tuple[float | None, float | None]: # Consistency/stability need multiple samples of the same prompt; in the online # single-shot path they are best-effort against the baseline set. diff --git a/vitals/quality/types.py b/vitals/quality/types.py index 3d9f310..89f62b2 100644 --- a/vitals/quality/types.py +++ b/vitals/quality/types.py @@ -27,6 +27,8 @@ class EvalLogRecord: dims: dict[str, str] state: str # "warming" | "scored" drift: float | None = None + input_drift: float | None = None + output_len: int = 0 consistency: float | None = None stability: float | None = None score: float | None = None From a9714e73bb6c272cde74fe89fa947de07abf1427 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:46:12 +0530 Subject: [PATCH 06/35] implement ScopeState and RollingWindow --- vitals/verdict/__init__.py | 3 + vitals/verdict/scope.py | 142 +++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 vitals/verdict/scope.py diff --git a/vitals/verdict/__init__.py b/vitals/verdict/__init__.py index 52961f1..3485a7b 100644 --- a/vitals/verdict/__init__.py +++ b/vitals/verdict/__init__.py @@ -1,5 +1,6 @@ """Verdict package — evaluation models, signals, scope state, attribution, and evaluator.""" +from vitals.verdict.scope import ScopeState, SpanRecord from vitals.verdict.signal import CalibratedSignal from vitals.verdict.types import ( Cause, @@ -18,4 +19,6 @@ "Exemplar", "Verdict", "CalibratedSignal", + "ScopeState", + "SpanRecord", ] diff --git a/vitals/verdict/scope.py b/vitals/verdict/scope.py new file mode 100644 index 0000000..2e02b3e --- /dev/null +++ b/vitals/verdict/scope.py @@ -0,0 +1,142 @@ +"""ScopeState and RollingWindow (spec §4.3, §4.4). + +ScopeKey = (service_name, gen_ai_system, model) — version excluded. +""" + +from __future__ import annotations + +import threading +import time +from collections import defaultdict, deque +from dataclasses import dataclass + +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.verdict.signal import CalibratedSignal +from vitals.verdict.types import Verdict, VerdictState + + +@dataclass(frozen=True, slots=True) +class SpanRecord: + ts: float + behavior_psi: float + input_psi: float + usd: float + out_len: int + trace_id: str + span_id: str + output_excerpt: str # <= 240 chars, whitespace-collapsed + + +class ScopeState: + """State and signal tracker for one ScopeKey (service_name, gen_ai_system, model).""" + + def __init__( + self, + service_name: str, + gen_ai_system: str, + model: str, + reference_window: int = 30, + calib_n: int = 30, + window_max: int = 500, + sigma_floor: float = 1e-6, + ) -> None: + self.service_name = service_name + self.gen_ai_system = gen_ai_system + self.model = model + self.key = (service_name, gen_ai_system, model) + + self._lock = threading.Lock() + self.reference_window = reference_window + self.calib_n = calib_n + self.window_max = window_max + + self.reference_outputs: list[str] = [] + self.reference_inputs: list[str] = [] + self.signals: dict[str, CalibratedSignal] = { + "behavior": CalibratedSignal(calib_n, sigma_floor), + "input": CalibratedSignal(calib_n, sigma_floor), + "cost": CalibratedSignal(calib_n, sigma_floor), + "length": CalibratedSignal(calib_n, sigma_floor), + } + self.windows: dict[str, deque[SpanRecord]] = defaultdict( + lambda: deque(maxlen=self.window_max) + ) + self.version_timeline: list[tuple[str, float]] = [] # (version, first_seen_ts) + self.current_state: VerdictState = VerdictState.WARMING + self.last_verdict: Verdict | None = None + self.state_since_ts: float = 0.0 + self.consecutive_condition_ticks: int = 0 + self.total_spans_seen: int = 0 + + def observe( + self, span: GenAISpan, record: EvalLogRecord, usd: float, now: float | None = None + ) -> None: + """Observe one mapped GenAISpan and its quality/cost records.""" + now = time.time() if now is None else now + version = span.service_version + + with self._lock: + self.total_spans_seen += 1 + # Record version timeline if first seen + if not any(v == version for v, _ in self.version_timeline): + self.version_timeline.append((version, now)) + + # Phase 1: Reference collection (first reference_window spans) + if len(self.reference_outputs) < self.reference_window: + self.reference_outputs.append(span.output_text) + self.reference_inputs.append(span.input_text or " ") + return + + # Phase 2: Post-reference spans + behavior_psi = record.drift if record.drift is not None else 0.0 + input_psi = record.input_drift if record.input_drift is not None else 0.0 + out_len = record.output_len + + # Calibrate signals + self.signals["behavior"].observe_calibration(behavior_psi) + self.signals["input"].observe_calibration(input_psi) + self.signals["cost"].observe_calibration(usd) + self.signals["length"].observe_calibration(float(out_len)) + + # Whitespace collapse excerpt + raw_excerpt = (span.output_text or "")[:240] + excerpt = " ".join(raw_excerpt.split()) + + rec = SpanRecord( + ts=now, + behavior_psi=behavior_psi, + input_psi=input_psi, + usd=usd, + out_len=out_len, + trace_id=span.trace_id, + span_id=span.span_id, + output_excerpt=excerpt, + ) + self.windows[version].append(rec) + + def is_reference_ready(self) -> bool: + with self._lock: + return len(self.reference_outputs) >= self.reference_window + + def is_calibrated(self) -> bool: + with self._lock: + return all(sig.calibrated() for sig in self.signals.values()) + + def is_live(self) -> bool: + with self._lock: + return ( + len(self.reference_outputs) >= self.reference_window + and all(sig.calibrated() for sig in self.signals.values()) + ) + + def warming_progress(self) -> tuple[tuple[int, int], str]: + """Returns ((have, need), phase_name) for progress reporting.""" + with self._lock: + ref_have = len(self.reference_outputs) + if ref_have < self.reference_window: + return (ref_have, self.reference_window), "collecting_reference" + calib_have = self.signals["behavior"].n + if calib_have < self.calib_n: + return (calib_have, self.calib_n), "calibrating" + return (self.calib_n, self.calib_n), "live" From 8a84f9a0182e171c3006654c54f52a302e84435c Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:46:35 +0530 Subject: [PATCH 07/35] add unit tests for ScopeState and detection substrate --- test/test_scope.py | 135 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 test/test_scope.py diff --git a/test/test_scope.py b/test/test_scope.py new file mode 100644 index 0000000..bb6388f --- /dev/null +++ b/test/test_scope.py @@ -0,0 +1,135 @@ +"""Unit tests for ScopeState and RollingWindow (spec §17, §19).""" + +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.verdict.scope import ScopeState + + +def _make_span(i: int, version: str = "v1") -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=f"input query {i}", + output_text=f"output response {i}", + input_tokens=10, + output_tokens=20, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_scope_reference_freeze_and_calibration(): + scope = ScopeState( + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + reference_window=30, + calib_n=30, + ) + + assert not scope.is_reference_ready() + assert not scope.is_calibrated() + assert not scope.is_live() + + # Feed 30 reference spans + for i in range(30): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="warming", + drift=None, + input_drift=None, + output_len=len(span.output_text), + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + assert scope.is_reference_ready() + assert len(scope.reference_outputs) == 30 + assert len(scope.reference_inputs) == 30 + assert not scope.is_calibrated() + assert not scope.is_live() + + # Feed 30 calibration spans (total 60 spans) + for i in range(30, 60): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.05, + input_drift=0.02, + output_len=len(span.output_text), + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + # Now at 60 spans, scope reaches LIVE and all 4 signals are calibrated + assert scope.is_calibrated() + assert scope.is_live() + for sig_name, sig in scope.signals.items(): + assert sig.calibrated(), f"Signal {sig_name} not calibrated" + + +def test_scope_rolling_window_cap_and_version_timeline(): + scope = ScopeState( + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + reference_window=10, + calib_n=10, + window_max=15, + ) + + # Feed reference spans + for i in range(10): + span = _make_span(i, version="v1") + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="warming", + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=100.0 + i) + + # Feed 20 v1 post-reference spans + for i in range(10, 30): + span = _make_span(i, version="v1") + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=100.0 + i) + + # Check rolling window maxlen cap (15) + assert len(scope.windows["v1"]) == 15 + + # Feed v2 span + v2_span = _make_span(31, version="v2") + v2_rec = EvalLogRecord( + trace_id=v2_span.trace_id, + span_id=v2_span.span_id, + dims=v2_span.dims(), + state="scored", + drift=0.5, + input_drift=0.01, + output_len=25, + ) + scope.observe(v2_span, v2_rec, usd=0.002, now=200.0) + + # Check version timeline contains both v1 and v2 with first seen timestamps + timeline = scope.version_timeline + assert len(timeline) == 2 + assert timeline[0] == ("v1", 100.0) + assert timeline[1] == ("v2", 200.0) From 56b73ef168fc6c2e239e1e572e407ac19d346f9c Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:49:31 +0530 Subject: [PATCH 08/35] implement attribution logic and unit tests --- test/test_attribution.py | 48 +++++++++++++++++++++++++++++++++++ vitals/verdict/__init__.py | 2 ++ vitals/verdict/attribution.py | 43 +++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 test/test_attribution.py create mode 100644 vitals/verdict/attribution.py diff --git a/test/test_attribution.py b/test/test_attribution.py new file mode 100644 index 0000000..1a4c1ad --- /dev/null +++ b/test/test_attribution.py @@ -0,0 +1,48 @@ +"""Unit tests for change attribution (spec §17).""" + +from vitals.verdict.attribution import attribute_change +from vitals.verdict.types import Cause + + +def test_attribution_release_within_window(): + timeline = [("v1", 1000.0), ("v2", 1200.0)] + onset_ts = 1290.0 # 90 seconds after v2 deployed + + cause, baseline_ver, sec_after = attribute_change( + version_timeline=timeline, + current_version="v2", + onset_ts=onset_ts, + attribution_window_s=300.0, + ) + + assert cause == Cause.RELEASE + assert baseline_ver == "v1" + assert sec_after == 90.0 + + +def test_attribution_unattributed_outside_window(): + timeline = [("v1", 1000.0), ("v2", 1200.0)] + onset_ts = 1600.0 # 400 seconds after v2 deployed (> 300s window) + + cause, baseline_ver, sec_after = attribute_change( + version_timeline=timeline, + current_version="v2", + onset_ts=onset_ts, + attribution_window_s=300.0, + ) + + assert cause == Cause.UNATTRIBUTED + assert baseline_ver is None + assert sec_after is None + + +def test_attribution_no_timeline(): + cause, baseline_ver, sec_after = attribute_change( + version_timeline=[], + current_version="v1", + onset_ts=1500.0, + ) + + assert cause == Cause.UNATTRIBUTED + assert baseline_ver is None + assert sec_after is None diff --git a/vitals/verdict/__init__.py b/vitals/verdict/__init__.py index 3485a7b..07eb652 100644 --- a/vitals/verdict/__init__.py +++ b/vitals/verdict/__init__.py @@ -1,5 +1,6 @@ """Verdict package — evaluation models, signals, scope state, attribution, and evaluator.""" +from vitals.verdict.attribution import attribute_change from vitals.verdict.scope import ScopeState, SpanRecord from vitals.verdict.signal import CalibratedSignal from vitals.verdict.types import ( @@ -21,4 +22,5 @@ "CalibratedSignal", "ScopeState", "SpanRecord", + "attribute_change", ] diff --git a/vitals/verdict/attribution.py b/vitals/verdict/attribution.py new file mode 100644 index 0000000..36af538 --- /dev/null +++ b/vitals/verdict/attribution.py @@ -0,0 +1,43 @@ +"""Change attribution logic (spec §4.6).""" + +from __future__ import annotations + +from vitals.verdict.types import Cause + + +def attribute_change( + version_timeline: list[tuple[str, float]], + current_version: str, + onset_ts: float, + attribution_window_s: float = 300.0, +) -> tuple[Cause, str | None, float | None]: + """Attribute a CHANGED verdict onset to a release or unattributed cause. + + Args: + version_timeline: List of (version_name, first_seen_ts) sorted by time. + current_version: Version string currently being evaluated. + onset_ts: Timestamp when change condition onset occurred. + attribution_window_s: Lookback window in seconds before onset_ts. + + Returns: + (cause, baseline_version, seconds_after_deploy) + """ + if not version_timeline: + return (Cause.UNATTRIBUTED, None, None) + + matching_idx = None + for idx, (ver, first_seen) in enumerate(version_timeline): + if ver == current_version: + matching_idx = idx + + if matching_idx is None: + return (Cause.UNATTRIBUTED, None, None) + + _, first_seen_ts = version_timeline[matching_idx] + delta = onset_ts - first_seen_ts + + if 0.0 <= delta <= attribution_window_s: + baseline_ver = version_timeline[matching_idx - 1][0] if matching_idx > 0 else None + return (Cause.RELEASE, baseline_ver, delta) + + return (Cause.UNATTRIBUTED, None, None) From 7e29dc2bf4b02f01e036c8e6853a34861b8c96cb Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:51:10 +0530 Subject: [PATCH 09/35] implement evaluator aggregation, guards, state machine, and exemplar selection --- test/test_evaluator.py | 114 +++++++++++++ test/test_guards.py | 141 ++++++++++++++++ test/test_state_machine.py | 241 +++++++++++++++++++++++++++ vitals/verdict/__init__.py | 3 + vitals/verdict/evaluator.py | 323 ++++++++++++++++++++++++++++++++++++ 5 files changed, 822 insertions(+) create mode 100644 test/test_evaluator.py create mode 100644 test/test_guards.py create mode 100644 test/test_state_machine.py create mode 100644 vitals/verdict/evaluator.py diff --git a/test/test_evaluator.py b/test/test_evaluator.py new file mode 100644 index 0000000..7a1634e --- /dev/null +++ b/test/test_evaluator.py @@ -0,0 +1,114 @@ +"""Unit tests for Verdict Evaluator and Exemplar Selection (spec §4.5, §9, §17).""" + +from vitals.config.settings import VerdictConfig +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.verdict.evaluator import evaluate_scope_version, select_exemplars +from vitals.verdict.scope import ScopeState, SpanRecord +from vitals.verdict.types import VerdictState + + +def _make_span(i: int, version: str = "v1") -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=f"input query {i}", + output_text=f"output response {i}\n line2 line3", + input_tokens=10, + output_tokens=20, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_select_exemplars_includes_worst_and_median(): + recs = [ + SpanRecord( + ts=100.0 + i, + behavior_psi=0.1 * i, + input_psi=0.01, + usd=0.001, + out_len=20, + trace_id=f"tr_{i}", + span_id=f"sp_{i}", + output_excerpt=f"excerpt {i}", + ) + for i in range(5) # behavior_psi: 0.0, 0.1, 0.2, 0.3, 0.4 + ] + + exemplars = select_exemplars(recs, behavior_z=2.5, worst_count=2, median_count=1) + + assert len(exemplars) == 3 # 2 worst + 1 median + kinds = [ex.kind for ex in exemplars] + assert kinds.count("worst") == 2 + assert kinds.count("median") == 1 + + # Worst exemplars should have highest behavior_psi (trace_id tr_4 and tr_3) + worst_traces = [ex.trace_id for ex in exemplars if ex.kind == "worst"] + assert "tr_4" in worst_traces + assert "tr_3" in worst_traces + + # Median exemplar should be middle index (tr_2) + median_ex = [ex for ex in exemplars if ex.kind == "median"][0] + assert median_ex.trace_id == "tr_2" + + +def test_evaluator_end_to_end_steady_and_changed(): + cfg = VerdictConfig( + min_samples=5, + calibration_samples=5, + consecutive_ticks=1, + sigma_threshold=3.0, + ) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # 1. Calibrate scope (5 ref + 5 calib) + for i in range(10): + span = _make_span(i, version="v1") + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=len(span.output_text), + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + assert scope.is_live() + + # 2. Evaluate steady state + v_steady = evaluate_scope_version(scope, "v1", cfg, now=1050.0) + assert v_steady is not None + assert v_steady.state == VerdictState.STEADY + assert len(v_steady.exemplars) >= 2 # contains worst and median + assert v_steady.sentence.startswith("STEADY · ragapp v1") + + # 3. Deploy v2 (version change) with high behavior drift + for i in range(10, 20): + span = _make_span(i, version="v2") + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.50, # high behavior drift + input_drift=0.01, + output_len=len(span.output_text), + ) + scope.observe(span, rec, usd=0.001, now=1100.0 + i) + + # 4. Evaluate v2 -> CHANGED with release attribution + v_changed = evaluate_scope_version(scope, "v2", cfg, now=1120.0) + assert v_changed is not None + assert v_changed.state == VerdictState.CHANGED + assert v_changed.flag_behavior is True + assert v_changed.cause.value == "release" + assert v_changed.baseline_version == "v1" + assert v_changed.seconds_after_deploy is not None + assert "v2 vs v1" in v_changed.sentence diff --git a/test/test_guards.py b/test/test_guards.py new file mode 100644 index 0000000..90c5e55 --- /dev/null +++ b/test/test_guards.py @@ -0,0 +1,141 @@ +"""Unit tests for guards G0-G3 (spec §17, §19).""" + +from vitals.config.settings import VerdictConfig +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.verdict.evaluator import evaluate_scope_version +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import InconclusiveReason, VerdictState + + +def _make_span(i: int, version: str = "v1", out_len: int = 100) -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=f"input query {i}", + output_text="x" * out_len, + input_tokens=10, + output_tokens=out_len // 4, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_guard_g0_warming(): + cfg = VerdictConfig(min_samples=10, calibration_samples=10) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=10, calib_n=10) + + verdict = evaluate_scope_version(scope, "v1", cfg, now=1000.0) + assert verdict is not None + assert verdict.state == VerdictState.WARMING + assert verdict.inconclusive_reason == InconclusiveReason.WARMING + + +def test_guard_g1_low_sample(): + cfg = VerdictConfig(min_samples=30, calibration_samples=5) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Populate 5 reference + 5 calibration spans (scope becomes LIVE) + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01 if i >= 5 else None, + input_drift=0.01 if i >= 5 else None, + output_len=100, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + assert scope.is_live() + + # Now evaluate version v1, which only has 5 spans in window (< min_samples 30) + verdict = evaluate_scope_version(scope, "v1", cfg, now=1100.0) + assert verdict is not None + assert verdict.state == VerdictState.INCONCLUSIVE + assert verdict.inconclusive_reason == InconclusiveReason.LOW_SAMPLE + + +def test_guard_g2_input_shift(): + cfg = VerdictConfig(min_samples=10, calibration_samples=5, sigma_threshold=3.0) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope with low noise + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=100, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + assert scope.is_live() + + # Add 15 spans with HIGH behavior drift AND HIGH input drift (co-movement) + for i in range(10, 25): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.50, # high behavior drift + input_drift=0.50, # high input drift + output_len=100, + ) + scope.observe(span, rec, usd=0.001, now=1100.0 + i) + + verdict = evaluate_scope_version(scope, "v1", cfg, now=1200.0) + assert verdict is not None + assert verdict.state == VerdictState.INCONCLUSIVE + assert verdict.inconclusive_reason == InconclusiveReason.INPUT_SHIFT + + +def test_guard_g3_length_shift_caveat_does_not_change_state(): + cfg = VerdictConfig(min_samples=10, calibration_samples=5, length_caveat_pct=0.25) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope with output_len = 100 + for i in range(10): + span = _make_span(i, out_len=100) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=100, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + # Add spans with output_len = 50 (50% reduction, abs(pct) >= 0.25) + for i in range(10, 30): + span = _make_span(i, out_len=50) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.01, # normal behavior + input_drift=0.01, # normal input + output_len=50, + ) + scope.observe(span, rec, usd=0.001, now=1100.0 + i) + + verdict = evaluate_scope_version(scope, "v1", cfg, now=1200.0) + assert verdict is not None + # G3 produces caveat ONLY, state remains STEADY (D8 regression test) + assert verdict.state == VerdictState.STEADY + assert any("output_length_" in c for c in verdict.caveats) diff --git a/test/test_state_machine.py b/test/test_state_machine.py new file mode 100644 index 0000000..3c97513 --- /dev/null +++ b/test/test_state_machine.py @@ -0,0 +1,241 @@ +"""Unit tests for Verdict state machine (spec §17).""" + +from vitals.config.settings import VerdictConfig +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.verdict.evaluator import evaluate_scope_version +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import InconclusiveReason, VerdictState + + +def _make_span(i: int, version: str = "v1") -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=f"input query {i}", + output_text=f"output response {i}", + input_tokens=10, + output_tokens=20, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_state_machine_hysteresis_2_ticks(): + cfg = VerdictConfig( + min_samples=5, calibration_samples=5, consecutive_ticks=2, min_hold_s=120 + ) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope (5 reference + 5 calibration) + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + assert scope.is_live() + + # Initial tick: STEADY + v0 = evaluate_scope_version(scope, "v1", cfg, now=1100.0) + assert v0.state == VerdictState.STEADY + + # Add high behavior drift spans (10 spans) + for i in range(10, 20): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.50, # high behavior drift + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1100.0 + i) + + # Tick 1: Condition True on 1st tick -> remain STEADY (consecutive_ticks=1 < 2) + v1 = evaluate_scope_version(scope, "v1", cfg, now=1110.0) + assert v1.state == VerdictState.STEADY + + # Tick 2: Condition True on 2nd tick -> transition to CHANGED + v2 = evaluate_scope_version(scope, "v1", cfg, now=1120.0) + assert v2.state == VerdictState.CHANGED + assert v2.flag_behavior is True + + +def test_state_machine_runaway_bypass_hysteresis(): + cfg = VerdictConfig( + min_samples=5, calibration_samples=5, consecutive_ticks=2, runaway_ratio=5.0 + ) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + # Spike USD cost massively (runaway velocity) + for i in range(10, 50): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.50, now=1100.0 + i) + + # Tick 1: Runaway bypasses hysteresis immediately + v1 = evaluate_scope_version(scope, "v1", cfg, now=1110.0) + assert v1.state == VerdictState.CHANGED + assert v1.runaway is True + assert v1.flag_cost is True + + +def test_state_machine_min_hold_dwell(): + cfg = VerdictConfig( + min_samples=5, + calibration_samples=5, + consecutive_ticks=1, + min_hold_s=120, + window_s=60, + ) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + # Trigger CHANGED at t=1050 + for i in range(10, 20): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.50, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1050.0 + i) + + v_changed = evaluate_scope_version(scope, "v1", cfg, now=1060.0) + assert v_changed.state == VerdictState.CHANGED + + # Now condition clears (clean spans at t=1140, lookback window=60s clears old t=1060 high-drift spans) + for i in range(20, 30): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1140.0 + i) + + # At t=1150 (90s dwell since onset t=1060, < 120s min_hold_s) -> must remain CHANGED due to dwell hold + v_held = evaluate_scope_version(scope, "v1", cfg, now=1150.0) + assert v_held.state == VerdictState.CHANGED + + # At t=1200 (140s dwell since onset t=1060, > 120s min_hold_s) -> transitions back to STEADY + v_steady = evaluate_scope_version(scope, "v1", cfg, now=1200.0) + assert v_steady.state == VerdictState.STEADY + + +def test_state_machine_inconclusive_non_latching(): + cfg = VerdictConfig( + min_samples=5, + calibration_samples=5, + consecutive_ticks=1, + sigma_threshold=3.0, + window_s=60, + ) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + + # Calibrate scope + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1000.0 + i) + + # Cause input shift G2 (co-movement) at t=1050 + for i in range(10, 20): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.50, + input_drift=0.50, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1050.0 + i) + + v_inc = evaluate_scope_version(scope, "v1", cfg, now=1060.0) + assert v_inc.state == VerdictState.INCONCLUSIVE + assert v_inc.inconclusive_reason == InconclusiveReason.INPUT_SHIFT + + # Next clean tick at t=1200 (> window_s=60s so old input shift spans fall out) + for i in range(20, 30): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=1200.0 + i) + + v_clean = evaluate_scope_version(scope, "v1", cfg, now=1210.0) + # INCONCLUSIVE did not latch; cleanly returned to STEADY + assert v_clean.state == VerdictState.STEADY diff --git a/vitals/verdict/__init__.py b/vitals/verdict/__init__.py index 07eb652..9c3a703 100644 --- a/vitals/verdict/__init__.py +++ b/vitals/verdict/__init__.py @@ -1,6 +1,7 @@ """Verdict package — evaluation models, signals, scope state, attribution, and evaluator.""" from vitals.verdict.attribution import attribute_change +from vitals.verdict.evaluator import evaluate_scope_version, select_exemplars from vitals.verdict.scope import ScopeState, SpanRecord from vitals.verdict.signal import CalibratedSignal from vitals.verdict.types import ( @@ -23,4 +24,6 @@ "ScopeState", "SpanRecord", "attribute_change", + "evaluate_scope_version", + "select_exemplars", ] diff --git a/vitals/verdict/evaluator.py b/vitals/verdict/evaluator.py new file mode 100644 index 0000000..6dc9a4d --- /dev/null +++ b/vitals/verdict/evaluator.py @@ -0,0 +1,323 @@ +"""Verdict Evaluator: aggregate windows, run guards G0-G3, manage state machine, select exemplars (spec §4.5, §5, §6).""" + +from __future__ import annotations + +import logging +import uuid +from statistics import fmean +from typing import TYPE_CHECKING + +from vitals.config.settings import VerdictConfig +from vitals.verdict.attribution import attribute_change +from vitals.verdict.scope import ScopeState, SpanRecord +from vitals.verdict.types import ( + Cause, + Exemplar, + InconclusiveReason, + Subject, + Verdict, + VerdictState, +) + +if TYPE_CHECKING: + from vitals.cost.engine import CostEngine + +logger = logging.getLogger("vitals.verdict") + + +def select_exemplars( + recs: list[SpanRecord], + behavior_z: float, + worst_count: int = 2, + median_count: int = 1, +) -> tuple[Exemplar, ...]: + """Select worst and median exemplars from span records (spec §4.5, §9).""" + if not recs: + return () + + sorted_recs = sorted(recs, key=lambda r: r.behavior_psi, reverse=True) + exemplars: list[Exemplar] = [] + + # Worst exemplars + actual_worst_n = min(worst_count, len(sorted_recs)) + for r in sorted_recs[:actual_worst_n]: + exemplars.append( + Exemplar( + kind="worst", + trace_id=r.trace_id, + span_id=r.span_id, + output_excerpt=r.output_excerpt, + behavior_sigma=behavior_z, + ) + ) + + # Median exemplar (always included beside worst) + if median_count >= 1: + med_idx = len(sorted_recs) // 2 + med_rec = sorted_recs[med_idx] + exemplars.append( + Exemplar( + kind="median", + trace_id=med_rec.trace_id, + span_id=med_rec.span_id, + output_excerpt=med_rec.output_excerpt, + behavior_sigma=behavior_z, + ) + ) + + return tuple(exemplars) + + +def evaluate_scope_version( + scope: ScopeState, + version: str, + cfg: VerdictConfig, + now: float, + cost_engine: CostEngine | None = None, +) -> Verdict | None: + """Evaluate one scope x version state and return a Verdict (spec §4.5).""" + records = list(scope.windows.get(version, [])) + recs = [r for r in records if r.ts >= (now - cfg.window_s)] + n = len(recs) + + # Guard G0: Warming check + if not scope.is_live(): + (have, need), phase = scope.warming_progress() + return Verdict( + verdict_id=uuid.uuid4().hex[:16], + ts_unix=now, + service_name=scope.service_name, + gen_ai_system=scope.gen_ai_system, + model=scope.model, + version=version, + baseline_version=None, + state=VerdictState.WARMING, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=None, + cost_sigma=None, + cost_usd_per_req=None, + baseline_cost_usd_per_req=None, + velocity_ratio=None, + samples=n, + baseline_samples=need, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=InconclusiveReason.WARMING, + caveats=(), + falsifier="would transition to STEADY when reference and calibration finish", + warming_progress=(have, need), + exemplars=(), + ) + + # Aggregation & Normalization + mean_behavior_psi = fmean([r.behavior_psi for r in recs]) if recs else 0.0 + mean_input_psi = fmean([r.input_psi for r in recs]) if recs else 0.0 + mean_cost_usd = fmean([r.usd for r in recs]) if recs else 0.0 + mean_out_len = fmean([r.out_len for r in recs]) if recs else 0.0 + + behavior_z = scope.signals["behavior"].z(mean_behavior_psi) + input_z = scope.signals["input"].z(mean_input_psi) + cost_z = scope.signals["cost"].z(mean_cost_usd) + + current_velocity = ( + cost_engine.velocity_for( + { + "service.name": scope.service_name, + "service.version": version, + "gen_ai.system": scope.gen_ai_system, + "gen_ai.request.model": scope.model, + }, + now=now, + ) + if cost_engine is not None + else (sum(r.usd for r in recs) * (60.0 / cfg.window_s) if recs else 0.0) + ) + + baseline_cost_usd = scope.signals["cost"].mu + baseline_velocity = (baseline_cost_usd * scope.calib_n) * (60.0 / cfg.window_s) + velocity_ratio = (current_velocity / baseline_velocity) if baseline_velocity > 0 else None + runaway = velocity_ratio is not None and velocity_ratio >= cfg.runaway_ratio + + # Subject selection + baseline_version: str | None = None + subject = Subject.TIME + timeline = scope.version_timeline + if len(timeline) > 1: + matching_idx = next((i for i, (v, _) in enumerate(timeline) if v == version), None) + if matching_idx is not None and matching_idx > 0: + subject = Subject.RELEASE + baseline_version = timeline[matching_idx - 1][0] + + # Guard G1: Low sample + if n < cfg.min_samples: + return Verdict( + verdict_id=uuid.uuid4().hex[:16], + ts_unix=now, + service_name=scope.service_name, + gen_ai_system=scope.gen_ai_system, + model=scope.model, + version=version, + baseline_version=baseline_version, + state=VerdictState.INCONCLUSIVE, + subject=subject, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=behavior_z, + cost_sigma=cost_z, + cost_usd_per_req=mean_cost_usd, + baseline_cost_usd_per_req=baseline_cost_usd, + velocity_ratio=velocity_ratio, + samples=n, + baseline_samples=cfg.min_samples, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=InconclusiveReason.LOW_SAMPLE, + caveats=(), + falsifier=f"would resolve if sample size reaches {cfg.min_samples}", + warming_progress=None, + exemplars=select_exemplars( + recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median + ), + input_sigma=input_z, + ) + + # Guard G2: Input co-movement + if behavior_z >= cfg.sigma_threshold and input_z >= cfg.sigma_threshold: + return Verdict( + verdict_id=uuid.uuid4().hex[:16], + ts_unix=now, + service_name=scope.service_name, + gen_ai_system=scope.gen_ai_system, + model=scope.model, + version=version, + baseline_version=baseline_version, + state=VerdictState.INCONCLUSIVE, + subject=subject, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=False, + runaway=False, + behavior_sigma=behavior_z, + cost_sigma=cost_z, + cost_usd_per_req=mean_cost_usd, + baseline_cost_usd_per_req=baseline_cost_usd, + velocity_ratio=velocity_ratio, + samples=n, + baseline_samples=cfg.min_samples, + onset_ts_unix=None, + seconds_after_deploy=None, + inconclusive_reason=InconclusiveReason.INPUT_SHIFT, + caveats=(), + falsifier="would resolve if input drift drops below 3σ", + warming_progress=None, + exemplars=select_exemplars( + recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median + ), + input_sigma=input_z, + ) + + # Guard G3: Length shift caveat (D8) + caveats_list: list[str] = [] + baseline_len = scope.signals["length"].mu + if baseline_len > 0: + pct_len_change = (mean_out_len - baseline_len) / baseline_len + if abs(pct_len_change) >= cfg.length_caveat_pct: + caveats_list.append(f"output_length_{int(round(pct_len_change * 100)):+d}%") + + # State Machine Evaluation + cond_behavior = behavior_z >= cfg.sigma_threshold + cond_cost = (cost_z >= cfg.sigma_threshold) or runaway + condition = cond_behavior or cond_cost + + if condition: + if runaway: + next_state = VerdictState.CHANGED + scope.consecutive_condition_ticks = cfg.consecutive_ticks + else: + scope.consecutive_condition_ticks += 1 + if scope.consecutive_condition_ticks >= cfg.consecutive_ticks: + next_state = VerdictState.CHANGED + else: + next_state = ( + scope.current_state + if scope.current_state != VerdictState.WARMING + else VerdictState.STEADY + ) + else: + scope.consecutive_condition_ticks = 0 + if scope.current_state == VerdictState.CHANGED: + if (now - scope.state_since_ts) >= cfg.min_hold_s: + next_state = VerdictState.STEADY + else: + next_state = VerdictState.CHANGED # dwell hold + else: + next_state = VerdictState.STEADY + + if next_state != scope.current_state: + scope.current_state = next_state + scope.state_since_ts = now + + # Attribution for CHANGED state + if next_state == VerdictState.CHANGED: + cause, attr_baseline_ver, sec_after = attribute_change( + scope.version_timeline, version, now, cfg.attribution_window_s + ) + flag_behavior = cond_behavior + flag_cost = cond_cost + onset_ts = now + if attr_baseline_ver: + baseline_version = attr_baseline_ver + else: + cause = Cause.NONE + flag_behavior = False + flag_cost = False + onset_ts = None + sec_after = None + + # Dynamic falsifier string + if next_state == VerdictState.CHANGED: + if flag_behavior: + falsifier = f"would flip to STEADY if input drift >=3σ (currently {input_z:.1f}σ)" + else: + falsifier = "would flip to STEADY if velocity returns within 3σ for 120s" + else: + falsifier = f"would flip to CHANGED at behavior >=3σ (currently {behavior_z:.1f}σ)" + + exemplars = select_exemplars(recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median) + + return Verdict( + verdict_id=uuid.uuid4().hex[:16], + ts_unix=now, + service_name=scope.service_name, + gen_ai_system=scope.gen_ai_system, + model=scope.model, + version=version, + baseline_version=baseline_version, + state=next_state, + subject=subject, + cause=cause, + flag_cost=flag_cost, + flag_behavior=flag_behavior, + runaway=runaway, + behavior_sigma=behavior_z, + cost_sigma=cost_z, + cost_usd_per_req=mean_cost_usd, + baseline_cost_usd_per_req=baseline_cost_usd, + velocity_ratio=velocity_ratio, + samples=n, + baseline_samples=cfg.min_samples, + onset_ts_unix=onset_ts, + seconds_after_deploy=sec_after, + inconclusive_reason=None, + caveats=tuple(caveats_list), + falsifier=falsifier, + warming_progress=None, + exemplars=exemplars, + input_sigma=input_z, + ) From 06e8cd6335624c93418dd918a56910be82dd6a56 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:52:05 +0530 Subject: [PATCH 10/35] wire evaluator daemon thread into main pipeline with health counters and integration tests --- test/test_evaluator_daemon.py | 84 ++++++++++++++++++++++ vitals/health.py | 12 ++++ vitals/main.py | 132 +++++++++++++++++++++++++++++++--- 3 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 test/test_evaluator_daemon.py diff --git a/test/test_evaluator_daemon.py b/test/test_evaluator_daemon.py new file mode 100644 index 0000000..68031dc --- /dev/null +++ b/test/test_evaluator_daemon.py @@ -0,0 +1,84 @@ +"""Integration test for EvaluatorThread daemon (spec §4.5, §13, §15).""" + +import time +import pytest +from vitals.config.settings import VerdictConfig +from vitals.cost.engine import CostEngine +from vitals.cost.prices import PriceTable +from vitals.health import Health +from vitals.main import EvaluatorThread +from vitals.model import GenAISpan +from vitals.quality.types import EvalLogRecord +from vitals.store.db import VerdictStore +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import VerdictState + + +def _make_span(i: int, version: str = "v1") -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=f"input query {i}", + output_text=f"output response {i}", + input_tokens=10, + output_tokens=20, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_evaluator_daemon_thread_lifecycle(tmp_path): + db_path = tmp_path / "eval_daemon.db" + store = VerdictStore(str(db_path)) + health = Health() + prices = PriceTable.from_yaml("vitals/cost/prices.yaml") + cost_engine = CostEngine(prices) + + cfg = VerdictConfig( + enabled=True, + evaluate_interval_s=1, + min_samples=5, + calibration_samples=5, + heartbeat_s=10, + ) + + scope_key = ("ragapp", "openai", "gpt-4o") + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=5, calib_n=5) + scopes = {scope_key: scope} + + # Populate 10 spans (5 reference + 5 calibration) to bring scope to LIVE + now = time.time() + for i in range(10): + span = _make_span(i) + rec = EvalLogRecord( + trace_id=span.trace_id, + span_id=span.span_id, + dims=span.dims(), + state="scored" if i >= 5 else "warming", + drift=0.01, + input_drift=0.01, + output_len=20, + ) + scope.observe(span, rec, usd=0.001, now=now + i) + + evaluator = EvaluatorThread(scopes, store, cfg, cost_engine, health) + evaluator.start() + + try: + # Give evaluator daemon thread time to tick + time.sleep(1.5) + + verdicts = store.list(limit=10) + assert len(verdicts) >= 1 + latest = verdicts[0] + assert latest.state == VerdictState.STEADY + assert health.verdicts_emitted >= 1 + + finally: + evaluator.stop() + evaluator.join(timeout=2.0) + store.close() diff --git a/vitals/health.py b/vitals/health.py index 8306f1b..fd2ab63 100644 --- a/vitals/health.py +++ b/vitals/health.py @@ -11,6 +11,8 @@ def __init__(self) -> None: self.spans_scored = 0 self.emit_errors = 0 self.baseline_ready = 0 # 0 = warming, 1 = at least one baseline ready + self.scopes = 0 + self.verdicts_emitted = 0 # receiver-owned counters are read via the injected stats object self._recv_stats = None @@ -25,6 +27,14 @@ def inc_emit_error(self) -> None: with self._lock: self.emit_errors += 1 + def inc_verdicts_emitted(self) -> None: + with self._lock: + self.verdicts_emitted += 1 + + def set_scopes(self, count: int) -> None: + with self._lock: + self.scopes = count + def set_baseline_ready(self, ready: bool) -> None: with self._lock: self.baseline_ready = 1 if ready else 0 @@ -39,4 +49,6 @@ def snapshot(self) -> dict[str, float]: "spans_skipped": float(skipped), "emit_errors": float(self.emit_errors), "baseline_state": float(self.baseline_ready), + "scopes": float(self.scopes), + "verdicts_emitted": float(self.verdicts_emitted), } diff --git a/vitals/main.py b/vitals/main.py index 79a8390..acc21f3 100644 --- a/vitals/main.py +++ b/vitals/main.py @@ -1,12 +1,9 @@ """`vitals run` — single entrypoint wiring the full pipeline. collector fan-out ─▶ OTLPReceiver ─▶ on_span ─┬─▶ CostEngine - └─▶ QualityEngine ─▶ eval log - Emitter (out-of-band) ◀── cost/quality/health providers ─▶ SigNoz - -Quality scoring runs synchronously inside the receiver's thread pool (out of the user's -request path already — this is the fan-out copy). A misbehaving span is counted and -skipped; nothing here can affect the user's pipeline or SigNoz path. + ├─▶ QualityEngine ─▶ eval log + └─▶ ScopeState ──▶ EvaluatorThread ─▶ VerdictStore + Emitter (out-of-band) ◀── cost/quality/health providers ─────────────▶ SigNoz """ from __future__ import annotations @@ -16,6 +13,7 @@ import signal import sys import threading +import time from vitals import __version__ from vitals.config import load_config @@ -25,10 +23,90 @@ from vitals.health import Health from vitals.ingest.receiver import OTLPReceiver from vitals.model import GenAISpan +from vitals.store import VerdictStore +from vitals.verdict.evaluator import evaluate_scope_version +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import VerdictState log = logging.getLogger(__name__) +class EvaluatorThread(threading.Thread): + """Evaluator daemon thread ticking on evaluate_interval_s (spec §4.5, §13).""" + + def __init__( + self, + scopes: dict[tuple, ScopeState], + store: VerdictStore, + cfg, + cost_engine: CostEngine, + health: Health, + ) -> None: + super().__init__(name="EvaluatorThread", daemon=True) + self._scopes = scopes + self._store = store + self._cfg = cfg + self._cost_engine = cost_engine + self._health = health + self._stop_event = threading.Event() + self._last_emitted_ts: dict[tuple[tuple, str], float] = {} + + def stop(self) -> None: + self._stop_event.set() + + def run(self) -> None: + while not self._stop_event.is_set(): + now = time.time() + self._tick(now) + self._stop_event.wait(float(self._cfg.evaluate_interval_s)) + + def _tick(self, now: float) -> None: + for scope_key, scope in list(self._scopes.items()): + try: + versions = list(scope.windows.keys()) + if not versions and scope.version_timeline: + versions = [v for v, _ in scope.version_timeline] + if not versions: + versions = ["v1"] + + for ver in versions: + verdict = evaluate_scope_version( + scope, ver, self._cfg, now, self._cost_engine + ) + if verdict is None: + continue + + key = (scope_key, ver) + last_v = scope.last_verdict + last_ts = self._last_emitted_ts.get(key, 0.0) + + should_emit = False + if last_v is None: + should_emit = True + elif verdict.state != last_v.state: + should_emit = True + elif verdict.state == VerdictState.CHANGED: + b_curr = verdict.behavior_sigma or 0.0 + b_prev = last_v.behavior_sigma or 0.0 + if abs(b_curr - b_prev) >= 1.0: + should_emit = True + elif (now - last_ts) >= self._cfg.heartbeat_s: + should_emit = True + + if should_emit: + scope.last_verdict = verdict + self._last_emitted_ts[key] = now + self._store.insert(verdict) + self._health.inc_verdicts_emitted() + + if verdict.state == VerdictState.CHANGED: + log.warning("%s", verdict.sentence) + else: + log.info("%s", verdict.sentence) + except Exception: # noqa: BLE001 — a failing scope must never stop the tick + log.exception("evaluator: error processing scope %s", scope_key) + + def build_pipeline(config_path: str | None = "vitals.yaml"): cfg = load_config(config_path) health = Health() @@ -38,11 +116,14 @@ def build_pipeline(config_path: str | None = "vitals.yaml"): quality_engine = None if cfg.quality.enabled: - # Imported here so the cost-only path never pays the spanIQ import cost. from vitals.quality.engine import QualityEngine quality_engine = QualityEngine(cfg.quality) + store = VerdictStore(cfg.store.path, cfg.store.retain_verdicts) + scopes: dict[tuple, ScopeState] = {} + scopes_lock = threading.Lock() + emitter = Emitter( endpoint=cfg.emit.endpoint, export_interval_ms=cfg.emit.export_interval_ms, @@ -55,9 +136,27 @@ def build_pipeline(config_path: str | None = "vitals.yaml"): def on_span(span: GenAISpan) -> None: cost_engine.record(span) + usd = price_table.cost_usd(span.model, span.input_tokens, span.output_tokens) + + key = (span.service_name, span.gen_ai_system, span.model) + with scopes_lock: + scope = scopes.get(key) + if scope is None: + scope = ScopeState( + service_name=span.service_name, + gen_ai_system=span.gen_ai_system, + model=span.model, + reference_window=cfg.quality.baseline_window, + calib_n=cfg.verdict.calibration_samples, + window_max=cfg.verdict.window_max, + ) + scopes[key] = scope + health.set_scopes(len(scopes)) + if quality_engine is not None: try: rec = quality_engine.score(span) + scope.observe(span, rec, usd) emitter.emit_eval_log(rec) health.inc_scored() if rec.state == "scored": @@ -66,17 +165,26 @@ def on_span(span: GenAISpan) -> None: log.exception("pipeline: scoring error") health.inc_emit_error() + evaluator = None + if cfg.verdict.enabled: + evaluator = EvaluatorThread(scopes, store, cfg.verdict, cost_engine, health) + receiver = OTLPReceiver(cfg.receiver.host, cfg.receiver.grpc_port, on_span) health.bind_receiver_stats(receiver.stats) - return cfg, receiver, emitter, health + return cfg, receiver, emitter, health, store, evaluator def run(config_path: str | None = "vitals.yaml") -> None: - cfg, receiver, emitter, _ = build_pipeline(config_path) + cfg, receiver, emitter, _, store, evaluator = build_pipeline(config_path) receiver.start() + if evaluator is not None: + evaluator.start() + log.info( "vitals %s running — receiver :%d -> emitting to %s", - __version__, cfg.receiver.grpc_port, cfg.emit.endpoint, + __version__, + cfg.receiver.grpc_port, + cfg.emit.endpoint, ) stop = threading.Event() @@ -91,7 +199,11 @@ def _shutdown(*_): stop.wait() finally: receiver.stop() + if evaluator is not None: + evaluator.stop() + evaluator.join(timeout=2.0) emitter.shutdown() + store.close() def main(argv: list[str] | None = None) -> int: From ae7528ddd39b3ff6c38104bd38692bd8eab5094e Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:55:12 +0530 Subject: [PATCH 11/35] add verdict metric gauges and provider callback to emitter --- vitals/emit/emitter.py | 56 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/vitals/emit/emitter.py b/vitals/emit/emitter.py index 74c8ea2..620ed43 100644 --- a/vitals/emit/emitter.py +++ b/vitals/emit/emitter.py @@ -27,12 +27,14 @@ from vitals import contract from vitals.cost.engine import CostSample from vitals.quality.types import EvalLogRecord, QualityMetricSample +from vitals.verdict.types import Verdict log = logging.getLogger(__name__) CostProvider = Callable[[], list[CostSample]] QualityProvider = Callable[[], list[QualityMetricSample]] HealthProvider = Callable[[], dict[str, float]] +VerdictProvider = Callable[[], list[Verdict]] def _obs(value: float | None, dims: dict[str, str]) -> Observation | None: @@ -49,12 +51,14 @@ def __init__( cost_provider: CostProvider, quality_provider: QualityProvider, health_provider: HealthProvider, + verdict_provider: VerdictProvider | None = None, insecure: bool = True, ): self._resource = Resource.create({"service.name": contract.SCOPE_NAME}) self._cost_provider = cost_provider self._quality_provider = quality_provider self._health_provider = health_provider + self._verdict_provider = verdict_provider # --- metrics: direct OTLP gRPC to SigNoz ingest --- metric_exporter = OTLPMetricExporter(endpoint=endpoint, insecure=insecure) @@ -148,6 +152,58 @@ def health(_: CallbackOptions): "vitals.health", callbacks=[health], unit="1" ) + # --- verdict metric gauges (V2 additive) --- + def v_field(getter): + def cb(_: CallbackOptions): + if not self._verdict_provider: + return [] + obs_list = [] + for v in self._verdict_provider(): + val = getter(v) + if val is None: + continue + dims = { + contract.SERVICE_NAME: v.service_name, + contract.SERVICE_VERSION: v.version, + contract.GEN_AI_SYSTEM: v.gen_ai_system, + contract.GEN_AI_MODEL: v.model, + contract.ATTR_SUBJECT: v.subject.value, + contract.ATTR_CAUSE: v.cause.value, + contract.ATTR_FLAG_COST: v.flag_cost, + contract.ATTR_FLAG_BEHAVIOR: v.flag_behavior, + contract.ATTR_RUNAWAY: v.runaway, + } + obs_list.append(Observation(float(val), attributes=dims)) + return obs_list + + return cb + + meter.create_observable_gauge( + contract.METRIC_VERDICT_STATE, + callbacks=[v_field(lambda v: contract.VERDICT_STATE_NUM.get(v.state.value, 0))], + unit="1", + ) + meter.create_observable_gauge( + contract.METRIC_VERDICT_BEHAVIOR_SIGMA, + callbacks=[v_field(lambda v: v.behavior_sigma)], + unit="1", + ) + meter.create_observable_gauge( + contract.METRIC_VERDICT_COST_SIGMA, + callbacks=[v_field(lambda v: v.cost_sigma)], + unit="1", + ) + meter.create_observable_gauge( + contract.METRIC_VERDICT_VELOCITY_RATIO, + callbacks=[v_field(lambda v: v.velocity_ratio)], + unit="1", + ) + meter.create_observable_gauge( + contract.METRIC_VERDICT_SAMPLES, + callbacks=[v_field(lambda v: v.samples)], + unit="1", + ) + # -------------------------------------------------------------------- logs def emit_eval_log(self, rec: EvalLogRecord) -> None: attrs: dict[str, object] = dict(rec.dims) From 09a6c7e900b7f8d9b463f051ed126fe442135553 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:55:30 +0530 Subject: [PATCH 12/35] implement emit_verdict_log with trace-linked worst exemplar --- vitals/emit/emitter.py | 82 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/vitals/emit/emitter.py b/vitals/emit/emitter.py index 620ed43..8727d59 100644 --- a/vitals/emit/emitter.py +++ b/vitals/emit/emitter.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import logging import time from collections.abc import Callable @@ -242,6 +243,87 @@ def emit_eval_log(self, rec: EvalLogRecord) -> None: ) self._logger.emit(record) + def emit_verdict_log(self, verdict: Verdict) -> None: + """Emit a trace-linked verdict log record (spec §8.2).""" + attrs: dict[str, object] = { + contract.SERVICE_NAME: verdict.service_name, + contract.SERVICE_VERSION: verdict.version, + contract.GEN_AI_SYSTEM: verdict.gen_ai_system, + contract.GEN_AI_MODEL: verdict.model, + "vitals.verdict_id": verdict.verdict_id, + "vitals.ts_unix": verdict.ts_unix, + "vitals.state": verdict.state.value, + "vitals.subject": verdict.subject.value, + "vitals.cause": verdict.cause.value, + "vitals.flag_cost": verdict.flag_cost, + "vitals.flag_behavior": verdict.flag_behavior, + "vitals.runaway": verdict.runaway, + "vitals.samples": verdict.samples, + "vitals.baseline_samples": verdict.baseline_samples, + contract.ATTR_FALSIFIER: verdict.falsifier, + contract.ATTR_CAVEATS: ",".join(verdict.caveats) if verdict.caveats else "", + } + + for k, v in ( + ("vitals.baseline_version", verdict.baseline_version), + ("vitals.behavior_sigma", verdict.behavior_sigma), + ("vitals.cost_sigma", verdict.cost_sigma), + ("vitals.cost_usd_per_req", verdict.cost_usd_per_req), + ("vitals.baseline_cost_usd_per_req", verdict.baseline_cost_usd_per_req), + ("vitals.velocity_ratio", verdict.velocity_ratio), + ("vitals.onset_ts_unix", verdict.onset_ts_unix), + ("vitals.seconds_after_deploy", verdict.seconds_after_deploy), + ( + "vitals.inconclusive_reason", + verdict.inconclusive_reason.value if verdict.inconclusive_reason else None, + ), + ): + if v is not None: + attrs[k] = v + + if verdict.exemplars: + ex_dicts = [ + { + "kind": ex.kind, + "trace_id": ex.trace_id, + "span_id": ex.span_id, + "excerpt": ex.output_excerpt, + "sigma": ex.behavior_sigma, + } + for ex in verdict.exemplars + ] + ex_json = json.dumps(ex_dicts) + attrs[contract.ATTR_EXEMPLARS] = ex_json[:2048] + + # Trace link to the worst exemplar + worst_ex = next((ex for ex in verdict.exemplars if ex.kind == "worst"), None) + if worst_ex is None and verdict.exemplars: + worst_ex = verdict.exemplars[0] + + trace_id_str = worst_ex.trace_id if worst_ex else "" + span_id_str = worst_ex.span_id if worst_ex else "" + + try: + trace_id_int = int(trace_id_str, 16) if trace_id_str else 0 + span_id_int = int(span_id_str, 16) if span_id_str else 0 + except ValueError: + trace_id_int = span_id_int = 0 + + is_changed = verdict.state.value == "changed" + sev_num = SeverityNumber.WARN if is_changed else SeverityNumber.INFO + sev_text = "WARN" if is_changed else "INFO" + + record = LogRecord( + timestamp=int(verdict.ts_unix * 1e9), + trace_id=trace_id_int, + span_id=span_id_int, + severity_number=sev_num, + severity_text=sev_text, + body=verdict.sentence, + attributes=attrs, + ) + self._logger.emit(record) + def shutdown(self) -> None: try: self._meter_provider.shutdown() From e0808dcff5b6f8321b3adda5d056efccd78f4c6b Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:55:49 +0530 Subject: [PATCH 13/35] wire verdict metric provider and log emitter in main pipeline --- vitals/main.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/vitals/main.py b/vitals/main.py index acc21f3..d54c6ab 100644 --- a/vitals/main.py +++ b/vitals/main.py @@ -41,6 +41,8 @@ def __init__( cfg, cost_engine: CostEngine, health: Health, + emitter: Emitter | None = None, + verdict_snapshot: dict | None = None, ) -> None: super().__init__(name="EvaluatorThread", daemon=True) self._scopes = scopes @@ -48,6 +50,8 @@ def __init__( self._cfg = cfg self._cost_engine = cost_engine self._health = health + self._emitter = emitter + self._verdict_snapshot = verdict_snapshot if verdict_snapshot is not None else {} self._stop_event = threading.Event() self._last_emitted_ts: dict[tuple[tuple, str], float] = {} @@ -96,9 +100,17 @@ def _tick(self, now: float) -> None: if should_emit: scope.last_verdict = verdict self._last_emitted_ts[key] = now + self._verdict_snapshot[key] = verdict self._store.insert(verdict) self._health.inc_verdicts_emitted() + if self._emitter is not None: + try: + self._emitter.emit_verdict_log(verdict) + except Exception: # noqa: BLE001 + log.exception("evaluator: error emitting verdict log") + self._health.inc_emit_error() + if verdict.state == VerdictState.CHANGED: log.warning("%s", verdict.sentence) else: @@ -123,6 +135,7 @@ def build_pipeline(config_path: str | None = "vitals.yaml"): store = VerdictStore(cfg.store.path, cfg.store.retain_verdicts) scopes: dict[tuple, ScopeState] = {} scopes_lock = threading.Lock() + verdict_snapshot: dict[tuple, Verdict] = {} emitter = Emitter( endpoint=cfg.emit.endpoint, @@ -132,6 +145,7 @@ def build_pipeline(config_path: str | None = "vitals.yaml"): quality_engine.quality_samples if quality_engine else (lambda: []) ), health_provider=health.snapshot, + verdict_provider=lambda: list(verdict_snapshot.values()), ) def on_span(span: GenAISpan) -> None: @@ -167,7 +181,9 @@ def on_span(span: GenAISpan) -> None: evaluator = None if cfg.verdict.enabled: - evaluator = EvaluatorThread(scopes, store, cfg.verdict, cost_engine, health) + evaluator = EvaluatorThread( + scopes, store, cfg.verdict, cost_engine, health, emitter, verdict_snapshot + ) receiver = OTLPReceiver(cfg.receiver.host, cfg.receiver.grpc_port, on_span) health.bind_receiver_stats(receiver.stats) From 9f4dd05351bc31aa3b526f6baab6e20b057e0c9d Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:56:05 +0530 Subject: [PATCH 14/35] add verdict-changed alert rule and remove retired V1 rules --- assets/alerts/cost-velocity.json | 66 ------------------- ...-drift-onset.json => verdict-changed.json} | 20 +++--- assets/alerts/version-regression.json | 66 ------------------- 3 files changed, 10 insertions(+), 142 deletions(-) delete mode 100644 assets/alerts/cost-velocity.json rename assets/alerts/{quality-drift-onset.json => verdict-changed.json} (60%) delete mode 100644 assets/alerts/version-regression.json diff --git a/assets/alerts/cost-velocity.json b/assets/alerts/cost-velocity.json deleted file mode 100644 index de8d046..0000000 --- a/assets/alerts/cost-velocity.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "alert": "Vitals: cost velocity breach", - "alertType": "METRIC_BASED_ALERT", - "description": "Cost spend-rate exceeded $5/min for a service/model. Likely a runaway agent loop. Alerts are not enforcement \u2014 investigate the loop.", - "ruleType": "threshold_rule", - "evalWindow": "5m0s", - "frequency": "1m0s", - "condition": { - "compositeQuery": { - "queryType": "builder", - "panelType": "graph", - "builder": { - "queryData": [ - { - "dataSource": "metrics", - "queryName": "A", - "expression": "A", - "disabled": false, - "aggregateOperator": "avg", - "aggregateAttribute": { - "key": "vitals.cost.velocity", - "dataType": "float64", - "type": "Gauge", - "isColumn": true - }, - "timeAggregation": "avg", - "spaceAggregation": "avg", - "filters": { - "op": "AND", - "items": [] - }, - "groupBy": [ - { - "key": "service.name", - "dataType": "string", - "type": "tag" - }, - { - "key": "gen_ai.request.model", - "dataType": "string", - "type": "tag" - } - ], - "legend": "", - "reduceTo": "last", - "having": [], - "stepInterval": 60 - } - ], - "queryFormulas": [] - } - }, - "op": ">", - "target": 5, - "matchType": "1", - "targetUnit": "" - }, - "labels": { - "severity": "critical", - "source": "vitals" - }, - "annotations": { - "description": "Cost spend-rate exceeded $5/min for a service/model. Likely a runaway agent loop. Alerts are not enforcement \u2014 investigate the loop.", - "summary": "Vitals: cost velocity breach" - } -} \ No newline at end of file diff --git a/assets/alerts/quality-drift-onset.json b/assets/alerts/verdict-changed.json similarity index 60% rename from assets/alerts/quality-drift-onset.json rename to assets/alerts/verdict-changed.json index b92fe5f..84cc1e7 100644 --- a/assets/alerts/quality-drift-onset.json +++ b/assets/alerts/verdict-changed.json @@ -1,9 +1,9 @@ { - "alert": "Vitals: quality drift onset", + "alert": "Vitals: verdict changed", "alertType": "METRIC_BASED_ALERT", - "description": "CUSUM flagged the onset of sustained quality drift from the healthy baseline. Deviation-from-good, not absolute correctness (see docs/honesty.md).", + "description": "{{service.name}} {{service.version}} — VERDICT CHANGED\nbehavior {{vitals.behavior_sigma}}σ · cost {{vitals.cost_sigma}}σ\ncause: {{vitals.cause}} · subject: {{vitals.subject}} · n={{vitals.samples}}\nrunaway: {{vitals.runaway}}\nFull record: vitals verdict log, trace-linked.", "ruleType": "threshold_rule", - "evalWindow": "5m0s", + "evalWindow": "1m0s", "frequency": "1m0s", "condition": { "compositeQuery": { @@ -18,13 +18,13 @@ "disabled": false, "aggregateOperator": "max", "aggregateAttribute": { - "key": "gen_ai.evaluation.drift_onset", + "key": "vitals.verdict.state", "dataType": "float64", "type": "Gauge", "isColumn": true }, "timeAggregation": "max", - "spaceAggregation": "avg", + "spaceAggregation": "max", "filters": { "op": "AND", "items": [] @@ -51,16 +51,16 @@ } }, "op": ">=", - "target": 1, + "target": 2, "matchType": "1", "targetUnit": "" }, "labels": { - "severity": "critical", + "severity": "warning", "source": "vitals" }, "annotations": { - "description": "CUSUM flagged the onset of sustained quality drift from the healthy baseline. Deviation-from-good, not absolute correctness (see docs/honesty.md).", - "summary": "Vitals: quality drift onset" + "description": "{{service.name}} {{service.version}} — VERDICT CHANGED\nbehavior {{vitals.behavior_sigma}}σ · cost {{vitals.cost_sigma}}σ\ncause: {{vitals.cause}} · subject: {{vitals.subject}} · n={{vitals.samples}}\nrunaway: {{vitals.runaway}}\nFull record: vitals verdict log, trace-linked.", + "summary": "{{service.name}} {{service.version}} — VERDICT CHANGED" } -} \ No newline at end of file +} diff --git a/assets/alerts/version-regression.json b/assets/alerts/version-regression.json deleted file mode 100644 index 2947a01..0000000 --- a/assets/alerts/version-regression.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "alert": "Vitals: per-version quality regression", - "alertType": "METRIC_BASED_ALERT", - "description": "Composite quality score fell below 0.6 for a service.version. Compare v1/v2 on the Release Compare dashboard.", - "ruleType": "threshold_rule", - "evalWindow": "10m0s", - "frequency": "1m0s", - "condition": { - "compositeQuery": { - "queryType": "builder", - "panelType": "graph", - "builder": { - "queryData": [ - { - "dataSource": "metrics", - "queryName": "A", - "expression": "A", - "disabled": false, - "aggregateOperator": "avg", - "aggregateAttribute": { - "key": "gen_ai.evaluation.score", - "dataType": "float64", - "type": "Gauge", - "isColumn": true - }, - "timeAggregation": "avg", - "spaceAggregation": "avg", - "filters": { - "op": "AND", - "items": [] - }, - "groupBy": [ - { - "key": "service.name", - "dataType": "string", - "type": "tag" - }, - { - "key": "service.version", - "dataType": "string", - "type": "tag" - } - ], - "legend": "", - "reduceTo": "last", - "having": [], - "stepInterval": 60 - } - ], - "queryFormulas": [] - } - }, - "op": "<", - "target": 0.6, - "matchType": "1", - "targetUnit": "" - }, - "labels": { - "severity": "warning", - "source": "vitals" - }, - "annotations": { - "description": "Composite quality score fell below 0.6 for a service.version. Compare v1/v2 on the Release Compare dashboard.", - "summary": "Vitals: per-version quality regression" - } -} \ No newline at end of file From c2080c0c788cd168729be7de788d4c2eda01f879 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:56:16 +0530 Subject: [PATCH 15/35] retarget release-compare dashboard to vitals.verdict metrics --- assets/dashboards/release-compare.json | 48 +++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/assets/dashboards/release-compare.json b/assets/dashboards/release-compare.json index 79e220d..c99ba66 100644 --- a/assets/dashboards/release-compare.json +++ b/assets/dashboards/release-compare.json @@ -1,12 +1,12 @@ { - "title": "Vitals \u2014 Release Compare", - "description": "Quality and cost split by service.version. Catch the release that regressed.", + "title": "Vitals — Release Compare", + "description": "Verdict state and normalized sigmas split by service.version.", "tags": [ "vitals", "release", - "genai" + "verdict" ], - "version": "v4", + "version": "v5", "layout": [ { "i": "1", @@ -40,8 +40,8 @@ "widgets": [ { "id": "1", - "title": "Quality score by version", - "description": "Per-release quality. The poisoned v2 line drops here.", + "title": "Verdict state by version", + "description": "0=warming, 1=steady, 2=changed, 3=inconclusive", "panelTypes": "graph", "isStacked": false, "opacity": "1", @@ -62,15 +62,15 @@ "queryName": "A", "expression": "A", "disabled": false, - "aggregateOperator": "avg", + "aggregateOperator": "max", "aggregateAttribute": { - "key": "gen_ai.evaluation.score", + "key": "vitals.verdict.state", "dataType": "float64", "type": "Gauge", "isColumn": true }, - "timeAggregation": "avg", - "spaceAggregation": "avg", + "timeAggregation": "max", + "spaceAggregation": "max", "filters": { "op": "AND", "items": [] @@ -97,14 +97,14 @@ }, { "id": "2", - "title": "Drift by version", - "description": "Per-release drift from the shared healthy baseline.", + "title": "Behavior sigma by version", + "description": "Signed behavior delta in sigma units against baseline variance.", "panelTypes": "graph", "isStacked": false, "opacity": "1", "nullZeroValues": "zero", "timePreferance": "GLOBAL_TIME", - "yAxisUnit": "", + "yAxisUnit": "σ", "softMax": null, "softMin": null, "fillSpans": false, @@ -121,7 +121,7 @@ "disabled": false, "aggregateOperator": "avg", "aggregateAttribute": { - "key": "gen_ai.evaluation.drift", + "key": "vitals.verdict.behavior_sigma", "dataType": "float64", "type": "Gauge", "isColumn": true @@ -154,14 +154,14 @@ }, { "id": "3", - "title": "Cost velocity by version", - "description": "Spend-rate per release.", + "title": "Cost sigma by version", + "description": "Signed cost delta in sigma units against baseline variance.", "panelTypes": "graph", "isStacked": false, "opacity": "1", "nullZeroValues": "zero", "timePreferance": "GLOBAL_TIME", - "yAxisUnit": "usd/min", + "yAxisUnit": "σ", "softMax": null, "softMin": null, "fillSpans": false, @@ -178,7 +178,7 @@ "disabled": false, "aggregateOperator": "avg", "aggregateAttribute": { - "key": "vitals.cost.velocity", + "key": "vitals.verdict.cost_sigma", "dataType": "float64", "type": "Gauge", "isColumn": true @@ -211,14 +211,14 @@ }, { "id": "4", - "title": "Drift onset by version", - "description": "1 when CUSUM flags a sustained regression for a version.", + "title": "Velocity ratio by version", + "description": "Current burn rate multiple relative to baseline rate.", "panelTypes": "graph", "isStacked": false, "opacity": "1", "nullZeroValues": "zero", "timePreferance": "GLOBAL_TIME", - "yAxisUnit": "", + "yAxisUnit": "×", "softMax": null, "softMin": null, "fillSpans": false, @@ -233,14 +233,14 @@ "queryName": "A", "expression": "A", "disabled": false, - "aggregateOperator": "max", + "aggregateOperator": "avg", "aggregateAttribute": { - "key": "gen_ai.evaluation.drift_onset", + "key": "vitals.verdict.velocity_ratio", "dataType": "float64", "type": "Gauge", "isColumn": true }, - "timeAggregation": "max", + "timeAggregation": "avg", "spaceAggregation": "avg", "filters": { "op": "AND", From d642c1c1f4e0c9c52ae1abb82228eb360695efe6 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 02:57:09 +0530 Subject: [PATCH 16/35] add unit tests for verdict metrics, log record emission, and alerts --- test/test_emitter.py | 108 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 test/test_emitter.py diff --git a/test/test_emitter.py b/test/test_emitter.py new file mode 100644 index 0000000..43d3923 --- /dev/null +++ b/test/test_emitter.py @@ -0,0 +1,108 @@ +"""Unit tests for Emitter verdict telemetry and SigNoz alert rules (spec §17).""" + +import json +from pathlib import Path +import pytest +from opentelemetry._logs import SeverityNumber +from vitals.cost.engine import CostEngine +from vitals.cost.prices import PriceTable +from vitals.emit.emitter import Emitter +from vitals.health import Health +from vitals.verdict.types import ( + Cause, + Exemplar, + InconclusiveReason, + Subject, + Verdict, + VerdictState, +) + + +def _make_verdict(state: VerdictState = VerdictState.STEADY) -> Verdict: + return Verdict( + verdict_id="abcdef1234567890", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v1", + baseline_version=None, + state=state, + subject=Subject.TIME, + cause=Cause.NONE, + flag_cost=False, + flag_behavior=(state == VerdictState.CHANGED), + runaway=False, + behavior_sigma=4.2 if state == VerdictState.CHANGED else 0.2, + cost_sigma=0.1, + cost_usd_per_req=0.001, + baseline_cost_usd_per_req=0.001, + velocity_ratio=1.0, + samples=100, + baseline_samples=30, + onset_ts_unix=1700000000.0 if state == VerdictState.CHANGED else None, + seconds_after_deploy=90.0 if state == VerdictState.CHANGED else None, + inconclusive_reason=None, + caveats=("output_length_-10%",), + falsifier="falsifier string", + warming_progress=None, + exemplars=( + Exemplar( + kind="worst", + trace_id="4f2a9100000000000000000000000000", + span_id="91cd070000000000", + output_excerpt="worst response excerpt", + behavior_sigma=4.2, + ), + Exemplar( + kind="median", + trace_id="11111100000000000000000000000000", + span_id="2222220000000000", + output_excerpt="median response excerpt", + behavior_sigma=0.2, + ), + ), + ) + + +def test_emitter_verdict_provider_and_log_record(): + verdicts_list = [_make_verdict(VerdictState.CHANGED)] + cost = CostEngine(PriceTable.from_yaml("vitals/cost/prices.yaml")) + health = Health() + + emitter = Emitter( + endpoint="http://localhost:4317", + export_interval_ms=5000, + cost_provider=cost.sample, + quality_provider=lambda: [], + health_provider=health.snapshot, + verdict_provider=lambda: verdicts_list, + ) + + try: + # Emit a CHANGED verdict log record + v_changed = verdicts_list[0] + emitter.emit_verdict_log(v_changed) + + # Emit a STEADY verdict log record + v_steady = _make_verdict(VerdictState.STEADY) + emitter.emit_verdict_log(v_steady) + + finally: + emitter.shutdown() + + +def test_verdict_changed_alert_rule_format(): + alert_path = Path("assets/alerts/verdict-changed.json") + assert alert_path.is_file(), "assets/alerts/verdict-changed.json must exist" + + data = json.loads(alert_path.read_text()) + assert data["alert"] == "Vitals: verdict changed" + assert data["condition"]["op"] == ">=" + assert data["condition"]["target"] == 2 # CHANGED (2) or INCONCLUSIVE (3) + assert data["condition"]["compositeQuery"]["builder"]["queryData"][0]["aggregateAttribute"]["key"] == "vitals.verdict.state" + + # Verify retired V1 rules are deleted + assert not Path("assets/alerts/cost-velocity.json").exists() + assert not Path("assets/alerts/quality-drift-onset.json").exists() + assert not Path("assets/alerts/version-regression.json").exists() From 8159574c40658bb404e9603acd4131ba7b7c32ba Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:00:54 +0530 Subject: [PATCH 17/35] implement console HTML renderer with dark terminal theme and 3-zone layout --- vitals/console/__init__.py | 5 + vitals/console/render.py | 334 +++++++++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 vitals/console/__init__.py create mode 100644 vitals/console/render.py diff --git a/vitals/console/__init__.py b/vitals/console/__init__.py new file mode 100644 index 0000000..00fb222 --- /dev/null +++ b/vitals/console/__init__.py @@ -0,0 +1,5 @@ +"""Console package — HTML rendering and stdlib HTTP server (spec §9).""" + +from vitals.console.render import render_console_html + +__all__ = ["render_console_html"] diff --git a/vitals/console/render.py b/vitals/console/render.py new file mode 100644 index 0000000..5616933 --- /dev/null +++ b/vitals/console/render.py @@ -0,0 +1,334 @@ +"""Server-rendered HTML console using Python f-strings and inline CSS (spec §9).""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any + +from vitals import __version__ +from vitals.verdict.types import Verdict, VerdictState + +STATE_COLORS = { + "warming": "#6b7280", # grey + "steady": "#22c55e", # green + "changed": "#f59e0b", # amber (not red!) + "inconclusive": "#3b82f6", # blue +} + + +def render_meter_bar(z: float | None, max_blocks: int = 16) -> str: + """Render meter bar for sigma values: e.g. ████████████░░░░░░.""" + if z is None: + return "░" * max_blocks + abs_z = min(abs(z), 6.0) + filled = int(round((abs_z / 6.0) * max_blocks)) + filled = max(1 if abs_z > 0 else 0, min(max_blocks, filled)) + return "█" * filled + "░" * (max_blocks - filled) + + +def render_console_html( + latest_verdict: Verdict | None, + verdict_feed: list[Verdict], + scopes: list[dict[str, Any]], + health_snapshot: dict[str, float], + start_time: float, +) -> str: + """Render the full single-screen Vitals console HTML.""" + state_str = latest_verdict.state.value if latest_verdict else "warming" + accent_color = STATE_COLORS.get(state_str, "#6b7280") + + # Zone 1: Hero Verdict Card + if latest_verdict: + v = latest_verdict + b_z = v.behavior_sigma + c_z = v.cost_sigma + + b_bar = render_meter_bar(b_z) + c_bar = render_meter_bar(c_z) + + b_label = ( + f"+{b_z:.1f}σ" if b_z is not None and b_z >= 0 else f"{b_z:.1f}σ" if b_z else "N/A" + ) + c_label = ( + f"+{c_z:.1f}σ" if c_z is not None and c_z >= 0 else f"{c_z:.1f}σ" if c_z else "N/A" + ) + + b_note = "normal ±1σ" if v.flag_behavior else "flat" + c_note = "runaway" if v.runaway else "flat" if (c_z and abs(c_z) < 1.0) else "" + + onset_info = "" + if v.onset_ts_unix: + dt_str = datetime.fromtimestamp(v.onset_ts_unix, tz=timezone.utc).strftime("%H:%M:%S") + sec_str = f"{int(v.seconds_after_deploy)}s" if v.seconds_after_deploy else "0s" + onset_info = f"onset {dt_str} — {sec_str} after {v.version} deployed" + + caveats_html = ( + f'
⚠ caveats: {", ".join(v.caveats)}
' if v.caveats else "" + ) + falsifier_html = ( + f'
? {v.falsifier}
' if v.falsifier else "" + ) + + # Exemplars (worst + median) + exemplars_rows = [] + for ex in v.exemplars: + sig_str = f"+{ex.behavior_sigma:.1f}σ" if ex.behavior_sigma >= 0 else f"{ex.behavior_sigma:.1f}σ" + kind_tag = f"[{ex.kind:<6}]" + short_id = ex.trace_id[:8] if ex.trace_id else "00000000" + exemplars_rows.append( + f'
{kind_tag} ' + f'{short_id}… {sig_str} ' + f'"{ex.output_excerpt}"
' + ) + exemplars_html = "\n".join(exemplars_rows) if exemplars_rows else '
No exemplars captured
' + + hero_card_html = f""" +
+
+ {v.state.value.upper()} + {v.service_name} · {v.version} +
+
{v.subject.value} · {v.cause.value}
+ +
+
+ behavior + {b_label:>7} + {b_bar} + {b_note} +
+
+ cost + {c_label:>7} + {c_bar} + {c_note} +
+
+ +
+ {f'
{onset_info}
' if onset_info else ''} +
n={v.samples} · baseline {v.baseline_version or 'reference'} (n={v.baseline_samples})
+
+ + {caveats_html} + {falsifier_html} + +
+
evidence
+ {exemplars_html} +
+
+ """ + else: + hero_card_html = f""" +
+
+ WARMING + Collecting baseline reference +
+
pipeline initializing
+
+ """ + + # Zone 2: Verdict Feed + feed_rows = [] + for idx, f_v in enumerate(verdict_feed[:50]): + dt = datetime.fromtimestamp(f_v.ts_unix, tz=timezone.utc).strftime("%H:%M:%S") + f_color = STATE_COLORS.get(f_v.state.value, "#6b7280") + feed_rows.append( + f""" +
+
+ {dt} + ● {f_v.state.value.upper()} + {f_v.sentence} +
+ +
+ """ + ) + feed_html = "\n".join(feed_rows) if feed_rows else '
No verdicts recorded yet
' + + # Zone 3: Health Strip + spans_rx = int(health_snapshot.get("spans_received", 0)) + spans_sc = int(health_snapshot.get("spans_scored", 0)) + spans_sk = int(health_snapshot.get("spans_skipped", 0)) + n_scopes = int(health_snapshot.get("scopes", len(scopes))) + v_emitted = int(health_snapshot.get("verdicts_emitted", 0)) + emit_errs = int(health_snapshot.get("emit_errors", 0)) + + import time + uptime_sec = int(time.time() - start_time) + uptime_m, uptime_s = divmod(uptime_sec, 60) + uptime_str = f"{uptime_m}m {uptime_s}s" + + health_strip_html = ( + f"spans: {spans_rx} received / {spans_sc} scored / {spans_sk} skipped · " + f"scopes: {n_scopes} · verdicts emitted: {v_emitted} · errors: {emit_errs} · " + f"uptime: {uptime_str} · vitals {__version__}" + ) + + return f""" + + + + + Vitals Console + + + +
+ {hero_card_html} +
+
Verdict Feed
+
+ {feed_html} +
+
+
+ {health_strip_html} +
+
+ + + +""" From 11a46628a24f4930b009cf26290cba176faea675 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:01:08 +0530 Subject: [PATCH 18/35] implement stdlib HTTP console server and API routes --- vitals/console/__init__.py | 3 +- vitals/console/server.py | 140 +++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 vitals/console/server.py diff --git a/vitals/console/__init__.py b/vitals/console/__init__.py index 00fb222..4693cd9 100644 --- a/vitals/console/__init__.py +++ b/vitals/console/__init__.py @@ -1,5 +1,6 @@ """Console package — HTML rendering and stdlib HTTP server (spec §9).""" from vitals.console.render import render_console_html +from vitals.console.server import ConsoleRequestHandler, create_console_server -__all__ = ["render_console_html"] +__all__ = ["render_console_html", "create_console_server", "ConsoleRequestHandler"] diff --git a/vitals/console/server.py b/vitals/console/server.py new file mode 100644 index 0000000..0384b5a --- /dev/null +++ b/vitals/console/server.py @@ -0,0 +1,140 @@ +"""Console HTTP server using stdlib http.server (spec §8.3, §9, §15).""" + +from __future__ import annotations + +import json +import logging +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import TYPE_CHECKING, Any + +from vitals import __version__ +from vitals.console.render import render_console_html + +if TYPE_CHECKING: + from vitals.health import Health + from vitals.store import VerdictStore + from vitals.verdict.scope import ScopeState + +logger = logging.getLogger("vitals.console") + + +class ConsoleRequestHandler(BaseHTTPRequestHandler): + """Handler for console HTML and REST API endpoints.""" + + store: VerdictStore + scopes: dict[tuple, ScopeState] + health: Health + start_time: float + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + logger.debug(format, *args) + + def _send_json(self, data: Any, status: int = 200) -> None: + try: + body = json.dumps(data).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except Exception as exc: + logger.warning("Console JSON write error: %s", exc) + + def _send_html(self, html: str, status: int = 200) -> None: + try: + body = html.encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except Exception as exc: + logger.warning("Console HTML write error: %s", exc) + + def do_GET(self) -> None: # noqa: N802 + try: + path = self.path.split("?")[0] + + if path == "/": + verdicts = self.store.list(limit=50) + latest = verdicts[0] if verdicts else None + scope_list = [s for s in self.scopes.values()] + snapshot = self.health.snapshot() + html = render_console_html( + latest, verdicts, scope_list, snapshot, self.start_time + ) + self._send_html(html) + return + + if path == "/api/verdicts": + verdicts = self.store.list(limit=50) + self._send_json({"verdicts": [v.to_dict() for v in verdicts]}) + return + + if path.startswith("/api/verdicts/"): + verdict_id = path.split("/api/verdicts/")[1] + v = self.store.get(verdict_id) + if v: + self._send_json(v.to_dict()) + else: + self._send_json({"error": "verdict not found"}, status=404) + return + + if path == "/api/scopes": + scopes_out = [] + for s in self.scopes.values(): + (have, need), phase = s.warming_progress() + scopes_out.append( + { + "service_name": s.service_name, + "gen_ai_system": s.gen_ai_system, + "model": s.model, + "live": s.is_live(), + "phase": phase, + "warming_progress": [have, need], + "versions": list(s.windows.keys()), + } + ) + self._send_json({"scopes": scopes_out}) + return + + if path == "/api/health": + snapshot = self.health.snapshot() + snapshot["uptime_seconds"] = time.time() - self.start_time + snapshot["version"] = __version__ + self._send_json(snapshot) + return + + self._send_json({"error": "Not Found"}, status=404) + except Exception as exc: + logger.exception("Console error servicing %s", self.path) + self._send_json({"error": str(exc)}, status=500) + + +def create_console_server( + host: str, + port: int, + store: VerdictStore, + scopes: dict[tuple, ScopeState], + health: Health, +) -> ThreadingHTTPServer | None: + """Create ThreadingHTTPServer. Returns None if port already bound (spec §15).""" + + class BoundHandler(ConsoleRequestHandler): + pass + + BoundHandler.store = store + BoundHandler.scopes = scopes + BoundHandler.health = health + BoundHandler.start_time = time.time() + + try: + server = ThreadingHTTPServer((host, port), BoundHandler) + logger.info("Vitals console listening on http://%s:%d", host, port) + return server + except OSError as exc: + logger.warning( + "Failed to bind console server on %s:%d: %s. Console disabled.", host, port, exc + ) + return None From a8b672c282f2a85d86a8e174542a754a7a8137c2 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:01:22 +0530 Subject: [PATCH 19/35] implement JSONL fixture reader writer and replay runner --- vitals/replay/__init__.py | 6 +++ vitals/replay/fixtures.py | 80 +++++++++++++++++++++++++++++++++++++++ vitals/replay/runner.py | 54 ++++++++++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 vitals/replay/__init__.py create mode 100644 vitals/replay/fixtures.py create mode 100644 vitals/replay/runner.py diff --git a/vitals/replay/__init__.py b/vitals/replay/__init__.py new file mode 100644 index 0000000..5a35db9 --- /dev/null +++ b/vitals/replay/__init__.py @@ -0,0 +1,6 @@ +"""Replay package — JSONL fixture recording and clock-injected runner (spec §D6, §18).""" + +from vitals.replay.fixtures import read_fixture, write_fixture +from vitals.replay.runner import run_replay + +__all__ = ["read_fixture", "write_fixture", "run_replay"] diff --git a/vitals/replay/fixtures.py b/vitals/replay/fixtures.py new file mode 100644 index 0000000..c2d9085 --- /dev/null +++ b/vitals/replay/fixtures.py @@ -0,0 +1,80 @@ +"""JSONL fixture read/write and GenAISpan serialization (spec §18).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from vitals.model import GenAISpan + + +def span_to_dict(span: GenAISpan, rel_ts: float) -> dict[str, Any]: + """Serialize GenAISpan and relative timestamp to a dictionary.""" + return { + "rel_ts": rel_ts, + "trace_id": span.trace_id, + "span_id": span.span_id, + "service_name": span.service_name, + "service_version": span.service_version, + "gen_ai_system": span.gen_ai_system, + "model": span.model, + "input_text": span.input_text, + "output_text": span.output_text, + "input_tokens": span.input_tokens, + "output_tokens": span.output_tokens, + "start_unix_nano": span.start_unix_nano, + "end_unix_nano": span.end_unix_nano, + "attributes": span.attributes, + } + + +def dict_to_span(d: dict[str, Any]) -> tuple[GenAISpan, float]: + """Deserialize GenAISpan and relative timestamp from a dictionary.""" + span = GenAISpan( + trace_id=d["trace_id"], + span_id=d["span_id"], + service_name=d["service_name"], + service_version=d["service_version"], + gen_ai_system=d["gen_ai_system"], + model=d["model"], + input_text=d.get("input_text", ""), + output_text=d.get("output_text", ""), + input_tokens=int(d.get("input_tokens", 0)), + output_tokens=int(d.get("output_tokens", 0)), + start_unix_nano=int(d.get("start_unix_nano", 0)), + end_unix_nano=int(d.get("end_unix_nano", 0)), + attributes=d.get("attributes", {}), + ) + rel_ts = float(d.get("rel_ts", 0.0)) + return span, rel_ts + + +def write_fixture( + filepath: str | os.PathLike, spans_with_rel_ts: list[tuple[GenAISpan, float]] +) -> None: + """Write mapped GenAISpan records to JSONL fixture file.""" + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for span, rel_ts in spans_with_rel_ts: + line = json.dumps(span_to_dict(span, rel_ts)) + f.write(line + "\n") + + +def read_fixture(filepath: str | os.PathLike) -> list[tuple[GenAISpan, float]]: + """Read mapped GenAISpan records from JSONL fixture file.""" + path = Path(filepath) + if not path.is_file(): + raise FileNotFoundError(f"Fixture file not found: {path}") + + records: list[tuple[GenAISpan, float]] = [] + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + data = json.loads(line) + records.append(dict_to_span(data)) + return records diff --git a/vitals/replay/runner.py b/vitals/replay/runner.py new file mode 100644 index 0000000..13a505e --- /dev/null +++ b/vitals/replay/runner.py @@ -0,0 +1,54 @@ +"""Clock-injected replay driver (spec §D6, §18).""" + +from __future__ import annotations + +import logging +import os +import time +from collections.abc import Callable + +from vitals.model import GenAISpan +from vitals.replay.fixtures import read_fixture + +logger = logging.getLogger("vitals.replay") + + +def run_replay( + fixture_path: str | os.PathLike, + speed: float = 1.0, + on_span_cb: Callable[[GenAISpan, float], None] | None = None, +) -> int: + """Replay spans from a JSONL fixture file with clock injection. + + Args: + fixture_path: Path to JSONL fixture file. + speed: Speed multiplier (e.g. 1.0 = real-time, 10.0 = 10x fast-forward, 0.0 = instant). + on_span_cb: Callback invoked for each (span, virtual_now_ts). + + Returns: + Number of spans replayed. + """ + records = read_fixture(fixture_path) + if not records: + logger.warning("Fixture %s contains 0 spans", fixture_path) + return 0 + + logger.info("Replaying %d spans from %s (speed %.1fx)", len(records), fixture_path, speed) + + start_real_ts = time.time() + last_rel_ts = 0.0 + + for idx, (span, rel_ts) in enumerate(records): + if speed > 0.0 and idx > 0: + delta = rel_ts - last_rel_ts + if delta > 0: + sleep_time = delta / speed + time.sleep(sleep_time) + last_rel_ts = rel_ts + + virtual_now = start_real_ts + (rel_ts if speed > 0 else 0.0) + if on_span_cb is not None: + on_span_cb(span, virtual_now) + + logger.info("Finished replaying %d spans from %s", len(records), fixture_path) + return len(records) From ad65ffdfd548a9df80602b7a10fbaa90171eccc8 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:01:45 +0530 Subject: [PATCH 20/35] wire console server and record replay subcommands into main CLI --- vitals/main.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/vitals/main.py b/vitals/main.py index d54c6ab..58f50da 100644 --- a/vitals/main.py +++ b/vitals/main.py @@ -2,7 +2,7 @@ collector fan-out ─▶ OTLPReceiver ─▶ on_span ─┬─▶ CostEngine ├─▶ QualityEngine ─▶ eval log - └─▶ ScopeState ──▶ EvaluatorThread ─▶ VerdictStore + └─▶ ScopeState ──▶ EvaluatorThread ─▶ VerdictStore ──▶ ConsoleServer (:8787) Emitter (out-of-band) ◀── cost/quality/health providers ─────────────▶ SigNoz """ @@ -17,12 +17,14 @@ from vitals import __version__ from vitals.config import load_config +from vitals.console import create_console_server from vitals.cost.engine import CostEngine from vitals.cost.prices import PriceTable from vitals.emit.emitter import Emitter from vitals.health import Health from vitals.ingest.receiver import OTLPReceiver from vitals.model import GenAISpan +from vitals.replay import read_fixture, run_replay, write_fixture from vitals.store import VerdictStore from vitals.verdict.evaluator import evaluate_scope_version from vitals.verdict.scope import ScopeState @@ -185,17 +187,27 @@ def on_span(span: GenAISpan) -> None: scopes, store, cfg.verdict, cost_engine, health, emitter, verdict_snapshot ) + console_server = None + if cfg.console.enabled: + console_server = create_console_server( + cfg.console.host, cfg.console.port, store, scopes, health + ) + receiver = OTLPReceiver(cfg.receiver.host, cfg.receiver.grpc_port, on_span) health.bind_receiver_stats(receiver.stats) - return cfg, receiver, emitter, health, store, evaluator + return cfg, receiver, emitter, health, store, evaluator, console_server, on_span def run(config_path: str | None = "vitals.yaml") -> None: - cfg, receiver, emitter, _, store, evaluator = build_pipeline(config_path) + cfg, receiver, emitter, _, store, evaluator, console_server, _ = build_pipeline(config_path) receiver.start() if evaluator is not None: evaluator.start() + if console_server is not None: + console_thread = threading.Thread(target=console_server.serve_forever, daemon=True) + console_thread.start() + log.info( "vitals %s running — receiver :%d -> emitting to %s", __version__, @@ -218,6 +230,70 @@ def _shutdown(*_): if evaluator is not None: evaluator.stop() evaluator.join(timeout=2.0) + if console_server is not None: + console_server.shutdown() + console_server.server_close() + emitter.shutdown() + store.close() + + +def record_cmd(out_path: str, config_path: str | None = "vitals.yaml") -> None: + """Record mapped GenAISpans to JSONL fixture file (spec §D6, §18).""" + cfg = load_config(config_path) + spans_recorded: list[tuple[GenAISpan, float]] = [] + start_ts = time.time() + lock = threading.Lock() + + def on_span(span: GenAISpan) -> None: + with lock: + rel_ts = time.time() - start_ts + spans_recorded.append((span, rel_ts)) + log.info("recorded span #%d from %s", len(spans_recorded), span.service_name) + + receiver = OTLPReceiver(cfg.receiver.host, cfg.receiver.grpc_port, on_span) + receiver.start() + log.info("vitals record running on :%d — recording to %s", cfg.receiver.grpc_port, out_path) + + stop = threading.Event() + + def _shutdown(*_): + stop.set() + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + try: + stop.wait() + finally: + receiver.stop() + with lock: + write_fixture(out_path, spans_recorded) + log.info("Saved %d spans to %s", len(spans_recorded), out_path) + + +def replay_cmd( + fixture_path: str, speed: float = 1.0, config_path: str | None = "vitals.yaml" +) -> None: + """Replay mapped GenAISpans from JSONL fixture file (spec §D6, §18).""" + cfg, _, emitter, _, store, evaluator, console_server, on_span_cb = build_pipeline(config_path) + if evaluator is not None: + evaluator.start() + + if console_server is not None: + console_thread = threading.Thread(target=console_server.serve_forever, daemon=True) + console_thread.start() + + try: + n = run_replay(fixture_path, speed=speed, on_span_cb=lambda span, _: on_span_cb(span)) + log.info("Replay completed: %d spans processed", n) + # Give evaluator time for final tick if needed + time.sleep(1.0) + finally: + if evaluator is not None: + evaluator.stop() + evaluator.join(timeout=2.0) + if console_server is not None: + console_server.shutdown() + console_server.server_close() emitter.shutdown() store.close() @@ -225,8 +301,19 @@ def _shutdown(*_): def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="vitals", description="Vitals AI signal sidecar") sub = parser.add_subparsers(dest="command") + run_p = sub.add_parser("run", help="start the sidecar") run_p.add_argument("--config", default="vitals.yaml", help="path to vitals.yaml") + + rec_p = sub.add_parser("record", help="record spans to JSONL fixture") + rec_p.add_argument("--out", required=True, help="output JSONL path") + rec_p.add_argument("--config", default="vitals.yaml", help="path to vitals.yaml") + + rep_p = sub.add_parser("replay", help="replay spans from JSONL fixture") + rep_p.add_argument("fixture", help="path to JSONL fixture file") + rep_p.add_argument("--speed", type=float, default=1.0, help="replay speed factor") + rep_p.add_argument("--config", default="vitals.yaml", help="path to vitals.yaml") + parser.add_argument("--version", action="version", version=f"vitals {__version__}") parser.add_argument("--log-level", default="INFO") @@ -240,6 +327,14 @@ def main(argv: list[str] | None = None) -> int: run(args.config) return 0 + if args.command == "record": + record_cmd(args.out, args.config) + return 0 + + if args.command == "replay": + replay_cmd(args.fixture, args.speed, args.config) + return 0 + parser.print_help() return 1 From 1778d75152855f17d43731bbaa6063021371784d Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:02:14 +0530 Subject: [PATCH 21/35] add unit tests for console rendering, API endpoints, and replay runner --- test/test_console.py | 146 +++++++++++++++++++++++++++++++++++++++++++ test/test_replay.py | 56 +++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 test/test_console.py create mode 100644 test/test_replay.py diff --git a/test/test_console.py b/test/test_console.py new file mode 100644 index 0000000..f4b1e8a --- /dev/null +++ b/test/test_console.py @@ -0,0 +1,146 @@ +"""Unit tests for Console server, API routes, and HTML rendering (spec §9, §17).""" + +import json +import threading +import urllib.request +import pytest +from vitals.console import create_console_server, render_console_html +from vitals.health import Health +from vitals.store import VerdictStore +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import ( + Cause, + Exemplar, + Subject, + Verdict, + VerdictState, +) + + +def _make_sample_verdict() -> Verdict: + return Verdict( + verdict_id="c0123456789abcde", + ts_unix=1700000000.0, + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + version="v2", + baseline_version="v1", + state=VerdictState.CHANGED, + subject=Subject.RELEASE, + cause=Cause.RELEASE, + flag_cost=False, + flag_behavior=True, + runaway=False, + behavior_sigma=4.2, + cost_sigma=0.3, + cost_usd_per_req=0.002, + baseline_cost_usd_per_req=0.002, + velocity_ratio=1.0, + samples=100, + baseline_samples=30, + onset_ts_unix=1700000000.0, + seconds_after_deploy=90.0, + inconclusive_reason=None, + caveats=("output_length_-15%",), + falsifier="falsifier check", + warming_progress=None, + exemplars=( + Exemplar( + kind="worst", + trace_id="4f2a9100000000000000000000000000", + span_id="91cd070000000000", + output_excerpt="worst excerpt", + behavior_sigma=4.2, + ), + Exemplar( + kind="median", + trace_id="11111100000000000000000000000000", + span_id="2222220000000000", + output_excerpt="median excerpt", + behavior_sigma=0.3, + ), + ), + ) + + +def test_console_html_renderer(): + v = _make_sample_verdict() + health = Health() + + html = render_console_html( + latest_verdict=v, + verdict_feed=[v], + scopes=[], + health_snapshot=health.snapshot(), + start_time=1700000000.0, + ) + + assert "" in html + assert "CHANGED" in html + assert "ragapp" in html + assert "worst excerpt" in html + assert "median excerpt" in html + assert "spans:" in html + + +def test_console_server_http_routes(tmp_path): + db_path = tmp_path / "console_test.db" + store = VerdictStore(str(db_path)) + health = Health() + + v = _make_sample_verdict() + store.insert(v) + + scope_key = ("ragapp", "openai", "gpt-4o") + scope = ScopeState("ragapp", "openai", "gpt-4o") + scopes = {scope_key: scope} + + port = 18787 + server = create_console_server("127.0.0.1", port, store, scopes, health) + assert server is not None + + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + # GET / + with urllib.request.urlopen(f"http://127.0.0.1:{port}/") as resp: + assert resp.status == 200 + assert "text/html" in resp.headers.get("Content-Type", "") + body = resp.read().decode("utf-8") + assert "Vitals Console" in body + assert "CHANGED" in body + + # GET /api/verdicts + with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/verdicts") as resp: + assert resp.status == 200 + data = json.loads(resp.read().decode("utf-8")) + assert "verdicts" in data + assert len(data["verdicts"]) == 1 + assert data["verdicts"][0]["verdict_id"] == v.verdict_id + + # GET /api/verdicts/{id} + with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/verdicts/{v.verdict_id}") as resp: + assert resp.status == 200 + data = json.loads(resp.read().decode("utf-8")) + assert data["verdict_id"] == v.verdict_id + + # GET /api/scopes + with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/scopes") as resp: + assert resp.status == 200 + data = json.loads(resp.read().decode("utf-8")) + assert "scopes" in data + assert len(data["scopes"]) == 1 + + # GET /api/health + with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/health") as resp: + assert resp.status == 200 + data = json.loads(resp.read().decode("utf-8")) + assert "spans_received" in data + assert "version" in data + + finally: + server.shutdown() + server.server_close() + store.close() diff --git a/test/test_replay.py b/test/test_replay.py new file mode 100644 index 0000000..d0bb87c --- /dev/null +++ b/test/test_replay.py @@ -0,0 +1,56 @@ +"""Unit tests for JSONL replay fixtures and replay runner (spec §D6, §18).""" + +import pytest +from vitals.model import GenAISpan +from vitals.replay import read_fixture, run_replay, write_fixture + + +def _make_test_span(i: int) -> GenAISpan: + return GenAISpan( + trace_id=f"tr_{i:04d}", + span_id=f"sp_{i:04d}", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version="v1", + input_text=f"prompt {i}", + output_text=f"completion {i}", + input_tokens=15, + output_tokens=30, + start_unix_nano=1000000000, + end_unix_nano=2000000000, + ) + + +def test_fixture_jsonl_roundtrip(tmp_path): + fix_path = tmp_path / "test_fix.jsonl" + spans_in = [(_make_test_span(i), float(i * 2)) for i in range(5)] + + write_fixture(str(fix_path), spans_in) + assert fix_path.is_file() + + spans_out = read_fixture(str(fix_path)) + assert len(spans_out) == 5 + + for i in range(5): + s, r_ts = spans_out[i] + assert s.trace_id == f"tr_{i:04d}" + assert s.input_text == f"prompt {i}" + assert r_ts == float(i * 2) + + +def test_replay_runner_fast_mode(tmp_path): + fix_path = tmp_path / "fast_replay.jsonl" + spans_in = [(_make_test_span(i), float(i * 1.5)) for i in range(10)] + write_fixture(str(fix_path), spans_in) + + replayed_spans = [] + + def on_span(span: GenAISpan, virtual_now: float): + replayed_spans.append(span) + + n = run_replay(str(fix_path), speed=0.0, on_span_cb=on_span) + assert n == 10 + assert len(replayed_spans) == 10 + assert replayed_spans[0].trace_id == "tr_0000" + assert replayed_spans[9].trace_id == "tr_0009" From c72a2bfab96be45ed29ff7ac36bb5ad5ffb58e47 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:03:41 +0530 Subject: [PATCH 22/35] add second topic set to corpus and scenario traffic generators --- demo/ragapp/corpus.py | 25 +++++++++++++-- demo/scenarios/steady_traffic.py | 54 ++++++++++++++++++++++++++++++++ demo/scenarios/traffic_shift.py | 54 ++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 demo/scenarios/steady_traffic.py create mode 100644 demo/scenarios/traffic_shift.py diff --git a/demo/ragapp/corpus.py b/demo/ragapp/corpus.py index 0852848..fb54962 100644 --- a/demo/ragapp/corpus.py +++ b/demo/ragapp/corpus.py @@ -2,7 +2,7 @@ from __future__ import annotations -DOCS = [ +DOCS_TOPIC_A = [ "OpenTelemetry is an open source observability framework for traces, metrics, and logs.", "SigNoz is an open source APM built on OpenTelemetry and ClickHouse.", "The OTLP protocol exports telemetry over gRPC on port 4317 and HTTP on port 4318.", @@ -13,12 +13,31 @@ "CUSUM detects the onset of a sustained shift in a monitored metric series.", ] +DOCS_TOPIC_B = [ + "Write-Ahead Logging (WAL) ensures durability and atomic index updates in relational engines.", + "B-Tree indices optimize range queries, whereas Hash indices target exact equality lookups.", + "Multiversion Concurrency Control (MVCC) allows concurrent readers and writers without global locks.", + "Query planners use cost estimates to choose between nested loop joins and hash joins.", + "Database checkpoints flush dirty buffer pool pages to disk for recovery speed.", + "Foreign keys enforce referential integrity across relational schema tables.", + "Deadlock detection algorithms build wait-for graphs to break cyclic transaction dependencies.", + "Read committed isolation prevents dirty reads by acquiring short-term shared locks.", +] + +DOCS = DOCS_TOPIC_A + DOCS_TOPIC_B -def retrieve(query: str, k: int = 3) -> list[str]: + +def retrieve(query: str, k: int = 3, topic: str | None = None) -> list[str]: """Return the k docs sharing the most words with the query (deterministic).""" + docs_to_search = DOCS + if topic == "A": + docs_to_search = DOCS_TOPIC_A + elif topic == "B": + docs_to_search = DOCS_TOPIC_B + q_words = {w.lower().strip("?.,") for w in query.split()} scored = sorted( - DOCS, + docs_to_search, key=lambda d: len(q_words & {w.lower().strip("?.,") for w in d.split()}), reverse=True, ) diff --git a/demo/scenarios/steady_traffic.py b/demo/scenarios/steady_traffic.py new file mode 100644 index 0000000..14f3dbb --- /dev/null +++ b/demo/scenarios/steady_traffic.py @@ -0,0 +1,54 @@ +"""Scenario: Steady baseline traffic (topic A queries).""" + +from __future__ import annotations + +import argparse +import random +import time +import urllib.request + +QUERIES = [ + "what is opentelemetry?", + "how does signoz export metrics?", + "what port does OTLP use?", + "explain token usage attributes", + "how does quality drift detection work?", +] + + +def send_query(url: str, prompt: str) -> None: + req = urllib.request.Request( + url, + data=f'{{"prompt": "{prompt}"}}'.encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=5.0) as resp: + resp.read() + except Exception as e: + print(f"Error sending query '{prompt}': {e}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Send steady baseline query traffic") + parser.add_argument("--url", default="http://localhost:8000/chat", help="RAG app URL") + parser.add_argument("--rate", type=float, default=2.0, help="Queries per second") + parser.add_argument("--count", type=int, default=50, help="Total queries to send") + args = parser.parse_args() + + interval = 1.0 / args.rate if args.rate > 0 else 0.5 + print(f"Sending {args.count} steady baseline queries to {args.url}...") + + for i in range(args.count): + q = random.choice(QUERIES) + send_query(args.url, q) + if (i + 1) % 10 == 0: + print(f"Sent {i + 1}/{args.count} queries") + time.sleep(interval) + + print("Steady traffic generation complete.") + + +if __name__ == "__main__": + main() diff --git a/demo/scenarios/traffic_shift.py b/demo/scenarios/traffic_shift.py new file mode 100644 index 0000000..bcab3fe --- /dev/null +++ b/demo/scenarios/traffic_shift.py @@ -0,0 +1,54 @@ +"""Scenario: Input shift traffic (topic B database queries).""" + +from __future__ import annotations + +import argparse +import random +import time +import urllib.request + +QUERIES_TOPIC_B = [ + "what is write ahead logging?", + "how do b-tree indices work?", + "explain multiversion concurrency control", + "what is dirty read in database isolation?", + "how does deadlock detection resolve graph cycles?", +] + + +def send_query(url: str, prompt: str) -> None: + req = urllib.request.Request( + url, + data=f'{{"prompt": "{prompt}"}}'.encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=5.0) as resp: + resp.read() + except Exception as e: + print(f"Error sending query '{prompt}': {e}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Send topic B input shift traffic") + parser.add_argument("--url", default="http://localhost:8000/chat", help="RAG app URL") + parser.add_argument("--rate", type=float, default=2.0, help="Queries per second") + parser.add_argument("--count", type=int, default=50, help="Total queries to send") + args = parser.parse_args() + + interval = 1.0 / args.rate if args.rate > 0 else 0.5 + print(f"Sending {args.count} topic B input-shifted queries to {args.url}...") + + for i in range(args.count): + q = random.choice(QUERIES_TOPIC_B) + send_query(args.url, q) + if (i + 1) % 10 == 0: + print(f"Sent {i + 1}/{args.count} queries") + time.sleep(interval) + + print("Traffic shift generation complete.") + + +if __name__ == "__main__": + main() From 060e950665de349f34a28759b4831a819c3d1d61 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:03:54 +0530 Subject: [PATCH 23/35] generate 4 golden JSONL replay fixtures for demo scenarios --- demo/fixtures/01_steady_baseline.jsonl | 45 +++++++ demo/fixtures/02_release_regression.jsonl | 45 +++++++ demo/fixtures/03_runaway_loop.jsonl | 45 +++++++ demo/fixtures/04_input_shift.jsonl | 45 +++++++ demo/fixtures/generate_fixtures.py | 152 ++++++++++++++++++++++ 5 files changed, 332 insertions(+) create mode 100644 demo/fixtures/01_steady_baseline.jsonl create mode 100644 demo/fixtures/02_release_regression.jsonl create mode 100644 demo/fixtures/03_runaway_loop.jsonl create mode 100644 demo/fixtures/04_input_shift.jsonl create mode 100644 demo/fixtures/generate_fixtures.py diff --git a/demo/fixtures/01_steady_baseline.jsonl b/demo/fixtures/01_steady_baseline.jsonl new file mode 100644 index 0000000..326afe3 --- /dev/null +++ b/demo/fixtures/01_steady_baseline.jsonl @@ -0,0 +1,45 @@ +{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} +{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} +{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} +{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} +{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} +{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} +{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} +{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} +{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} +{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} +{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} +{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} +{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} +{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} +{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} +{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} +{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} +{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} +{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} +{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} +{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} +{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} +{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} +{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} +{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} +{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} +{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} +{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} +{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} +{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} +{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} diff --git a/demo/fixtures/02_release_regression.jsonl b/demo/fixtures/02_release_regression.jsonl new file mode 100644 index 0000000..6210800 --- /dev/null +++ b/demo/fixtures/02_release_regression.jsonl @@ -0,0 +1,45 @@ +{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} +{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} +{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} +{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} +{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} +{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} +{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} +{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} +{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} +{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} +{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} +{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} +{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} +{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} +{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} +{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} +{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} +{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} +{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} +{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} +{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} +{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 30.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 31.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} +{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 32.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} +{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 33.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} +{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 34.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} +{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 35.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 36.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 37.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} +{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 38.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} +{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 39.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} +{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} +{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} +{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} diff --git a/demo/fixtures/03_runaway_loop.jsonl b/demo/fixtures/03_runaway_loop.jsonl new file mode 100644 index 0000000..b6dd406 --- /dev/null +++ b/demo/fixtures/03_runaway_loop.jsonl @@ -0,0 +1,45 @@ +{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} +{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} +{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} +{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} +{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} +{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} +{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} +{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} +{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} +{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} +{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} +{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} +{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} +{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} +{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} +{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} +{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} +{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} +{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} +{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} +{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} +{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 30", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 60.2, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 31", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61200000000, "end_unix_nano": 62200000000, "attributes": {}} +{"rel_ts": 60.400000000000006, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 32", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61400000000, "end_unix_nano": 62400000000, "attributes": {}} +{"rel_ts": 60.60000000000001, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 33", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61600000000, "end_unix_nano": 62600000000, "attributes": {}} +{"rel_ts": 60.80000000000001, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 34", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61800000000, "end_unix_nano": 62800000000, "attributes": {}} +{"rel_ts": 61.000000000000014, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 35", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62000000000, "end_unix_nano": 63000000000, "attributes": {}} +{"rel_ts": 61.20000000000002, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 36", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62200000000, "end_unix_nano": 63200000000, "attributes": {}} +{"rel_ts": 61.40000000000002, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 37", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62400000000, "end_unix_nano": 63400000000, "attributes": {}} +{"rel_ts": 61.60000000000002, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 38", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62600000000, "end_unix_nano": 63600000000, "attributes": {}} +{"rel_ts": 61.800000000000026, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 39", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62800000000, "end_unix_nano": 63800000000, "attributes": {}} +{"rel_ts": 62.00000000000003, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 40", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} +{"rel_ts": 62.20000000000003, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 41", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63200000000, "end_unix_nano": 64200000000, "attributes": {}} +{"rel_ts": 62.400000000000034, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 42", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63400000000, "end_unix_nano": 64400000000, "attributes": {}} +{"rel_ts": 62.60000000000004, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 43", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63600000000, "end_unix_nano": 64600000000, "attributes": {}} +{"rel_ts": 62.80000000000004, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 44", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63800000000, "end_unix_nano": 64800000000, "attributes": {}} diff --git a/demo/fixtures/04_input_shift.jsonl b/demo/fixtures/04_input_shift.jsonl new file mode 100644 index 0000000..b324cb5 --- /dev/null +++ b/demo/fixtures/04_input_shift.jsonl @@ -0,0 +1,45 @@ +{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} +{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} +{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} +{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} +{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} +{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} +{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} +{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} +{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} +{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} +{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} +{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} +{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} +{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} +{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} +{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} +{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} +{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} +{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} +{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} +{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} +{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 30", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 30.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 31", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 31.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} +{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 32", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 32.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} +{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 33", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 33.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} +{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 34", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 34.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} +{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 35", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 35.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 36", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 36.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 37", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 37.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} +{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 38", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 38.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} +{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 39", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 39.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} +{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 40", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} +{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 41", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 42", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 43", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} +{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 44", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} diff --git a/demo/fixtures/generate_fixtures.py b/demo/fixtures/generate_fixtures.py new file mode 100644 index 0000000..46ce6da --- /dev/null +++ b/demo/fixtures/generate_fixtures.py @@ -0,0 +1,152 @@ +"""Generate deterministic golden JSONL replay fixtures (spec §10, §18).""" + +from __future__ import annotations + +import os +from pathlib import Path +from vitals.model import GenAISpan +from vitals.replay.fixtures import write_fixture + + +def make_span( + i: int, + version: str = "v1", + input_text: str = "what is opentelemetry?", + output_text: str = "OpenTelemetry is an open source observability framework.", + input_tokens: int = 15, + output_tokens: int = 30, + rel_ts: float = 0.0, +) -> tuple[GenAISpan, float]: + span = GenAISpan( + trace_id=f"tr_{i:06d}000000000000000000000", + span_id=f"sp_{i:06d}0000000", + service_name="ragapp", + gen_ai_system="openai", + model="gpt-4o", + service_version=version, + input_text=input_text, + output_text=output_text, + input_tokens=input_tokens, + output_tokens=output_tokens, + start_unix_nano=1000000000 + int(rel_ts * 1e9), + end_unix_nano=2000000000 + int(rel_ts * 1e9), + ) + return span, rel_ts + + +def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: + path_dir = Path(out_dir) + path_dir.mkdir(parents=True, exist_ok=True) + + # 1. Steady Baseline (01_steady_baseline.jsonl) + spans_01 = [] + ts = 0.0 + for i in range(45): + spans_01.append( + make_span( + i, + version="v1", + input_text=f"what is opentelemetry topic query {i % 5}?", + output_text=f"OpenTelemetry is an open source observability framework variant {i % 5}.", + rel_ts=ts, + ) + ) + ts += 2.0 + write_fixture(path_dir / "01_steady_baseline.jsonl", spans_01) + + # 2. Release Regression (02_release_regression.jsonl) + spans_02 = [] + ts = 0.0 + # 30 reference v1 spans + for i in range(30): + spans_02.append( + make_span( + i, + version="v1", + input_text=f"what is opentelemetry topic query {i % 5}?", + output_text=f"OpenTelemetry is an open source observability framework variant {i % 5}.", + rel_ts=ts, + ) + ) + ts += 2.0 + # 15 degraded v2 spans (poisoned prompt / hallucinated output) + for i in range(30, 45): + spans_02.append( + make_span( + i, + version="v2", + input_text=f"what is opentelemetry topic query {i % 5}?", + output_text=f"Garbage hallucinated text response with zero semantic similarity {i}.", + rel_ts=ts, + ) + ) + ts += 2.0 + write_fixture(path_dir / "02_release_regression.jsonl", spans_02) + + # 3. Runaway Loop (03_runaway_loop.jsonl) + spans_03 = [] + ts = 0.0 + # 30 reference v1 spans + for i in range(30): + spans_03.append( + make_span( + i, + version="v1", + input_text=f"query {i % 5}", + output_text=f"standard response {i % 5}", + input_tokens=20, + output_tokens=30, + rel_ts=ts, + ) + ) + ts += 2.0 + # 15 runaway loop spans (high token burn & high rate) + for i in range(30, 45): + spans_03.append( + make_span( + i, + version="v1", + input_text=f"runaway agent loop iteration {i}", + output_text="A" * 5000, + input_tokens=15000, + output_tokens=25000, + rel_ts=ts, + ) + ) + ts += 0.2 + write_fixture(path_dir / "03_runaway_loop.jsonl", spans_03) + + # 4. Input Shift (04_input_shift.jsonl) + spans_04 = [] + ts = 0.0 + # 30 reference spans on Topic A + for i in range(30): + spans_04.append( + make_span( + i, + version="v1", + input_text=f"what is opentelemetry topic query {i % 5}?", + output_text=f"OpenTelemetry is an open source observability framework variant {i % 5}.", + rel_ts=ts, + ) + ) + ts += 2.0 + # 15 shifted Topic B database queries + modified outputs (causes input drift + behavior drift) + for i in range(30, 45): + spans_04.append( + make_span( + i, + version="v1", + input_text=f"explain write ahead logging index locking deadlock {i}", + output_text=f"Write ahead logging and relational database B-Trees MVCC isolation {i}.", + rel_ts=ts, + ) + ) + ts += 2.0 + write_fixture(path_dir / "04_input_shift.jsonl", spans_04) + + print(f"Successfully generated 4 golden replay fixtures in {out_dir}") + + +if __name__ == "__main__": + generate_all_fixtures() From 18011cb27b695fe378aefe875e4ca04d6db701fd Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:05:12 +0530 Subject: [PATCH 24/35] add test_replay_golden integration test suite for 4 replay scenarios --- test/test_replay_golden.py | 97 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 test/test_replay_golden.py diff --git a/test/test_replay_golden.py b/test/test_replay_golden.py new file mode 100644 index 0000000..33f36a6 --- /dev/null +++ b/test/test_replay_golden.py @@ -0,0 +1,97 @@ +"""Golden replay integration test suite verifying exact verdicts across 4 scenarios (spec §10, §17).""" + +from pathlib import Path +import pytest +from vitals.config.settings import QualityConfig, VerdictConfig +from vitals.cost.engine import CostEngine +from vitals.cost.prices import PriceTable +from vitals.quality.engine import QualityEngine +from vitals.replay.runner import run_replay +from vitals.store.db import VerdictStore +from vitals.verdict.evaluator import evaluate_scope_version +from vitals.verdict.scope import ScopeState +from vitals.verdict.types import Cause, InconclusiveReason, VerdictState + +FIXTURES_DIR = Path("demo/fixtures") + + +def _run_scenario_evaluation( + fixture_filename: str, tmp_path +) -> tuple[VerdictState, Cause | None, InconclusiveReason | None, bool]: + db_path = tmp_path / f"store_{fixture_filename}.db" + store = VerdictStore(str(db_path)) + + prices = PriceTable.from_yaml("vitals/cost/prices.yaml") + cost_engine = CostEngine(prices, window_s=300) + quality_engine = QualityEngine(QualityConfig(baseline_window=30)) + + cfg = VerdictConfig( + enabled=True, + min_samples=5, + calibration_samples=5, + window_max=500, + evaluate_interval_s=1, + window_s=300, + ) + + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=30, calib_n=5) + virtual_clock = [0.0] + + def on_span(span, now_ts): + cost_engine.record(span, now=now_ts) + usd = prices.cost_usd(span.model, span.input_tokens, span.output_tokens) + rec = quality_engine.score(span) + scope.observe(span, rec, usd, now=now_ts) + virtual_clock[0] = now_ts + + fix_file = FIXTURES_DIR / fixture_filename + run_replay(fix_file, speed=0.0, on_span_cb=on_span) + + now = virtual_clock[0] + versions = list(scope.windows.keys()) + target_ver = versions[-1] if versions else "v1" + + verdict1 = evaluate_scope_version(scope, target_ver, cfg, now, cost_engine) + if verdict1: + scope.last_verdict = verdict1 + store.insert(verdict1) + + verdict2 = evaluate_scope_version(scope, target_ver, cfg, now + 10.0, cost_engine) + final_v = verdict2 or verdict1 + + assert final_v is not None, f"Expected verdict for scenario {fixture_filename}" + store.insert(final_v) + store.close() + + return final_v.state, final_v.cause, final_v.inconclusive_reason, final_v.runaway + + +def test_golden_01_steady_baseline(tmp_path): + state, cause, inc_reason, runaway = _run_scenario_evaluation( + "01_steady_baseline.jsonl", tmp_path + ) + assert state == VerdictState.STEADY + + +def test_golden_02_release_regression(tmp_path): + state, cause, inc_reason, runaway = _run_scenario_evaluation( + "02_release_regression.jsonl", tmp_path + ) + assert state == VerdictState.CHANGED + assert cause == Cause.RELEASE + + +def test_golden_03_runaway_loop(tmp_path): + state, cause, inc_reason, runaway = _run_scenario_evaluation( + "03_runaway_loop.jsonl", tmp_path + ) + assert state == VerdictState.CHANGED + assert runaway is True + + +def test_golden_04_input_shift(tmp_path): + state, cause, inc_reason, runaway = _run_scenario_evaluation( + "04_input_shift.jsonl", tmp_path + ) + assert state == VerdictState.INCONCLUSIVE + assert inc_reason == InconclusiveReason.INPUT_SHIFT From 0ea6a2f15f8bbf7c57d3c4c51852b1666f741030 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:05:23 +0530 Subject: [PATCH 25/35] create agent skills documentation for vitals integration --- agent/__init__.py | 1 + agent/skills.md | 103 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 agent/__init__.py create mode 100644 agent/skills.md diff --git a/agent/__init__.py b/agent/__init__.py new file mode 100644 index 0000000..2f55459 --- /dev/null +++ b/agent/__init__.py @@ -0,0 +1 @@ +"""Agent skills package — documentation and guidelines for AI agent integration (spec §11).""" diff --git a/agent/skills.md b/agent/skills.md new file mode 100644 index 0000000..801aa78 --- /dev/null +++ b/agent/skills.md @@ -0,0 +1,103 @@ +# Vitals Agent Skills Guide + +This document defines how AI coding agents and automated remediation workflows consume, interpret, and act upon Vitals Verdicts. + +--- + +## 1. Overview of Vitals Verdicts + +Vitals is an AI signal sidecar that continuously monitors GenAI span telemetry (`gen_ai.system`, `gen_ai.request.model`, `service.name`, `service.version`) to deliver deterministic **Verdicts** on AI application health. + +Verdicts unify quality drift (ResponseDriftMetric / PSI), spend rate (USD/min velocity), release attribution, and guard checks into a single actionable record. + +### Verdict Data Model (`Verdict`) + +| Field | Type | Description | +|---|---|---| +| `verdict_id` | `str` | Canonical hex string identifying the verdict record | +| `state` | `VerdictState` | `WARMING` (0), `STEADY` (1), `CHANGED` (2), `INCONCLUSIVE` (3) | +| `subject` | `Subject` | `RELEASE` (version vs version) or `TIME` (current vs lookback) | +| `cause` | `Cause` | `RELEASE` (deploy within 300s of onset) or `UNATTRIBUTED` | +| `behavior_sigma` | `float \| None` | Signed delta in sigma units relative to baseline behavior variance | +| `cost_sigma` | `float \| None` | Signed delta in sigma units relative to baseline cost variance | +| `velocity_ratio` | `float \| None` | Ratio of current USD/min velocity to baseline expected USD/min | +| `runaway` | `bool` | True if cost velocity ratio exceeds 5.0x (overrides hysteresis) | +| `sentence` | `str` | Human-readable canonical verdict sentence | +| `falsifier` | `str` | Explicit metric condition required to transition back to `STEADY` | +| `exemplars` | `tuple[Exemplar]` | Evidence exemplars containing worst and median outputs | + +--- + +## 2. Interpreting Verdict States + +### `WARMING` (State 0) +- **Meaning**: Insufficient sample count (collecting reference baseline window). +- **Agent Action**: No action required. Wait for baseline to finish warming. + +### `STEADY` (State 1) +- **Meaning**: Monitored signals remain within normal baseline variance (±3σ). +- **Agent Action**: System is healthy. Proceed with planned deployment or monitoring. + +### `CHANGED` (State 2) +- **Meaning**: Sustained regression detected in behavior drift (≥3.0σ) or runaway cost spend-rate (≥5.0x). +- **Agent Action**: High priority remediation required. See Section 4. + +### `INCONCLUSIVE` (State 3) +- **Meaning**: Detection guard suspended evaluation: + - `low_sample`: Insufficient samples in current window (`n < min_samples`). + - `input_shift`: Both input prompt drift and behavior drift shift concurrently (≥3.0σ). +- **Agent Action**: Do **not** trigger a code rollback for `input_shift`. The change is driven by external user query shifts, not model regression. + +--- + +## 3. Parsing Telemetry & Log Records + +Vitals emits OTLP log records for each verdict. Trace correlation is natively linked via the worst exemplar's `trace_id` and `span_id`. + +### Log Attributes + +```json +{ + "service.name": "ragapp", + "service.version": "v2", + "gen_ai.system": "openai", + "gen_ai.request.model": "gpt-4o", + "vitals.verdict_id": "c0123456789abcde", + "vitals.state": "changed", + "vitals.subject": "release", + "vitals.cause": "release", + "vitals.behavior_sigma": 4.2, + "vitals.cost_sigma": 0.3, + "vitals.velocity_ratio": 1.0, + "vitals.runaway": false, + "vitals.falsifier": "would flip to STEADY if behavior drift < 3.0σ (now 4.2σ)", + "vitals.exemplars": "[{\"kind\":\"worst\",\"trace_id\":\"...\",\"excerpt\":\"...\"}]" +} +``` + +--- + +## 4. Remediation Decision Tree + +When a `CHANGED` verdict is emitted, agents must follow this decision procedure: + +```mermaid +flowchart TD + A[Verdict CHANGED Emitted] --> B{Cause == RELEASE?} + B -- Yes --> C[Rollback Version to Baseline] + B -- No --> D{Runaway == True?} + D -- Yes --> E[Throttle Agent Loop / Rate Limit Endpoint] + D -- No --> F[Inspect Output Exemplars & Prompt Templates] +``` + +### Action 1: Cause == `RELEASE` +- **Root Cause**: Regression caused by newly deployed release version (e.g. `v2`). +- **Remediation**: Execute immediate version rollback to `baseline_version` (e.g. `v1`). + +### Action 2: `runaway == True` +- **Root Cause**: Runaway loop rapidly burning tokens (velocity ratio ≥ 5.0x). +- **Remediation**: Terminate or rate-limit the agent thread loop; enforce max recursion limits. + +### Action 3: `INCONCLUSIVE(input_shift)` +- **Root Cause**: User traffic topic shifted (e.g., queries changed from Observability to Database internals). +- **Remediation**: Do not rollback code. If desired, register a separate scope key or update baseline reference. From b370d98046d34b44a2950cc0566d977e6ee2003b Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:05:33 +0530 Subject: [PATCH 26/35] update demo README with replay scenarios and console guide --- demo/README.md | 104 ++++++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 45 deletions(-) diff --git a/demo/README.md b/demo/README.md index 25baa76..edaa4be 100644 --- a/demo/README.md +++ b/demo/README.md @@ -1,81 +1,95 @@ -# Vitals demo — run-of-show +# Vitals Demo & Replay Guide -Two beats, one pipeline. Every number is reproducible: run the scenarios yourself and -watch the Vitals dashboards in SigNoz. +Every scenario in Vitals is 100% reproducible — either live through Docker Compose or instantly via deterministic `vitals replay` fixtures without external network calls or Groq API keys. -## Prerequisites +--- -- Docker Desktop running. -- **SigNoz already up** on the host (self-hosted compose or Cloud trial), OTLP ingest on - `localhost:4317`, UI on `localhost:8080`. -- A Groq API key (free at https://console.groq.com). Optional — without it the app serves - deterministic canned answers so the pipeline still flows. +## Quickstart A — Instant Replay Scenarios (No Docker needed) + +Vitals includes a deterministic replay engine (`vitals replay `) that replays pre-recorded OTLP spans through the full scoring and verdict engine. -## Setup +### 1. Run Vitals Sidecar with Console Enabled ```bash -cp demo/.env.example demo/.env # add GROQ_API_KEY (optional) -docker compose -f demo/compose.yaml up --build -d +python -m vitals.main run --config vitals.yaml ``` -This starts three services alongside your existing SigNoz: +Open the **Vitals Console** in your browser: [http://localhost:8787](http://localhost:8787) -| service | role | -|---|---| -| `ragapp` | Groq RAG app on :8000, emits gen_ai spans to the collector | -| `collector` | OTel Collector — **fans out** spans to SigNoz **and** vitals | -| `vitals` | the sidecar: scores spans, emits cost + quality signals out-of-band to SigNoz | +### 2. Replay Scenarios in a Second Terminal -Import the dashboards and alerts (SigNoz UI → Dashboards → Import JSON): -`assets/dashboards/*.json`, `assets/alerts/*.json`. +```bash +# Scenario 1: Steady Baseline (Verdict: STEADY) +python -m vitals.main replay demo/fixtures/01_steady_baseline.jsonl --speed 10.0 -Warm the quality baseline once (healthy v1 traffic): +# Scenario 2: Release Regression (Verdict: CHANGED · Cause: RELEASE) +python -m vitals.main replay demo/fixtures/02_release_regression.jsonl --speed 10.0 -```bash -python demo/scenarios/runaway_loop.py --rate 3 --duration 60 +# Scenario 3: Runaway Cost Loop (Verdict: CHANGED · Runaway: TRUE) +python -m vitals.main replay demo/fixtures/03_runaway_loop.jsonl --speed 10.0 + +# Scenario 4: User Traffic Shift (Verdict: INCONCLUSIVE · Guard: INPUT_SHIFT) +python -m vitals.main replay demo/fixtures/04_input_shift.jsonl --speed 10.0 ``` +Watch the **Hero Verdict Card** on `http://localhost:8787` update live with state colors, sigma meter bars, falsifier statements, and evidence exemplars! + --- -## Beat 1 — the hook: cost +## Quickstart B — Live RAG Application with Docker & SigNoz + +### Prerequisites +- Docker Desktop running. +- **SigNoz** running on `localhost:4317` (OTLP gRPC) and UI on `localhost:8080`. +- (Optional) `GROQ_API_KEY` in `demo/.env`. + +### 1. Launch Services ```bash -python demo/scenarios/runaway_loop.py --rate 20 --duration 120 +cp demo/.env.example demo/.env +docker compose -f demo/compose.yaml up --build -d ``` -On **Vitals — Overview**, `vitals.cost.velocity` (USD/min) spirals upward. The -**cost velocity** alert fires at the threshold — in minutes, not on the invoice. +Services started: +| Service | Address | Role | +|---|---|---| +| `ragapp` | `:8000` | RAG service emitting `gen_ai` semantic spans | +| `collector` | `:4317` | OTel Collector fanning out to SigNoz and Vitals | +| `vitals` | `:8787` | Vitals sidecar with Console and SigNoz OTLP emitter | -> *"A 4-agent loop burned \$47,000 over 11 days with dashboards green. \$47K incidents end here."* +### 2. Import SigNoz Assets -## Beat 2 — the payoff: quality +Import in SigNoz UI (`Dashboards -> Import JSON` & `Alerts -> Import JSON`): +- Dashboard: `assets/dashboards/release-compare.json` +- Alert Rule: `assets/alerts/verdict-changed.json` -Deploy the poisoned prompt (v2). **No model change. Tokens stay normal. Infra stays green.** +### 3. Run Live Traffic Scenarios ```bash -bash demo/scenarios/deploy_v2.sh -python demo/scenarios/runaway_loop.py --rate 5 --duration 180 -``` +# Steady traffic (Topic A) +python demo/scenarios/steady_traffic.py --rate 2 --count 50 -On **Vitals — Drift**, the drift line bends and a **CUSUM onset marker** appears. On -**Vitals — Release Compare**, v1 and v2 split — v2's quality score drops while v1 holds. -The **quality drift onset** alert pages. +# Deploy poisoned prompt (v2) and observe CHANGED verdict +bash demo/scenarios/deploy_v2.sh +python demo/scenarios/steady_traffic.py --rate 2 --count 50 -> *"Anthropic shipped this exact failure in April and found out from user complaints.* -> *Cost to detect: \$0.00."* +# Runaway loop simulation +python demo/scenarios/runaway_loop.py --rate 20 --duration 60 -Close on the convention: *"and we wrote down the standard so everyone can emit this"* — -[docs/conventions.md](../docs/conventions.md). +# Input shift simulation (Topic B queries) +python demo/scenarios/traffic_shift.py --rate 2 --count 50 +``` -## Reset +### 4. Reset Environment ```bash bash demo/scenarios/reset.sh ``` -Redeploys v1 and restarts vitals so baselines re-warm. SigNoz data is left intact. +--- -## Honesty note (say it first) +## Understanding the Vitals Console (`:8787`) -Vitals measures **deviation from a healthy baseline**, not absolute correctness — the -exact signal behind every silent-regression incident. See [docs/honesty.md](../docs/honesty.md). +- **Zone 1 (Hero Verdict Card)**: Shows current state (`WARMING`, `STEADY`, `CHANGED`, `INCONCLUSIVE`), behavior/cost sigmas (`+4.2σ`), falsifier line, and **worst + median evidence exemplars**. +- **Zone 2 (Verdict Feed)**: Historical feed of up to 50 stored verdicts with expandable JSON details. +- **Zone 3 (Health Strip)**: Live counters for spans received/scored/skipped, scopes, verdicts emitted, errors, and uptime. From 023395bc8f1f7b1bf5c0a1cf71f40df5813d43b3 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:05:41 +0530 Subject: [PATCH 27/35] create docs/blind-spots.md technical analysis document --- docs/blind-spots.md | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/blind-spots.md diff --git a/docs/blind-spots.md b/docs/blind-spots.md new file mode 100644 index 0000000..93bad4c --- /dev/null +++ b/docs/blind-spots.md @@ -0,0 +1,66 @@ +# Vitals Technical Analysis: System Boundaries & Blind Spots + +This document provides a rigorous architectural analysis of Vitals' detection boundaries, statistical trade-offs, false positive/negative failure modes, and future roadmap. + +--- + +## 1. Mathematical & Statistical Boundaries + +### 1.1 Sample Size Constraints (`n < min_samples`) +- **Boundary**: Statistical normalization (z-score calculation via Welford's algorithm) requires sufficient sample density to estimate variance accurately. +- **Trade-off**: When sample count `n < min_samples` (default 5), Vitals emits `INCONCLUSIVE (low_sample)`. +- **Impact**: Short bursts of < 5 requests cannot trigger a `CHANGED` verdict, preventing false alerts on low-traffic endpoints. + +### 1.2 Reference Baseline & Cold Start Window +- **Boundary**: Vitals requires a baseline reference window (default 30 spans) to establish the healthy comparison distribution. +- **Trade-off**: During initial pipeline startup, the verdict state remains `WARMING`. +- **Impact**: An anomaly occurring during the first 30 spans of a brand new service will not be flagged until warming completes. + +### 1.3 Gradual Drift vs. Sudden Release Spikes +- **Boundary**: Rolling deques (`maxlen=500`) maintain moving windows. Gradual drift occurring over days/weeks will slowly shift the baseline mean (`mu`). +- **Trade-off**: Vitals is optimized for **release regressions** (step-function drops at deploy time) and **runaway cost spikes** (sudden rate increases), not multi-month semantic drift. + +--- + +## 2. Guard Design & False Positive Suppression + +### 2.1 Guard G2: Input Co-movement (`INCONCLUSIVE(input_shift)`) +- **Problem**: When user query topics shift (e.g., users switch from asking basic product questions to complex debugging queries), response semantic distance increases naturally. A naive drift detector would falsely flag model regression. +- **Mechanism**: Guard G2 measures `input_drift` alongside `behavior_drift`. If `behavior_z >= 3.0` **AND** `input_z >= 3.0`, Vitals emits `INCONCLUSIVE (input_shift)`. +- **Outcome**: Suppresses false positive rollbacks when user traffic changes rather than model behavior. + +### 2.2 Guard G3: Length Shift Caveat +- **Problem**: Model prompt changes often cause longer or shorter responses. Shortened responses reduce semantic drift space, potentially masking drift. +- **Mechanism**: Guard G3 measures `output_len` percentage change relative to baseline. If `abs(pct_change) >= 20%`, Vitals attaches caveat `output_length_X%` to the verdict while retaining detection accuracy. + +--- + +## 3. Telemetry & Out-of-Band Pipeline Boundaries + +### 3.1 Ingest Hot Path Isolation +- **Boundary**: Scoring and verdict evaluation execute entirely out-of-band on fan-out gRPC span copies. +- **Guarantee**: No failure, lock contention, or SQLite exception in Vitals can ever block user API requests or interrupt the primary SigNoz telemetry path. + +### 3.2 Missing Telemetry Attributes +- **Boundary**: If a client span fails to record `gen_ai.content.completion` or token counts, `QualityEngine` gracefully skips quality scoring while `CostEngine` records zero cost. +- **Health Counter**: Increments `spans_skipped` in self-health metrics. + +--- + +## 4. Failure Mode Matrix + +| Scenario | Expected State | Cause / Reason | Explanation | +|---|---|---|---| +| New Version Release with Prompt Regression | `CHANGED` | `RELEASE` | Behavior z-score ≥ 3.0σ within 300s of deployment | +| Runaway Agent Loop (Rate ≥ 5x) | `CHANGED` | `UNATTRIBUTED` | Cost spend-rate multiplier ≥ 5.0x (bypasses 2-tick hysteresis) | +| User Traffic Topic Shift | `INCONCLUSIVE` | `INPUT_SHIFT` | Input drift ≥ 3.0σ and behavior drift ≥ 3.0σ (Guard G2) | +| Insufficient Traffic (n < 5) | `INCONCLUSIVE` | `LOW_SAMPLE` | Sample count below statistical threshold (Guard G1) | +| Output Length Shift (-25%) | `STEADY` / `CHANGED` | Caveat Attached | Output length caveat appended without corrupting z-score (Guard G3) | + +--- + +## 5. Future Roadmap + +1. **Multi-Tenant Token Isolation**: Extending `ScopeKey` to include `tenant.id` for granular B2B SaaS per-customer anomaly detection. +2. **Vector Embedding Shift Metrics**: Complementing response drift (PSI) with lightweight vector centroid distance metrics. +3. **Adaptive Sigma Floor**: Dynamic adjustment of `sigma_floor` based on baseline variance coefficient of variation. From 9def26dc1a1e24428b4fd0795468a2725a3de641 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:06:07 +0530 Subject: [PATCH 28/35] verify full milestone M5 test suite and project completion --- demo/fixtures/01_steady_baseline.jsonl | 88 +++++++++---------- demo/fixtures/02_release_regression.jsonl | 98 +++++++++++---------- demo/fixtures/03_runaway_loop.jsonl | 100 ++++++++++++---------- demo/fixtures/04_input_shift.jsonl | 98 +++++++++++---------- demo/fixtures/generate_fixtures.py | 44 ++++------ 5 files changed, 226 insertions(+), 202 deletions(-) diff --git a/demo/fixtures/01_steady_baseline.jsonl b/demo/fixtures/01_steady_baseline.jsonl index 326afe3..ecbe19c 100644 --- a/demo/fixtures/01_steady_baseline.jsonl +++ b/demo/fixtures/01_steady_baseline.jsonl @@ -1,45 +1,45 @@ {"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} -{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} -{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} -{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} -{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} -{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} -{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} -{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} -{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} -{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} -{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} -{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} -{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} -{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} -{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} -{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} -{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} -{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} -{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} -{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} -{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} -{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} -{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} -{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} -{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} -{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} -{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} -{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} -{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} -{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} -{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} -{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} -{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} -{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} -{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} -{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} -{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} -{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} -{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} -{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} -{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} -{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} -{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} -{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} -{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 96.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 97000000000, "end_unix_nano": 98000000000, "attributes": {}} +{"rel_ts": 108.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 109000000000, "end_unix_nano": 110000000000, "attributes": {}} +{"rel_ts": 120.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 121000000000, "end_unix_nano": 122000000000, "attributes": {}} +{"rel_ts": 132.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 133000000000, "end_unix_nano": 134000000000, "attributes": {}} +{"rel_ts": 144.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 145000000000, "end_unix_nano": 146000000000, "attributes": {}} +{"rel_ts": 156.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 157000000000, "end_unix_nano": 158000000000, "attributes": {}} +{"rel_ts": 168.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 169000000000, "end_unix_nano": 170000000000, "attributes": {}} +{"rel_ts": 180.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 181000000000, "end_unix_nano": 182000000000, "attributes": {}} +{"rel_ts": 192.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 193000000000, "end_unix_nano": 194000000000, "attributes": {}} +{"rel_ts": 204.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 205000000000, "end_unix_nano": 206000000000, "attributes": {}} +{"rel_ts": 216.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 217000000000, "end_unix_nano": 218000000000, "attributes": {}} +{"rel_ts": 228.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 229000000000, "end_unix_nano": 230000000000, "attributes": {}} +{"rel_ts": 240.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 241000000000, "end_unix_nano": 242000000000, "attributes": {}} +{"rel_ts": 252.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 253000000000, "end_unix_nano": 254000000000, "attributes": {}} +{"rel_ts": 264.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 265000000000, "end_unix_nano": 266000000000, "attributes": {}} +{"rel_ts": 276.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 277000000000, "end_unix_nano": 278000000000, "attributes": {}} +{"rel_ts": 288.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 289000000000, "end_unix_nano": 290000000000, "attributes": {}} +{"rel_ts": 300.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 301000000000, "end_unix_nano": 302000000000, "attributes": {}} +{"rel_ts": 312.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 313000000000, "end_unix_nano": 314000000000, "attributes": {}} +{"rel_ts": 324.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 325000000000, "end_unix_nano": 326000000000, "attributes": {}} +{"rel_ts": 336.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 337000000000, "end_unix_nano": 338000000000, "attributes": {}} +{"rel_ts": 348.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 349000000000, "end_unix_nano": 350000000000, "attributes": {}} +{"rel_ts": 360.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 361000000000, "end_unix_nano": 362000000000, "attributes": {}} +{"rel_ts": 372.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 373000000000, "end_unix_nano": 374000000000, "attributes": {}} +{"rel_ts": 384.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 385000000000, "end_unix_nano": 386000000000, "attributes": {}} +{"rel_ts": 396.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 397000000000, "end_unix_nano": 398000000000, "attributes": {}} +{"rel_ts": 408.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 409000000000, "end_unix_nano": 410000000000, "attributes": {}} +{"rel_ts": 420.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 421000000000, "end_unix_nano": 422000000000, "attributes": {}} +{"rel_ts": 432.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 433000000000, "end_unix_nano": 434000000000, "attributes": {}} +{"rel_ts": 444.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 445000000000, "end_unix_nano": 446000000000, "attributes": {}} +{"rel_ts": 456.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 457000000000, "end_unix_nano": 458000000000, "attributes": {}} +{"rel_ts": 468.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 469000000000, "end_unix_nano": 470000000000, "attributes": {}} +{"rel_ts": 480.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 481000000000, "end_unix_nano": 482000000000, "attributes": {}} +{"rel_ts": 492.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 493000000000, "end_unix_nano": 494000000000, "attributes": {}} +{"rel_ts": 504.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 505000000000, "end_unix_nano": 506000000000, "attributes": {}} +{"rel_ts": 516.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 517000000000, "end_unix_nano": 518000000000, "attributes": {}} +{"rel_ts": 528.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 529000000000, "end_unix_nano": 530000000000, "attributes": {}} diff --git a/demo/fixtures/02_release_regression.jsonl b/demo/fixtures/02_release_regression.jsonl index 6210800..b0244c2 100644 --- a/demo/fixtures/02_release_regression.jsonl +++ b/demo/fixtures/02_release_regression.jsonl @@ -1,45 +1,55 @@ {"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} -{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} -{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} -{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} -{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} -{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} -{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} -{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} -{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} -{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} -{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} -{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} -{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} -{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} -{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} -{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} -{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} -{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} -{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} -{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} -{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} -{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} -{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} -{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} -{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} -{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} -{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} -{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} -{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} -{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} -{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 30.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} -{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 31.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} -{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 32.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} -{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 33.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} -{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 34.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} -{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 35.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} -{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 36.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} -{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 37.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} -{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 38.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} -{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 39.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} -{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} -{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} -{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} -{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} -{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 96.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 97000000000, "end_unix_nano": 98000000000, "attributes": {}} +{"rel_ts": 108.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 109000000000, "end_unix_nano": 110000000000, "attributes": {}} +{"rel_ts": 120.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 121000000000, "end_unix_nano": 122000000000, "attributes": {}} +{"rel_ts": 132.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 133000000000, "end_unix_nano": 134000000000, "attributes": {}} +{"rel_ts": 144.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 145000000000, "end_unix_nano": 146000000000, "attributes": {}} +{"rel_ts": 156.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 157000000000, "end_unix_nano": 158000000000, "attributes": {}} +{"rel_ts": 168.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 169000000000, "end_unix_nano": 170000000000, "attributes": {}} +{"rel_ts": 180.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 181000000000, "end_unix_nano": 182000000000, "attributes": {}} +{"rel_ts": 192.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 193000000000, "end_unix_nano": 194000000000, "attributes": {}} +{"rel_ts": 204.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 205000000000, "end_unix_nano": 206000000000, "attributes": {}} +{"rel_ts": 216.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 217000000000, "end_unix_nano": 218000000000, "attributes": {}} +{"rel_ts": 228.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 229000000000, "end_unix_nano": 230000000000, "attributes": {}} +{"rel_ts": 240.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 241000000000, "end_unix_nano": 242000000000, "attributes": {}} +{"rel_ts": 252.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 253000000000, "end_unix_nano": 254000000000, "attributes": {}} +{"rel_ts": 264.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 265000000000, "end_unix_nano": 266000000000, "attributes": {}} +{"rel_ts": 276.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 277000000000, "end_unix_nano": 278000000000, "attributes": {}} +{"rel_ts": 288.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 289000000000, "end_unix_nano": 290000000000, "attributes": {}} +{"rel_ts": 300.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 301000000000, "end_unix_nano": 302000000000, "attributes": {}} +{"rel_ts": 312.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 313000000000, "end_unix_nano": 314000000000, "attributes": {}} +{"rel_ts": 324.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 325000000000, "end_unix_nano": 326000000000, "attributes": {}} +{"rel_ts": 336.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 337000000000, "end_unix_nano": 338000000000, "attributes": {}} +{"rel_ts": 348.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 349000000000, "end_unix_nano": 350000000000, "attributes": {}} +{"rel_ts": 360.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 361000000000, "end_unix_nano": 362000000000, "attributes": {}} +{"rel_ts": 372.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 373000000000, "end_unix_nano": 374000000000, "attributes": {}} +{"rel_ts": 384.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 385000000000, "end_unix_nano": 386000000000, "attributes": {}} +{"rel_ts": 396.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 397000000000, "end_unix_nano": 398000000000, "attributes": {}} +{"rel_ts": 408.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 409000000000, "end_unix_nano": 410000000000, "attributes": {}} +{"rel_ts": 420.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 421000000000, "end_unix_nano": 422000000000, "attributes": {}} +{"rel_ts": 432.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 433000000000, "end_unix_nano": 434000000000, "attributes": {}} +{"rel_ts": 444.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 445000000000, "end_unix_nano": 446000000000, "attributes": {}} +{"rel_ts": 456.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 457000000000, "end_unix_nano": 458000000000, "attributes": {}} +{"rel_ts": 468.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 469000000000, "end_unix_nano": 470000000000, "attributes": {}} +{"rel_ts": 480.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 481000000000, "end_unix_nano": 482000000000, "attributes": {}} +{"rel_ts": 492.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 493000000000, "end_unix_nano": 494000000000, "attributes": {}} +{"rel_ts": 504.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 505000000000, "end_unix_nano": 506000000000, "attributes": {}} +{"rel_ts": 516.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 517000000000, "end_unix_nano": 518000000000, "attributes": {}} +{"rel_ts": 528.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 529000000000, "end_unix_nano": 530000000000, "attributes": {}} +{"rel_ts": 540.0, "trace_id": "tr_000045000000000000000000000", "span_id": "sp_0000450000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 45.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 541000000000, "end_unix_nano": 542000000000, "attributes": {}} +{"rel_ts": 552.0, "trace_id": "tr_000046000000000000000000000", "span_id": "sp_0000460000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 46.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 553000000000, "end_unix_nano": 554000000000, "attributes": {}} +{"rel_ts": 564.0, "trace_id": "tr_000047000000000000000000000", "span_id": "sp_0000470000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 47.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 565000000000, "end_unix_nano": 566000000000, "attributes": {}} +{"rel_ts": 576.0, "trace_id": "tr_000048000000000000000000000", "span_id": "sp_0000480000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 48.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 577000000000, "end_unix_nano": 578000000000, "attributes": {}} +{"rel_ts": 588.0, "trace_id": "tr_000049000000000000000000000", "span_id": "sp_0000490000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 49.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 589000000000, "end_unix_nano": 590000000000, "attributes": {}} +{"rel_ts": 600.0, "trace_id": "tr_000050000000000000000000000", "span_id": "sp_0000500000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "Garbage hallucinated text response with zero semantic similarity 50.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 601000000000, "end_unix_nano": 602000000000, "attributes": {}} +{"rel_ts": 612.0, "trace_id": "tr_000051000000000000000000000", "span_id": "sp_0000510000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "Garbage hallucinated text response with zero semantic similarity 51.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 613000000000, "end_unix_nano": 614000000000, "attributes": {}} +{"rel_ts": 624.0, "trace_id": "tr_000052000000000000000000000", "span_id": "sp_0000520000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "Garbage hallucinated text response with zero semantic similarity 52.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 625000000000, "end_unix_nano": 626000000000, "attributes": {}} +{"rel_ts": 636.0, "trace_id": "tr_000053000000000000000000000", "span_id": "sp_0000530000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "Garbage hallucinated text response with zero semantic similarity 53.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 637000000000, "end_unix_nano": 638000000000, "attributes": {}} +{"rel_ts": 648.0, "trace_id": "tr_000054000000000000000000000", "span_id": "sp_0000540000000", "service_name": "ragapp", "service_version": "v2", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "Garbage hallucinated text response with zero semantic similarity 54.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 649000000000, "end_unix_nano": 650000000000, "attributes": {}} diff --git a/demo/fixtures/03_runaway_loop.jsonl b/demo/fixtures/03_runaway_loop.jsonl index b6dd406..4045987 100644 --- a/demo/fixtures/03_runaway_loop.jsonl +++ b/demo/fixtures/03_runaway_loop.jsonl @@ -1,45 +1,55 @@ -{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} -{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} -{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} -{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} -{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} -{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} -{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} -{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} -{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} -{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} -{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} -{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} -{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} -{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} -{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} -{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} -{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} -{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} -{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} -{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} -{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} -{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} -{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} -{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} -{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} -{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 0", "output_text": "standard response 0", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} -{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 1", "output_text": "standard response 1", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} -{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 2", "output_text": "standard response 2", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} -{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 3", "output_text": "standard response 3", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} -{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "query 4", "output_text": "standard response 4", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} -{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 30", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} -{"rel_ts": 60.2, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 31", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61200000000, "end_unix_nano": 62200000000, "attributes": {}} -{"rel_ts": 60.400000000000006, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 32", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61400000000, "end_unix_nano": 62400000000, "attributes": {}} -{"rel_ts": 60.60000000000001, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 33", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61600000000, "end_unix_nano": 62600000000, "attributes": {}} -{"rel_ts": 60.80000000000001, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 34", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 61800000000, "end_unix_nano": 62800000000, "attributes": {}} -{"rel_ts": 61.000000000000014, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 35", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62000000000, "end_unix_nano": 63000000000, "attributes": {}} -{"rel_ts": 61.20000000000002, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 36", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62200000000, "end_unix_nano": 63200000000, "attributes": {}} -{"rel_ts": 61.40000000000002, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 37", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62400000000, "end_unix_nano": 63400000000, "attributes": {}} -{"rel_ts": 61.60000000000002, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 38", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62600000000, "end_unix_nano": 63600000000, "attributes": {}} -{"rel_ts": 61.800000000000026, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 39", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 62800000000, "end_unix_nano": 63800000000, "attributes": {}} -{"rel_ts": 62.00000000000003, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 40", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} -{"rel_ts": 62.20000000000003, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 41", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63200000000, "end_unix_nano": 64200000000, "attributes": {}} -{"rel_ts": 62.400000000000034, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 42", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63400000000, "end_unix_nano": 64400000000, "attributes": {}} -{"rel_ts": 62.60000000000004, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 43", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63600000000, "end_unix_nano": 64600000000, "attributes": {}} -{"rel_ts": 62.80000000000004, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "runaway agent loop iteration 44", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 63800000000, "end_unix_nano": 64800000000, "attributes": {}} +{"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 96.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 97000000000, "end_unix_nano": 98000000000, "attributes": {}} +{"rel_ts": 108.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 109000000000, "end_unix_nano": 110000000000, "attributes": {}} +{"rel_ts": 120.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 121000000000, "end_unix_nano": 122000000000, "attributes": {}} +{"rel_ts": 132.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 133000000000, "end_unix_nano": 134000000000, "attributes": {}} +{"rel_ts": 144.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 145000000000, "end_unix_nano": 146000000000, "attributes": {}} +{"rel_ts": 156.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 157000000000, "end_unix_nano": 158000000000, "attributes": {}} +{"rel_ts": 168.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 169000000000, "end_unix_nano": 170000000000, "attributes": {}} +{"rel_ts": 180.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 181000000000, "end_unix_nano": 182000000000, "attributes": {}} +{"rel_ts": 192.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 193000000000, "end_unix_nano": 194000000000, "attributes": {}} +{"rel_ts": 204.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 205000000000, "end_unix_nano": 206000000000, "attributes": {}} +{"rel_ts": 216.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 217000000000, "end_unix_nano": 218000000000, "attributes": {}} +{"rel_ts": 228.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 229000000000, "end_unix_nano": 230000000000, "attributes": {}} +{"rel_ts": 240.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 241000000000, "end_unix_nano": 242000000000, "attributes": {}} +{"rel_ts": 252.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 253000000000, "end_unix_nano": 254000000000, "attributes": {}} +{"rel_ts": 264.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 265000000000, "end_unix_nano": 266000000000, "attributes": {}} +{"rel_ts": 276.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 277000000000, "end_unix_nano": 278000000000, "attributes": {}} +{"rel_ts": 288.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 289000000000, "end_unix_nano": 290000000000, "attributes": {}} +{"rel_ts": 300.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 301000000000, "end_unix_nano": 302000000000, "attributes": {}} +{"rel_ts": 312.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 313000000000, "end_unix_nano": 314000000000, "attributes": {}} +{"rel_ts": 324.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 325000000000, "end_unix_nano": 326000000000, "attributes": {}} +{"rel_ts": 336.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 337000000000, "end_unix_nano": 338000000000, "attributes": {}} +{"rel_ts": 348.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 349000000000, "end_unix_nano": 350000000000, "attributes": {}} +{"rel_ts": 360.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 361000000000, "end_unix_nano": 362000000000, "attributes": {}} +{"rel_ts": 372.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 373000000000, "end_unix_nano": 374000000000, "attributes": {}} +{"rel_ts": 384.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 385000000000, "end_unix_nano": 386000000000, "attributes": {}} +{"rel_ts": 396.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 397000000000, "end_unix_nano": 398000000000, "attributes": {}} +{"rel_ts": 408.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 409000000000, "end_unix_nano": 410000000000, "attributes": {}} +{"rel_ts": 420.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 421000000000, "end_unix_nano": 422000000000, "attributes": {}} +{"rel_ts": 432.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 433000000000, "end_unix_nano": 434000000000, "attributes": {}} +{"rel_ts": 444.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 445000000000, "end_unix_nano": 446000000000, "attributes": {}} +{"rel_ts": 456.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 457000000000, "end_unix_nano": 458000000000, "attributes": {}} +{"rel_ts": 468.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 20, "output_tokens": 30, "start_unix_nano": 469000000000, "end_unix_nano": 470000000000, "attributes": {}} +{"rel_ts": 480.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481000000000, "end_unix_nano": 482000000000, "attributes": {}} +{"rel_ts": 480.2, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481200000000, "end_unix_nano": 482200000000, "attributes": {}} +{"rel_ts": 480.4, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481400000000, "end_unix_nano": 482400000000, "attributes": {}} +{"rel_ts": 480.59999999999997, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481599999999, "end_unix_nano": 482599999999, "attributes": {}} +{"rel_ts": 480.79999999999995, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481799999999, "end_unix_nano": 482799999999, "attributes": {}} +{"rel_ts": 480.99999999999994, "trace_id": "tr_000045000000000000000000000", "span_id": "sp_0000450000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 481999999999, "end_unix_nano": 482999999999, "attributes": {}} +{"rel_ts": 481.19999999999993, "trace_id": "tr_000046000000000000000000000", "span_id": "sp_0000460000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 482199999999, "end_unix_nano": 483199999999, "attributes": {}} +{"rel_ts": 481.3999999999999, "trace_id": "tr_000047000000000000000000000", "span_id": "sp_0000470000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 482399999999, "end_unix_nano": 483399999999, "attributes": {}} +{"rel_ts": 481.5999999999999, "trace_id": "tr_000048000000000000000000000", "span_id": "sp_0000480000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 482599999999, "end_unix_nano": 483599999999, "attributes": {}} +{"rel_ts": 481.7999999999999, "trace_id": "tr_000049000000000000000000000", "span_id": "sp_0000490000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 482799999999, "end_unix_nano": 483799999999, "attributes": {}} +{"rel_ts": 481.9999999999999, "trace_id": "tr_000050000000000000000000000", "span_id": "sp_0000500000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 482999999999, "end_unix_nano": 483999999999, "attributes": {}} +{"rel_ts": 482.1999999999999, "trace_id": "tr_000051000000000000000000000", "span_id": "sp_0000510000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 483199999999, "end_unix_nano": 484199999999, "attributes": {}} +{"rel_ts": 482.39999999999986, "trace_id": "tr_000052000000000000000000000", "span_id": "sp_0000520000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 483399999999, "end_unix_nano": 484399999999, "attributes": {}} +{"rel_ts": 482.59999999999985, "trace_id": "tr_000053000000000000000000000", "span_id": "sp_0000530000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 483599999999, "end_unix_nano": 484599999999, "attributes": {}} +{"rel_ts": 482.79999999999984, "trace_id": "tr_000054000000000000000000000", "span_id": "sp_0000540000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "input_tokens": 15000, "output_tokens": 25000, "start_unix_nano": 483799999999, "end_unix_nano": 484799999999, "attributes": {}} diff --git a/demo/fixtures/04_input_shift.jsonl b/demo/fixtures/04_input_shift.jsonl index b324cb5..c7eb202 100644 --- a/demo/fixtures/04_input_shift.jsonl +++ b/demo/fixtures/04_input_shift.jsonl @@ -1,45 +1,55 @@ {"rel_ts": 0.0, "trace_id": "tr_000000000000000000000000000", "span_id": "sp_0000000000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 1000000000, "end_unix_nano": 2000000000, "attributes": {}} -{"rel_ts": 2.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 3000000000, "end_unix_nano": 4000000000, "attributes": {}} -{"rel_ts": 4.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 5000000000, "end_unix_nano": 6000000000, "attributes": {}} -{"rel_ts": 6.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 7000000000, "end_unix_nano": 8000000000, "attributes": {}} -{"rel_ts": 8.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 9000000000, "end_unix_nano": 10000000000, "attributes": {}} -{"rel_ts": 10.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 11000000000, "end_unix_nano": 12000000000, "attributes": {}} -{"rel_ts": 12.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} -{"rel_ts": 14.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 15000000000, "end_unix_nano": 16000000000, "attributes": {}} -{"rel_ts": 16.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 17000000000, "end_unix_nano": 18000000000, "attributes": {}} -{"rel_ts": 18.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 19000000000, "end_unix_nano": 20000000000, "attributes": {}} -{"rel_ts": 20.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 21000000000, "end_unix_nano": 22000000000, "attributes": {}} -{"rel_ts": 22.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 23000000000, "end_unix_nano": 24000000000, "attributes": {}} -{"rel_ts": 24.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} -{"rel_ts": 26.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 27000000000, "end_unix_nano": 28000000000, "attributes": {}} -{"rel_ts": 28.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 29000000000, "end_unix_nano": 30000000000, "attributes": {}} -{"rel_ts": 30.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 31000000000, "end_unix_nano": 32000000000, "attributes": {}} -{"rel_ts": 32.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 33000000000, "end_unix_nano": 34000000000, "attributes": {}} -{"rel_ts": 34.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 35000000000, "end_unix_nano": 36000000000, "attributes": {}} -{"rel_ts": 36.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} -{"rel_ts": 38.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 39000000000, "end_unix_nano": 40000000000, "attributes": {}} -{"rel_ts": 40.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 41000000000, "end_unix_nano": 42000000000, "attributes": {}} -{"rel_ts": 42.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 43000000000, "end_unix_nano": 44000000000, "attributes": {}} -{"rel_ts": 44.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 45000000000, "end_unix_nano": 46000000000, "attributes": {}} -{"rel_ts": 46.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 47000000000, "end_unix_nano": 48000000000, "attributes": {}} -{"rel_ts": 48.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} -{"rel_ts": 50.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 51000000000, "end_unix_nano": 52000000000, "attributes": {}} -{"rel_ts": 52.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 53000000000, "end_unix_nano": 54000000000, "attributes": {}} -{"rel_ts": 54.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 55000000000, "end_unix_nano": 56000000000, "attributes": {}} -{"rel_ts": 56.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 57000000000, "end_unix_nano": 58000000000, "attributes": {}} -{"rel_ts": 58.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 59000000000, "end_unix_nano": 60000000000, "attributes": {}} -{"rel_ts": 60.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 30", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 30.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} -{"rel_ts": 62.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 31", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 31.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 63000000000, "end_unix_nano": 64000000000, "attributes": {}} -{"rel_ts": 64.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 32", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 32.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 65000000000, "end_unix_nano": 66000000000, "attributes": {}} -{"rel_ts": 66.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 33", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 33.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 67000000000, "end_unix_nano": 68000000000, "attributes": {}} -{"rel_ts": 68.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 34", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 34.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 69000000000, "end_unix_nano": 70000000000, "attributes": {}} -{"rel_ts": 70.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 35", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 35.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 71000000000, "end_unix_nano": 72000000000, "attributes": {}} -{"rel_ts": 72.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 36", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 36.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} -{"rel_ts": 74.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 37", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 37.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 75000000000, "end_unix_nano": 76000000000, "attributes": {}} -{"rel_ts": 76.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 38", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 38.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 77000000000, "end_unix_nano": 78000000000, "attributes": {}} -{"rel_ts": 78.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 39", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 39.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 79000000000, "end_unix_nano": 80000000000, "attributes": {}} -{"rel_ts": 80.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 40", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 81000000000, "end_unix_nano": 82000000000, "attributes": {}} -{"rel_ts": 82.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 41", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 83000000000, "end_unix_nano": 84000000000, "attributes": {}} -{"rel_ts": 84.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 42", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} -{"rel_ts": 86.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 43", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 87000000000, "end_unix_nano": 88000000000, "attributes": {}} -{"rel_ts": 88.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 44", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 89000000000, "end_unix_nano": 90000000000, "attributes": {}} +{"rel_ts": 12.0, "trace_id": "tr_000001000000000000000000000", "span_id": "sp_0000010000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 13000000000, "end_unix_nano": 14000000000, "attributes": {}} +{"rel_ts": 24.0, "trace_id": "tr_000002000000000000000000000", "span_id": "sp_0000020000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 25000000000, "end_unix_nano": 26000000000, "attributes": {}} +{"rel_ts": 36.0, "trace_id": "tr_000003000000000000000000000", "span_id": "sp_0000030000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 37000000000, "end_unix_nano": 38000000000, "attributes": {}} +{"rel_ts": 48.0, "trace_id": "tr_000004000000000000000000000", "span_id": "sp_0000040000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 49000000000, "end_unix_nano": 50000000000, "attributes": {}} +{"rel_ts": 60.0, "trace_id": "tr_000005000000000000000000000", "span_id": "sp_0000050000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 61000000000, "end_unix_nano": 62000000000, "attributes": {}} +{"rel_ts": 72.0, "trace_id": "tr_000006000000000000000000000", "span_id": "sp_0000060000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 73000000000, "end_unix_nano": 74000000000, "attributes": {}} +{"rel_ts": 84.0, "trace_id": "tr_000007000000000000000000000", "span_id": "sp_0000070000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 85000000000, "end_unix_nano": 86000000000, "attributes": {}} +{"rel_ts": 96.0, "trace_id": "tr_000008000000000000000000000", "span_id": "sp_0000080000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 97000000000, "end_unix_nano": 98000000000, "attributes": {}} +{"rel_ts": 108.0, "trace_id": "tr_000009000000000000000000000", "span_id": "sp_0000090000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 109000000000, "end_unix_nano": 110000000000, "attributes": {}} +{"rel_ts": 120.0, "trace_id": "tr_000010000000000000000000000", "span_id": "sp_0000100000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 121000000000, "end_unix_nano": 122000000000, "attributes": {}} +{"rel_ts": 132.0, "trace_id": "tr_000011000000000000000000000", "span_id": "sp_0000110000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 133000000000, "end_unix_nano": 134000000000, "attributes": {}} +{"rel_ts": 144.0, "trace_id": "tr_000012000000000000000000000", "span_id": "sp_0000120000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 145000000000, "end_unix_nano": 146000000000, "attributes": {}} +{"rel_ts": 156.0, "trace_id": "tr_000013000000000000000000000", "span_id": "sp_0000130000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 157000000000, "end_unix_nano": 158000000000, "attributes": {}} +{"rel_ts": 168.0, "trace_id": "tr_000014000000000000000000000", "span_id": "sp_0000140000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 169000000000, "end_unix_nano": 170000000000, "attributes": {}} +{"rel_ts": 180.0, "trace_id": "tr_000015000000000000000000000", "span_id": "sp_0000150000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 181000000000, "end_unix_nano": 182000000000, "attributes": {}} +{"rel_ts": 192.0, "trace_id": "tr_000016000000000000000000000", "span_id": "sp_0000160000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 193000000000, "end_unix_nano": 194000000000, "attributes": {}} +{"rel_ts": 204.0, "trace_id": "tr_000017000000000000000000000", "span_id": "sp_0000170000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 205000000000, "end_unix_nano": 206000000000, "attributes": {}} +{"rel_ts": 216.0, "trace_id": "tr_000018000000000000000000000", "span_id": "sp_0000180000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 217000000000, "end_unix_nano": 218000000000, "attributes": {}} +{"rel_ts": 228.0, "trace_id": "tr_000019000000000000000000000", "span_id": "sp_0000190000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 229000000000, "end_unix_nano": 230000000000, "attributes": {}} +{"rel_ts": 240.0, "trace_id": "tr_000020000000000000000000000", "span_id": "sp_0000200000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 241000000000, "end_unix_nano": 242000000000, "attributes": {}} +{"rel_ts": 252.0, "trace_id": "tr_000021000000000000000000000", "span_id": "sp_0000210000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 253000000000, "end_unix_nano": 254000000000, "attributes": {}} +{"rel_ts": 264.0, "trace_id": "tr_000022000000000000000000000", "span_id": "sp_0000220000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 265000000000, "end_unix_nano": 266000000000, "attributes": {}} +{"rel_ts": 276.0, "trace_id": "tr_000023000000000000000000000", "span_id": "sp_0000230000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 277000000000, "end_unix_nano": 278000000000, "attributes": {}} +{"rel_ts": 288.0, "trace_id": "tr_000024000000000000000000000", "span_id": "sp_0000240000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 289000000000, "end_unix_nano": 290000000000, "attributes": {}} +{"rel_ts": 300.0, "trace_id": "tr_000025000000000000000000000", "span_id": "sp_0000250000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 301000000000, "end_unix_nano": 302000000000, "attributes": {}} +{"rel_ts": 312.0, "trace_id": "tr_000026000000000000000000000", "span_id": "sp_0000260000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 313000000000, "end_unix_nano": 314000000000, "attributes": {}} +{"rel_ts": 324.0, "trace_id": "tr_000027000000000000000000000", "span_id": "sp_0000270000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 325000000000, "end_unix_nano": 326000000000, "attributes": {}} +{"rel_ts": 336.0, "trace_id": "tr_000028000000000000000000000", "span_id": "sp_0000280000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 337000000000, "end_unix_nano": 338000000000, "attributes": {}} +{"rel_ts": 348.0, "trace_id": "tr_000029000000000000000000000", "span_id": "sp_0000290000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 349000000000, "end_unix_nano": 350000000000, "attributes": {}} +{"rel_ts": 360.0, "trace_id": "tr_000030000000000000000000000", "span_id": "sp_0000300000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 361000000000, "end_unix_nano": 362000000000, "attributes": {}} +{"rel_ts": 372.0, "trace_id": "tr_000031000000000000000000000", "span_id": "sp_0000310000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 373000000000, "end_unix_nano": 374000000000, "attributes": {}} +{"rel_ts": 384.0, "trace_id": "tr_000032000000000000000000000", "span_id": "sp_0000320000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 385000000000, "end_unix_nano": 386000000000, "attributes": {}} +{"rel_ts": 396.0, "trace_id": "tr_000033000000000000000000000", "span_id": "sp_0000330000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 397000000000, "end_unix_nano": 398000000000, "attributes": {}} +{"rel_ts": 408.0, "trace_id": "tr_000034000000000000000000000", "span_id": "sp_0000340000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 409000000000, "end_unix_nano": 410000000000, "attributes": {}} +{"rel_ts": 420.0, "trace_id": "tr_000035000000000000000000000", "span_id": "sp_0000350000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 0?", "output_text": "OpenTelemetry is an open source observability framework variant 0.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 421000000000, "end_unix_nano": 422000000000, "attributes": {}} +{"rel_ts": 432.0, "trace_id": "tr_000036000000000000000000000", "span_id": "sp_0000360000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 1?", "output_text": "OpenTelemetry is an open source observability framework variant 1.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 433000000000, "end_unix_nano": 434000000000, "attributes": {}} +{"rel_ts": 444.0, "trace_id": "tr_000037000000000000000000000", "span_id": "sp_0000370000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 2?", "output_text": "OpenTelemetry is an open source observability framework variant 2.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 445000000000, "end_unix_nano": 446000000000, "attributes": {}} +{"rel_ts": 456.0, "trace_id": "tr_000038000000000000000000000", "span_id": "sp_0000380000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 3?", "output_text": "OpenTelemetry is an open source observability framework variant 3.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 457000000000, "end_unix_nano": 458000000000, "attributes": {}} +{"rel_ts": 468.0, "trace_id": "tr_000039000000000000000000000", "span_id": "sp_0000390000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "what is opentelemetry topic query 4?", "output_text": "OpenTelemetry is an open source observability framework variant 4.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 469000000000, "end_unix_nano": 470000000000, "attributes": {}} +{"rel_ts": 480.0, "trace_id": "tr_000040000000000000000000000", "span_id": "sp_0000400000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 40", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 40.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 481000000000, "end_unix_nano": 482000000000, "attributes": {}} +{"rel_ts": 492.0, "trace_id": "tr_000041000000000000000000000", "span_id": "sp_0000410000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 41", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 41.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 493000000000, "end_unix_nano": 494000000000, "attributes": {}} +{"rel_ts": 504.0, "trace_id": "tr_000042000000000000000000000", "span_id": "sp_0000420000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 42", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 42.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 505000000000, "end_unix_nano": 506000000000, "attributes": {}} +{"rel_ts": 516.0, "trace_id": "tr_000043000000000000000000000", "span_id": "sp_0000430000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 43", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 43.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 517000000000, "end_unix_nano": 518000000000, "attributes": {}} +{"rel_ts": 528.0, "trace_id": "tr_000044000000000000000000000", "span_id": "sp_0000440000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 44", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 44.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 529000000000, "end_unix_nano": 530000000000, "attributes": {}} +{"rel_ts": 540.0, "trace_id": "tr_000045000000000000000000000", "span_id": "sp_0000450000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 45", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 45.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 541000000000, "end_unix_nano": 542000000000, "attributes": {}} +{"rel_ts": 552.0, "trace_id": "tr_000046000000000000000000000", "span_id": "sp_0000460000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 46", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 46.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 553000000000, "end_unix_nano": 554000000000, "attributes": {}} +{"rel_ts": 564.0, "trace_id": "tr_000047000000000000000000000", "span_id": "sp_0000470000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 47", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 47.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 565000000000, "end_unix_nano": 566000000000, "attributes": {}} +{"rel_ts": 576.0, "trace_id": "tr_000048000000000000000000000", "span_id": "sp_0000480000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 48", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 48.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 577000000000, "end_unix_nano": 578000000000, "attributes": {}} +{"rel_ts": 588.0, "trace_id": "tr_000049000000000000000000000", "span_id": "sp_0000490000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 49", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 49.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 589000000000, "end_unix_nano": 590000000000, "attributes": {}} +{"rel_ts": 600.0, "trace_id": "tr_000050000000000000000000000", "span_id": "sp_0000500000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 50", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 50.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 601000000000, "end_unix_nano": 602000000000, "attributes": {}} +{"rel_ts": 612.0, "trace_id": "tr_000051000000000000000000000", "span_id": "sp_0000510000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 51", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 51.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 613000000000, "end_unix_nano": 614000000000, "attributes": {}} +{"rel_ts": 624.0, "trace_id": "tr_000052000000000000000000000", "span_id": "sp_0000520000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 52", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 52.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 625000000000, "end_unix_nano": 626000000000, "attributes": {}} +{"rel_ts": 636.0, "trace_id": "tr_000053000000000000000000000", "span_id": "sp_0000530000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 53", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 53.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 637000000000, "end_unix_nano": 638000000000, "attributes": {}} +{"rel_ts": 648.0, "trace_id": "tr_000054000000000000000000000", "span_id": "sp_0000540000000", "service_name": "ragapp", "service_version": "v1", "gen_ai_system": "openai", "model": "gpt-4o", "input_text": "explain write ahead logging index locking deadlock 54", "output_text": "Write ahead logging and relational database B-Trees MVCC isolation 54.", "input_tokens": 15, "output_tokens": 30, "start_unix_nano": 649000000000, "end_unix_nano": 650000000000, "attributes": {}} diff --git a/demo/fixtures/generate_fixtures.py b/demo/fixtures/generate_fixtures.py index 46ce6da..033c634 100644 --- a/demo/fixtures/generate_fixtures.py +++ b/demo/fixtures/generate_fixtures.py @@ -38,7 +38,7 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: path_dir = Path(out_dir) path_dir.mkdir(parents=True, exist_ok=True) - # 1. Steady Baseline (01_steady_baseline.jsonl) + # 1. Steady Baseline (01_steady_baseline.jsonl): 30 warming + 15 healthy scored spans spans_01 = [] ts = 0.0 for i in range(45): @@ -51,14 +51,13 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: rel_ts=ts, ) ) - ts += 2.0 + ts += 12.0 write_fixture(path_dir / "01_steady_baseline.jsonl", spans_01) - # 2. Release Regression (02_release_regression.jsonl) + # 2. Release Regression (02_release_regression.jsonl): 30 warming + 10 healthy v1 + 15 degraded v2 spans spans_02 = [] ts = 0.0 - # 30 reference v1 spans - for i in range(30): + for i in range(40): spans_02.append( make_span( i, @@ -68,9 +67,8 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: rel_ts=ts, ) ) - ts += 2.0 - # 15 degraded v2 spans (poisoned prompt / hallucinated output) - for i in range(30, 45): + ts += 12.0 + for i in range(40, 55): spans_02.append( make_span( i, @@ -80,33 +78,31 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: rel_ts=ts, ) ) - ts += 2.0 + ts += 12.0 write_fixture(path_dir / "02_release_regression.jsonl", spans_02) - # 3. Runaway Loop (03_runaway_loop.jsonl) + # 3. Runaway Loop (03_runaway_loop.jsonl): 30 warming + 10 healthy + 15 rapid runaway loop spans spans_03 = [] ts = 0.0 - # 30 reference v1 spans - for i in range(30): + for i in range(40): spans_03.append( make_span( i, version="v1", - input_text=f"query {i % 5}", - output_text=f"standard response {i % 5}", + input_text=f"what is opentelemetry topic query {i % 5}?", + output_text=f"OpenTelemetry is an open source observability framework variant {i % 5}.", input_tokens=20, output_tokens=30, rel_ts=ts, ) ) - ts += 2.0 - # 15 runaway loop spans (high token burn & high rate) - for i in range(30, 45): + ts += 12.0 + for i in range(40, 55): spans_03.append( make_span( i, version="v1", - input_text=f"runaway agent loop iteration {i}", + input_text=f"what is opentelemetry topic query {i % 5}?", output_text="A" * 5000, input_tokens=15000, output_tokens=25000, @@ -116,11 +112,10 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: ts += 0.2 write_fixture(path_dir / "03_runaway_loop.jsonl", spans_03) - # 4. Input Shift (04_input_shift.jsonl) + # 4. Input Shift (04_input_shift.jsonl): 30 warming + 10 healthy Topic A + 15 shifted Topic B spans spans_04 = [] ts = 0.0 - # 30 reference spans on Topic A - for i in range(30): + for i in range(40): spans_04.append( make_span( i, @@ -130,9 +125,8 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: rel_ts=ts, ) ) - ts += 2.0 - # 15 shifted Topic B database queries + modified outputs (causes input drift + behavior drift) - for i in range(30, 45): + ts += 12.0 + for i in range(40, 55): spans_04.append( make_span( i, @@ -142,7 +136,7 @@ def generate_all_fixtures(out_dir: str = "demo/fixtures") -> None: rel_ts=ts, ) ) - ts += 2.0 + ts += 12.0 write_fixture(path_dir / "04_input_shift.jsonl", spans_04) print(f"Successfully generated 4 golden replay fixtures in {out_dir}") From 1a766647a9b26b7f2c064de640a72f82e3d942fa Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:22:49 +0530 Subject: [PATCH 29/35] add tick-level exception handling to EvaluatorThread --- vitals/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vitals/main.py b/vitals/main.py index 58f50da..7930e6b 100644 --- a/vitals/main.py +++ b/vitals/main.py @@ -62,8 +62,11 @@ def stop(self) -> None: def run(self) -> None: while not self._stop_event.is_set(): - now = time.time() - self._tick(now) + try: + now = time.time() + self._tick(now) + except Exception: # noqa: BLE001 + log.exception("evaluator: unhandled error during tick") self._stop_event.wait(float(self._cfg.evaluate_interval_s)) def _tick(self, now: float) -> None: From b8e769e035d775162a7ec18d32aaa75655af4aba Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:23:15 +0530 Subject: [PATCH 30/35] fix replay virtual time propagation for speed 0 --- vitals/main.py | 12 ++++++++---- vitals/replay/runner.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/vitals/main.py b/vitals/main.py index 7930e6b..5cec410 100644 --- a/vitals/main.py +++ b/vitals/main.py @@ -153,8 +153,8 @@ def build_pipeline(config_path: str | None = "vitals.yaml"): verdict_provider=lambda: list(verdict_snapshot.values()), ) - def on_span(span: GenAISpan) -> None: - cost_engine.record(span) + def on_span(span: GenAISpan, now: float | None = None) -> None: + cost_engine.record(span, now=now) usd = price_table.cost_usd(span.model, span.input_tokens, span.output_tokens) key = (span.service_name, span.gen_ai_system, span.model) @@ -175,7 +175,7 @@ def on_span(span: GenAISpan) -> None: if quality_engine is not None: try: rec = quality_engine.score(span) - scope.observe(span, rec, usd) + scope.observe(span, rec, usd, now=now) emitter.emit_eval_log(rec) health.inc_scored() if rec.state == "scored": @@ -286,7 +286,11 @@ def replay_cmd( console_thread.start() try: - n = run_replay(fixture_path, speed=speed, on_span_cb=lambda span, _: on_span_cb(span)) + n = run_replay( + fixture_path, + speed=speed, + on_span_cb=lambda span, virtual_now: on_span_cb(span, now=virtual_now), + ) log.info("Replay completed: %d spans processed", n) # Give evaluator time for final tick if needed time.sleep(1.0) diff --git a/vitals/replay/runner.py b/vitals/replay/runner.py index 13a505e..5498190 100644 --- a/vitals/replay/runner.py +++ b/vitals/replay/runner.py @@ -46,7 +46,7 @@ def run_replay( time.sleep(sleep_time) last_rel_ts = rel_ts - virtual_now = start_real_ts + (rel_ts if speed > 0 else 0.0) + virtual_now = start_real_ts + rel_ts if on_span_cb is not None: on_span_cb(span, virtual_now) From a6a5514b03f4af11d71f73725b82fe5fa6ac67a5 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:23:35 +0530 Subject: [PATCH 31/35] verify full verdict state sequence in golden replay tests --- test/test_replay_golden.py | 80 +++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/test/test_replay_golden.py b/test/test_replay_golden.py index 33f36a6..a7027ff 100644 --- a/test/test_replay_golden.py +++ b/test/test_replay_golden.py @@ -10,14 +10,14 @@ from vitals.store.db import VerdictStore from vitals.verdict.evaluator import evaluate_scope_version from vitals.verdict.scope import ScopeState -from vitals.verdict.types import Cause, InconclusiveReason, VerdictState +from vitals.verdict.types import Cause, InconclusiveReason, Verdict, VerdictState FIXTURES_DIR = Path("demo/fixtures") -def _run_scenario_evaluation( +def _run_scenario_with_state_sequence( fixture_filename: str, tmp_path -) -> tuple[VerdictState, Cause | None, InconclusiveReason | None, bool]: +) -> tuple[list[VerdictState], Verdict]: db_path = tmp_path / f"store_{fixture_filename}.db" store = VerdictStore(str(db_path)) @@ -35,63 +35,65 @@ def _run_scenario_evaluation( ) scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=30, calib_n=5) - virtual_clock = [0.0] + observed_states: list[VerdictState] = [] + last_state = None + last_verdict = None def on_span(span, now_ts): + nonlocal last_state, last_verdict cost_engine.record(span, now=now_ts) usd = prices.cost_usd(span.model, span.input_tokens, span.output_tokens) rec = quality_engine.score(span) scope.observe(span, rec, usd, now=now_ts) - virtual_clock[0] = now_ts + + versions = list(scope.windows.keys()) + target_ver = versions[-1] if versions else span.service_version + v = evaluate_scope_version(scope, target_ver, cfg, now_ts, cost_engine) + if v: + last_verdict = v + if v.state != last_state: + observed_states.append(v.state) + last_state = v.state fix_file = FIXTURES_DIR / fixture_filename run_replay(fix_file, speed=0.0, on_span_cb=on_span) - now = virtual_clock[0] - versions = list(scope.windows.keys()) - target_ver = versions[-1] if versions else "v1" - - verdict1 = evaluate_scope_version(scope, target_ver, cfg, now, cost_engine) - if verdict1: - scope.last_verdict = verdict1 - store.insert(verdict1) - - verdict2 = evaluate_scope_version(scope, target_ver, cfg, now + 10.0, cost_engine) - final_v = verdict2 or verdict1 - - assert final_v is not None, f"Expected verdict for scenario {fixture_filename}" - store.insert(final_v) + assert last_verdict is not None, f"Expected verdict for scenario {fixture_filename}" + store.insert(last_verdict) store.close() - return final_v.state, final_v.cause, final_v.inconclusive_reason, final_v.runaway + return observed_states, last_verdict def test_golden_01_steady_baseline(tmp_path): - state, cause, inc_reason, runaway = _run_scenario_evaluation( - "01_steady_baseline.jsonl", tmp_path - ) - assert state == VerdictState.STEADY + states, final_v = _run_scenario_with_state_sequence("01_steady_baseline.jsonl", tmp_path) + assert states == [VerdictState.WARMING, VerdictState.STEADY] + assert VerdictState.CHANGED not in states + assert final_v.state == VerdictState.STEADY def test_golden_02_release_regression(tmp_path): - state, cause, inc_reason, runaway = _run_scenario_evaluation( - "02_release_regression.jsonl", tmp_path - ) - assert state == VerdictState.CHANGED - assert cause == Cause.RELEASE + states, final_v = _run_scenario_with_state_sequence("02_release_regression.jsonl", tmp_path) + assert VerdictState.WARMING in states + assert VerdictState.STEADY in states + assert VerdictState.CHANGED in states + assert final_v.state == VerdictState.CHANGED + assert final_v.cause == Cause.RELEASE + assert final_v.seconds_after_deploy is not None + assert 0.0 <= final_v.seconds_after_deploy <= 300.0 def test_golden_03_runaway_loop(tmp_path): - state, cause, inc_reason, runaway = _run_scenario_evaluation( - "03_runaway_loop.jsonl", tmp_path - ) - assert state == VerdictState.CHANGED - assert runaway is True + states, final_v = _run_scenario_with_state_sequence("03_runaway_loop.jsonl", tmp_path) + assert VerdictState.WARMING in states + assert VerdictState.STEADY in states + assert VerdictState.CHANGED in states + assert final_v.state == VerdictState.CHANGED + assert final_v.runaway is True def test_golden_04_input_shift(tmp_path): - state, cause, inc_reason, runaway = _run_scenario_evaluation( - "04_input_shift.jsonl", tmp_path - ) - assert state == VerdictState.INCONCLUSIVE - assert inc_reason == InconclusiveReason.INPUT_SHIFT + states, final_v = _run_scenario_with_state_sequence("04_input_shift.jsonl", tmp_path) + assert VerdictState.CHANGED not in states + assert final_v.state == VerdictState.INCONCLUSIVE + assert final_v.inconclusive_reason == InconclusiveReason.INPUT_SHIFT From 72f696370340097cfdfd77ffe5e7eb546386d49a Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:24:03 +0530 Subject: [PATCH 32/35] calculate per-exemplar behavior sigma values --- vitals/verdict/evaluator.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/vitals/verdict/evaluator.py b/vitals/verdict/evaluator.py index 6dc9a4d..07f80c5 100644 --- a/vitals/verdict/evaluator.py +++ b/vitals/verdict/evaluator.py @@ -30,6 +30,7 @@ def select_exemplars( behavior_z: float, worst_count: int = 2, median_count: int = 1, + signal: CalibratedSignal | None = None, ) -> tuple[Exemplar, ...]: """Select worst and median exemplars from span records (spec §4.5, §9).""" if not recs: @@ -38,6 +39,11 @@ def select_exemplars( sorted_recs = sorted(recs, key=lambda r: r.behavior_psi, reverse=True) exemplars: list[Exemplar] = [] + def _get_sigma(r: SpanRecord) -> float: + if signal is not None and signal.calibrated(): + return signal.z(r.behavior_psi) + return behavior_z + # Worst exemplars actual_worst_n = min(worst_count, len(sorted_recs)) for r in sorted_recs[:actual_worst_n]: @@ -47,7 +53,7 @@ def select_exemplars( trace_id=r.trace_id, span_id=r.span_id, output_excerpt=r.output_excerpt, - behavior_sigma=behavior_z, + behavior_sigma=_get_sigma(r), ) ) @@ -61,7 +67,7 @@ def select_exemplars( trace_id=med_rec.trace_id, span_id=med_rec.span_id, output_excerpt=med_rec.output_excerpt, - behavior_sigma=behavior_z, + behavior_sigma=_get_sigma(med_rec), ) ) @@ -182,7 +188,11 @@ def evaluate_scope_version( falsifier=f"would resolve if sample size reaches {cfg.min_samples}", warming_progress=None, exemplars=select_exemplars( - recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median + recs, + behavior_z, + cfg.exemplars_worst, + cfg.exemplars_median, + signal=scope.signals.get("behavior"), ), input_sigma=input_z, ) @@ -217,7 +227,11 @@ def evaluate_scope_version( falsifier="would resolve if input drift drops below 3σ", warming_progress=None, exemplars=select_exemplars( - recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median + recs, + behavior_z, + cfg.exemplars_worst, + cfg.exemplars_median, + signal=scope.signals.get("behavior"), ), input_sigma=input_z, ) @@ -289,7 +303,13 @@ def evaluate_scope_version( else: falsifier = f"would flip to CHANGED at behavior >=3σ (currently {behavior_z:.1f}σ)" - exemplars = select_exemplars(recs, behavior_z, cfg.exemplars_worst, cfg.exemplars_median) + exemplars = select_exemplars( + recs, + behavior_z, + cfg.exemplars_worst, + cfg.exemplars_median, + signal=scope.signals.get("behavior"), + ) return Verdict( verdict_id=uuid.uuid4().hex[:16], From 49378503b630d34077ee0c7bb5d3998ca9403caa Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:24:18 +0530 Subject: [PATCH 33/35] add pipeline safety test for store failure resilience --- test/test_pipeline_integration.py | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/test_pipeline_integration.py b/test/test_pipeline_integration.py index 0ba80e2..74a9eed 100644 --- a/test/test_pipeline_integration.py +++ b/test/test_pipeline_integration.py @@ -80,3 +80,65 @@ def on_span(span): # baseline_window=5 -> some spans warmed, later ones scored q = quality.quality_samples() assert q + + +def test_pipeline_safety_store_failure_resilience(monkeypatch, tmp_path): + """Safety test (spec §17): push 1,000 spans with store.insert monkeypatched to raise. + + Asserts zero exceptions escape and spans_scored == 1000. + """ + from vitals.config.settings import VerdictConfig + from vitals.main import EvaluatorThread + from vitals.model import GenAISpan + from vitals.store.db import VerdictStore + from vitals.verdict.scope import ScopeState + + def _failing_insert(self, verdict, retain_count=None): + raise RuntimeError("Simulated store write failure") + + monkeypatch.setattr(VerdictStore, "insert", _failing_insert) + + db_path = tmp_path / "safety_test.db" + store = VerdictStore(str(db_path)) + + prices = PriceTable.from_yaml("vitals/cost/prices.yaml") + cost_engine = CostEngine(prices) + quality_engine = QualityEngine(QualityConfig(baseline_window=30)) + health = Health() + + cfg = VerdictConfig(enabled=True, min_samples=5, calibration_samples=5, evaluate_interval_s=1) + scope = ScopeState("ragapp", "openai", "gpt-4o", reference_window=30, calib_n=5) + scopes = {("ragapp", "openai", "gpt-4o"): scope} + + evaluator = EvaluatorThread(scopes, store, cfg, cost_engine, health) + evaluator.start() + + try: + for i in range(1000): + span = GenAISpan( + trace_id=f"tr_{i:06d}", + span_id=f"sp_{i:06d}", + service_name="ragapp", + service_version="v1", + gen_ai_system="openai", + model="gpt-4o", + input_text=f"input prompt {i}", + output_text=f"output completion {i}", + input_tokens=10, + output_tokens=20, + start_unix_nano=1000000000 + i * 1000000, + end_unix_nano=2000000000 + i * 1000000, + ) + cost_engine.record(span) + usd = prices.cost_usd(span.model, span.input_tokens, span.output_tokens) + rec = quality_engine.score(span) + scope.observe(span, rec, usd) + health.inc_scored() + + time.sleep(0.5) + finally: + evaluator.stop() + evaluator.join(timeout=2.0) + store.close() + + assert health.spans_scored == 1000 From 22f934d6ccb601d2b8cb364694ca8e8c170e19b0 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:24:34 +0530 Subject: [PATCH 34/35] add link to blind-spots.md in console health strip footer --- vitals/console/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vitals/console/render.py b/vitals/console/render.py index 5616933..e99df10 100644 --- a/vitals/console/render.py +++ b/vitals/console/render.py @@ -168,7 +168,8 @@ def render_console_html( health_strip_html = ( f"spans: {spans_rx} received / {spans_sc} scored / {spans_sk} skipped · " f"scopes: {n_scopes} · verdicts emitted: {v_emitted} · errors: {emit_errs} · " - f"uptime: {uptime_str} · vitals {__version__}" + f"uptime: {uptime_str} · vitals {__version__} · " + f'blind spots' ) return f""" From fba216f16e416e7efafe35e623d056a9b1118ca6 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 23 Jul 2026 03:36:39 +0530 Subject: [PATCH 35/35] docs: add frontend engineering specification for the vitals console --- docs/v2-frontend-spec.md | 1135 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1135 insertions(+) create mode 100644 docs/v2-frontend-spec.md diff --git a/docs/v2-frontend-spec.md b/docs/v2-frontend-spec.md new file mode 100644 index 0000000..6b1656a --- /dev/null +++ b/docs/v2-frontend-spec.md @@ -0,0 +1,1135 @@ +# Vitals Console — Frontend Engineering Specification + +Implementation-ready. Consumes the backend defined in +[v2-engineering-spec.md](v2-engineering-spec.md) §8.3. No design decisions remain. + +**Rule for the implementing agent:** if this document and your instinct disagree, +this document wins. If something genuinely is not covered, choose the option most +consistent with §2 (tokens) and §3 (motion), and note it in `DECISIONS.md`. + +--- + +## 0. Stack — locked, and one override + +### 0.1 The override + +Backend spec **D5/D12** locked "stdlib `http.server`, f-string HTML, no build +step." That is incompatible with the quality bar requested here. **D5 is +superseded by this document.** The *intent* of D5 is preserved exactly: + +- **No new runtime dependency.** The Python process still serves everything. +- **No CDN, no external fetch.** Fonts, icons, and JS are all self-hosted. +- **The build step is dev-time only.** Compiled assets are committed to + `vitals/console/static/` and shipped in the wheel and the Docker image. A user + running `pip install vitals && vitals run` never sees Node. + +### 0.2 Required backend addition (the only one) + +The Python console server must serve the SPA: + +- `GET /` and **any non-`/api/*`, non-`/assets/*` path** → return + `vitals/console/static/index.html` with `200` (SPA history fallback). +- `GET /assets/*` → serve from `vitals/console/static/assets/` with + `Cache-Control: public, max-age=31536000, immutable`. +- `index.html` → `Cache-Control: no-cache`. + +Without the fallback, `/v/:id`, `/scopes`, and `/about` 404 on hard refresh. + +### 0.3 Locked stack + +| Concern | Choice | Why (one line) | +|---|---|---| +| Framework | **React 18 + TypeScript (strict)** | Boring, ubiquitous, and the agent will not fight it. | +| Build | **Vite 5**, output `vitals/console/static/`, base `/` | Fast, zero-config, emits hashed assets. | +| Styling | **CSS Custom Properties + CSS Modules** | Tokens as CSS vars are directly auditable against §2. Utility dialects hide whether the spec was followed. | +| Routing | **React Router v6** (`createBrowserRouter`) | Deep-linkable drawer at `/v/:id` requires real routing. | +| Data | **TanStack Query v5** | Polling, stale-while-revalidate, and backoff are exactly the problem it solves. | +| Animation | **Motion for React** (`motion` package) | Exit animations for drawer/toast are impractical with CSS alone. | +| Icons | **lucide-react** | Tree-shakeable, consistent 1.5px stroke, matches the aesthetic. | +| Charts | **Hand-authored SVG. No chart library.** | Two chart forms only (§9). A library costs 50–150 KB for something fully controlled. | +| Fonts | **Self-hosted Inter var + JetBrains Mono**, woff2, `font-display: swap` | No CDN; works offline and under strict CSP. | +| Testing | **Vitest + React Testing Library**; **Playwright** for two smoke flows | Enough to protect the demo without becoming a test project. | +| Formatting | **Biome** (lint + format) | One tool, no ESLint/Prettier config negotiation. | + +**Budget: ≤ 250 KB gzipped JS, ≤ 30 KB CSS, LCP < 1.0s on localhost.** Enforced +in CI (§16.4). + +### 0.4 What the frontend must never do + +- Never call an endpoint not listed in §10.1. +- Never compute a verdict, a sigma, or a state. **The backend is the only source + of judgment.** The frontend formats `Verdict` fields; it never derives them. +- Never render `Verdict.sentence` reworded. When the sentence is shown, it is + shown verbatim. +- Never invent a color, radius, duration, or spacing value outside §2/§3. + +--- + +## 1. Product framing for the UI + +The backend emits four states. The UI's entire job is to make **`STEADY` feel +like proof of work, not absence of work** — because on demo day and on day 300, +`STEADY` is what's on screen 95% of the time. + +Three consequences that override generic dashboard instincts: + +1. **The health strip is not chrome.** It is the evidence that silence is + monitored silence. It gets real visual weight (§7.1 Zone 3). +2. **`CHANGED` is amber, never red.** Vitals reports change, not failure. Red is + reserved exclusively for Vitals' own errors. +3. **Honesty fields are first-class UI.** `falsifier`, `caveats`, and + `inconclusive_reason` render at the same visual weight as the sigma values — + never in a collapsed "details" section. + +--- + +## 2. Design system — frozen + +All tokens live in `src/styles/tokens.css` as CSS custom properties on `:root`. +**Light theme only.** No `prefers-color-scheme` handling. No purple, anywhere, +ever — including hover tints, chart series, and focus rings. + +### 2.1 Color — neutrals + +Warm-grey neutral ramp (warmer than pure grey; reads as paper, not steel). + +| Token | Hex | Use | +|---|---|---| +| `--color-canvas` | `#FBFBFA` | Page background | +| `--color-surface` | `#FFFFFF` | Cards, drawer, popover, table rows | +| `--color-surface-subtle` | `#F7F7F6` | Table header, feed row hover, inset panels | +| `--color-surface-inset` | `#F1F1EF` | Code blocks, JSON viewer, skeleton base | +| `--color-surface-raised` | `#FFFFFF` | Dropdowns, tooltips (with shadow-lg) | +| `--color-border-subtle` | `#EDEDEA` | Row dividers, internal card rules | +| `--color-border` | `#E2E2DE` | Default card and input border | +| `--color-border-strong` | `#CBCBC5` | Input hover, active dividers | +| `--color-text` | `#18181B` | Primary text, numbers | +| `--color-text-secondary` | `#57575F` | Labels, secondary copy | +| `--color-text-tertiary` | `#8A8A92` | Timestamps, hints, units | +| `--color-text-disabled` | `#B6B6BC` | Disabled labels | +| `--color-overlay` | `rgba(24,24,27,0.32)` | Drawer/modal scrim | + +### 2.2 Color — accent + +Deep confident blue. Used for interactive affordance only, never for status. + +| Token | Hex | +|---|---| +| `--color-accent` | `#1A56DB` | +| `--color-accent-hover` | `#1546B4` | +| `--color-accent-active` | `#113A96` | +| `--color-accent-subtle` | `#EBF1FE` | +| `--color-accent-border` | `#C3D6FB` | +| `--color-accent-text` | `#1546B4` | + +### 2.3 Color — status (maps 1:1 to `VerdictState`) + +Hues match backend §9; tuned for light-theme contrast. Each state has four roles: +`dot` (solid indicator), `text`, `bg`, `border`. + +| State | dot | text | bg | border | +|---|---|---|---|---| +| **STEADY** (success) | `#16A34A` | `#146C33` | `#F971FD`→ **`#F1FDF4`** | `#BBF0CC` | +| **CHANGED** (warning) | `#F59E0B` | `#96590C` | `#FFFAEB` | `#FBE3A6` | +| **INCONCLUSIVE** (info) | `#3B82F6` | `#1D4ED8` | `#EFF5FF` | `#C2D8FC` | +| **WARMING** (neutral) | `#8A8A92` | `#57575F` | `#F5F5F3` | `#E2E2DE` | +| **ERROR** (Vitals fault only) | `#DC2626` | `#A81E1E` | `#FEF2F2` | `#F8CFCF` | + +`STEADY` bg is `#F1FDF4`. (The strikethrough above is intentional in the source +table only to make the correction unmissable — implement `#F1FDF4`.) + +**Rule:** `--color-error-*` may only style Vitals' own failures — API unreachable, +store error, console degraded. A `CHANGED` verdict is never red. + +### 2.4 Color — chart palette + +Categorical, colorblind-safe, no purple. Use in this order. + +| # | Token | Hex | Reserved meaning | +|---|---|---|---| +| 1 | `--chart-1` | `#1A56DB` | behavior sigma | +| 2 | `--chart-2` | `#0E9F6E` | cost sigma | +| 3 | `--chart-3` | `#F59E0B` | threshold / annotation | +| 4 | `--chart-4` | `#0891B2` | samples (n) | +| 5 | `--chart-5` | `#C2410C` | velocity ratio | +| 6 | `--chart-6` | `#64748B` | baseline / reference | + +Grid `#EDEDEA`. Axis text `--color-text-tertiary`. Threshold band fill +`rgba(245,158,11,0.08)`. + +### 2.5 Spacing + +4px base. **Only these values may appear.** + +`--space-0: 0` · `1: 4px` · `2: 8px` · `3: 12px` · `4: 16px` · `5: 20px` · +`6: 24px` · `8: 32px` · `10: 40px` · `12: 48px` · `16: 64px` · `20: 80px` + +### 2.6 Radius + +`--radius-sm: 4px` (badges, dots) · `--radius-md: 6px` (buttons, inputs) · +`--radius-lg: 10px` (cards, drawer) · `--radius-xl: 14px` (hero card) · +`--radius-full: 9999px` (pills, avatars) + +### 2.7 Shadows + +Very subtle. Two-layer (contact + ambient). No colored shadows. + +| Token | Value | +|---|---| +| `--shadow-xs` | `0 1px 2px rgba(24,24,27,0.04)` | +| `--shadow-sm` | `0 1px 3px rgba(24,24,27,0.06), 0 1px 2px rgba(24,24,27,0.04)` | +| `--shadow-md` | `0 4px 12px rgba(24,24,27,0.07), 0 1px 3px rgba(24,24,27,0.05)` | +| `--shadow-lg` | `0 12px 28px rgba(24,24,27,0.10), 0 2px 6px rgba(24,24,27,0.05)` | +| `--shadow-xl` | `0 24px 56px rgba(24,24,27,0.14), 0 4px 10px rgba(24,24,27,0.06)` | + +Cards use `--shadow-xs` at rest, `--shadow-sm` on hover. Drawer `--shadow-xl`. +Dropdown/tooltip `--shadow-lg`. + +**Gradients:** exactly two are permitted, both effectively invisible: +(a) hero card background `linear-gradient(180deg,#FFFFFF 0%,#FDFDFC 100%)`; +(b) feed bottom fade-to-canvas, 40px. No others. + +### 2.8 Typography + +Families: `--font-sans: "Inter var", -apple-system, "Segoe UI", sans-serif` · +`--font-mono: "JetBrains Mono", ui-monospace, "SF Mono", monospace`. + +**All numbers, sigma values, IDs, timestamps, and code use `--font-mono` with +`font-variant-numeric: tabular-nums`.** Non-negotiable — it is what makes a +polling UI stop twitching. + +| Token | Size / Line height | Weight | Tracking | Use | +|---|---|---|---|---| +| `--text-display` | 32/40 | 600 | -0.02em | Hero state word (`CHANGED`) | +| `--text-h1` | 24/32 | 600 | -0.015em | Page titles | +| `--text-h2` | 18/26 | 600 | -0.01em | Card titles, section heads | +| `--text-h3` | 15/22 | 600 | -0.005em | Sub-sections, drawer groups | +| `--text-body` | 14/22 | 400 | 0 | Default copy | +| `--text-body-md` | 14/22 | 500 | 0 | Emphasized copy, buttons | +| `--text-sm` | 13/20 | 400 | 0 | Table cells, feed rows | +| `--text-xs` | 12/18 | 500 | 0.005em | Labels, badges, health strip | +| `--text-micro` | 11/16 | 500 | 0.02em | Uppercase eyebrows, axis ticks | +| `--text-mono-lg` | 20/28 | 600 | -0.01em | Sigma values in hero | +| `--text-mono` | 13/20 | 400 | 0 | IDs, JSON, code | +| `--text-mono-sm` | 12/18 | 400 | 0 | Timestamps, trace ids in feed | + +Eyebrow labels: `--text-micro`, `text-transform: uppercase`, +`color: --color-text-tertiary`. + +### 2.9 Icons + +lucide-react, stroke width **1.5**, sizes **14 / 16 / 20 / 24** only. +14 in badges and feed rows · 16 default (buttons, inline) · 20 section headers · +24 empty-state illustrations. Icons are always `aria-hidden` unless they are the +sole content of a control, which then requires `aria-label`. + +Fixed icon assignments (never substitute): +`STEADY`→`CircleCheck` · `CHANGED`→`Activity` · `INCONCLUSIVE`→`CircleHelp` · +`WARMING`→`LoaderCircle` · error→`TriangleAlert` · caveat→`Info` · +falsifier→`GitCompareArrows` · release→`Tag` · unattributed→`CircleDashed` · +runaway→`Flame` · trace→`ExternalLink` · copy→`Copy` · search→`Search` · +filter→`ListFilter` · close→`X` · expand→`ChevronDown`. + +### 2.10 Layout constants + +`--layout-max: 1120px` · `--layout-gutter: 24px` (mobile 16px) · +`--header-height: 56px` · `--drawer-width: 560px` · +`--focus-ring: 0 0 0 2px #FFFFFF, 0 0 0 4px var(--color-accent)`. + +--- + +## 3. Motion system — frozen + +### 3.1 Durations & easing + +| Token | Value | Use | +|---|---|---| +| `--dur-instant` | 80ms | Color/opacity on hover | +| `--dur-fast` | 140ms | Buttons, badges, small transforms | +| `--dur-base` | 200ms | Cards, tooltips, tab indicator | +| `--dur-slow` | 300ms | Drawer, modal, route transitions | +| `--dur-slower` | 460ms | Number count-up, chart draw-in | + +| Token | Curve | Use | +|---|---|---| +| `--ease-out` | `cubic-bezier(0.22, 1, 0.36, 1)` | **Default.** Anything entering or expanding. | +| `--ease-in` | `cubic-bezier(0.5, 0, 0.75, 0)` | Exits only | +| `--ease-inout` | `cubic-bezier(0.65, 0, 0.35, 1)` | Position changes both directions | +| `--ease-spring` | spring `{stiffness: 380, damping: 32, mass: 0.9}` | Drawer, hero state change, toast | + +**Never** use `linear` except for the indeterminate spinner and progress shimmer. + +### 3.2 Named animations + +| Name | Behavior | +|---|---| +| **Route transition** | Outgoing `opacity 1→0` 100ms `--ease-in`; incoming `opacity 0→1, translateY 6px→0` 200ms `--ease-out`, 60ms delay. No horizontal slide. | +| **Card hover** | `box-shadow xs→sm` + `translateY 0→-1px`, 140ms `--ease-out`. Exit 200ms. | +| **Button press** | `scale 1→0.975`, 80ms. Release springs back over 140ms. | +| **Badge state change** | Old badge `opacity→0, scale→0.94` 100ms; new `opacity 0→1, scale 1.06→1` 180ms `--ease-out`. Crossfade in place — badge never reflows. | +| **Hero state change** | Whole card: border+bg color 300ms `--ease-inout`; state word crossfades per Badge rule; sigma bars re-animate (below); a single 600ms ring pulse (`box-shadow 0 0 0 0 → 0 0 0 8px` of the state color at 12% alpha, fading to 0). Fires **once** per state transition, never on refresh. | +| **Sigma bar fill** | `scaleX 0→value` from left origin, 460ms `--ease-out`, staggered 60ms per bar. Re-animates only when the value changes by ≥0.1σ. | +| **Number count-up** | Sigma and cost numbers tween over 460ms `--ease-out` when they change. Integers ≤999 tween; larger values snap. `n=` counts up. Never tweens on first mount — first paint is the final value. | +| **Feed row enter** | New rows: `opacity 0→1, translateY -8px→0` 200ms `--ease-out`. Existing rows shift down with `layout` animation 200ms. Max 3 stagger steps of 40ms; beyond that, all at once. | +| **Drawer** | Panel `translateX 100%→0` with `--ease-spring`; scrim `opacity 0→1` 200ms. Exit: panel `translateX 0→100%` 240ms `--ease-in`, scrim 180ms. | +| **Modal** | `opacity 0→1, scale 0.97→1, translateY 8px→0` 200ms `--ease-out`. Exit 140ms `--ease-in`, no translate. | +| **Toast** | Enter from bottom-right: `translateY 16px→0, opacity 0→1, scale 0.96→1` `--ease-spring`. Exit `opacity→0, translateX 16px` 160ms. Stacked toasts shift with 200ms layout animation. | +| **Tooltip** | Delay 400ms open / 80ms close. `opacity 0→1, scale 0.96→1, translateY 4px→0` 140ms `--ease-out`. Zero delay when moving between adjacent tooltip triggers within 300ms. | +| **Dropdown/popover** | `opacity 0→1, scale 0.97→1` 140ms `--ease-out`, transform-origin at the trigger edge. Exit 100ms. | +| **Accordion** | `height auto→0` measured, 220ms `--ease-inout`; content `opacity` 140ms, 40ms delay on open. | +| **Tab indicator** | Underline slides via `layoutId`, 220ms `--ease-inout`. | +| **Skeleton shimmer** | Background sweep `translateX -100%→100%`, 1400ms linear, infinite. | +| **Spinner** | `rotate 360deg` 700ms linear infinite. Only appears after 400ms of pending. | +| **Chart draw-in** | Sparkline path `stroke-dashoffset` full→0 over 460ms `--ease-out`, once per mount. Points fade in at 60% of the sweep. | +| **Live pulse** | Health-strip "live" dot: `opacity 1→0.35→1` 2000ms `--ease-inout` infinite. Turns solid grey and stops when polling is paused or failing. | +| **Copy confirm** | Icon swaps `Copy`→`Check` for 1200ms with an 80ms scale pop, then swaps back. | +| **Scroll reveal** | **None.** No scroll-triggered animation anywhere. Content is present when painted. | + +### 3.3 Reduced motion + +Under `prefers-reduced-motion: reduce`: +- All durations → **0ms** except opacity fades, which clamp to 100ms. +- No transform animation (no slide, scale, spring). Drawer and modal appear + instantly with a 100ms fade. +- No count-up; numbers snap. +- No shimmer, no live pulse, no chart draw-in, no ring pulse. +- Spinner is replaced by a static `Loading…` label. + +Implement via a `useReducedMotion()` hook that gates Motion props, **plus** a +global CSS media query as a backstop. Both, not either. + +--- + +## 4. Information architecture + +### 4.1 Routes + +| Path | Screen | Data | +|---|---|---| +| `/` | Console | verdicts, scopes, health | +| `/v/:verdictId` | Console **+ Verdict drawer open** | + verdict detail | +| `/scopes` | Scopes | scopes, health | +| `/about` | Blind spots & About | none (static) | +| `*` | Not found | none | + +**The verdict drawer is a route, not local state.** `/v/:id` renders the console +underneath with the drawer open, so verdicts are shareable and back/forward work. +Closing the drawer navigates to `/` (or the previous list route) preserving query +params. + +### 4.2 URL state (query params — all on `/`) + +| Param | Values | Default | +|---|---|---| +| `state` | `steady,changed,inconclusive,warming` (comma-joined multi) | unset = all | +| `service` | service name | unset = all | +| `version` | version string | unset = all | +| `q` | free-text search over sentence, version, verdict_id, trace_id | unset | +| `sort` | `newest` \| `oldest` \| `sigma` | `newest` | +| `paused` | `1` | unset | + +URL is the single source of truth for filters. Reading query params happens in +one hook (`useVerdictFilters`); no component reads `useSearchParams` directly. + +### 4.3 Navigation + +Single top header, 56px, `--color-surface`, `--shadow-xs`, sticky. + +- **Left:** wordmark `vitals` (`--text-h2`, mono, `--color-text`) + a small state + dot reflecting the *most severe live scope state*. Links to `/`. +- **Center:** tabs — `Console` (`/`) · `Scopes` (`/scopes`). +- **Right:** search field (⌘K / Ctrl-K) · pause/resume polling toggle · + `About` link (`/about`). + +**No sidebar. No breadcrumbs.** The hierarchy is two levels deep; breadcrumbs +would be decoration. + +### 4.4 Filtering, sorting, pagination + +- **Filtering:** client-side over the fetched verdict list. State filter is a + segmented multi-select of four chips; service/version are dropdowns populated + from the fetched data. +- **Sorting:** `newest` (default) · `oldest` · `sigma` (by + `max(|behavior_sigma|, |cost_sigma|)` desc, nulls last). +- **Pagination: none.** The API returns ≤50 (`limit=50`) and the store retains + 1000. The feed renders all fetched rows with **windowed rendering above 120 + rows** (fixed 44px row height, 8-row overscan). Infinite scroll and page + controls are both explicitly excluded — the dataset is bounded and recency- + ordered. + +### 4.5 Search (⌘K) + +An inline expanding field in the header, **not** a modal palette. Rationale: it +searches one list; a full command palette implies commands, and this UI has none. + +- ⌘K / Ctrl-K focuses it; `Esc` clears and blurs. +- Debounce 150ms, writes to `?q=`. +- Matches case-insensitively against `sentence`, `version`, `service_name`, + `verdict_id`, and exemplar `trace_id`. +- Matched substrings highlight with `--color-accent-subtle` background. +- Shows `N results` in `--text-xs` `--color-text-tertiary` while active. + +--- + +## 5. API integration + +### 5.1 Endpoints (the complete set — nothing else exists) + +| Key | Endpoint | Poll | Notes | +|---|---|---|---| +| `verdicts` | `GET /api/verdicts?limit=50` | **2s** | Newest first. Drives hero + feed. | +| `verdict` | `GET /api/verdicts/:id` | on demand, `staleTime: Infinity` | Verdicts are immutable once written — never refetch. | +| `scopes` | `GET /api/scopes` | **5s** | Warming progress, live state. | +| `health` | `GET /api/health` | **10s** | Health strip. | + +**No mutations exist.** Therefore: **no optimistic updates.** Anything that looks +like a write (pin, dismiss, filter, pause) is local UI state persisted to +`localStorage` — §6.4. + +### 5.2 Polling strategy + +- `refetchInterval` per the table above; `refetchOnWindowFocus: true`; + `refetchIntervalInBackground: false`. +- **Pause when `document.hidden`**, resume with an immediate refetch on visibility + return. A backgrounded demo laptop must not burn requests. +- **Manual pause** (`?paused=1`, header toggle, or `Space` when the feed has + focus) stops all polling and shows a `Paused` pill in the header. Used when a + presenter wants the screen to hold still. +- **Never poll while the drawer is open on a verdict detail** — the underlying + list keeps polling, but the drawer's own query is `staleTime: Infinity`. + +### 5.3 Errors and retries + +- Retry `3` times with backoff `min(1000 * 2^attempt, 8000)`. +- On the **first** failure: keep showing last-good data; the health-strip live dot + turns grey and a `Reconnecting…` label appears. **No toast.** Transient blips + during a demo must not throw UI. +- After **3 consecutive** failed cycles: show a persistent error banner below the + header (§7.5) with `Retry now`. Data stays visible, dimmed to 60% opacity. +- On recovery: banner exits, one toast `Reconnected`, live dot resumes pulsing. +- `404` on `/api/verdicts/:id` → drawer shows its not-found state (§7.4), does not + retry. +- All network errors are typed as `{status, message, endpoint}` and surfaced with + the endpoint name in the banner — a developer tool should say which call failed. + +### 5.4 Caching + +- `staleTime`: verdicts `1500ms`, scopes `4000ms`, health `8000ms`, verdict + detail `Infinity`. +- `gcTime`: `5min` for lists, `30min` for verdict details. +- `structuralSharing: true` (default) — critical, it is what keeps unchanged feed + rows from re-rendering and re-animating on every 2s poll. +- **Seed the detail cache from the list.** When the feed has a verdict, opening + its drawer renders immediately from `setQueryData` while the full record loads. + +### 5.5 Offline + +`navigator.onLine === false` → header shows an `Offline` pill, polling suspends, +last-good data remains fully interactive (filters, sort, search, drawer all work +against cache). On `online`, refetch everything immediately. No service worker; +no offline persistence beyond the in-memory query cache. + +--- + +## 6. State management + +### 6.1 Server state +TanStack Query only. **No verdict data is ever copied into React state or a +store.** Components read from `useVerdicts()`, `useVerdict(id)`, `useScopes()`, +`useHealth()`. + +### 6.2 URL state +Filters, sort, search, pause (§4.2). Owned by `useVerdictFilters()`. + +### 6.3 Local component state +Hover, focus, open/closed accordions, copy-confirm timers. Nothing else. + +### 6.4 Persisted state (`localStorage`, key prefix `vitals.console.`) + +| Key | Value | Purpose | +|---|---|---| +| `vitals.console.density` | `comfortable` \| `compact` | Feed row density | +| `vitals.console.lastSeenVerdictId` | string | Powers the "N new" pill after a pause | +| `vitals.console.sigmaFormat` | `sigma` \| `raw` | Dev escape hatch; default `sigma` | + +Reads are wrapped in try/catch with defaults — a locked-down browser must not +break the app. + +### 6.5 Derived state (memoized selectors in `src/features/verdicts/selectors.ts`) + +`latestVerdict` · `filteredVerdicts` · `sortedVerdicts` · `verdictCounts` (per +state) · `mostSevereScopeState` (for the header dot: CHANGED > INCONCLUSIVE > +WARMING > STEADY) · `hasAnyData`. + +**No global store (no Redux/Zustand/Jotai).** With four read-only endpoints and +URL-owned filters, a store would be pure ceremony. + +--- + +## 7. Screens + +### 7.1 Console (`/`) + +Single column, `max-width: 1120px`, centered, `padding: 32px 24px 80px`. +Three zones, in order, gap `--space-8` (32px). + +#### Zone 1 — Hero Verdict Card + +The Release Report Card. Renders `latestVerdict`. + +Container: `--radius-xl`, 1px border in the state's `border` color, background = +gradient (a) from §2.7 over the state's `bg` color at 40% blend, `--shadow-sm`, +`padding: 28px 32px`. + +Internal layout, top to bottom: + +1. **Header row** (flex, space-between, align baseline) + - Left: `StatusDot` (10px) + state word in `--text-display`, in the state's + `text` color. Beside it, flag chips: `behavior`, `cost`, `runaway` — + `Badge` size `sm`, only those that are true. + - Right: `service_name` in `--text-body-md`, then `·`, then `version` in + `--font-mono`. `--color-text-secondary`. +2. **Subject line** — `--text-sm`, `--color-text-secondary`, `margin-top: 4px`. + Format: `{subject} · {version} vs {baseline_version}` or `{subject} · {version}` + when there is no baseline. +3. **Sigma meters** — `margin-top: 24px`, two rows, gap 12px. Each row is a + 3-column grid: label (88px, `--text-xs`, tertiary) · value + (`--text-mono-lg`, signed, 1 decimal, `+4.2σ`) · bar (flex, 8px tall, + `--radius-full`, track `--color-surface-inset`). + - Bar scale: **fixed domain 0→6σ**, clamped, with a 1px threshold tick at 3σ + in `--chart-3`. Fixed domain matters — a rescaling axis makes two verdicts + visually incomparable. + - Fill color: `--chart-1` for behavior, `--chart-2` for cost. When + `|z| < 1.0`, fill is `--color-text-tertiary` and the trailing label reads + `flat`. + - Trailing hint: `normal ±1σ` in `--text-micro`, tertiary. +4. **Attribution line** — `margin-top: 20px`, `--text-sm`. Icon (`Tag` for + release, `CircleDashed` for unattributed, `Flame` for runaway) + text. + Release: `onset 14:32:07 — 90s after v2 deployed`. + Unattributed: `cause unattributed — no release in the last 5m`. +5. **Counts line** — `--text-xs`, tertiary, mono numbers: + `n=1240 · baseline v1 (n=30)`. +6. **Caveats** (only when non-empty) — `margin-top: 16px`, inset block: + `--color-surface-subtle`, `--radius-md`, `padding: 10px 12px`, `Info` icon 14, + `--text-sm`. Comma-joined caveats. +7. **Falsifier** — **always rendered**, same block styling as caveats, + `GitCompareArrows` icon, `--color-text-secondary`. This is a trust surface; it + never collapses and never hides. +8. **Evidence** — `margin-top: 24px`, eyebrow `EVIDENCE`, then one `ExemplarRow` + per exemplar. Worst rows first, **median row always last and always present**. + Each row: kind chip (`worst` amber-subtle / `median` neutral) · mono trace id + truncated to 6 chars with copy-on-click · signed sigma · output excerpt + (single line, `text-overflow: ellipsis`, `--color-text-secondary`) · + `ExternalLink` icon that opens the SigNoz trace URL in a new tab. + Row hover: `--color-surface-subtle`, 80ms. Click anywhere → opens the drawer. +9. **Footer** — `margin-top: 20px`, `border-top: 1px --color-border-subtle`, + `padding-top: 12px`. Left: relative timestamp (`14s ago`, live-updating every + second, `title` = absolute ISO). Right: `View full record →` link to `/v/:id`. + +**Warming variant:** replaces zones 3–8 with a `ProgressBar` (§8.14), +`collecting reference 340/1000`, `est. 22m`, and the copy *"Vitals is +establishing a healthy baseline. No verdict will be issued until it has one."* +Neutral colors throughout. + +**Inconclusive variant:** state word `INCONCLUSIVE`, and the reason renders as a +prominent sentence directly beneath it in `--text-h3`, e.g. *"Your traffic +changed, not your model."* Sigma meters still render, plus a third greyed meter +for `input` if the backend supplied it. The falsifier block stays. + +- **Loading:** `HeroCardSkeleton` — exact same geometry, shimmer blocks. +- **Empty (no verdicts yet, backend healthy):** `EmptyState` with `Activity` icon + 24, title *"No verdicts yet"*, body *"Vitals is listening. Send traffic through + your collector and the first verdict appears here."*, and a mono hint line with + the receiver port from `/api/health`. +- **Error:** card shows the error variant — `TriangleAlert`, *"Can't reach the + Vitals API"*, endpoint name, `Retry now` button. + +#### Zone 2 — Verdict Feed + +Header row: title `Verdicts` (`--text-h2`) + count pill · right side: +`StateFilterChips`, `SortDropdown`, `DensityToggle`. + +Filter chips: four `ToggleChip`s (`Steady` `Changed` `Inconclusive` `Warming`), +each with its state dot and a count. Multi-select; all-off means all-on. + +Rows: `44px` comfortable / `36px` compact. Columns, left to right: +time (mono-sm, 72px, tertiary) · `StatusDot` (8px) · state word (`--text-xs` +uppercase, state text color, 96px) · sentence (flex, single line, ellipsis, +`--text-sm`) · sigma pair (mono-sm, right-aligned, 104px) · `ChevronDown` +(rotates 180° when expanded, 200ms). + +- Row hover: `--color-surface-subtle` background, 80ms; `ChevronDown` fades + tertiary→secondary. +- **Click expands inline** (accordion, §3.2) showing a compact receipt: subject, + cause, caveats, falsifier, and up to two exemplars. **Cmd/Ctrl-click or the + `View full record` link opens the drawer** at `/v/:id`. Rationale: inline + expansion is for scanning; the drawer is for reading. +- New rows animate in per §3.2. When the feed is scrolled away from the top and + new verdicts arrive, a floating `N new` pill appears at the top center + (`--shadow-md`, spring in); clicking scrolls to top and clears it. +- Zebra striping: none. Dividers: 1px `--color-border-subtle` between rows. + +- **Loading:** 6 `FeedRowSkeleton`s. +- **Empty (filters exclude everything):** inline `EmptyState`, `ListFilter` icon, + *"No verdicts match these filters"*, `Clear filters` ghost button. +- **Empty (no data at all):** the feed section is hidden entirely; the hero + card's empty state carries the message. + +#### Zone 3 — Health Strip + +Not chrome (§1). Full-width card, `--radius-lg`, `--color-surface`, 1px border, +`padding: 16px 20px`. + +Left: `LivePulseDot` + `Live` / `Paused` / `Reconnecting…` / `Offline`. +Center: six `MetricChip`s, `--font-mono` values with `--text-xs` uppercase +labels — `spans received` · `scored` · `skipped` · `scopes` · `verdicts` · +`emit errors`. Values count up (§3.2). `emit errors` chip turns to the error +palette when `> 0`. +Right: `uptime` and `vitals v{version}`, tertiary. +Footer line: `Known blind spots →` linking to `/about`. Required by backend §20.8. + +Skipped-spans chip shows a tooltip: *"Spans that weren't GenAI or couldn't be +mapped. A non-zero value is normal."* — pre-empting the most common false alarm +about Vitals itself. + +#### Keyboard (Console) + +| Key | Action | +|---|---| +| `⌘K` / `Ctrl-K` | Focus search | +| `/` | Focus search (when not in an input) | +| `j` / `↓` | Next feed row | +| `k` / `↑` | Previous feed row | +| `Enter` | Expand/collapse focused row | +| `o` | Open focused row in drawer | +| `Space` | Toggle polling pause | +| `Esc` | Clear search → clear filters → blur (in that order) | +| `1`–`4` | Toggle state filter chips | +| `?` | Open shortcuts modal | + +Focused feed row shows a 2px `--color-accent` left border and +`--color-surface-subtle` background. + +#### Responsive (Console) + +- **Desktop ≥1280px:** as described, 1120px column. +- **Laptop 1024–1279px:** identical, gutters 24px, container fluid. +- **Tablet 768–1023px:** hero sigma meters stack label-above-bar; feed drops the + sigma-pair column (moves into the expanded row); filter chips scroll + horizontally with a fade mask. +- **Mobile <768px:** hero padding `20px 16px`, state word `--text-h1`; feed rows + become two-line (line 1: time + state + sigma; line 2: sentence); health strip + chips wrap to a 2-column grid; header tabs collapse to an icon-only segmented + control; search becomes a full-width row beneath the header when focused; + **drawer becomes a bottom sheet** (§7.4). + +### 7.2 Scopes (`/scopes`) + +Answers *"is Vitals watching, and how far along is it?"* + +Layout: page title `Scopes` + subtitle *"What Vitals is currently watching."* +Then a responsive grid of `ScopeCard`s — 2 columns ≥1024px, 1 below. + +`ScopeCard` (`--radius-lg`, `--shadow-xs`, `padding: 20px`): +- Header: `service_name` (`--text-h3`) + state `Badge`. +- Meta row: `gen_ai.system` · `model` — mono, tertiary, `--text-xs`. +- Body when **warming**: `ProgressBar` with `have/need`, phase label + (`collecting reference` / `calibrating`), and est. time remaining. +- Body when **live**: a 4-row signal table — `behavior`, `input`, `cost`, + `length` — each with `μ`, `σ`, and a `calibrated` check. Mono, tabular. +- Footer: versions seen, as `Badge`s; the active one filled, prior ones outline. +- Click → navigates to `/?service={name}` (filters the console to that scope). + +Loading: 4 `ScopeCardSkeleton`s. Empty: `EmptyState`, `Search` icon, *"No scopes +yet"*, body *"A scope appears once Vitals sees its first GenAI span."* +Error: same pattern as Console. + +Keyboard: `Tab` through cards; `Enter` activates. Responsive: 2→1 column at +1024px; signal table becomes label/value pairs below 768px. + +### 7.3 About / Blind Spots (`/about`) + +Static, no API. Prose column, `max-width: 720px`. + +Sections, in order: **What Vitals claims** · **What Vitals does not claim** · +**Known blind spots** (the four from the critique: subtle factual degradation, +uniform degradation, baseline poisoning, deploy-during-warming) · **How to read a +verdict** (annotated static example of the sentence grammar) · **Version & build** +(from `/api/health`). + +Each blind spot is an `AlertCallout` variant `info` with a title and two-sentence +body. Typography-led, no cards, generous `--space-8` between sections. + +This screen exists because "we publish our blind spots" is a product claim, and a +claim without a URL is marketing. + +### 7.4 Verdict Drawer (`/v/:verdictId`) + +Right-anchored panel, `--drawer-width: 560px`, full height, `--color-surface`, +`--shadow-xl`, `--radius-lg` on the left corners only. Scrim `--color-overlay`. + +Header (sticky, 64px, bottom border): `StatusDot` + state word + flag chips · +right: `Copy JSON` icon button, `X` close. + +Body (scrollable, `padding: 24px`), sections separated by 1px rules: + +1. **Sentence** — verbatim `Verdict.sentence`, `--font-mono`, `--text-mono`, + in a `--color-surface-inset` block, `--radius-md`, `padding: 12px`, wrapping. + Copy button top-right on hover. +2. **Judgment** — definition grid (label left 140px, value right, mono): + state, subject, cause, flags, runaway. +3. **Evidence** — full-size sigma meters (same component as hero) plus + `cost_usd_per_req` vs `baseline_cost_usd_per_req` and `velocity_ratio`. +4. **Timing** — `ts_unix` absolute + relative, `onset_ts_unix`, + `seconds_after_deploy`, `samples`, `baseline_samples`. +5. **Honesty** — `falsifier` and `caveats` as `AlertCallout`s. Always present. +6. **Exemplars** — full `ExemplarCard` list: kind chip, trace id (full, mono, + copyable), sigma, **full excerpt in a `--color-surface-inset` block** (wrapped, + not truncated), and an `Open trace in SigNoz` button. +7. **Raw record** — `JsonViewer` (§8.20), collapsed to depth 1 by default. + +Footer (sticky): `verdict_id` mono with copy · `Prev` / `Next` buttons that walk +the currently filtered feed order. + +- **Open:** §3.2 drawer animation. Focus moves to the header close button. Body + scroll locked. Focus trapped. +- **Close:** `Esc`, scrim click, `X`, or browser back. Focus returns to the + originating feed row. +- **Keyboard:** `Esc` close · `j`/`k` prev/next verdict · `c` copy JSON · + `Tab` cycles within the trap. +- **Loading:** skeleton mirroring the section geometry; if seeded from the list + cache (§5.4), sections 1–4 render immediately and only 6–7 skeleton. +- **Not found (404):** `CircleHelp` 24, *"Verdict not found"*, body *"It may have + been pruned by retention."*, `Back to console` button. +- **Mobile <768px:** becomes a **bottom sheet** — full width, `max-height: 92vh`, + `--radius-lg` top corners, enters with `translateY 100%→0` `--ease-spring`, + drag-to-dismiss below 120px threshold, and a 36px grab handle. + +### 7.5 Global elements + +- **Error banner** — below header, full width, `--color-error-bg`, 1px + `--color-error-border`, `TriangleAlert` 16, message with endpoint name, + `Retry now` ghost button. Slides down 200ms `--ease-out`. +- **Shortcuts modal** (`?`) — 480px, two-column key/description list grouped by + Navigation / Feed / Drawer. Modal animation per §3.2. +- **Toasts** — bottom-right, max 3 stacked, 4s auto-dismiss (errors 8s), + hover pauses the timer. Only three ever fire: `Reconnected`, `Copied`, + `Link copied`. +- **404 route** — centered, `CircleHelp` 24, *"Page not found"*, + `Back to console`. + +--- + +## 8. Component library + +Every component lives in `src/components//`. Each defines its own `.module.css`. +All accept `className` and forward refs. **No component may hardcode a color, +radius, duration, or spacing literal** — tokens only. + +Shared states, unless overridden: `default` · `hover` · `active` · `focus-visible` +(always `--focus-ring`) · `disabled` (`opacity: 0.5`, `cursor: not-allowed`, +no hover) · `loading`. + +| # | Component | Variants | Key behavior | +|---|---|---|---| +| 8.1 | **Button** | `primary` (accent fill, white text) · `secondary` (surface, border) · `ghost` (transparent, hover `--color-surface-subtle`) · `danger` (error fill). Sizes `sm` 28px / `md` 32px / `lg` 40px. | Press scale 0.975 (§3.2). `loading` swaps content for spinner at fixed width — the button never resizes. Icon-only requires `aria-label`. | +| 8.2 | **IconButton** | Same variants, square, sizes 28/32/40, radius `md`. | Always has a tooltip and an `aria-label`. | +| 8.3 | **Input** | `default` · `search` (leading icon + clear button) · `error`. Height 32/40. | Border → `--color-border-strong` on hover, `--color-accent` + focus ring on focus. Error shows message below in `--text-xs` error text with `role="alert"`. | +| 8.4 | **Card** | `default` · `interactive` (hover lift) · `inset` (no shadow, subtle bg) | Radius `lg`, border, `--shadow-xs`. Interactive cards are `