diff --git a/CHANGELOG.md b/CHANGELOG.md index 2195a63..1ed6369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **Pre-flight now validates Foundry project reachability independently from + Application Insights credential discovery.** Projects whose attached + Application Insights connection uses `ProjectManagedIdentity` no longer + produce a false Foundry warning or require API Key credentials, while an + explicit `APPLICATIONINSIGHTS_CONNECTION_STRING` still takes precedence. + ## [0.8.7] - 2026-08-10 ### Added diff --git a/src/agentops/services/preflight.py b/src/agentops/services/preflight.py index cfa2339..7a583ed 100644 --- a/src/agentops/services/preflight.py +++ b/src/agentops/services/preflight.py @@ -172,7 +172,7 @@ def _check_azure_cli() -> PreflightCheck: def _check_foundry_project() -> PreflightCheck: - """`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` reachable + App Insights wired.""" + """Check that `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` is reachable.""" started = time.time() endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") if not endpoint: @@ -189,7 +189,7 @@ def _check_foundry_project() -> PreflightCheck: ) try: from agentops.utils.foundry_discovery import ( - resolve_appinsights_connection_with_reason, + check_foundry_project_reachable_with_reason, ) except ImportError: return PreflightCheck( @@ -199,20 +199,20 @@ def _check_foundry_project() -> PreflightCheck: message="agentops.utils.foundry_discovery not available.", duration_seconds=time.time() - started, ) - conn, reason = resolve_appinsights_connection_with_reason(endpoint) - if conn: + reachable, reason = check_foundry_project_reachable_with_reason(endpoint) + if reachable: return PreflightCheck( name="foundry_project", display_name="Foundry project", status="ok", - message="Project reachable; App Insights auto-discovered.", + message="Project reachable.", duration_seconds=time.time() - started, ) return PreflightCheck( name="foundry_project", display_name="Foundry project", status="warn", - message=f"Discovery failed — {reason}", + message=f"Reachability check failed — {reason}", remediation=( "Confirm the signed-in identity has Reader on the project " "resource group, then re-run." @@ -221,9 +221,7 @@ def _check_foundry_project() -> PreflightCheck: ) -def _check_application_insights_env( - foundry_check: Optional[PreflightCheck] = None, -) -> PreflightCheck: +def _check_application_insights_env() -> PreflightCheck: """Heads-up when neither env var nor Foundry discovery yields a connection string. The production telemetry tile will stay grey.""" started = time.time() @@ -256,45 +254,14 @@ def _check_application_insights_env( duration_seconds=time.time() - started, ) - if foundry_check and foundry_check.status == "ok": - return PreflightCheck( - name="app_insights", - display_name="Application Insights", - status="ok", - message="Resolved via Foundry discovery.", - duration_seconds=time.time() - started, - ) - - if ( - foundry_check - and foundry_check.status == "warn" - and "discovery failed" in foundry_check.message.lower() - and "returned no application insights connection" - not in foundry_check.message.lower() - ): - return PreflightCheck( - name="app_insights", - display_name="Application Insights", - status="warn", - message=( - "Could not verify the Foundry App Insights connection because " - "Foundry discovery did not complete." - ), - remediation=( - "Fix the Foundry project warning above, or set " - "APPLICATIONINSIGHTS_CONNECTION_STRING explicitly." - ), - duration_seconds=time.time() - started, - ) - - # Try Foundry discovery as a fallback (uses the same cached helper). endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") if endpoint: try: from agentops.utils.foundry_discovery import ( + PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON, resolve_appinsights_connection_with_reason, ) - conn, _ = resolve_appinsights_connection_with_reason(endpoint) + conn, reason = resolve_appinsights_connection_with_reason(endpoint) if conn: return PreflightCheck( name="app_insights", @@ -303,6 +270,27 @@ def _check_application_insights_env( message="Resolved via Foundry discovery.", duration_seconds=time.time() - started, ) + if reason == PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON: + return PreflightCheck( + name="app_insights", + display_name="Application Insights", + status="ok", + message=PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON, + duration_seconds=time.time() - started, + ) + if reason and "returned no application insights connection" not in reason.lower(): + return PreflightCheck( + name="app_insights", + display_name="Application Insights", + status="warn", + message=f"Connection-string discovery failed — {reason}", + remediation=( + "Confirm the signed-in identity can read the Foundry " + "project, or set APPLICATIONINSIGHTS_CONNECTION_STRING " + "explicitly." + ), + duration_seconds=time.time() - started, + ) except ImportError: pass return PreflightCheck( @@ -345,9 +333,8 @@ def run_preflight( # Foundry / App Insights probes are advisory; they help the user # understand *why* certain sources will be silent rather than # blocking the run. - foundry_check = _check_foundry_project() - checks.append(foundry_check) - checks.append(_check_application_insights_env(foundry_check)) + checks.append(_check_foundry_project()) + checks.append(_check_application_insights_env()) return PreflightReport(checks=checks) @@ -373,7 +360,7 @@ def format_report( ✓ Workspace /repo/path ✓ Azure authentication ARM token acquired (expires in 89 min) - ✓ Foundry project Project reachable; App Insights auto-discovered. + ✓ Foundry project Project reachable. ⚠ Application Insights No connection string available; production telemetry will be empty. → Wire App Insights in Foundry or set APPLICATIONINSIGHTS_CONNECTION_STRING. diff --git a/src/agentops/utils/foundry_discovery.py b/src/agentops/utils/foundry_discovery.py index f46cf0e..8737b3a 100644 --- a/src/agentops/utils/foundry_discovery.py +++ b/src/agentops/utils/foundry_discovery.py @@ -22,6 +22,15 @@ log = logging.getLogger(__name__) +PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON = ( + "Foundry Application Insights connection uses ProjectManagedIdentity; " + "API Key credentials are not required." +) +_PROJECT_MANAGED_IDENTITY_APPINSIGHTS_ERROR = ( + "Application Insights connection does not use API Key credentials." +) +_NO_APPINSIGHTS_CONNECTION_ERROR = "No Application Insights connection found." + # Per-process cache so the cockpit does not re-query Foundry on every # page load. Successful results are remembered for a long window @@ -96,6 +105,56 @@ def _summarize_discovery_exception(exc: Exception, *, context: str) -> str: return f"{context} failed ({type(exc).__name__}: {snippet})." +def check_foundry_project_reachable_with_reason( + project_endpoint: str, +) -> Tuple[bool, Optional[str]]: + """Return whether *project_endpoint* is reachable with the current identity. + + The reachability probe lists project connections without requesting their + credentials. This keeps project validation independent from Application + Insights connection-string discovery, which is unavailable for + ProjectManagedIdentity connections by design. + """ + if not project_endpoint: + return False, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" + + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError: + return ( + False, + "azure-ai-projects / azure-identity not installed in the cockpit's " + "Python environment. Install with " + "`pip install azure-ai-projects azure-identity`.", + ) + + try: + credential = DefaultAzureCredential( + exclude_developer_cli_credential=True, + process_timeout=30, + ) + client = AIProjectClient( + endpoint=project_endpoint, + credential=credential, + ) + connections = getattr(client, "connections", None) + list_connections = getattr(connections, "list", None) + if not callable(list_connections): + return ( + False, + "AIProjectClient has no connections.list helper " + "(azure-ai-projects too old).", + ) + next(iter(list_connections()), None) + except Exception as exc: # noqa: BLE001 + return False, _summarize_discovery_exception( + exc, + context="Foundry project reachability check", + ) + return True, None + + def resolve_appinsights_connection_with_reason( project_endpoint: str, ) -> Tuple[Optional[str], Optional[str]]: @@ -183,7 +242,21 @@ def resolve_appinsights_connection_with_reason( _store(project_endpoint, value, None) return value, None - if last_exc is not None: + if ( + isinstance(last_exc, ValueError) + and str(last_exc) == _PROJECT_MANAGED_IDENTITY_APPINSIGHTS_ERROR + ): + reason = PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON + elif ( + type(last_exc).__name__ == "ResourceNotFoundError" + and str(last_exc) == _NO_APPINSIGHTS_CONNECTION_ERROR + ): + reason = ( + "Foundry returned no Application Insights connection. Wire " + "one in: Project details \u2192 Connected resources \u2192 " + "Add connection \u2192 Application Insights." + ) + elif last_exc is not None: reason = _summarize_discovery_exception( last_exc, context="Foundry telemetry discovery", diff --git a/tests/unit/test_foundry_discovery.py b/tests/unit/test_foundry_discovery.py index acc07d5..235ba1a 100644 --- a/tests/unit/test_foundry_discovery.py +++ b/tests/unit/test_foundry_discovery.py @@ -244,6 +244,91 @@ def test_with_reason_surfaces_telemetry_call_failure(): assert "Reader on the Foundry project resource group" in reason +def test_with_reason_accepts_project_managed_identity_connection(): + fake_telemetry = mock.MagicMock(spec=["get_application_insights_connection_string"]) + fake_telemetry.get_application_insights_connection_string.side_effect = ValueError( + "Application Insights connection does not use API Key credentials." + ) + fake_client = mock.MagicMock() + fake_client.telemetry = fake_telemetry + fake_projects_mod = mock.MagicMock() + fake_projects_mod.AIProjectClient.return_value = fake_client + fake_identity_mod = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + {"azure.ai.projects": fake_projects_mod, "azure.identity": fake_identity_mod}, + ): + from agentops.utils.foundry_discovery import ( + PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON, + resolve_appinsights_connection_with_reason, + ) + + conn, reason = resolve_appinsights_connection_with_reason( + "https://x.services.ai.azure.com/api/projects/project-managed-identity" + ) + + assert conn is None + assert reason == PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON + + +def test_with_reason_reports_missing_app_insights_connection(): + class ResourceNotFoundError(Exception): + pass + + fake_telemetry = mock.MagicMock(spec=["get_application_insights_connection_string"]) + fake_telemetry.get_application_insights_connection_string.side_effect = ( + ResourceNotFoundError("No Application Insights connection found.") + ) + fake_client = mock.MagicMock() + fake_client.telemetry = fake_telemetry + fake_projects_mod = mock.MagicMock() + fake_projects_mod.AIProjectClient.return_value = fake_client + fake_identity_mod = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + {"azure.ai.projects": fake_projects_mod, "azure.identity": fake_identity_mod}, + ): + from agentops.utils.foundry_discovery import ( + resolve_appinsights_connection_with_reason, + ) + + conn, reason = resolve_appinsights_connection_with_reason( + "https://x.services.ai.azure.com/api/projects/no-app-insights" + ) + + assert conn is None + assert reason and "returned no Application Insights connection" in reason + + +def test_project_reachability_does_not_request_connection_credentials(): + fake_connections = mock.MagicMock() + fake_connections.list.return_value = iter([]) + fake_client = mock.MagicMock() + fake_client.connections = fake_connections + fake_projects_mod = mock.MagicMock() + fake_projects_mod.AIProjectClient.return_value = fake_client + fake_identity_mod = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + {"azure.ai.projects": fake_projects_mod, "azure.identity": fake_identity_mod}, + ): + from agentops.utils.foundry_discovery import ( + check_foundry_project_reachable_with_reason, + ) + + reachable, reason = check_foundry_project_reachable_with_reason( + "https://x.services.ai.azure.com/api/projects/reachable" + ) + + assert reachable is True + assert reason is None + fake_connections.list.assert_called_once_with() + assert not fake_client.telemetry.get_application_insights_connection_string.called + + def test_successful_discovery_is_cached_in_process(): """A second call must reuse the cached connection string instead of invoking the SDK again.""" diff --git a/tests/unit/test_preflight.py b/tests/unit/test_preflight.py index de84a2d..a6bb98e 100644 --- a/tests/unit/test_preflight.py +++ b/tests/unit/test_preflight.py @@ -64,6 +64,28 @@ def test_foundry_project_skip_when_env_missing(monkeypatch) -> None: assert c.status == "skip" +def test_foundry_project_reachability_is_independent_from_app_insights( + monkeypatch, +) -> None: + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/p", + ) + with mock.patch( + "agentops.utils.foundry_discovery." + "check_foundry_project_reachable_with_reason", + return_value=(True, None), + ), mock.patch( + "agentops.utils.foundry_discovery." + "resolve_appinsights_connection_with_reason", + side_effect=AssertionError("Foundry reachability must not request telemetry"), + ): + c = _check_foundry_project() + + assert c.status == "ok" + assert c.message == "Project reachable." + + def test_application_insights_ok_when_env_var_set(monkeypatch) -> None: monkeypatch.setenv( "APPLICATIONINSIGHTS_CONNECTION_STRING", @@ -73,6 +95,52 @@ def test_application_insights_ok_when_env_var_set(monkeypatch) -> None: assert c.status == "ok" +def test_application_insights_explicit_connection_skips_foundry_discovery( + monkeypatch, +) -> None: + monkeypatch.setenv( + "APPLICATIONINSIGHTS_CONNECTION_STRING", + "InstrumentationKey=11111111-2222-3333-4444-555555555555", + ) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/p", + ) + with mock.patch( + "agentops.utils.foundry_discovery." + "resolve_appinsights_connection_with_reason", + side_effect=AssertionError("Explicit configuration must win"), + ): + c = _check_application_insights_env() + + assert c.status == "ok" + assert c.message == "APPLICATIONINSIGHTS_CONNECTION_STRING is set." + + +def test_application_insights_accepts_project_managed_identity( + monkeypatch, +) -> None: + from agentops.utils.foundry_discovery import ( + PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON, + ) + + monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/p", + ) + with mock.patch( + "agentops.utils.foundry_discovery." + "resolve_appinsights_connection_with_reason", + return_value=(None, PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON), + ): + c = _check_application_insights_env() + + assert c.status == "ok" + assert c.message == PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON + + def test_application_insights_warns_when_env_var_is_invalid(monkeypatch) -> None: monkeypatch.setenv( "APPLICATIONINSIGHTS_CONNECTION_STRING", @@ -96,19 +164,21 @@ def test_application_insights_warns_when_unconfigured(monkeypatch) -> None: def test_application_insights_warns_when_foundry_auth_blocks_discovery(monkeypatch) -> None: monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) - c = _check_application_insights_env( - PreflightCheck( - name="foundry_project", - display_name="Foundry project", - status="warn", - message=( - "Discovery failed - Foundry authentication failed while reading " - "telemetry metadata." - ), - ) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/p", ) + with mock.patch( + "agentops.utils.foundry_discovery." + "resolve_appinsights_connection_with_reason", + return_value=( + None, + "Foundry authentication failed while reading telemetry metadata.", + ), + ): + c = _check_application_insights_env() assert c.status == "warn" - assert "could not verify" in c.message.lower() + assert "connection-string discovery failed" in c.message.lower() assert "no connection string available" not in c.message.lower() assert "APPLICATIONINSIGHTS_CONNECTION_STRING" in c.remediation