From 57bdd81d647ff79536138ac3f5276faa72c8da1a Mon Sep 17 00:00:00 2001 From: "Nikhil Chitlur Navakiran (from Dev Box)" Date: Fri, 26 Jun 2026 13:39:39 -0600 Subject: [PATCH] feat(observability): emit guardrail findings as span events on ETW DTO path The ETW DTO logging path (A365EtwLogger.LogApplyGuardrail -> ApplyGuardrailDataBuilder -> BaseData -> ExportFormatter.FormatLogData) previously dropped guardrail findings entirely, so they were invisible to downstream consumers despite being emitted on the Activity span path. Add an optional findings parameter that flows through to a generic Events carrier on BaseData and is emitted by FormatLogData. The event JSON is pinned to match the Activity path's finding events exactly (same event name, attribute keys, value types), enforced by a cross-path consistency test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Runtime/Common/ExportFormatter.cs | 7 +- src/Observability/Runtime/DTOs/BaseData.cs | 12 ++ .../Builders/ApplyGuardrailDataBuilder.cs | 49 +++++++- .../Runtime/DTOs/Builders/BaseDataBuilder.cs | 11 ++ .../Runtime/Etw/A365EtwLogger.cs | 56 ++++----- .../Runtime/Etw/IA365EtwLogger.cs | 5 +- .../Common/ExportFormatterEventsTests.cs | 112 +++++++++++++++++ .../DTOs/BaseDataTests.cs | 35 ++++++ .../ApplyGuardrailDataBuilderTests.cs | 114 ++++++++++++++++++ .../Etw/EtwLoggerTests.cs | 62 ++++++++++ 10 files changed, 430 insertions(+), 33 deletions(-) create mode 100644 src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Common/ExportFormatterEventsTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/Builders/ApplyGuardrailDataBuilderTests.cs diff --git a/src/Observability/Runtime/Common/ExportFormatter.cs b/src/Observability/Runtime/Common/ExportFormatter.cs index a69fdd66..db659b6a 100644 --- a/src/Observability/Runtime/Common/ExportFormatter.cs +++ b/src/Observability/Runtime/Common/ExportFormatter.cs @@ -152,7 +152,8 @@ public string FormatLogData(IDictionary data) Kind = data.TryGetValue("SpanKind", out var spanKindObj) && spanKindObj != null ? spanKindObj : SpanKindConstants.Client, Status = data.TryGetValue("Status", out var statusObj) && statusObj != null ? statusObj - : new Dictionary { { "code", 0 }, { "message", "" } } + : new Dictionary { { "code", 0 }, { "message", "" } }, + Events = data.TryGetValue("Events", out var eventsObj) ? eventsObj : null }; return SerializePayload(payload); @@ -367,7 +368,7 @@ internal sealed class EtwResourceSpan { [JsonPropertyName("resource")] public OtlpResource? Resource { get; set; } - + [JsonPropertyName("scopeSpan")] public EtwScopeSpan ScopeSpan { get; set; } = new EtwScopeSpan(); } @@ -376,7 +377,7 @@ internal sealed class EtwScopeSpan { [JsonPropertyName("scope")] public InstrumentationScope? Scope { get; set; } - + [JsonPropertyName("span")] public OtlpSpan Span { get; set; } = new OtlpSpan(); } diff --git a/src/Observability/Runtime/DTOs/BaseData.cs b/src/Observability/Runtime/DTOs/BaseData.cs index 9c016497..bb5ca251 100644 --- a/src/Observability/Runtime/DTOs/BaseData.cs +++ b/src/Observability/Runtime/DTOs/BaseData.cs @@ -92,6 +92,13 @@ public BaseData( /// public string? StatusMessage { get; set; } + /// + /// Gets the span events for the operation. Each event is a dictionary with OTLP-style keys + /// (timeUnixNano, name, attributes), matching the Activity export path. + /// Empty by default; populated by builders that emit events (e.g. apply_guardrail findings). + /// + public List> Events { get; } = new List>(); + /// /// Gets the duration of the operation if both start and end times are provided. /// @@ -124,6 +131,11 @@ public BaseData( } }; + if (Events.Count > 0) + { + dict["Events"] = Events; + } + return dict; } } diff --git a/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs index fd234fb2..0c9d964f 100644 --- a/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs @@ -30,6 +30,7 @@ public class ApplyGuardrailDataBuilder : BaseDataBuilder /// Optional span kind override. /// Optional trace ID for distributed tracing. /// Optional exception describing a failure; sets an OTel error status and the error.type attribute. + /// Optional security findings to emit as span events on the apply_guardrail span. /// An ApplyGuardrailData object containing all telemetry data. public static ApplyGuardrailData Build( GuardrailDetails guardrailDetails, @@ -44,11 +45,55 @@ public static ApplyGuardrailData Build( IDictionary? extraAttributes = null, string? spanKind = null, string? traceId = null, - Exception? error = null) + Exception? error = null, + IEnumerable? findings = null) { var attributes = BuildAttributes(guardrailDetails, agentDetails, conversationId, channel, callerDetails, extraAttributes); - return ApplyStatus(new ApplyGuardrailData(parentSpanId, attributes, startTime, endTime, spanId, spanKind, traceId), error); + var data = ApplyStatus(new ApplyGuardrailData(parentSpanId, attributes, startTime, endTime, spanId, spanKind, traceId), error); + + AddFindingEvents(data, findings); + + return data; + } + + private static void AddFindingEvents(ApplyGuardrailData data, IEnumerable? findings) + { + if (findings == null) + { + return; + } + + foreach (var finding in findings) + { + if (finding == null) + { + continue; + } + + var eventAttributes = new Dictionary + { + { OpenTelemetryConstants.GenAiSecurityRiskCategoryKey, finding.RiskCategory }, + { OpenTelemetryConstants.GenAiSecurityRiskSeverityKey, finding.RiskSeverity } + }; + + AddIfNotNull(eventAttributes, OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey, finding.PolicyDecisionType); + AddIfNotNull(eventAttributes, OpenTelemetryConstants.GenAiSecurityPolicyIdKey, finding.PolicyId); + AddIfNotNull(eventAttributes, OpenTelemetryConstants.GenAiSecurityPolicyNameKey, finding.PolicyName); + AddIfNotNull(eventAttributes, OpenTelemetryConstants.GenAiSecurityPolicyVersionKey, finding.PolicyVersion); + if (finding.RiskScore.HasValue) + { + eventAttributes[OpenTelemetryConstants.GenAiSecurityRiskScoreKey] = finding.RiskScore.Value; + } + AddIfNotNull(eventAttributes, OpenTelemetryConstants.GenAiSecurityRiskMetadataKey, finding.RiskMetadata); + + data.Events.Add(new Dictionary + { + { "timeUnixNano", ToUnixNanos(DateTimeOffset.UtcNow) }, + { "name", OpenTelemetryConstants.GenAiSecurityFindingEventName }, + { "attributes", eventAttributes } + }); + } } private static Dictionary BuildAttributes( diff --git a/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs index 7b9bcafb..bee77a2d 100644 --- a/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs @@ -168,6 +168,17 @@ protected static void AddIfNotNull(IDictionary attributes, stri } } + /// + /// Converts a timestamp to Unix epoch nanoseconds, matching the Activity export path's + /// conversion in ExportFormatter. Used to stamp span events on the ETW DTO path. + /// + protected static ulong ToUnixNanos(DateTimeOffset timestamp) + { + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var ns = (timestamp.UtcDateTime - unixEpoch).Ticks * 100; + return (ulong)ns; + } + /// /// Adds extra attributes to the attributes dictionary. /// Extra attributes cannot override keys already set by the builder on this span. diff --git a/src/Observability/Runtime/Etw/A365EtwLogger.cs b/src/Observability/Runtime/Etw/A365EtwLogger.cs index 1c0df7c9..22c7ce5e 100644 --- a/src/Observability/Runtime/Etw/A365EtwLogger.cs +++ b/src/Observability/Runtime/Etw/A365EtwLogger.cs @@ -37,14 +37,14 @@ public A365EtwLogger(ILoggerFactory factory) /// public void LogInferenceCall( - InferenceCallDetails inferenceCallDetails, - AgentDetails agentDetails, - string conversationId, - string[]? inputMessages, - string[]? outputMessages, - DateTimeOffset? startTime, - DateTimeOffset? endTime, - string? spanId, + InferenceCallDetails inferenceCallDetails, + AgentDetails agentDetails, + string conversationId, + string[]? inputMessages, + string[]? outputMessages, + DateTimeOffset? startTime, + DateTimeOffset? endTime, + string? spanId, string? parentSpanId, Channel? channel, CallerDetails? callerDetails, @@ -77,16 +77,16 @@ public void LogInferenceCall( /// public void LogInvokeAgent( - InvokeAgentScopeDetails invokeAgentScopeDetails, - AgentDetails agentDetails, - string conversationId, - Request? request, - CallerDetails? callerDetails, - string[]? inputMessages, - string[]? outputMessages, - DateTimeOffset? startTime, - DateTimeOffset? endTime, - string? spanId, + InvokeAgentScopeDetails invokeAgentScopeDetails, + AgentDetails agentDetails, + string conversationId, + Request? request, + CallerDetails? callerDetails, + string[]? inputMessages, + string[]? outputMessages, + DateTimeOffset? startTime, + DateTimeOffset? endTime, + string? spanId, string? parentSpanId, string? traceId, Exception? error = null) @@ -117,13 +117,13 @@ public void LogInvokeAgent( /// public void LogToolCall( - ToolCallDetails toolCallDetails, - AgentDetails agentDetails, - string conversationId, - string? responseContent, - DateTimeOffset? startTime, - DateTimeOffset? endTime, - string? spanId, + ToolCallDetails toolCallDetails, + AgentDetails agentDetails, + string conversationId, + string? responseContent, + DateTimeOffset? startTime, + DateTimeOffset? endTime, + string? spanId, string? parentSpanId, Channel? channel, CallerDetails? callerDetails, @@ -201,7 +201,8 @@ public void LogApplyGuardrail( Channel? channel = null, CallerDetails? callerDetails = null, string? traceId = null, - Exception? error = null) + Exception? error = null, + IEnumerable? findings = null) { var data = ApplyGuardrailDataBuilder.Build( guardrailDetails, @@ -214,7 +215,8 @@ public void LogApplyGuardrail( channel, callerDetails: callerDetails, traceId: traceId, - error: error); + error: error, + findings: findings); logger.Log( LogLevel.Information, diff --git a/src/Observability/Runtime/Etw/IA365EtwLogger.cs b/src/Observability/Runtime/Etw/IA365EtwLogger.cs index bb4b5dde..411284f2 100644 --- a/src/Observability/Runtime/Etw/IA365EtwLogger.cs +++ b/src/Observability/Runtime/Etw/IA365EtwLogger.cs @@ -3,6 +3,7 @@ using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; using System; +using System.Collections.Generic; namespace Microsoft.Agents.A365.Observability.Runtime.Etw { @@ -143,6 +144,7 @@ public void LogOutput( /// Optional details of the caller. /// Optional trace ID for distributed tracing. /// Optional exception describing a failure; sets an OTel error status and the error.type attribute. + /// Optional security findings to emit as span events on the apply_guardrail span. public void LogApplyGuardrail( GuardrailDetails guardrailDetails, AgentDetails agentDetails, @@ -154,6 +156,7 @@ public void LogApplyGuardrail( Channel? channel = null, CallerDetails? callerDetails = null, string? traceId = null, - Exception? error = null); + Exception? error = null, + IEnumerable? findings = null); } } diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Common/ExportFormatterEventsTests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Common/ExportFormatterEventsTests.cs new file mode 100644 index 00000000..8e4a8650 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Common/ExportFormatterEventsTests.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Linq; +using System.Text.Json; +using FluentAssertions; +using Microsoft.Agents.A365.Observability.Runtime.DTOs; +using Microsoft.Agents.A365.Observability.Runtime.DTOs.Builders; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tests.Common; + +public partial class ExportFormatterTests +{ + [TestMethod] + public void FormatLogData_NoEvents_OmitsEventsField() + { + // Arrange + var data = new InvokeAgentData( + new Dictionary { { "key", "val" } }, + spanId: "span-1"); + var formatter = CreateFormatter(); + + // Act + var json = formatter.FormatLogData(data.ToDictionary()); + + // Assert + using var doc = JsonDocument.Parse(json); + doc.RootElement.TryGetProperty("Events", out _).Should().BeFalse(); + } + + [TestMethod] + public void FormatLogData_WithFindings_EmitsEventsArray() + { + // Arrange + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + var findings = new[] + { + new GuardrailFinding("prompt_injection", GuardrailRiskSeverity.High, policyDecisionType: "deny") + }; + var data = ApplyGuardrailDataBuilder.Build(details, TestAgentDetails, "conv-1", "parent-1", findings: findings); + var formatter = CreateFormatter(); + + // Act + var json = formatter.FormatLogData(data.ToDictionary()); + + // Assert + using var doc = JsonDocument.Parse(json); + var events = doc.RootElement.GetProperty("Events"); + events.GetArrayLength().Should().Be(1); + var evt = events[0]; + evt.GetProperty("name").GetString().Should().Be(OpenTelemetryConstants.GenAiSecurityFindingEventName); + evt.GetProperty("timeUnixNano").GetUInt64().Should().BeGreaterThan(0); + var attrs = evt.GetProperty("attributes"); + attrs.GetProperty(OpenTelemetryConstants.GenAiSecurityRiskCategoryKey).GetString().Should().Be("prompt_injection"); + attrs.GetProperty(OpenTelemetryConstants.GenAiSecurityRiskSeverityKey).GetString().Should().Be("high"); + attrs.GetProperty(OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey).GetString().Should().Be("deny"); + } + + [TestMethod] + public void FindingEvent_DtoPath_StructurallyMatches_ActivityPath() + { + // Arrange — one finding with all fields populated. + var finding = new GuardrailFinding( + riskCategory: "sensitive_info_disclosure", + riskSeverity: GuardrailRiskSeverity.High, + policyDecisionType: "deny", + policyId: "policy_pii_v2", + policyName: "PII Policy", + policyVersion: "2.0", + riskScore: 0.92, + riskMetadata: new[] { "pattern:ssn", "count:2" }); + + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + + var resource = CreateResource(); + var formatter = CreateFormatter(); + + // Activity path: RecordFinding -> FormatSingle + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, TestAgentDetails); + scope.RecordFinding(finding); + }); + using var activityDoc = JsonDocument.Parse(formatter.FormatSingle(activity, resource)); + var activityEvent = activityDoc.RootElement + .GetProperty("resourceSpan") + .GetProperty("scopeSpan") + .GetProperty("span") + .GetProperty("events")[0]; + + // DTO path: ApplyGuardrailDataBuilder -> FormatLogData + var data = ApplyGuardrailDataBuilder.Build(details, TestAgentDetails, "conv-1", "parent-1", findings: new[] { finding }); + using var dtoDoc = JsonDocument.Parse(formatter.FormatLogData(data.ToDictionary())); + var dtoEvent = dtoDoc.RootElement.GetProperty("Events")[0]; + + // Assert — same event name and same attributes (timeUnixNano intentionally ignored). + dtoEvent.GetProperty("name").GetString() + .Should().Be(activityEvent.GetProperty("name").GetString()); + + AttributeMap(dtoEvent.GetProperty("attributes")) + .Should().BeEquivalentTo(AttributeMap(activityEvent.GetProperty("attributes"))); + } + + private static Dictionary AttributeMap(JsonElement attributes) => + attributes.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetRawText()); +} diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/BaseDataTests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/BaseDataTests.cs index fb9c5331..242f0118 100644 --- a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/BaseDataTests.cs +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/BaseDataTests.cs @@ -230,5 +230,40 @@ public void Status_ErrorInToDictionary_EmitsCodeAndMessage() status["code"].Should().Be(2); status["message"].Should().Be("kaboom"); } + + [TestMethod] + public void Events_DefaultsToEmpty() + { + var data = new TestData(); + data.Events.Should().BeEmpty(); + } + + [TestMethod] + public void Events_NotIncludedInToDictionary_WhenEmpty() + { + var data = new TestData(); + var dict = data.ToDictionary(); + dict.Should().NotContainKey("Events"); + } + + [TestMethod] + public void Events_IncludedInToDictionary_WhenPopulated() + { + var data = new TestData(); + var evt = new Dictionary + { + { "timeUnixNano", 123UL }, + { "name", "finding" }, + { "attributes", new Dictionary { { "k", "v" } } } + }; + data.Events.Add(evt); + + var dict = data.ToDictionary(); + + dict.Should().ContainKey("Events"); + var events = dict["Events"].Should().BeAssignableTo>>().Subject; + events.Should().HaveCount(1); + events[0].Should().BeSameAs(evt); + } } } diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/Builders/ApplyGuardrailDataBuilderTests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/Builders/ApplyGuardrailDataBuilderTests.cs new file mode 100644 index 00000000..c4a7cd88 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/DTOs/Builders/ApplyGuardrailDataBuilderTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.Observability.Runtime.DTOs.Builders; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tests.DTOs.Builders +{ + [TestClass] + public class ApplyGuardrailDataBuilderTests + { + private static GuardrailDetails Details() => new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + + private static AgentDetails Agent() => new AgentDetails("agent-id"); + + [TestMethod] + public void Build_NoFindings_ProducesNoEvents() + { + var data = ApplyGuardrailDataBuilder.Build(Details(), Agent(), "conv-1", "parent-1"); + + data.Events.Should().BeEmpty(); + data.ToDictionary().Should().NotContainKey("Events"); + } + + [TestMethod] + public void Build_SingleFinding_ProducesOneEvent_WithRequiredShape() + { + var findings = new[] + { + new GuardrailFinding( + riskCategory: "prompt_injection", + riskSeverity: GuardrailRiskSeverity.High) + }; + + var data = ApplyGuardrailDataBuilder.Build(Details(), Agent(), "conv-1", "parent-1", findings: findings); + + data.Events.Should().HaveCount(1); + var evt = data.Events[0]; + evt["name"].Should().Be(OpenTelemetryConstants.GenAiSecurityFindingEventName); + ((ulong)evt["timeUnixNano"]!).Should().BeGreaterThan(0); + + var attrs = evt["attributes"].Should().BeAssignableTo>().Subject; + attrs[OpenTelemetryConstants.GenAiSecurityRiskCategoryKey].Should().Be("prompt_injection"); + attrs[OpenTelemetryConstants.GenAiSecurityRiskSeverityKey].Should().Be("high"); + } + + [TestMethod] + public void Build_OptionalFieldsOmitted_WhenNull() + { + var findings = new[] + { + new GuardrailFinding( + riskCategory: "toxicity", + riskSeverity: GuardrailRiskSeverity.Low) + }; + + var data = ApplyGuardrailDataBuilder.Build(Details(), Agent(), "conv-1", "parent-1", findings: findings); + + var attrs = (IDictionary)data.Events[0]["attributes"]!; + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey); + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityPolicyIdKey); + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityPolicyNameKey); + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityPolicyVersionKey); + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityRiskScoreKey); + attrs.Should().NotContainKey(OpenTelemetryConstants.GenAiSecurityRiskMetadataKey); + } + + [TestMethod] + public void Build_AllFields_PreservesValueTypes() + { + var findings = new[] + { + new GuardrailFinding( + riskCategory: "sensitive_info_disclosure", + riskSeverity: GuardrailRiskSeverity.High, + policyDecisionType: "deny", + policyId: "policy_pii_v2", + policyName: "PII Policy", + policyVersion: "2.0", + riskScore: 0.92, + riskMetadata: new[] { "pattern:ssn", "count:2" }) + }; + + var data = ApplyGuardrailDataBuilder.Build(Details(), Agent(), "conv-1", "parent-1", findings: findings); + + var attrs = (IDictionary)data.Events[0]["attributes"]!; + attrs[OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey].Should().Be("deny"); + attrs[OpenTelemetryConstants.GenAiSecurityPolicyIdKey].Should().Be("policy_pii_v2"); + attrs[OpenTelemetryConstants.GenAiSecurityPolicyNameKey].Should().Be("PII Policy"); + attrs[OpenTelemetryConstants.GenAiSecurityPolicyVersionKey].Should().Be("2.0"); + attrs[OpenTelemetryConstants.GenAiSecurityRiskScoreKey].Should().BeOfType().And.Be(0.92); + attrs[OpenTelemetryConstants.GenAiSecurityRiskMetadataKey].Should().BeOfType() + .Which.Should().BeEquivalentTo(new[] { "pattern:ssn", "count:2" }); + } + + [TestMethod] + public void Build_MultipleFindings_ProducesMultipleEvents() + { + var findings = new[] + { + new GuardrailFinding("pii", GuardrailRiskSeverity.Medium), + new GuardrailFinding("toxicity", GuardrailRiskSeverity.Low) + }; + + var data = ApplyGuardrailDataBuilder.Build(Details(), Agent(), "conv-1", "parent-1", findings: findings); + + data.Events.Should().HaveCount(2); + } + } +} diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Etw/EtwLoggerTests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Etw/EtwLoggerTests.cs index 26bf5aca..937c8a97 100644 --- a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Etw/EtwLoggerTests.cs +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Etw/EtwLoggerTests.cs @@ -118,5 +118,67 @@ public void Logs_Output_Event() var root = JsonDocument.Parse(payloadStr!).RootElement; Assert.AreEqual(OpenTelemetryConstants.OperationNames.OutputMessages.ToString(), root.GetProperty("Name").GetString()); } + + [TestMethod] + public void Logs_ApplyGuardrail_Event_WithFindings() + { + // Arrange + using var listener = new TestEventListener(); + listener.EnableEvents(EtwEventSource.Log, EventLevel.Informational); + using var provider = BuildProvider(); + var etwLogger = provider.GetRequiredService>(); + var agentDetails = new AgentDetails("agent-id", agentName: "agent-name"); + var guardrailDetails = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + var findings = new[] + { + new GuardrailFinding("prompt_injection", GuardrailRiskSeverity.High, policyDecisionType: "deny") + }; + + // Act + etwLogger.LogApplyGuardrail(guardrailDetails, agentDetails, "conv-guardrail-1", "parent-1", findings: findings); + + // Assert + var evt = listener.Events.FirstOrDefault(e => e.EventId == 2000); + Assert.IsNotNull(evt); + var payloadStr = evt!.Payload![0] as string; + Assert.IsNotNull(payloadStr); + var root = JsonDocument.Parse(payloadStr!).RootElement; + Assert.AreEqual(OpenTelemetryConstants.OperationNames.ApplyGuardrail.ToString(), root.GetProperty("Name").GetString()); + + var events = root.GetProperty("Events"); + Assert.AreEqual(1, events.GetArrayLength()); + var finding = events[0]; + Assert.AreEqual(OpenTelemetryConstants.GenAiSecurityFindingEventName, finding.GetProperty("name").GetString()); + var attrs = finding.GetProperty("attributes"); + Assert.AreEqual("prompt_injection", attrs.GetProperty(OpenTelemetryConstants.GenAiSecurityRiskCategoryKey).GetString()); + Assert.AreEqual("high", attrs.GetProperty(OpenTelemetryConstants.GenAiSecurityRiskSeverityKey).GetString()); + } + + [TestMethod] + public void Logs_ApplyGuardrail_Event_NoFindings_OmitsEvents() + { + // Arrange + using var listener = new TestEventListener(); + listener.EnableEvents(EtwEventSource.Log, EventLevel.Informational); + using var provider = BuildProvider(); + var etwLogger = provider.GetRequiredService>(); + var agentDetails = new AgentDetails("agent-id", agentName: "agent-name"); + var guardrailDetails = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Allow); + + // Act + etwLogger.LogApplyGuardrail(guardrailDetails, agentDetails, "conv-guardrail-2", "parent-1"); + + // Assert + var evt = listener.Events.FirstOrDefault(e => e.EventId == 2000); + Assert.IsNotNull(evt); + var payloadStr = evt!.Payload![0] as string; + Assert.IsNotNull(payloadStr); + var root = JsonDocument.Parse(payloadStr!).RootElement; + Assert.IsFalse(root.TryGetProperty("Events", out _)); + } } }