Skip to content

Add agent diagnostic telemetry vocabulary #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ANcpLua wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add agent diagnostic telemetry vocabulary #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/Qyl.Telemetry.SemanticConventions.Analyzers/SemconvRegistryFacts.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,15 @@ internal static class SemconvRegistryFacts
["process.vpid"] = SemconvAttributeValueKind.Integer,
["process.working_directory"] = SemconvAttributeValueKind.String,
["profile.frame.type"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.check.count"] = SemconvAttributeValueKind.Integer,
["qyl.agent.diagnostic.check.failed_count"] = SemconvAttributeValueKind.Integer,
["qyl.agent.diagnostic.extension.id"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.format.version"] = SemconvAttributeValueKind.Integer,
["qyl.agent.diagnostic.outcome"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.phase"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.probe.id"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.snapshot.id"] = SemconvAttributeValueKind.String,
["qyl.agent.diagnostic.variable.count"] = SemconvAttributeValueKind.Integer,
["qyl.capability.id"] = SemconvAttributeValueKind.String,
["qyl.capability.kind"] = SemconvAttributeValueKind.String,
["qyl.exception.source"] = SemconvAttributeValueKind.String,
Expand All @@ -806,6 +815,11 @@ internal static class SemconvRegistryFacts
["qyl.mcp.test_case.id"] = SemconvAttributeValueKind.String,
["qyl.mcp.tool.name"] = SemconvAttributeValueKind.String,
["qyl.project.id"] = SemconvAttributeValueKind.String,
["qyl.workflow.agent.id"] = SemconvAttributeValueKind.String,
["qyl.workflow.attempt.id"] = SemconvAttributeValueKind.String,
["qyl.workflow.event.id"] = SemconvAttributeValueKind.String,
["qyl.workflow.run.id"] = SemconvAttributeValueKind.String,
["qyl.workflow.tool_call.id"] = SemconvAttributeValueKind.String,
["qyl.workspace.id"] = SemconvAttributeValueKind.String,
["rpc.connect_rpc.error_code"] = SemconvAttributeValueKind.String,
["rpc.grpc.status_code"] = SemconvAttributeValueKind.String,
Expand Down Expand Up @@ -1137,6 +1151,8 @@ private static readonly (string Prefix, SemconvAttributeValueKind Kind)[] Templa
["process.paging.fault_type"] = new[] { "major", "minor" },
["process.state"] = new[] { "defunct", "running", "sleeping", "stopped" },
["profile.frame.type"] = new[] { "beam", "cpython", "dotnet", "go", "jvm", "kernel", "luajit", "native", "perl", "php", "ruby", "rust", "v8js" },
["qyl.agent.diagnostic.outcome"] = new[] { "fail", "not_evaluated", "pass", "unknown" },
["qyl.agent.diagnostic.phase"] = new[] { "checkpoint", "error", "input", "output" },
["rpc.connect_rpc.error_code"] = new[] { "aborted", "already_exists", "cancelled", "data_loss", "deadline_exceeded", "failed_precondition", "internal", "invalid_argument", "not_found", "out_of_range", "permission_denied", "resource_exhausted", "unauthenticated", "unavailable", "unimplemented", "unknown" },
["rpc.grpc.status_code"] = new[] { "0", "1", "10", "11", "12", "13", "14", "15", "16", "2", "3", "4", "5", "6", "7", "8", "9" },
["rpc.message.type"] = new[] { "RECEIVED", "SENT" },
Expand Down Expand Up @@ -1190,7 +1206,7 @@ private static readonly (string Prefix, SemconvAttributeValueKind Kind)[] Templa
new(new[] { "Qyl.Collector", "Qyl.Instrumentation.ErrorCapture", "Qyl.OpenTelemetry.AutoInstrumentation", "Qyl.OpenTelemetry.AutoInstrumentation.Database", "Qyl.OpenTelemetry.AutoInstrumentation.NServiceBus", "System.Runtime" }, StringComparer.Ordinal);

private static readonly HashSet<string> KnownEventNames =
new(new[] { "app.crash", "app.jank", "app.screen.click", "app.widget.click", "az.resource.log", "azure.resource.log", "browser.web_vital", "db.client.operation.exception", "device.app.lifecycle", "exception", "faas.invocation.exception", "feature_flag.evaluation", "gen_ai.client.inference.operation.details", "gen_ai.client.operation.exception", "gen_ai.evaluation.result", "http.client.request.exception", "http.server.request.exception", "messaging.create.exception", "messaging.process.exception", "messaging.receive.exception", "messaging.send.exception", "messaging.settle.exception", "qyl.http.client", "qyl.rpc.grpc", "rpc.client.call.exception", "rpc.message", "rpc.server.call.exception", "session.end", "session.start" }, StringComparer.Ordinal);
new(new[] { "app.crash", "app.jank", "app.screen.click", "app.widget.click", "az.resource.log", "azure.resource.log", "browser.web_vital", "db.client.operation.exception", "device.app.lifecycle", "exception", "faas.invocation.exception", "feature_flag.evaluation", "gen_ai.client.inference.operation.details", "gen_ai.client.operation.exception", "gen_ai.evaluation.result", "http.client.request.exception", "http.server.request.exception", "messaging.create.exception", "messaging.process.exception", "messaging.receive.exception", "messaging.send.exception", "messaging.settle.exception", "qyl.agent.diagnostic.snapshot", "qyl.http.client", "qyl.rpc.grpc", "rpc.client.call.exception", "rpc.message", "rpc.server.call.exception", "session.end", "session.start" }, StringComparer.Ordinal);

internal static readonly SemconvSpanRule[] SpanRules =
{
Expand Down
122 changes: 122 additions & 0 deletions src/Qyl.Telemetry.SemanticConventions.Incubating/Attributes/Qyl/QylAttributes.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,103 @@ namespace Qyl.Telemetry.SemanticConventions.Incubating.Attributes.Qyl;
/// <summary>Qyl Attributes (incubating).</summary>
public static class QylAttributes
{
/// <summary>
/// Number of machine-readable checks in an agent diagnostic state frame.
/// </summary>
public const string AgentDiagnosticCheckCount = "qyl.agent.diagnostic.check.count";

/// <summary>
/// Number of failed checks in an agent diagnostic state frame.
/// </summary>
public const string AgentDiagnosticCheckFailedCount = "qyl.agent.diagnostic.check.failed_count";

/// <summary>
/// Stable discriminator of the machine-readable agent diagnostic extension.
/// </summary>
public const string AgentDiagnosticExtensionId = "qyl.agent.diagnostic.extension.id";

/// <summary>
/// Version of the machine-readable agent diagnostic state-frame format.
/// </summary>
public const string AgentDiagnosticFormatVersion = "qyl.agent.diagnostic.format.version";

/// <summary>
/// Deterministically computed aggregate outcome of an agent diagnostic state frame.
/// </summary>
public const string AgentDiagnosticOutcome = "qyl.agent.diagnostic.outcome";

/// <summary>
/// Values for the <c>AgentDiagnosticOutcome</c> attribute.
/// </summary>
public static class AgentDiagnosticOutcomeValues
{
/// <summary>
/// At least one evaluated diagnostic check failed.
/// </summary>
public const string Fail = "fail";

/// <summary>
/// The state frame contained no diagnostic checks.
/// </summary>
public const string NotEvaluated = "not_evaluated";

/// <summary>
/// Every evaluated diagnostic check passed.
/// </summary>
public const string Pass = "pass";

/// <summary>
/// No failed check was observed, but at least one check could not be evaluated.
/// </summary>
public const string Unknown = "unknown";
}

/// <summary>
/// Program boundary at which an agent diagnostic state frame was captured.
/// </summary>
public const string AgentDiagnosticPhase = "qyl.agent.diagnostic.phase";

/// <summary>
/// Values for the <c>AgentDiagnosticPhase</c> attribute.
/// </summary>
public static class AgentDiagnosticPhaseValues
{
/// <summary>
/// State captured at an explicit verification checkpoint.
/// </summary>
public const string Checkpoint = "checkpoint";

/// <summary>
/// State captured on an error path.
/// </summary>
public const string Error = "error";

/// <summary>
/// State captured before the probed operation.
/// </summary>
public const string Input = "input";

/// <summary>
/// State captured after the probed operation.
/// </summary>
public const string Output = "output";
}

/// <summary>
/// Bounded machine identifier of the diagnostic probe that captured the state frame.
/// </summary>
public const string AgentDiagnosticProbeId = "qyl.agent.diagnostic.probe.id";

/// <summary>
/// Stable identifier of one immutable agent diagnostic state frame.
/// </summary>
public const string AgentDiagnosticSnapshotId = "qyl.agent.diagnostic.snapshot.id";

/// <summary>
/// Number of dynamic variables described by an agent diagnostic state frame.
/// </summary>
public const string AgentDiagnosticVariableCount = "qyl.agent.diagnostic.variable.count";

/// <summary>
/// Stable identifier of the qyl capability that produced the telemetry.
/// </summary>
Expand Down Expand Up @@ -78,6 +175,31 @@ public static class QylAttributes
/// </summary>
public const string ProjectId = "qyl.project.id";

/// <summary>
/// Stable workflow agent identifier mirrored into telemetry for journal correlation.
/// </summary>
public const string WorkflowAgentId = "qyl.workflow.agent.id";

/// <summary>
/// Stable workflow attempt identifier mirrored into telemetry for journal correlation.
/// </summary>
public const string WorkflowAttemptId = "qyl.workflow.attempt.id";

/// <summary>
/// Stable immutable workflow journal event identifier mirrored into telemetry for correlation.
/// </summary>
public const string WorkflowEventId = "qyl.workflow.event.id";

/// <summary>
/// Stable workflow run identifier mirrored into telemetry for journal correlation.
/// </summary>
public const string WorkflowRunId = "qyl.workflow.run.id";

/// <summary>
/// Stable workflow tool-call identifier mirrored into telemetry for journal correlation.
/// </summary>
public const string WorkflowToolCallId = "qyl.workflow.tool_call.id";

/// <summary>
/// qyl workspace identifier the telemetry belongs to.
/// </summary>
Expand Down
11 changes: 8 additions & 3 deletions src/Qyl.Telemetry.SemanticConventions.Incubating/Names/QylTelemetryNames.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,22 @@ public static class Scopes
}

/// <summary>
/// Names of the qyl-owned <c>DiagnosticSource</c> bridge events.
/// Names of qyl-owned <c>Activity</c> and <c>DiagnosticSource</c> events.
/// </summary>
public static class Events
{
/// <summary>
/// The <c>qyl.http.client</c> bridge event.
/// The <c>qyl.agent.diagnostic.snapshot</c> telemetry event.
/// </summary>
public const string QylAgentDiagnosticSnapshot = "qyl.agent.diagnostic.snapshot";

/// <summary>
/// The <c>qyl.http.client</c> telemetry event.
/// </summary>
public const string QylHttpClient = "qyl.http.client";

/// <summary>
/// The <c>qyl.rpc.grpc</c> bridge event.
/// The <c>qyl.rpc.grpc</c> telemetry event.
/// </summary>
public const string QylRpcGrpc = "qyl.rpc.grpc";
}
Expand Down
141 changes: 140 additions & 1 deletion src/Qyl.Telemetry.SemanticConventions.SourceGeneration/Resources/qyl-registry.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
{
"description": "qyl-owned vocabulary, first-class next to the pinned upstream registries (qyl architecture, loop 1). Attributes here are qyl.* names the platform emits and the collector must recognize; scope_names are the ActivitySource/Meter names qyl constructs; event_names are qyl-owned DiagnosticSource bridge event names. Consumed by emit_analyzer_registry.py (QYL0200/QYL0201 allowlists). Entries under qyl.mcp.* are the experimental staging namespace: deletion-targeted on every upstream registry bump that lands an equivalent.",
"description": "qyl-owned vocabulary, first-class next to the pinned upstream registries (qyl architecture, loop 1). Attributes here are qyl.* names the platform emits and the collector must recognize; scope_names are the ActivitySource/Meter names qyl constructs; event_names are qyl-owned Activity or DiagnosticSource event names. Consumed by emit_analyzer_registry.py (QYL0200/QYL0201 allowlists). Entries under qyl.mcp.* are the experimental staging namespace: deletion-targeted on every upstream registry bump that lands an equivalent.",
"attributes": [
{
"key": "qyl.agent.diagnostic.extension.id",
"type": "string",
"stability": "development",
"brief": "Stable discriminator of the machine-readable agent diagnostic extension."
},
{
"key": "qyl.capability.id",
"type": "string",
Expand All @@ -25,6 +31,138 @@
"stability": "development",
"brief": "qyl auto-instrumentation domain that emitted the span or metric."
},
{
"key": "qyl.agent.diagnostic.snapshot.id",
"type": "string",
"stability": "development",
"brief": "Stable identifier of one immutable agent diagnostic state frame."
},
{
"key": "qyl.agent.diagnostic.format.version",
"type": "int",
"stability": "development",
"brief": "Version of the machine-readable agent diagnostic state-frame format."
},
{
"key": "qyl.agent.diagnostic.probe.id",
"type": "string",
"stability": "development",
"brief": "Bounded machine identifier of the diagnostic probe that captured the state frame."
},
{
"key": "qyl.agent.diagnostic.phase",
"type": {
"members": [
{
"id": "input",
"value": "input",
"stability": "development",
"brief": "State captured before the probed operation."
},
{
"id": "output",
"value": "output",
"stability": "development",
"brief": "State captured after the probed operation."
},
{
"id": "error",
"value": "error",
"stability": "development",
"brief": "State captured on an error path."
},
{
"id": "checkpoint",
"value": "checkpoint",
"stability": "development",
"brief": "State captured at an explicit verification checkpoint."
}
]
},
"stability": "development",
"brief": "Program boundary at which an agent diagnostic state frame was captured."
},
{
"key": "qyl.agent.diagnostic.outcome",
"type": {
"members": [
{
"id": "pass",
"value": "pass",
"stability": "development",
"brief": "Every evaluated diagnostic check passed."
},
{
"id": "fail",
"value": "fail",
"stability": "development",
"brief": "At least one evaluated diagnostic check failed."
},
{
"id": "unknown",
"value": "unknown",
"stability": "development",
"brief": "No failed check was observed, but at least one check could not be evaluated."
},
{
"id": "not_evaluated",
"value": "not_evaluated",
"stability": "development",
"brief": "The state frame contained no diagnostic checks."
}
]
},
"stability": "development",
"brief": "Deterministically computed aggregate outcome of an agent diagnostic state frame."
},
{
"key": "qyl.agent.diagnostic.variable.count",
"type": "int",
"stability": "development",
"brief": "Number of dynamic variables described by an agent diagnostic state frame."
},
{
"key": "qyl.agent.diagnostic.check.count",
"type": "int",
"stability": "development",
"brief": "Number of machine-readable checks in an agent diagnostic state frame."
},
{
"key": "qyl.agent.diagnostic.check.failed_count",
"type": "int",
"stability": "development",
"brief": "Number of failed checks in an agent diagnostic state frame."
},
{
"key": "qyl.workflow.run.id",
"type": "string",
"stability": "development",
"brief": "Stable workflow run identifier mirrored into telemetry for journal correlation."
},
{
"key": "qyl.workflow.event.id",
"type": "string",
"stability": "development",
"brief": "Stable immutable workflow journal event identifier mirrored into telemetry for correlation."
},
{
"key": "qyl.workflow.attempt.id",
"type": "string",
"stability": "development",
"brief": "Stable workflow attempt identifier mirrored into telemetry for journal correlation."
},
{
"key": "qyl.workflow.agent.id",
"type": "string",
"stability": "development",
"brief": "Stable workflow agent identifier mirrored into telemetry for journal correlation."
},
{
"key": "qyl.workflow.tool_call.id",
"type": "string",
"stability": "development",
"brief": "Stable workflow tool-call identifier mirrored into telemetry for journal correlation."
},
{
"key": "qyl.project.id",
"type": "string",
Expand Down Expand Up @@ -96,6 +234,7 @@
"System.Runtime"
],
"event_names": [
"qyl.agent.diagnostic.snapshot",
"qyl.http.client",
"qyl.rpc.grpc"
]
Expand Down
6 changes: 3 additions & 3 deletions src/Qyl.Telemetry.SemanticConventions.SourceGeneration/scripts/emit_attributes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ def attr_ident(a):


# --------------------------------------------------------------------------- #
# qyl-owned scope + bridge event names #
# qyl-owned scope + telemetry event names #
# --------------------------------------------------------------------------- #
NAMES_NAMESPACE = "Qyl.Telemetry.SemanticConventions.Incubating.Names"
NAMES_CLASS = "QylTelemetryNames"
Expand Down Expand Up @@ -700,15 +700,15 @@ def emit_names_file(qyl_registry) -> str:
w.line(f"{pad}}}")
w.line()

w.summary(pad, ["Names of the qyl-owned <c>DiagnosticSource</c> bridge events."])
w.summary(pad, ["Names of qyl-owned <c>Activity</c> and <c>DiagnosticSource</c> events."])
w.line(f"{pad}public static class Events")
w.line(f"{pad}{{")
first = True
for ident, value in events:
if not first:
w.line()
first = False
w.summary(ipad, [f"The <c>{escape_attribute(value)}</c> bridge event."])
w.summary(ipad, [f"The <c>{escape_attribute(value)}</c> telemetry event."])
w.line(f"{ipad}public const string {ident} = {csharp_string(value)};")
w.line(f"{pad}}}")

Expand Down
Loading
Loading