From 3e43062c01cf0b4fd4000ce8a9309133d98094a8 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Sun, 12 Jul 2026 09:04:26 -0600 Subject: [PATCH 1/2] observability: add kubernetesResourceSnapshots Kusto table Add a dedicated Kusto table for Kubernetes resource snapshots emitted by the mgmt-agent's ResourceWatcher and PodWatcher controllers. This upgrades raw container logs into a structured table with top-level columns for apiVersion, objectKind, namespace, name, uid, and the full object, making resource state queries as easy as 'kubectl get'. A new 'snapshotType' structured log field is added to resource and pod event log lines. Fluent Bit routes logs where snapshotType is 'kubernetes' to the new table while preserving all existing log flows unchanged. The PodWatcher now sets TypeMeta on a shallow copy of the pod before serializing, so apiVersion/kind appear in the JSON output without mutating the shared informer cache object. --- .../modules/logs/kusto/main.bicep | 1 + .../tables/kubernetesResourceSnapshots.kql | 30 +++++++++++++ ...elmTemplate_dev_westus3_mgmt_1_arobit.yaml | 42 ++++++++++++++++++- ...HelmTemplate_dev_westus3_svc_1_arobit.yaml | 42 ++++++++++++++++++- mgmt-agent/pkg/controller/podwatcher.go | 12 ++++-- mgmt-agent/pkg/controller/resourcewatcher.go | 1 + .../deploy/templates/forwarder-configmap.yaml | 34 ++++++++++++++- ...plate_helmtest_buffering_disabled_svc.yaml | 32 +++++++++++++- ...mTemplate_helmtest_kusto_disabled_svc.yaml | 42 ++++++++++++++++++- ...e_helmtest_kusto_unlimited_memory_svc.yaml | 42 ++++++++++++++++++- ..._helmtest_mdsd_and_kusto_enabled_mgmt.yaml | 42 ++++++++++++++++++- ...e_helmtest_mdsd_and_kusto_enabled_svc.yaml | 42 ++++++++++++++++++- 12 files changed, 343 insertions(+), 19 deletions(-) create mode 100644 dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql diff --git a/dev-infrastructure/modules/logs/kusto/main.bicep b/dev-infrastructure/modules/logs/kusto/main.bicep index d0fbae85596..030fc600cc7 100644 --- a/dev-infrastructure/modules/logs/kusto/main.bicep +++ b/dev-infrastructure/modules/logs/kusto/main.bicep @@ -59,6 +59,7 @@ var allServiceLogsTablesKQL = { kubernetesEvents: loadTextContent('tables/kubernetesEvents.kql') aksEvents: loadTextContent('tables/aksEvents.kql') systemdLogs: loadTextContent('tables/systemdLogs.kql') + resourceSnapshots: loadTextContent('tables/kubernetesResourceSnapshots.kql') } var allCustomerLogsTablesKQL = { diff --git a/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql b/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql new file mode 100644 index 00000000000..46388195929 --- /dev/null +++ b/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql @@ -0,0 +1,30 @@ +.create-merge table kubernetesResourceSnapshots ( + timestamp: datetime, + environment: string, + region: string, + cluster: string, + event: string, + apiVersion: string, + objectKind: string, + namespace: string, + name: string, + uid: string, + object: dynamic +) + +.create-or-alter table kubernetesResourceSnapshots ingestion json mapping 'ingestionMapping' +``` +[ + {"column":"timestamp","Properties":{"path":"$.timestamp"}}, + {"column":"environment","Properties":{"path":"$.log.environment"}}, + {"column":"region","Properties":{"path":"$.log.region"}}, + {"column":"cluster","Properties":{"path":"$.log.cluster"}}, + {"column":"event","Properties":{"path":"$.log.event"}}, + {"column":"apiVersion","Properties":{"path":"$.log.object.apiVersion"}}, + {"column":"objectKind","Properties":{"path":"$.log.object.kind"}}, + {"column":"namespace","Properties":{"path":"$.log.namespace"}}, + {"column":"name","Properties":{"path":"$.log.name"}}, + {"column":"uid","Properties":{"path":"$.log.object.metadata.uid"}}, + {"column":"object","Properties":{"path":"$.log.object"}} +] +``` diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml index cf8d188e8e0..e99ba3dff25 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -328,6 +340,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -507,7 +545,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 037cb06975e55da7b9e58781c9ef1e8f9f238bc4b5241202812884ee76667b93 + checksum/configmap: caa2a0302f1e88d2104c1d27fbe82c37496c4acc43201519bc538c479fb74fc5 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml index 4a47e386b21..0af08663aad 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +316,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +521,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: dadd4d46b94263a3545e2e0cbfa77a848648dd0b35259ce52d79c7a9c46fdd81 + checksum/configmap: a0db58c4b90f0e339fa95969c990c6dbb3116e0625ad0b5b807f311ad83fe79e spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/mgmt-agent/pkg/controller/podwatcher.go b/mgmt-agent/pkg/controller/podwatcher.go index 3c308346c2d..ee700ed251d 100644 --- a/mgmt-agent/pkg/controller/podwatcher.go +++ b/mgmt-agent/pkg/controller/podwatcher.go @@ -154,12 +154,16 @@ func containerStateType(state corev1.ContainerState) string { } // logPodEvent logs a pod event with structured key-value pairs including -// the full pod object. +// the full pod object. A shallow copy is made so that setting TypeMeta +// (which typed informers leave empty) does not mutate the shared cache object. func logPodEvent(eventType string, pod *corev1.Pod) { + podCopy := *pod + podCopy.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod")) klog.InfoS("pod event", + "snapshotType", "kubernetes", "event", eventType, - "namespace", pod.Namespace, - "name", pod.Name, - "object", pod, + "namespace", podCopy.Namespace, + "name", podCopy.Name, + "object", &podCopy, ) } diff --git a/mgmt-agent/pkg/controller/resourcewatcher.go b/mgmt-agent/pkg/controller/resourcewatcher.go index a699e731f1e..6c755f963af 100644 --- a/mgmt-agent/pkg/controller/resourcewatcher.go +++ b/mgmt-agent/pkg/controller/resourcewatcher.go @@ -245,6 +245,7 @@ func logResourceEvent(ctx context.Context, eventType string, gvr schema.GroupVer return } logger.Info("resource event", + "snapshotType", "kubernetes", "event", eventType, "gvr", gvr.String(), "namespace", u.GetNamespace(), diff --git a/observability/arobit/deploy/templates/forwarder-configmap.yaml b/observability/arobit/deploy/templates/forwarder-configmap.yaml index d05f64ebcd4..7dac57b2296 100644 --- a/observability/arobit/deploy/templates/forwarder-configmap.yaml +++ b/observability/arobit/deploy/templates/forwarder-configmap.yaml @@ -191,6 +191,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -198,11 +199,23 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + {{- if and .Values.forwarder.mdsd.enabled (eq .Values.forwarder.clusterType "mgmt") }} filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation @@ -448,6 +461,25 @@ data: {{- if .Values.forwarder.kusto.buffering }} {{- template "buffer_settings" . }} {{- end }} + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id {{ .Values.forwarder.tenantId }} + client_id {{ .Values.forwarder.msiClientId }} + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint {{ .Values.forwarder.kusto.ingestionEndpoint }} + Database_Name {{ .Values.forwarder.kusto.serviceLogsDatabase }} + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "{{ .Values.forwarder.kusto.buffer_dir }}" +{{- if .Values.forwarder.kusto.buffering }} + {{- template "buffer_settings" . }} +{{- end }} + [OUTPUT] Match kubernetes.* Name azure_kusto diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml index 23f2b3f1976..6f90a508ea1 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -263,6 +275,22 @@ data: ingestion_mapping_reference ingestionMapping azure_kusto_buffer_key clustersServiceLogs buffer_dir "/var/kusto" + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -412,7 +440,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: cedbb2c4a08219c8f60e68876cf5b2a1312059e19ae296f2f2b302eabb1e3cd0 + checksum/configmap: 5110c975037c707e5700f48c2690edee0842247750d67d20c7dfd8a47d813158 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml index fde7d6221c5..372cd1faabe 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml @@ -184,6 +184,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -191,11 +192,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -303,6 +315,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -482,7 +520,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 2dbcfcb8ccec730e935c42c2cc3a4972ce31fec6b50ebf7b3bb38a4de9cd1a5b + checksum/configmap: a2162ef96adde241c8649122b89119ed9cdb3b1adeeb9b3cc3fb2aa7a1a5e2c1 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml index a44bf666629..0244055f4ce 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +316,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +521,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: c96b02b3aaaf6ea5f610e506ef371b49d48d71b9d4995a17a9aeb919ccff2222 + checksum/configmap: 6c54a3ec7bae8329cca6f3ced7f992e4e57e923c46f8daaa5430fd91795276c6 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml index 4bc9d7dc364..775d8956778 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml @@ -187,6 +187,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -194,11 +195,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation [FILTER] @@ -475,6 +487,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -695,7 +733,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: fb6352469086fc4a644ff8c675f4491c595eab0b0989c6685aa59f396abbbe53 + checksum/configmap: 904e073efc29553c1d7b81201642bd9a3471e2ad605f7fe218905c8bb7b1dc3e spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml index c4a66d75593..54d4952f3e3 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,22 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +316,32 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +521,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: ea9b20dc6ce649a7e19a3731ea1c175f3a3065a1b912da5fef263e15dc15f53e + checksum/configmap: 2d90355f4be9445e67a511900b91b684b4628a4823f41d947f97fff41a886ee0 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' From f6387a0c403cb6c346554c17d03c99e773ff74d8 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Sun, 12 Jul 2026 09:05:04 -0600 Subject: [PATCH 2/2] observability: add cosmosResourceSnapshots Kusto table Add a dedicated Kusto table for Cosmos DB resource snapshots emitted by the backend's data-dump controllers. Each dump site now emits a 'snapshotType' field (value 'cosmos') and an 'objectMetadata' struct containing per-document identity: cosmosContainer, subscriptionID, resourceGroup, resourceType, resourceName, and resourceID. Fluent Bit routes logs where snapshotType is 'cosmos' from both the backend and frontend containers to the new table, while preserving all existing log flows unchanged. The cosmosContainer field distinguishes document sources: 'resources', 'operations', 'billing', 'kubeApplier', and 'fleet'. Signed-off-by: Steve Kuznetsov --- .../dump_management_cluster.go | 3 ++ .../dump_subscription_non_cluster.go | 3 ++ .../modules/logs/kusto/main.bicep | 1 + .../kusto/tables/cosmosResourceSnapshots.kql | 30 +++++++++++ ...elmTemplate_dev_westus3_mgmt_1_arobit.yaml | 49 +++++++++++++++++- ...HelmTemplate_dev_westus3_svc_1_arobit.yaml | 49 +++++++++++++++++- internal/serverutils/dump_data.go | 50 +++++++++++++++++++ .../deploy/templates/forwarder-configmap.yaml | 39 +++++++++++++++ ...plate_helmtest_buffering_disabled_svc.yaml | 39 ++++++++++++++- ...mTemplate_helmtest_kusto_disabled_svc.yaml | 49 +++++++++++++++++- ...e_helmtest_kusto_unlimited_memory_svc.yaml | 49 +++++++++++++++++- ..._helmtest_mdsd_and_kusto_enabled_mgmt.yaml | 49 +++++++++++++++++- ...e_helmtest_mdsd_and_kusto_enabled_svc.yaml | 49 +++++++++++++++++- 13 files changed, 452 insertions(+), 7 deletions(-) create mode 100644 dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql diff --git a/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go b/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go index 4878bd98942..43ee81e7fe8 100644 --- a/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go +++ b/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-HCP/internal/database" dbinformers "github.com/Azure/ARO-HCP/internal/database/informers" dblisters "github.com/Azure/ARO-HCP/internal/database/listers" + "github.com/Azure/ARO-HCP/internal/serverutils" "github.com/Azure/ARO-HCP/internal/utils" ) @@ -69,7 +70,9 @@ func (c *managementClusterDataDump) SyncOnce(ctx context.Context, key controller } logger.Info(fmt.Sprintf("dumping resourceID %v", mc.CosmosMetadata.ResourceID), + "snapshotType", "cosmos", "currentResourceID", mc.CosmosMetadata.ResourceID.String(), + "objectMetadata", serverutils.ObjectMetadataForResourceID("fleet", mc.CosmosMetadata.ResourceID), "content", mc, ) diff --git a/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go b/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go index 44e90043301..20235d27582 100644 --- a/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go +++ b/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-HCP/backend/pkg/listers" controllerutil "github.com/Azure/ARO-HCP/internal/controllerutils" "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/serverutils" "github.com/Azure/ARO-HCP/internal/utils" ) @@ -75,7 +76,9 @@ func (c *subscriptionNonClusterDataDump) SyncOnce(ctx context.Context, key contr } logger.Info(fmt.Sprintf("dumping resourceID %v", key.GetResourceID()), + "snapshotType", "cosmos", "currentResourceID", key.GetResourceID().String(), + "objectMetadata", serverutils.ObjectMetadataForResourceID("resources", key.GetResourceID()), "content", subscription, ) diff --git a/dev-infrastructure/modules/logs/kusto/main.bicep b/dev-infrastructure/modules/logs/kusto/main.bicep index 030fc600cc7..ec15614766e 100644 --- a/dev-infrastructure/modules/logs/kusto/main.bicep +++ b/dev-infrastructure/modules/logs/kusto/main.bicep @@ -60,6 +60,7 @@ var allServiceLogsTablesKQL = { aksEvents: loadTextContent('tables/aksEvents.kql') systemdLogs: loadTextContent('tables/systemdLogs.kql') resourceSnapshots: loadTextContent('tables/kubernetesResourceSnapshots.kql') + cosmosResourceSnapshots: loadTextContent('tables/cosmosResourceSnapshots.kql') } var allCustomerLogsTablesKQL = { diff --git a/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql b/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql new file mode 100644 index 00000000000..9e37ba272b4 --- /dev/null +++ b/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql @@ -0,0 +1,30 @@ +.create-merge table cosmosResourceSnapshots ( + timestamp: datetime, + environment: string, + region: string, + cluster: string, + cosmosContainer: string, + subscriptionID: string, + resourceGroup: string, + resourceType: string, + resourceName: string, + resourceID: string, + content: dynamic +) + +.create-or-alter table cosmosResourceSnapshots ingestion json mapping 'ingestionMapping' +``` +[ + {"column":"timestamp","Properties":{"path":"$.timestamp"}}, + {"column":"environment","Properties":{"path":"$.log.environment"}}, + {"column":"region","Properties":{"path":"$.log.region"}}, + {"column":"cluster","Properties":{"path":"$.log.cluster"}}, + {"column":"cosmosContainer","Properties":{"path":"$.log.objectMetadata.cosmosContainer"}}, + {"column":"subscriptionID","Properties":{"path":"$.log.objectMetadata.subscriptionID"}}, + {"column":"resourceGroup","Properties":{"path":"$.log.objectMetadata.resourceGroup"}}, + {"column":"resourceType","Properties":{"path":"$.log.objectMetadata.resourceType"}}, + {"column":"resourceName","Properties":{"path":"$.log.objectMetadata.resourceName"}}, + {"column":"resourceID","Properties":{"path":"$.log.objectMetadata.resourceID"}}, + {"column":"content","Properties":{"path":"$.log.content"}} +] +``` diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml index e99ba3dff25..022f467b6ba 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml @@ -209,6 +209,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -366,6 +388,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -545,7 +592,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: caa2a0302f1e88d2104c1d27fbe82c37496c4acc43201519bc538c479fb74fc5 + checksum/configmap: 7e5331bafe7c892ba671104e27e8b78ecbd570950ed30b691c598c3337f1f404 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml index 0af08663aad..aad10331956 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml @@ -209,6 +209,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -342,6 +364,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -521,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: a0db58c4b90f0e339fa95969c990c6dbb3116e0625ad0b5b807f311ad83fe79e + checksum/configmap: 7d62e084ce38da5386f0334e7c207089acd67818e8cabab1a83c3e5894cdc1fa spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/internal/serverutils/dump_data.go b/internal/serverutils/dump_data.go index dc8ef8ecb54..484a3965c37 100644 --- a/internal/serverutils/dump_data.go +++ b/internal/serverutils/dump_data.go @@ -31,6 +31,46 @@ import ( const RedactStr = "REDACTED" +// ObjectMetadata provides per-document identity for the cosmosResourceSnapshots +// Kusto table. It is emitted as a structured log field alongside the document content. +type ObjectMetadata struct { + CosmosContainer string `json:"cosmosContainer"` + SubscriptionID string `json:"subscriptionID"` + ResourceGroup string `json:"resourceGroup"` + ResourceType string `json:"resourceType"` + ResourceName string `json:"resourceName"` + ResourceID string `json:"resourceID"` +} + +func objectMetadataForTypedDocument(container string, doc *database.TypedDocument) ObjectMetadata { + if doc == nil || doc.ResourceID == nil { + return ObjectMetadata{CosmosContainer: container} + } + return ObjectMetadata{ + CosmosContainer: container, + SubscriptionID: doc.ResourceID.SubscriptionID, + ResourceGroup: doc.ResourceID.ResourceGroupName, + ResourceType: doc.ResourceType, + ResourceName: doc.ResourceID.Name, + ResourceID: doc.ResourceID.String(), + } +} + +// ObjectMetadataForResourceID builds ObjectMetadata from an ARM resource ID. +func ObjectMetadataForResourceID(container string, resourceID *azcorearm.ResourceID) ObjectMetadata { + if resourceID == nil { + return ObjectMetadata{CosmosContainer: container} + } + return ObjectMetadata{ + CosmosContainer: container, + SubscriptionID: resourceID.SubscriptionID, + ResourceGroup: resourceID.ResourceGroupName, + ResourceType: resourceID.ResourceType.String(), + ResourceName: resourceID.Name, + ResourceID: resourceID.String(), + } +} + // DumpDataToLogger writes a structured-log entry for every document related // to resourceID. It covers three storage layers: // @@ -71,7 +111,9 @@ func DumpDataToLogger( return utils.TrackError(err) } logger.Info(fmt.Sprintf("dumping resourceID %v", startingCosmosRecord.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(startingCosmosRecord.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("resources", startingCosmosRecord), "content", startingCosmosRecord, ) @@ -87,7 +129,9 @@ func DumpDataToLogger( continue } logger.Info(fmt.Sprintf("dumping resourceID %v", typedDocument.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(typedDocument.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("resources", typedDocument), "content", typedDocument, ) } @@ -105,7 +149,9 @@ func DumpDataToLogger( currOperationTarget := strings.ToLower(operation.ExternalID.String()) if strings.HasPrefix(currOperationTarget, resourceIDString) { logger.Info(fmt.Sprintf("dumping resourceID %v", operation.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(operation.ResourceID), + "objectMetadata", ObjectMetadataForResourceID("operations", operation.ResourceID), "content", operation, ) } @@ -174,7 +220,9 @@ func dumpKubeApplierData( } for _, doc := range desireIterator.Items(ctx) { mcLogger.Info(fmt.Sprintf("dumping kube-applier resourceID %v", doc.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(doc.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("kubeApplier", doc), "content", doc, ) } @@ -220,7 +268,9 @@ func DumpBillingToLogger(ctx context.Context, resourcesDBClient database.Resourc } logger.Info(fmt.Sprintf("dumping billing document for resourceID %v", billingDoc.ResourceID), + "snapshotType", "cosmos", "currentResourceID", billingDoc.ResourceID.String(), + "objectMetadata", ObjectMetadataForResourceID("billing", billingDoc.ResourceID), "content", billingDoc, ) diff --git a/observability/arobit/deploy/templates/forwarder-configmap.yaml b/observability/arobit/deploy/templates/forwarder-configmap.yaml index 7dac57b2296..0fb16e030f5 100644 --- a/observability/arobit/deploy/templates/forwarder-configmap.yaml +++ b/observability/arobit/deploy/templates/forwarder-configmap.yaml @@ -216,6 +216,27 @@ data: Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem {{- if and .Values.forwarder.mdsd.enabled (eq .Values.forwarder.clusterType "mgmt") }} filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation @@ -480,6 +501,24 @@ data: {{- template "buffer_settings" . }} {{- end }} + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id {{ .Values.forwarder.tenantId }} + client_id {{ .Values.forwarder.msiClientId }} + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint {{ .Values.forwarder.kusto.ingestionEndpoint }} + Database_Name {{ .Values.forwarder.kusto.serviceLogsDatabase }} + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "{{ .Values.forwarder.kusto.buffer_dir }}" +{{- if .Values.forwarder.kusto.buffering }} + {{- template "buffer_settings" . }} +{{- end }} + [OUTPUT] Match kubernetes.* Name azure_kusto diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml index 6f90a508ea1..1a6d6e1cab0 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml @@ -209,6 +209,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -291,6 +313,21 @@ data: azure_kusto_buffer_key resourceSnapshots buffer_dir "/var/kusto" + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -440,7 +477,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 5110c975037c707e5700f48c2690edee0842247750d67d20c7dfd8a47d813158 + checksum/configmap: fc8ecf2d00f28090a8f57c5943fc8f7055bc991f55c268e65aba5fe772259349 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml index 372cd1faabe..78d45cceea9 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml @@ -208,6 +208,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -341,6 +363,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -520,7 +567,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: a2162ef96adde241c8649122b89119ed9cdb3b1adeeb9b3cc3fb2aa7a1a5e2c1 + checksum/configmap: cc0b6dbe4578bdcf3109961a5201933c5110a126d198bec409c6b17389f17d67 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml index 0244055f4ce..f7958a0645c 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml @@ -209,6 +209,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -342,6 +364,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -521,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 6c54a3ec7bae8329cca6f3ced7f992e4e57e923c46f8daaa5430fd91795276c6 + checksum/configmap: f0d28b45ea337e1efbc7dbf1c8a737e99967001d684cd97645001ef5e0d8b48a spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml index 775d8956778..34093563604 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml @@ -211,6 +211,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation [FILTER] @@ -513,6 +535,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -733,7 +780,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 904e073efc29553c1d7b81201642bd9a3471e2ad605f7fe218905c8bb7b1dc3e + checksum/configmap: 8a5e701d3d44ef5eea580e724c2b4543bcb94fccf28527f5e2e078d7c7a2d65c spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml index 54d4952f3e3..994c550ba23 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml @@ -209,6 +209,28 @@ data: Emitter_Name re_emitted_resource_snapshot_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -342,6 +364,31 @@ data: delete_on_max_upload_error true io_timeout 60s + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -521,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 2d90355f4be9445e67a511900b91b684b4628a4823f41d947f97fff41a886ee0 + checksum/configmap: 9e17b716826b865057dd85af96718d04123d41e1428f52bde37ba5093820d60e spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder'