diff --git a/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go b/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go index 4878bd98942..43ee81e7fe8 100644 --- a/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go +++ b/backend/pkg/controllers/datadumpcontrollers/dump_management_cluster.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-HCP/internal/database" dbinformers "github.com/Azure/ARO-HCP/internal/database/informers" dblisters "github.com/Azure/ARO-HCP/internal/database/listers" + "github.com/Azure/ARO-HCP/internal/serverutils" "github.com/Azure/ARO-HCP/internal/utils" ) @@ -69,7 +70,9 @@ func (c *managementClusterDataDump) SyncOnce(ctx context.Context, key controller } logger.Info(fmt.Sprintf("dumping resourceID %v", mc.CosmosMetadata.ResourceID), + "snapshotType", "cosmos", "currentResourceID", mc.CosmosMetadata.ResourceID.String(), + "objectMetadata", serverutils.ObjectMetadataForResourceID("fleet", mc.CosmosMetadata.ResourceID), "content", mc, ) diff --git a/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go b/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go index 44e90043301..20235d27582 100644 --- a/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go +++ b/backend/pkg/controllers/datadumpcontrollers/dump_subscription_non_cluster.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-HCP/backend/pkg/listers" controllerutil "github.com/Azure/ARO-HCP/internal/controllerutils" "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/serverutils" "github.com/Azure/ARO-HCP/internal/utils" ) @@ -75,7 +76,9 @@ func (c *subscriptionNonClusterDataDump) SyncOnce(ctx context.Context, key contr } logger.Info(fmt.Sprintf("dumping resourceID %v", key.GetResourceID()), + "snapshotType", "cosmos", "currentResourceID", key.GetResourceID().String(), + "objectMetadata", serverutils.ObjectMetadataForResourceID("resources", key.GetResourceID()), "content", subscription, ) diff --git a/dev-infrastructure/modules/logs/kusto/main.bicep b/dev-infrastructure/modules/logs/kusto/main.bicep index d0fbae85596..ec15614766e 100644 --- a/dev-infrastructure/modules/logs/kusto/main.bicep +++ b/dev-infrastructure/modules/logs/kusto/main.bicep @@ -59,6 +59,8 @@ var allServiceLogsTablesKQL = { kubernetesEvents: loadTextContent('tables/kubernetesEvents.kql') aksEvents: loadTextContent('tables/aksEvents.kql') systemdLogs: loadTextContent('tables/systemdLogs.kql') + resourceSnapshots: loadTextContent('tables/kubernetesResourceSnapshots.kql') + cosmosResourceSnapshots: loadTextContent('tables/cosmosResourceSnapshots.kql') } var allCustomerLogsTablesKQL = { diff --git a/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql b/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql new file mode 100644 index 00000000000..9e37ba272b4 --- /dev/null +++ b/dev-infrastructure/modules/logs/kusto/tables/cosmosResourceSnapshots.kql @@ -0,0 +1,30 @@ +.create-merge table cosmosResourceSnapshots ( + timestamp: datetime, + environment: string, + region: string, + cluster: string, + cosmosContainer: string, + subscriptionID: string, + resourceGroup: string, + resourceType: string, + resourceName: string, + resourceID: string, + content: dynamic +) + +.create-or-alter table cosmosResourceSnapshots ingestion json mapping 'ingestionMapping' +``` +[ + {"column":"timestamp","Properties":{"path":"$.timestamp"}}, + {"column":"environment","Properties":{"path":"$.log.environment"}}, + {"column":"region","Properties":{"path":"$.log.region"}}, + {"column":"cluster","Properties":{"path":"$.log.cluster"}}, + {"column":"cosmosContainer","Properties":{"path":"$.log.objectMetadata.cosmosContainer"}}, + {"column":"subscriptionID","Properties":{"path":"$.log.objectMetadata.subscriptionID"}}, + {"column":"resourceGroup","Properties":{"path":"$.log.objectMetadata.resourceGroup"}}, + {"column":"resourceType","Properties":{"path":"$.log.objectMetadata.resourceType"}}, + {"column":"resourceName","Properties":{"path":"$.log.objectMetadata.resourceName"}}, + {"column":"resourceID","Properties":{"path":"$.log.objectMetadata.resourceID"}}, + {"column":"content","Properties":{"path":"$.log.content"}} +] +``` diff --git a/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql b/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql new file mode 100644 index 00000000000..46388195929 --- /dev/null +++ b/dev-infrastructure/modules/logs/kusto/tables/kubernetesResourceSnapshots.kql @@ -0,0 +1,30 @@ +.create-merge table kubernetesResourceSnapshots ( + timestamp: datetime, + environment: string, + region: string, + cluster: string, + event: string, + apiVersion: string, + objectKind: string, + namespace: string, + name: string, + uid: string, + object: dynamic +) + +.create-or-alter table kubernetesResourceSnapshots ingestion json mapping 'ingestionMapping' +``` +[ + {"column":"timestamp","Properties":{"path":"$.timestamp"}}, + {"column":"environment","Properties":{"path":"$.log.environment"}}, + {"column":"region","Properties":{"path":"$.log.region"}}, + {"column":"cluster","Properties":{"path":"$.log.cluster"}}, + {"column":"event","Properties":{"path":"$.log.event"}}, + {"column":"apiVersion","Properties":{"path":"$.log.object.apiVersion"}}, + {"column":"objectKind","Properties":{"path":"$.log.object.kind"}}, + {"column":"namespace","Properties":{"path":"$.log.namespace"}}, + {"column":"name","Properties":{"path":"$.log.name"}}, + {"column":"uid","Properties":{"path":"$.log.object.metadata.uid"}}, + {"column":"object","Properties":{"path":"$.log.object"}} +] +``` diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml index cf8d188e8e0..022f467b6ba 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_arobit.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -328,6 +362,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -507,7 +592,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 037cb06975e55da7b9e58781c9ef1e8f9f238bc4b5241202812884ee76667b93 + checksum/configmap: 7e5331bafe7c892ba671104e27e8b78ecbd570950ed30b691c598c3337f1f404 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml index 4a47e386b21..aad10331956 100644 --- a/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml +++ b/dev-infrastructure/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_arobit.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +338,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: dadd4d46b94263a3545e2e0cbfa77a848648dd0b35259ce52d79c7a9c46fdd81 + checksum/configmap: 7d62e084ce38da5386f0334e7c207089acd67818e8cabab1a83c3e5894cdc1fa spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/internal/serverutils/dump_data.go b/internal/serverutils/dump_data.go index dc8ef8ecb54..484a3965c37 100644 --- a/internal/serverutils/dump_data.go +++ b/internal/serverutils/dump_data.go @@ -31,6 +31,46 @@ import ( const RedactStr = "REDACTED" +// ObjectMetadata provides per-document identity for the cosmosResourceSnapshots +// Kusto table. It is emitted as a structured log field alongside the document content. +type ObjectMetadata struct { + CosmosContainer string `json:"cosmosContainer"` + SubscriptionID string `json:"subscriptionID"` + ResourceGroup string `json:"resourceGroup"` + ResourceType string `json:"resourceType"` + ResourceName string `json:"resourceName"` + ResourceID string `json:"resourceID"` +} + +func objectMetadataForTypedDocument(container string, doc *database.TypedDocument) ObjectMetadata { + if doc == nil || doc.ResourceID == nil { + return ObjectMetadata{CosmosContainer: container} + } + return ObjectMetadata{ + CosmosContainer: container, + SubscriptionID: doc.ResourceID.SubscriptionID, + ResourceGroup: doc.ResourceID.ResourceGroupName, + ResourceType: doc.ResourceType, + ResourceName: doc.ResourceID.Name, + ResourceID: doc.ResourceID.String(), + } +} + +// ObjectMetadataForResourceID builds ObjectMetadata from an ARM resource ID. +func ObjectMetadataForResourceID(container string, resourceID *azcorearm.ResourceID) ObjectMetadata { + if resourceID == nil { + return ObjectMetadata{CosmosContainer: container} + } + return ObjectMetadata{ + CosmosContainer: container, + SubscriptionID: resourceID.SubscriptionID, + ResourceGroup: resourceID.ResourceGroupName, + ResourceType: resourceID.ResourceType.String(), + ResourceName: resourceID.Name, + ResourceID: resourceID.String(), + } +} + // DumpDataToLogger writes a structured-log entry for every document related // to resourceID. It covers three storage layers: // @@ -71,7 +111,9 @@ func DumpDataToLogger( return utils.TrackError(err) } logger.Info(fmt.Sprintf("dumping resourceID %v", startingCosmosRecord.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(startingCosmosRecord.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("resources", startingCosmosRecord), "content", startingCosmosRecord, ) @@ -87,7 +129,9 @@ func DumpDataToLogger( continue } logger.Info(fmt.Sprintf("dumping resourceID %v", typedDocument.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(typedDocument.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("resources", typedDocument), "content", typedDocument, ) } @@ -105,7 +149,9 @@ func DumpDataToLogger( currOperationTarget := strings.ToLower(operation.ExternalID.String()) if strings.HasPrefix(currOperationTarget, resourceIDString) { logger.Info(fmt.Sprintf("dumping resourceID %v", operation.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(operation.ResourceID), + "objectMetadata", ObjectMetadataForResourceID("operations", operation.ResourceID), "content", operation, ) } @@ -174,7 +220,9 @@ func dumpKubeApplierData( } for _, doc := range desireIterator.Items(ctx) { mcLogger.Info(fmt.Sprintf("dumping kube-applier resourceID %v", doc.ResourceID), + "snapshotType", "cosmos", "currentResourceID", resourceIDToString(doc.ResourceID), + "objectMetadata", objectMetadataForTypedDocument("kubeApplier", doc), "content", doc, ) } @@ -220,7 +268,9 @@ func DumpBillingToLogger(ctx context.Context, resourcesDBClient database.Resourc } logger.Info(fmt.Sprintf("dumping billing document for resourceID %v", billingDoc.ResourceID), + "snapshotType", "cosmos", "currentResourceID", billingDoc.ResourceID.String(), + "objectMetadata", ObjectMetadataForResourceID("billing", billingDoc.ResourceID), "content", billingDoc, ) diff --git a/mgmt-agent/pkg/controller/podwatcher.go b/mgmt-agent/pkg/controller/podwatcher.go index 3c308346c2d..ee700ed251d 100644 --- a/mgmt-agent/pkg/controller/podwatcher.go +++ b/mgmt-agent/pkg/controller/podwatcher.go @@ -154,12 +154,16 @@ func containerStateType(state corev1.ContainerState) string { } // logPodEvent logs a pod event with structured key-value pairs including -// the full pod object. +// the full pod object. A shallow copy is made so that setting TypeMeta +// (which typed informers leave empty) does not mutate the shared cache object. func logPodEvent(eventType string, pod *corev1.Pod) { + podCopy := *pod + podCopy.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod")) klog.InfoS("pod event", + "snapshotType", "kubernetes", "event", eventType, - "namespace", pod.Namespace, - "name", pod.Name, - "object", pod, + "namespace", podCopy.Namespace, + "name", podCopy.Name, + "object", &podCopy, ) } diff --git a/mgmt-agent/pkg/controller/resourcewatcher.go b/mgmt-agent/pkg/controller/resourcewatcher.go index a699e731f1e..6c755f963af 100644 --- a/mgmt-agent/pkg/controller/resourcewatcher.go +++ b/mgmt-agent/pkg/controller/resourcewatcher.go @@ -245,6 +245,7 @@ func logResourceEvent(ctx context.Context, eventType string, gvr schema.GroupVer return } logger.Info("resource event", + "snapshotType", "kubernetes", "event", eventType, "gvr", gvr.String(), "namespace", u.GetNamespace(), diff --git a/observability/arobit/deploy/templates/forwarder-configmap.yaml b/observability/arobit/deploy/templates/forwarder-configmap.yaml index d05f64ebcd4..0fb16e030f5 100644 --- a/observability/arobit/deploy/templates/forwarder-configmap.yaml +++ b/observability/arobit/deploy/templates/forwarder-configmap.yaml @@ -191,6 +191,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -198,11 +199,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem {{- if and .Values.forwarder.mdsd.enabled (eq .Values.forwarder.clusterType "mgmt") }} filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation @@ -448,6 +482,43 @@ data: {{- if .Values.forwarder.kusto.buffering }} {{- template "buffer_settings" . }} {{- end }} + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id {{ .Values.forwarder.tenantId }} + client_id {{ .Values.forwarder.msiClientId }} + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint {{ .Values.forwarder.kusto.ingestionEndpoint }} + Database_Name {{ .Values.forwarder.kusto.serviceLogsDatabase }} + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "{{ .Values.forwarder.kusto.buffer_dir }}" +{{- if .Values.forwarder.kusto.buffering }} + {{- template "buffer_settings" . }} +{{- end }} + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id {{ .Values.forwarder.tenantId }} + client_id {{ .Values.forwarder.msiClientId }} + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint {{ .Values.forwarder.kusto.ingestionEndpoint }} + Database_Name {{ .Values.forwarder.kusto.serviceLogsDatabase }} + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "{{ .Values.forwarder.kusto.buffer_dir }}" +{{- if .Values.forwarder.kusto.buffering }} + {{- template "buffer_settings" . }} +{{- end }} + [OUTPUT] Match kubernetes.* Name azure_kusto diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml index 23f2b3f1976..1a6d6e1cab0 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_buffering_disabled_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -263,6 +297,37 @@ data: ingestion_mapping_reference ingestionMapping azure_kusto_buffer_key clustersServiceLogs buffer_dir "/var/kusto" + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -412,7 +477,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: cedbb2c4a08219c8f60e68876cf5b2a1312059e19ae296f2f2b302eabb1e3cd0 + checksum/configmap: fc8ecf2d00f28090a8f57c5943fc8f7055bc991f55c268e65aba5fe772259349 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml index fde7d6221c5..78d45cceea9 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_disabled_svc.yaml @@ -184,6 +184,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -191,11 +192,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -303,6 +337,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -482,7 +567,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: 2dbcfcb8ccec730e935c42c2cc3a4972ce31fec6b50ebf7b3bb38a4de9cd1a5b + checksum/configmap: cc0b6dbe4578bdcf3109961a5201933c5110a126d198bec409c6b17389f17d67 spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml index a44bf666629..f7958a0645c 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_kusto_unlimited_memory_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +338,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: c96b02b3aaaf6ea5f610e506ef371b49d48d71b9d4995a17a9aeb919ccff2222 + checksum/configmap: f0d28b45ea337e1efbc7dbf1c8a737e99967001d684cd97645001ef5e0d8b48a spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml index 4bc9d7dc364..34093563604 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_mgmt.yaml @@ -187,6 +187,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -194,11 +195,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem filter-shoebox.conf: | # Route control plane logs from OCM namespaces for Shoebox transformation [FILTER] @@ -475,6 +509,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -695,7 +780,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: fb6352469086fc4a644ff8c675f4491c595eab0b0989c6685aa59f396abbbe53 + checksum/configmap: 8a5e701d3d44ef5eea580e724c2b4543bcb94fccf28527f5e2e078d7c7a2d65c spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder' diff --git a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml index c4a66d75593..994c550ba23 100644 --- a/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml +++ b/observability/arobit/testdata/zz_fixture_TestHelmTemplate_helmtest_mdsd_and_kusto_enabled_svc.yaml @@ -185,6 +185,7 @@ data: Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true + Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true Emitter_Name re_emitted_container_name_router Emitter_Mem_Buf_Limit 20M Emitter_Storage.type filesystem @@ -192,11 +193,44 @@ data: [FILTER] # Parse the log key and make it a json object again under $ Name Parser - Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs + Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs Key_Name log Parser containerlogs Preserve_Key On Reserve_Data On + + [FILTER] + # Route mgmt-agent logs with snapshotType=kubernetes + # to a dedicated tag for the kubernetesResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.resource_snapshot_router + Match mgmt-agent.logs + Rule $snapshotType ^kubernetes$ resource-snapshots.logs true + Emitter_Name re_emitted_resource_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route backend logs with snapshotType=cosmos + # to a dedicated tag for the cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router + Match aro-hcp-backend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem + + [FILTER] + # Route frontend logs with snapshotType=cosmos + # to the same cosmosResourceSnapshots Kusto table. + Name rewrite_tag + Alias filter.cosmos_snapshot_router_frontend + Match aro-hcp-frontend.logs + Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true + Emitter_Name re_emitted_cosmos_snapshot_router_frontend + Emitter_Mem_Buf_Limit 20M + Emitter_Storage.type filesystem output.conf: | [OUTPUT] Name prometheus_exporter @@ -304,6 +338,57 @@ data: scheduler_max_retries 9 delete_on_max_upload_error true io_timeout 60s + + [OUTPUT] + Match resource-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name kubernetesResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key resourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + + [OUTPUT] + Match cosmos-snapshots.logs + Name azure_kusto + Retry_Limit 5 + auth_type workload_identity + tenant_id __tenantId__ + client_id __msiClientId__ + workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE} + Ingestion_Endpoint __kustoDataIngestionUri__ + Database_Name ServiceLogs + Table_Name cosmosResourceSnapshots + ingestion_mapping_reference ingestionMapping + azure_kusto_buffer_key cosmosResourceSnapshots + buffer_dir "/var/kusto" + buffering_enabled true + upload_timeout 1m + upload_file_size 16M + buffer_file_delete_early false + unify_tag true + store_dir_limit_size 1.8GB + blob_uri_length 128 + scheduler_max_retries 9 + delete_on_max_upload_error true + io_timeout 60s + [OUTPUT] Match kubernetes.* Name azure_kusto @@ -483,7 +568,7 @@ spec: app: arobit-forwarder azure.workload.identity/use: "true" annotations: - checksum/configmap: ea9b20dc6ce649a7e19a3731ea1c175f3a3065a1b912da5fef263e15dc15f53e + checksum/configmap: 9e17b716826b865057dd85af96718d04123d41e1428f52bde37ba5093820d60e spec: priorityClassName: service-lifecycle-critical serviceAccountName: 'arobit-forwarder'