Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/Azure/ARO-HCP/internal/database"
dbinformers "github.com/Azure/ARO-HCP/internal/database/informers"
dblisters "github.com/Azure/ARO-HCP/internal/database/listers"
"github.com/Azure/ARO-HCP/internal/serverutils"
"github.com/Azure/ARO-HCP/internal/utils"
)

Expand Down Expand Up @@ -69,7 +70,9 @@ func (c *managementClusterDataDump) SyncOnce(ctx context.Context, key controller
}

logger.Info(fmt.Sprintf("dumping resourceID %v", mc.CosmosMetadata.ResourceID),
"snapshotType", "cosmos",
"currentResourceID", mc.CosmosMetadata.ResourceID.String(),
"objectMetadata", serverutils.ObjectMetadataForResourceID("fleet", mc.CosmosMetadata.ResourceID),
"content", mc,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/Azure/ARO-HCP/backend/pkg/listers"
controllerutil "github.com/Azure/ARO-HCP/internal/controllerutils"
"github.com/Azure/ARO-HCP/internal/database"
"github.com/Azure/ARO-HCP/internal/serverutils"
"github.com/Azure/ARO-HCP/internal/utils"
)

Expand Down Expand Up @@ -75,7 +76,9 @@ func (c *subscriptionNonClusterDataDump) SyncOnce(ctx context.Context, key contr
}

logger.Info(fmt.Sprintf("dumping resourceID %v", key.GetResourceID()),
"snapshotType", "cosmos",
"currentResourceID", key.GetResourceID().String(),
"objectMetadata", serverutils.ObjectMetadataForResourceID("resources", key.GetResourceID()),
"content", subscription,
)

Expand Down
2 changes: 2 additions & 0 deletions dev-infrastructure/modules/logs/kusto/main.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ var allServiceLogsTablesKQL = {
kubernetesEvents: loadTextContent('tables/kubernetesEvents.kql')
aksEvents: loadTextContent('tables/aksEvents.kql')
systemdLogs: loadTextContent('tables/systemdLogs.kql')
resourceSnapshots: loadTextContent('tables/kubernetesResourceSnapshots.kql')
cosmosResourceSnapshots: loadTextContent('tables/cosmosResourceSnapshots.kql')
}

var allCustomerLogsTablesKQL = {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
.create-merge table cosmosResourceSnapshots (
timestamp: datetime,
environment: string,
region: string,
cluster: string,
cosmosContainer: string,
subscriptionID: string,
resourceGroup: string,
resourceType: string,
resourceName: string,
resourceID: string,
content: dynamic
)

.create-or-alter table cosmosResourceSnapshots ingestion json mapping 'ingestionMapping'
```
[
{"column":"timestamp","Properties":{"path":"$.timestamp"}},
{"column":"environment","Properties":{"path":"$.log.environment"}},
{"column":"region","Properties":{"path":"$.log.region"}},
{"column":"cluster","Properties":{"path":"$.log.cluster"}},
{"column":"cosmosContainer","Properties":{"path":"$.log.objectMetadata.cosmosContainer"}},
{"column":"subscriptionID","Properties":{"path":"$.log.objectMetadata.subscriptionID"}},
{"column":"resourceGroup","Properties":{"path":"$.log.objectMetadata.resourceGroup"}},
{"column":"resourceType","Properties":{"path":"$.log.objectMetadata.resourceType"}},
{"column":"resourceName","Properties":{"path":"$.log.objectMetadata.resourceName"}},
{"column":"resourceID","Properties":{"path":"$.log.objectMetadata.resourceID"}},
{"column":"content","Properties":{"path":"$.log.content"}}
]
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
.create-merge table kubernetesResourceSnapshots (
timestamp: datetime,
environment: string,
region: string,
cluster: string,
event: string,
apiVersion: string,
objectKind: string,
namespace: string,
Comment thread
stevekuznetsov marked this conversation as resolved.
name: string,
uid: string,
object: dynamic
)

.create-or-alter table kubernetesResourceSnapshots ingestion json mapping 'ingestionMapping'
```
[
{"column":"timestamp","Properties":{"path":"$.timestamp"}},
{"column":"environment","Properties":{"path":"$.log.environment"}},
{"column":"region","Properties":{"path":"$.log.region"}},
{"column":"cluster","Properties":{"path":"$.log.cluster"}},
{"column":"event","Properties":{"path":"$.log.event"}},
{"column":"apiVersion","Properties":{"path":"$.log.object.apiVersion"}},
{"column":"objectKind","Properties":{"path":"$.log.object.kind"}},
Comment thread
stevekuznetsov marked this conversation as resolved.
{"column":"namespace","Properties":{"path":"$.log.namespace"}},
{"column":"name","Properties":{"path":"$.log.name"}},
{"column":"uid","Properties":{"path":"$.log.object.metadata.uid"}},
{"column":"object","Properties":{"path":"$.log.object"}}
]
```
Original file line number Diff line number Diff line change
Expand Up @@ -185,18 +185,52 @@ data:
Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true
Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true
Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true
Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true
Emitter_Name re_emitted_container_name_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem
[FILTER]
# Parse the log key and make it a json object again under $
Name Parser
Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs
Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs
Key_Name log
Parser containerlogs
Preserve_Key On
Reserve_Data On
[FILTER]
# Route mgmt-agent logs with snapshotType=kubernetes
# to a dedicated tag for the kubernetesResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.resource_snapshot_router
Match mgmt-agent.logs
Rule $snapshotType ^kubernetes$ resource-snapshots.logs true
Emitter_Name re_emitted_resource_snapshot_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem
[FILTER]
# Route backend logs with snapshotType=cosmos
# to a dedicated tag for the cosmosResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.cosmos_snapshot_router
Match aro-hcp-backend.logs
Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true
Emitter_Name re_emitted_cosmos_snapshot_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem
[FILTER]
# Route frontend logs with snapshotType=cosmos
# to the same cosmosResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.cosmos_snapshot_router_frontend
Match aro-hcp-frontend.logs
Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true
Emitter_Name re_emitted_cosmos_snapshot_router_frontend
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem
output.conf: |
[OUTPUT]
Name prometheus_exporter
Expand Down Expand Up @@ -328,6 +362,57 @@ data:
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s
[OUTPUT]
Match resource-snapshots.logs
Name azure_kusto
Retry_Limit 5
auth_type workload_identity
tenant_id __tenantId__
client_id __msiClientId__
workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE}
Ingestion_Endpoint __kustoDataIngestionUri__
Database_Name ServiceLogs
Table_Name kubernetesResourceSnapshots
ingestion_mapping_reference ingestionMapping
azure_kusto_buffer_key resourceSnapshots
buffer_dir "/var/kusto"
buffering_enabled true
upload_timeout 1m
upload_file_size 16M
buffer_file_delete_early false
unify_tag true
store_dir_limit_size 1.8GB
blob_uri_length 128
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s
[OUTPUT]
Match cosmos-snapshots.logs
Name azure_kusto
Retry_Limit 5
auth_type workload_identity
tenant_id __tenantId__
client_id __msiClientId__
workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE}
Ingestion_Endpoint __kustoDataIngestionUri__
Database_Name ServiceLogs
Table_Name cosmosResourceSnapshots
ingestion_mapping_reference ingestionMapping
azure_kusto_buffer_key cosmosResourceSnapshots
buffer_dir "/var/kusto"
buffering_enabled true
upload_timeout 1m
upload_file_size 16M
buffer_file_delete_early false
unify_tag true
store_dir_limit_size 1.8GB
blob_uri_length 128
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s
[OUTPUT]
Match kubernetes.*
Name azure_kusto
Expand Down Expand Up @@ -507,7 +592,7 @@ spec:
app: arobit-forwarder
azure.workload.identity/use: "true"
annotations:
checksum/configmap: 037cb06975e55da7b9e58781c9ef1e8f9f238bc4b5241202812884ee76667b93
checksum/configmap: 7e5331bafe7c892ba671104e27e8b78ecbd570950ed30b691c598c3337f1f404
spec:
priorityClassName: service-lifecycle-critical
serviceAccountName: 'arobit-forwarder'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,18 +185,52 @@ data:
Rule $kubernetes['container_name'] ^clusters-service* clusters-service.logs true
Rule $kubernetes['container_name'] ^fleet-controller* fleet-controller.logs true
Rule $kubernetes['container_name'] ^kube-events* kube-events.logs true
Rule $kubernetes['container_name'] ^mgmt-agent-controller* mgmt-agent.logs true
Emitter_Name re_emitted_container_name_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem

[FILTER]
# Parse the log key and make it a json object again under $
Name Parser
Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller)\.logs
Match_regex (aro-hcp-(frontend|backend)|kube-events|clusters-service|fleet-controller|mgmt-agent)\.logs
Key_Name log
Parser containerlogs
Preserve_Key On
Reserve_Data On

[FILTER]
# Route mgmt-agent logs with snapshotType=kubernetes
# to a dedicated tag for the kubernetesResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.resource_snapshot_router
Match mgmt-agent.logs
Rule $snapshotType ^kubernetes$ resource-snapshots.logs true
Emitter_Name re_emitted_resource_snapshot_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem

[FILTER]
# Route backend logs with snapshotType=cosmos
# to a dedicated tag for the cosmosResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.cosmos_snapshot_router
Match aro-hcp-backend.logs
Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true
Emitter_Name re_emitted_cosmos_snapshot_router
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem

[FILTER]
# Route frontend logs with snapshotType=cosmos
# to the same cosmosResourceSnapshots Kusto table.
Name rewrite_tag
Alias filter.cosmos_snapshot_router_frontend
Match aro-hcp-frontend.logs
Rule $snapshotType ^cosmos$ cosmos-snapshots.logs true
Emitter_Name re_emitted_cosmos_snapshot_router_frontend
Emitter_Mem_Buf_Limit 20M
Emitter_Storage.type filesystem
output.conf: |
[OUTPUT]
Name prometheus_exporter
Expand Down Expand Up @@ -304,6 +338,57 @@ data:
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s

[OUTPUT]
Match resource-snapshots.logs
Name azure_kusto
Retry_Limit 5
auth_type workload_identity
tenant_id __tenantId__
client_id __msiClientId__
workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE}
Ingestion_Endpoint __kustoDataIngestionUri__
Database_Name ServiceLogs
Table_Name kubernetesResourceSnapshots
ingestion_mapping_reference ingestionMapping
azure_kusto_buffer_key resourceSnapshots
buffer_dir "/var/kusto"
buffering_enabled true
upload_timeout 1m
upload_file_size 16M
buffer_file_delete_early false
unify_tag true
store_dir_limit_size 1.8GB
blob_uri_length 128
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s

[OUTPUT]
Match cosmos-snapshots.logs
Name azure_kusto
Retry_Limit 5
auth_type workload_identity
tenant_id __tenantId__
client_id __msiClientId__
workload_identity_token_file ${AZURE_FEDERATED_TOKEN_FILE}
Ingestion_Endpoint __kustoDataIngestionUri__
Database_Name ServiceLogs
Table_Name cosmosResourceSnapshots
ingestion_mapping_reference ingestionMapping
azure_kusto_buffer_key cosmosResourceSnapshots
buffer_dir "/var/kusto"
buffering_enabled true
upload_timeout 1m
upload_file_size 16M
buffer_file_delete_early false
unify_tag true
store_dir_limit_size 1.8GB
blob_uri_length 128
scheduler_max_retries 9
delete_on_max_upload_error true
io_timeout 60s

[OUTPUT]
Match kubernetes.*
Name azure_kusto
Expand Down Expand Up @@ -483,7 +568,7 @@ spec:
app: arobit-forwarder
azure.workload.identity/use: "true"
annotations:
checksum/configmap: dadd4d46b94263a3545e2e0cbfa77a848648dd0b35259ce52d79c7a9c46fdd81
checksum/configmap: 7d62e084ce38da5386f0334e7c207089acd67818e8cabab1a83c3e5894cdc1fa
spec:
priorityClassName: service-lifecycle-critical
serviceAccountName: 'arobit-forwarder'
Expand Down
Loading