From f9423860a5997a20db3d5f4024afc6ede8b76499 Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 14 Apr 2026 10:29:12 -0700 Subject: [PATCH 01/12] monitoring: add adx grafana datasource rollout --- config/config.schema.json | 9 + config/config.yaml | 2 + .../configurations/kusto.tmpl.bicepparam | 2 + dev-infrastructure/geography-pipeline.yaml | 161 ++++++++++++++++++ .../modules/logs/kusto/cluster.bicep | 1 + .../modules/logs/kusto/main.bicep | 28 ++- dev-infrastructure/templates/kusto.bicep | 7 + docs/monitoring.md | 39 +++++ 8 files changed, 248 insertions(+), 1 deletion(-) diff --git a/config/config.schema.json b/config/config.schema.json index 61b03dbf449..1782f92ec25 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -2786,6 +2786,15 @@ "grafanaName": { "type": "string" }, + "adxDatasourceEnabled": { + "type": "boolean", + "description": "Whether to provision ADX datasources in Managed Grafana" + }, + "adxDatasourceGeographies": { + "type": "string", + "pattern": "^$|^[a-z0-9-]+(,[a-z0-9-]+)*$", + "description": "Comma-separated geography allowlist for optional targeted ADX datasource provisioning. Empty means all geographies." + }, "grafanaMajorVersion": { "type": "string" }, diff --git a/config/config.yaml b/config/config.yaml index 4f2b838d82c..1a4c5e224f3 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -345,6 +345,8 @@ defaults: # Monitoring monitoring: grafanaName: "arohcp-{{ .ctx.environment }}" + adxDatasourceEnabled: false + adxDatasourceGeographies: "" # Format: # Multiline string using '>-' YAML block scalar # One item per line, formatted as: UUID/PrincipalType/RoleName diff --git a/dev-infrastructure/configurations/kusto.tmpl.bicepparam b/dev-infrastructure/configurations/kusto.tmpl.bicepparam index 280b15e90ea..c2b0d566a0d 100644 --- a/dev-infrastructure/configurations/kusto.tmpl.bicepparam +++ b/dev-infrastructure/configurations/kusto.tmpl.bicepparam @@ -7,6 +7,8 @@ param tier = '{{ .kusto.tier }}' param kustoName = '{{ .kusto.kustoName }}' +param grafanaResourceId = '__grafanaResourceId__' + param manageInstance = {{ .kusto.manageInstance }} param serviceLogsDatabase = '{{ .kusto.serviceLogsDatabase }}' diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 833d32988f0..9b286c66571 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -2,6 +2,16 @@ $schema: pipeline.schema.v1 rolloutName: Geography Rollout serviceGroup: Microsoft.Azure.ARO.HCP.Geography resourceGroups: +- name: global + resourceGroup: '{{ .global.rg }}' + subscription: '{{ .global.subscription.key }}' + steps: + - name: output + action: ARM + template: templates/output-global.bicep + parameters: configurations/output-global.tmpl.bicepparam + deploymentLevel: ResourceGroup + outputOnly: true - name: kusto-infra resourceGroup: '{{ .kusto.rg }}' subscription: '{{ .global.subscription.key }}' @@ -38,3 +48,154 @@ resourceGroups: template: templates/kusto.bicep parameters: configurations/kusto.tmpl.bicepparam deploymentLevel: ResourceGroup + variables: + - name: grafanaResourceId + input: + resourceGroup: global + step: output + name: grafanaResourceId + dependsOn: + - resourceGroup: global + step: output + - name: adx-datasource + action: Shell + omitFromServiceGroupCompletion: true + command: | + set -eu + + if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then + echo "ADX datasource provisioning disabled, skipping" + exit 0 + fi + + if [ -z "${ADX_CLUSTER_URL}" ]; then + echo "No Kusto cluster URI available (manageInstance is likely false), skipping" + exit 0 + fi + + GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) + ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') + EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" + + case "${KUSTO_NAME}" in + "${EXPECTED_KUSTO_PREFIX}"*) + GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} + ;; + *) + echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" + exit 1 + ;; + esac + + if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then + NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') + + if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then + echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" + exit 1 + fi + + case ",${NORMALIZED_GEOS}," in + *,"${NORMALIZED_GEO}",*) ;; + *) + echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" + exit 0 + ;; + esac + fi + + DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" + + az resource wait \ + --custom "properties.provisioningState=='Succeeded'" \ + --ids "${GRAFANA_RESOURCE_ID}" \ + --api-version 2024-10-01 + + DEFINITION=$(jq -nc \ + --arg name "${DATASOURCE_NAME}" \ + --arg clusterUrl "${ADX_CLUSTER_URL}" \ + --arg defaultDatabase "${ADX_DATABASE}" \ + '{ + name: $name, + type: "grafana-azure-data-explorer-datasource", + access: "proxy", + jsonData: { + clusterUrl: $clusterUrl, + defaultDatabase: $defaultDatabase, + dataConsistency: "strongconsistency", + azureCredentials: { + authType: "msi" + } + } + }') + + if az grafana data-source show \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" >/dev/null 2>&1; then + echo "Datasource ${DATASOURCE_NAME} exists, updating" + az grafana data-source update \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" \ + --definition "${DEFINITION}" + else + echo "Datasource ${DATASOURCE_NAME} not found, creating" + az grafana data-source create \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --definition "${DEFINITION}" + fi + + az grafana data-source show \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" \ + --query "{name:name, type:type, uid:uid}" \ + -o table + automatedRetry: + errorContainsAny: + - "is invalid as it is being provisioned with state" + - "503 Server Error: Service Unavailable for url" + - "(CreateOrUpdateConflict) Operation conflict occurred for workspace" + - "InvalidMonitorAccountIntegration" + maximumRetryCount: 5 + durationBetweenRetries: 2m + shellIdentity: + input: + resourceGroup: global + step: output + name: globalMSIId + variables: + - name: ADX_CLUSTER_URL + input: + resourceGroup: kusto-infra + step: deploy + name: kustoUri + - name: ADX_DATABASE + configRef: kusto.serviceLogsDatabase + - name: KUSTO_NAME + configRef: kusto.kustoName + - name: GRAFANA_NAME + configRef: monitoring.grafanaName + - name: GRAFANA_RG + configRef: global.rg + - name: GRAFANA_RESOURCE_ID + input: + resourceGroup: global + step: output + name: grafanaResourceId + - name: ADX_DATASOURCE_ENABLED + configRef: monitoring.adxDatasourceEnabled + - name: ADX_DATASOURCE_GEOGRAPHIES + configRef: monitoring.adxDatasourceGeographies + dependsOn: + - resourceGroup: global + step: output + - resourceGroup: kusto-infra + step: deploy diff --git a/dev-infrastructure/modules/logs/kusto/cluster.bicep b/dev-infrastructure/modules/logs/kusto/cluster.bicep index e81d4f891f3..75c2160a77b 100644 --- a/dev-infrastructure/modules/logs/kusto/cluster.bicep +++ b/dev-infrastructure/modules/logs/kusto/cluster.bicep @@ -120,3 +120,4 @@ resource kusto 'Microsoft.Kusto/clusters@2024-04-13' = { output id string = kusto.id output name string = kusto.name +output uri string = kusto.properties.uri diff --git a/dev-infrastructure/modules/logs/kusto/main.bicep b/dev-infrastructure/modules/logs/kusto/main.bicep index 8deea211c2a..b9ccaca6e1a 100644 --- a/dev-infrastructure/modules/logs/kusto/main.bicep +++ b/dev-infrastructure/modules/logs/kusto/main.bicep @@ -1,3 +1,5 @@ +import * as res from '../../resource.bicep' + @description('Azure Region Location') param location string = resourceGroup().location @@ -44,15 +46,26 @@ param crossClusterServiceLogsScript string = '' @description('Optional cross-cluster HostedControlPlaneLogs Kusto script content.') @secure() param crossClusterHostedControlPlaneLogsScript string = '' + +@description('Optional: Grafana resource ID for database-level Viewer access') +param grafanaResourceId string = '' + var db = { serviceLogs: serviceLogsDatabase hostedControlPlaneLogs: hostedControlPlaneLogsDatabase } var databases = [db.serviceLogs, db.hostedControlPlaneLogs] +var hasGrafana = grafanaResourceId != '' +var grafanaRef = hasGrafana ? res.grafanaRefFromId(grafanaResourceId) : null var dummyScript = '.create-or-alter function with (docstring = \'dummy function to run last and to remove permission\') dummyFunction() {print \'dummy\'}' +resource grafana 'Microsoft.Dashboard/grafana@2024-10-01' existing = if (hasGrafana) { + name: grafanaRef!.name + scope: resourceGroup(grafanaRef!.resourceGroup.subscriptionId, grafanaRef!.resourceGroup.name) +} + var allServiceLogsTablesKQL = { backendLogs: loadTextContent('tables/backendLogs.kql') containerlogs: loadTextContent('tables/containerLogs.kql') @@ -194,7 +207,18 @@ module crossClusterHostedControlPlaneLogsQueryScript 'script.bicep' = if (deploy ] } -// 6. Remove the caller principal +// 6. Grafana service logs access +module grafanaServiceLogsAccess 'grant-access.bicep' = if (hasGrafana) { + name: 'grafana-serviceLogs-viewer' + params: { + kustoName: kustoName + databaseName: db.serviceLogs + readAccessPrincipalIds: [grafana.identity.principalId] + } + dependsOn: [serviceLogsTables] +} + +// 7. Remove the caller principal // THIS MUST BE THE LAST SCRIPT TO RUN module removePermission 'script.bicep' = [ for (database, i) in databases: { @@ -213,9 +237,11 @@ module removePermission 'script.bicep' = [ hostedControlPlaneLogsTables crossClusterServiceLogsQueryScript crossClusterHostedControlPlaneLogsQueryScript + grafanaServiceLogsAccess ] } ] // Outputs mirror original contract output id string = cluster.outputs.id +output kustoUri string = cluster.outputs.uri diff --git a/dev-infrastructure/templates/kusto.bicep b/dev-infrastructure/templates/kusto.bicep index f3a5a90c10a..47d18f6476e 100644 --- a/dev-infrastructure/templates/kusto.bicep +++ b/dev-infrastructure/templates/kusto.bicep @@ -44,6 +44,10 @@ param crossClusterServiceLogsScript string = '' @description('Optional cross-cluster HostedControlPlaneLogs Kusto script content.') @secure() param crossClusterHostedControlPlaneLogsScript string = '' + +@description('Optional: Grafana resource ID for database-level Viewer access') +param grafanaResourceId string = '' + module kusto '../modules/logs/kusto/main.bicep' = if (manageInstance) { name: 'kusto-${location}' params: { @@ -62,5 +66,8 @@ module kusto '../modules/logs/kusto/main.bicep' = if (manageInstance) { enableAutoScale: enableAutoScale crossClusterServiceLogsScript: crossClusterServiceLogsScript crossClusterHostedControlPlaneLogsScript: crossClusterHostedControlPlaneLogsScript + grafanaResourceId: grafanaResourceId } } + +output kustoUri string = manageInstance ? kusto.outputs.kustoUri : '' diff --git a/docs/monitoring.md b/docs/monitoring.md index 3d653042d92..18e8aeebb94 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -90,6 +90,45 @@ A single **Azure Managed Grafana** instance is deployed globally and configured - Region-agnostic dashboard experience - Consolidated alerting and monitoring workflows +### ADX / Kusto datasources + +Managed Grafana can also expose Azure Data Explorer datasources for ARO-HCP Kusto clusters. These datasources follow the same single-workspace-per-environment model as the Azure Monitor datasources above. + +- **Provisioning scope**: Geography pipeline, once per managed Kusto cluster +- **Datasource name**: `kusto--` (for example `kusto-int-uk`) +- **Target database**: `ServiceLogs` +- **Authentication**: Grafana workspace system-assigned managed identity +- **RBAC**: Database-level `Viewer` on `ServiceLogs` only + +Provisioning is controlled by: + +- `monitoring.adxDatasourceEnabled` +- `monitoring.adxDatasourceGeographies` + +When `adxDatasourceEnabled` is `true`, the geography pipeline creates or updates the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. + +During rollout, validate at least one datasource during the environment bake window with: + +1. Grafana UI +2. Data sources +3. `kusto--` +4. **Save & Test** + +### Kusto datasource lifecycle and teardown + +`adxDatasourceEnabled: false` is a provisioning gate, not a deletion signal. Disabling it stops future create or update attempts, but it does **not** remove an existing datasource. + +If a geography's Kusto cluster is intentionally decommissioned (`kusto.manageInstance` transitions to `false`), remove the corresponding datasource explicitly: + +```bash +az grafana data-source delete \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --data-source "kusto--" +``` + +This manual delete is required because the pipeline intentionally skips datasource changes when no `kustoUri` is available. That fail-closed behavior prevents accidental deletion caused by transient deployment or output issues. + ### Regional Azure Monitor Workspace Each region contains **two Azure Monitor Workspaces (AMW)**: From ede7c3287b73a8af27cd4a716d87df868af04bb2 Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 14 Apr 2026 11:58:30 -0700 Subject: [PATCH 02/12] monitoring: clean up kusto grafana datasources --- .../delete.kusto.instance.pipeline.yaml | 8 ++ .../cleanup/delete.kusto.instance.sh | 127 +++++++++++++----- docs/monitoring.md | 10 +- 3 files changed, 111 insertions(+), 34 deletions(-) diff --git a/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml b/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml index e6e99935194..22074cabe65 100644 --- a/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml +++ b/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml @@ -45,8 +45,16 @@ resourceGroups: configRef: kusto.rg - name: KUSTO_INSTANCE configRef: kusto.kustoName + - name: GRAFANA_RESOURCE_ID + input: + resourceGroup: global + step: output + name: grafanaResourceId shellIdentity: input: resourceGroup: global step: output name: globalMSIId + dependsOn: + - resourceGroup: global + step: output diff --git a/dev-infrastructure/cleanup/delete.kusto.instance.sh b/dev-infrastructure/cleanup/delete.kusto.instance.sh index cce52051ac3..5fb8e1da9f6 100755 --- a/dev-infrastructure/cleanup/delete.kusto.instance.sh +++ b/dev-infrastructure/cleanup/delete.kusto.instance.sh @@ -4,14 +4,15 @@ set -o errexit set -o nounset set -o pipefail -# Usage: RESOURCE_GROUP= KUSTO_INSTANCE= [DRY_RUN=true] ./delete.kusto.instance.sh +# Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh # -# Deletes all specific Kusto instance +# Deletes a specific Kusto instance and its matching Grafana datasource # # Environment variables: -# RESOURCE_GROUP - Name of the resource group to clean up (required) -# KUSTO_INSTANCE - Name of the Kusto instance to clean up (required) -# DRY_RUN - Set to 'true' to preview actions without deleting (optional, default: false) +# RESOURCE_GROUP - Name of the resource group to clean up (required) +# KUSTO_INSTANCE - Name of the Kusto instance to clean up (required) +# GRAFANA_RESOURCE_ID - Resource ID of the shared Grafana instance (required) +# DRY_RUN - Set to 'true' to preview actions without deleting (optional, default: false) if [[ -z "${RESOURCE_GROUP:-}" ]]; then echo "Error: RESOURCE_GROUP environment variable is required" @@ -21,7 +22,13 @@ fi if [[ -z "${KUSTO_INSTANCE:-}" ]]; then echo "Error: KUSTO_INSTANCE environment variable is required" - echo "Usage: KUSTO_INSTANCE= [DRY_RUN=true] ./delete.kusto.instance.sh" + echo "Usage: KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" + exit 1 +fi + +if [[ -z "${GRAFANA_RESOURCE_ID:-}" ]]; then + echo "Error: GRAFANA_RESOURCE_ID environment variable is required" + echo "Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" exit 1 fi @@ -32,12 +39,6 @@ if [[ "$DRY_RUN" == "true" ]]; then echo "🔍 DRY RUN MODE - No resources will actually be deleted" fi -# Check if resource group exists -if [[ "$(az group exists --name "$RESOURCE_GROUP" --output tsv 2>/dev/null)" != "true" ]]; then - echo "⚠️ Resource group '$RESOURCE_GROUP' does not exist" - exit 0 -fi - # Function to log actions log() { local level="$1" @@ -60,11 +61,6 @@ list_all_resources() { --output table 2>/dev/null || log ERROR "Failed to list resources" } -# List all resources for debugging -if [[ "$DRY_RUN" == "true" ]]; then - list_all_resources -fi - # Function to check if resource has locks has_locks() { local resource_id="$1" @@ -82,25 +78,94 @@ get_kusto_instance_id() { --output tsv 2>/dev/null || true } -log STEP "Removing Kusto instance" -kusto_instance_id=$(get_kusto_instance_id) -if [[ -n "$kusto_instance_id" ]]; then - if has_locks "$kusto_instance_id"; then - log WARN "Skipping locked Kusto instance: $kusto_instance_id" - continue +parse_grafana_context() { + GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) + GRAFANA_RG=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f5) + GRAFANA_NAME=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f9) + ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') + EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" + + case "${KUSTO_INSTANCE}" in + "${EXPECTED_KUSTO_PREFIX}"*) + GEO_SHORT_ID=${KUSTO_INSTANCE#"${EXPECTED_KUSTO_PREFIX}"} + ;; + *) + log ERROR "Unexpected Kusto instance '${KUSTO_INSTANCE}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" + exit 1 + ;; + esac + + DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" +} + +delete_grafana_datasource() { + log STEP "Removing Grafana datasource" + az resource wait \ + --custom "properties.provisioningState=='Succeeded'" \ + --ids "${GRAFANA_RESOURCE_ID}" \ + --api-version 2024-10-01 + + local existing_datasource + existing_datasource=$(az grafana data-source list \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --query "[?name=='${DATASOURCE_NAME}'].name | [0]" \ + --output tsv) + + if [[ -z "${existing_datasource}" ]]; then + log INFO "Grafana datasource '${DATASOURCE_NAME}' not found, nothing to delete" + return 0 fi if [[ "$DRY_RUN" == "true" ]]; then - log INFO "[DRY RUN] Would delete Kusto instance: $(basename "$kusto_instance_id")" - else - log INFO "Deleting Kusto instance: $(basename "$kusto_instance_id")" - if az resource delete --ids "$kusto_instance_id" --output none 2>/dev/null; then - log SUCCESS "Deleted Kusto instance: $(basename "$kusto_instance_id")" + log INFO "[DRY RUN] Would delete Grafana datasource: ${DATASOURCE_NAME}" + return 0 + fi + + log INFO "Deleting Grafana datasource: ${DATASOURCE_NAME}" + az grafana data-source delete \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" \ + --output none + log SUCCESS "Deleted Grafana datasource: ${DATASOURCE_NAME}" +} + +parse_grafana_context + +resource_group_exists=$(az group exists --name "$RESOURCE_GROUP" --output tsv 2>/dev/null || echo "false") + +# List all resources for debugging +if [[ "$DRY_RUN" == "true" && "$resource_group_exists" == "true" ]]; then + list_all_resources +fi + +if [[ "$resource_group_exists" == "true" ]]; then + log STEP "Removing Kusto instance" + kusto_instance_id=$(get_kusto_instance_id) + if [[ -n "$kusto_instance_id" ]]; then + if has_locks "$kusto_instance_id"; then + log WARN "Skipping locked Kusto instance: $kusto_instance_id" + exit 0 + fi + + if [[ "$DRY_RUN" == "true" ]]; then + log INFO "[DRY RUN] Would delete Kusto instance: $(basename "$kusto_instance_id")" else - log ERROR "Failed to delete Kusto instance: $(basename "$kusto_instance_id")" + log INFO "Deleting Kusto instance: $(basename "$kusto_instance_id")" + if az resource delete --ids "$kusto_instance_id" --output none 2>/dev/null; then + log SUCCESS "Deleted Kusto instance: $(basename "$kusto_instance_id")" + else + log ERROR "Failed to delete Kusto instance: $(basename "$kusto_instance_id")" + fi fi + else + log WARN "Kusto instance '$KUSTO_INSTANCE' not found in resource group '$RESOURCE_GROUP', continuing with datasource cleanup" fi else - log ERROR "Kusto instance '$KUSTO_INSTANCE' not found in resource group '$RESOURCE_GROUP'" - exit 1 + log WARN "Resource group '$RESOURCE_GROUP' does not exist, skipping Kusto instance deletion" fi + +delete_grafana_datasource diff --git a/docs/monitoring.md b/docs/monitoring.md index 18e8aeebb94..3354dd46029 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -116,9 +116,13 @@ During rollout, validate at least one datasource during the environment bake win ### Kusto datasource lifecycle and teardown -`adxDatasourceEnabled: false` is a provisioning gate, not a deletion signal. Disabling it stops future create or update attempts, but it does **not** remove an existing datasource. +`adxDatasourceEnabled: false` is a provisioning gate, not a deletion signal. Disabling it stops future create or update attempts, but it does **not** remove an existing datasource by itself. -If a geography's Kusto cluster is intentionally decommissioned (`kusto.manageInstance` transitions to `false`), remove the corresponding datasource explicitly: +If a geography's Kusto cluster is intentionally decommissioned through the `Microsoft.Azure.ARO.HCP.Kusto.Delete` cleanup rollout, that cleanup flow deletes the corresponding `kusto--` datasource from the shared Grafana workspace. + +The regular geography pipeline still skips datasource changes when no `kustoUri` is available. That fail-closed behavior prevents accidental deletion caused by transient deployment or output issues during normal rollout or provisioning retries. + +If a datasource needs to be removed outside the cleanup flow, delete it explicitly: ```bash az grafana data-source delete \ @@ -127,7 +131,7 @@ az grafana data-source delete \ --data-source "kusto--" ``` -This manual delete is required because the pipeline intentionally skips datasource changes when no `kustoUri` is available. That fail-closed behavior prevents accidental deletion caused by transient deployment or output issues. +This manual delete is only needed for out-of-band cleanup. Normal Kusto decommission should use the cleanup rollout so the cluster teardown and datasource teardown stay aligned. ### Regional Azure Monitor Workspace From c13744037c8617a9400ad58d412b3afdb8632c5a Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 14 Apr 2026 14:38:58 -0700 Subject: [PATCH 03/12] monitoring: harden kusto cleanup grafana wait --- dev-infrastructure/cleanup/delete.kusto.instance.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dev-infrastructure/cleanup/delete.kusto.instance.sh b/dev-infrastructure/cleanup/delete.kusto.instance.sh index 5fb8e1da9f6..ea8ba6f6973 100755 --- a/dev-infrastructure/cleanup/delete.kusto.instance.sh +++ b/dev-infrastructure/cleanup/delete.kusto.instance.sh @@ -16,7 +16,7 @@ set -o pipefail if [[ -z "${RESOURCE_GROUP:-}" ]]; then echo "Error: RESOURCE_GROUP environment variable is required" - echo "Usage: RESOURCE_GROUP= [DRY_RUN=true] ./delete.kusto.instance.sh" + echo "Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" exit 1 fi @@ -100,10 +100,14 @@ parse_grafana_context() { delete_grafana_datasource() { log STEP "Removing Grafana datasource" - az resource wait \ + if ! az resource wait \ --custom "properties.provisioningState=='Succeeded'" \ --ids "${GRAFANA_RESOURCE_ID}" \ - --api-version 2024-10-01 + --api-version 2024-10-01 \ + --timeout 300; then + log ERROR "Failed waiting for Grafana resource '${GRAFANA_RESOURCE_ID}' to reach provisioningState 'Succeeded' within 300 seconds" + exit 1 + fi local existing_datasource existing_datasource=$(az grafana data-source list \ From 4725fbb627fa2c7ede04599f592fcd78a2e7f73b Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 14 Apr 2026 15:13:38 -0700 Subject: [PATCH 04/12] monitoring: fail adx datasource shell pipelines --- dev-infrastructure/geography-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 9b286c66571..603fff23356 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -61,7 +61,7 @@ resourceGroups: action: Shell omitFromServiceGroupCompletion: true command: | - set -eu + set -euo pipefail if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then echo "ADX datasource provisioning disabled, skipping" From 07579fa1bd71dccb747f9c20366488a7744118e3 Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 14 Apr 2026 17:01:06 -0700 Subject: [PATCH 05/12] monitoring: externalize adx datasource shell step --- dev-infrastructure/geography-pipeline.yaml | 100 +---------------- .../scripts/add-kusto-grafana-datasource.sh | 101 ++++++++++++++++++ 2 files changed, 103 insertions(+), 98 deletions(-) create mode 100755 dev-infrastructure/scripts/add-kusto-grafana-datasource.sh diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 603fff23356..2925d7a3f2c 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -60,104 +60,8 @@ resourceGroups: - name: adx-datasource action: Shell omitFromServiceGroupCompletion: true - command: | - set -euo pipefail - - if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then - echo "ADX datasource provisioning disabled, skipping" - exit 0 - fi - - if [ -z "${ADX_CLUSTER_URL}" ]; then - echo "No Kusto cluster URI available (manageInstance is likely false), skipping" - exit 0 - fi - - GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) - ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') - EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" - - case "${KUSTO_NAME}" in - "${EXPECTED_KUSTO_PREFIX}"*) - GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} - ;; - *) - echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" - exit 1 - ;; - esac - - if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then - NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') - - if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then - echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" - exit 1 - fi - - case ",${NORMALIZED_GEOS}," in - *,"${NORMALIZED_GEO}",*) ;; - *) - echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" - exit 0 - ;; - esac - fi - - DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" - - az resource wait \ - --custom "properties.provisioningState=='Succeeded'" \ - --ids "${GRAFANA_RESOURCE_ID}" \ - --api-version 2024-10-01 - - DEFINITION=$(jq -nc \ - --arg name "${DATASOURCE_NAME}" \ - --arg clusterUrl "${ADX_CLUSTER_URL}" \ - --arg defaultDatabase "${ADX_DATABASE}" \ - '{ - name: $name, - type: "grafana-azure-data-explorer-datasource", - access: "proxy", - jsonData: { - clusterUrl: $clusterUrl, - defaultDatabase: $defaultDatabase, - dataConsistency: "strongconsistency", - azureCredentials: { - authType: "msi" - } - } - }') - - if az grafana data-source show \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" >/dev/null 2>&1; then - echo "Datasource ${DATASOURCE_NAME} exists, updating" - az grafana data-source update \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" \ - --definition "${DEFINITION}" - else - echo "Datasource ${DATASOURCE_NAME} not found, creating" - az grafana data-source create \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --definition "${DEFINITION}" - fi - - az grafana data-source show \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" \ - --query "{name:name, type:type, uid:uid}" \ - -o table + command: ./add-kusto-grafana-datasource.sh + workingDir: ./scripts automatedRetry: errorContainsAny: - "is invalid as it is being provisioned with state" diff --git a/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh b/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh new file mode 100755 index 00000000000..afecca67a2d --- /dev/null +++ b/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then + echo "ADX datasource provisioning disabled, skipping" + exit 0 +fi + +if [ -z "${ADX_CLUSTER_URL}" ]; then + echo "No Kusto cluster URI available (manageInstance is likely false), skipping" + exit 0 +fi + +GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) +ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') +EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" + +case "${KUSTO_NAME}" in + "${EXPECTED_KUSTO_PREFIX}"*) + GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} + ;; + *) + echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" + exit 1 + ;; +esac + +if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then + NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') + + if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then + echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" + exit 1 + fi + + case ",${NORMALIZED_GEOS}," in + *,"${NORMALIZED_GEO}",*) ;; + *) + echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" + exit 0 + ;; + esac +fi + +DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" + +az resource wait \ + --custom "properties.provisioningState=='Succeeded'" \ + --ids "${GRAFANA_RESOURCE_ID}" \ + --api-version 2024-10-01 + +DEFINITION=$(jq -nc \ + --arg name "${DATASOURCE_NAME}" \ + --arg clusterUrl "${ADX_CLUSTER_URL}" \ + --arg defaultDatabase "${ADX_DATABASE}" \ + '{ + name: $name, + type: "grafana-azure-data-explorer-datasource", + access: "proxy", + jsonData: { + clusterUrl: $clusterUrl, + defaultDatabase: $defaultDatabase, + dataConsistency: "strongconsistency", + azureCredentials: { + authType: "msi" + } + } + }') + +if az grafana data-source show \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" >/dev/null 2>&1; then + echo "Datasource ${DATASOURCE_NAME} exists, updating" + az grafana data-source update \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" \ + --definition "${DEFINITION}" +else + echo "Datasource ${DATASOURCE_NAME} not found, creating" + az grafana data-source create \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --definition "${DEFINITION}" +fi + +az grafana data-source show \ + --name "${GRAFANA_NAME}" \ + --resource-group "${GRAFANA_RG}" \ + --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ + --data-source "${DATASOURCE_NAME}" \ + --query "{name:name, type:type, uid:uid}" \ + -o table From d8fe1678a6face0a051162997468ab8d57ccc31e Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 28 Apr 2026 11:40:27 -0700 Subject: [PATCH 06/12] monitoring: use grafanactl for ADX datasources Replace the temporary shell datasource script with a direct grafanactl invocation and keep cleanup datasource deletion out of this rollout scope. Depends on Azure/ARO-Tools#228. --- .../delete.kusto.instance.pipeline.yaml | 8 -- .../cleanup/delete.kusto.instance.sh | 133 +++++------------- dev-infrastructure/geography-pipeline.yaml | 70 +++++++-- .../modules/logs/kusto/main.bicep | 2 +- .../scripts/add-kusto-grafana-datasource.sh | 101 ------------- dev-infrastructure/templates/kusto.bicep | 2 +- docs/monitoring.md | 8 +- 7 files changed, 99 insertions(+), 225 deletions(-) delete mode 100755 dev-infrastructure/scripts/add-kusto-grafana-datasource.sh diff --git a/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml b/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml index 22074cabe65..e6e99935194 100644 --- a/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml +++ b/dev-infrastructure/cleanup/delete.kusto.instance.pipeline.yaml @@ -45,16 +45,8 @@ resourceGroups: configRef: kusto.rg - name: KUSTO_INSTANCE configRef: kusto.kustoName - - name: GRAFANA_RESOURCE_ID - input: - resourceGroup: global - step: output - name: grafanaResourceId shellIdentity: input: resourceGroup: global step: output name: globalMSIId - dependsOn: - - resourceGroup: global - step: output diff --git a/dev-infrastructure/cleanup/delete.kusto.instance.sh b/dev-infrastructure/cleanup/delete.kusto.instance.sh index ea8ba6f6973..77c36450242 100755 --- a/dev-infrastructure/cleanup/delete.kusto.instance.sh +++ b/dev-infrastructure/cleanup/delete.kusto.instance.sh @@ -4,31 +4,24 @@ set -o errexit set -o nounset set -o pipefail -# Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh +# Usage: RESOURCE_GROUP= KUSTO_INSTANCE= [DRY_RUN=true] ./delete.kusto.instance.sh # -# Deletes a specific Kusto instance and its matching Grafana datasource +# Deletes all specific Kusto instance # # Environment variables: -# RESOURCE_GROUP - Name of the resource group to clean up (required) -# KUSTO_INSTANCE - Name of the Kusto instance to clean up (required) -# GRAFANA_RESOURCE_ID - Resource ID of the shared Grafana instance (required) -# DRY_RUN - Set to 'true' to preview actions without deleting (optional, default: false) +# RESOURCE_GROUP - Name of the resource group to clean up (required) +# KUSTO_INSTANCE - Name of the Kusto instance to clean up (required) +# DRY_RUN - Set to 'true' to preview actions without deleting (optional, default: false) if [[ -z "${RESOURCE_GROUP:-}" ]]; then echo "Error: RESOURCE_GROUP environment variable is required" - echo "Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" + echo "Usage: RESOURCE_GROUP= [DRY_RUN=true] ./delete.kusto.instance.sh" exit 1 fi if [[ -z "${KUSTO_INSTANCE:-}" ]]; then echo "Error: KUSTO_INSTANCE environment variable is required" - echo "Usage: KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" - exit 1 -fi - -if [[ -z "${GRAFANA_RESOURCE_ID:-}" ]]; then - echo "Error: GRAFANA_RESOURCE_ID environment variable is required" - echo "Usage: RESOURCE_GROUP= KUSTO_INSTANCE= GRAFANA_RESOURCE_ID= [DRY_RUN=true] ./delete.kusto.instance.sh" + echo "Usage: KUSTO_INSTANCE= [DRY_RUN=true] ./delete.kusto.instance.sh" exit 1 fi @@ -39,6 +32,12 @@ if [[ "$DRY_RUN" == "true" ]]; then echo "🔍 DRY RUN MODE - No resources will actually be deleted" fi +# Check if resource group exists +if [[ "$(az group exists --name "$RESOURCE_GROUP" --output tsv 2>/dev/null)" != "true" ]]; then + echo "⚠️ Resource group '$RESOURCE_GROUP' does not exist" + exit 0 +fi + # Function to log actions log() { local level="$1" @@ -61,6 +60,11 @@ list_all_resources() { --output table 2>/dev/null || log ERROR "Failed to list resources" } +# List all resources for debugging +if [[ "$DRY_RUN" == "true" ]]; then + list_all_resources +fi + # Function to check if resource has locks has_locks() { local resource_id="$1" @@ -78,98 +82,25 @@ get_kusto_instance_id() { --output tsv 2>/dev/null || true } -parse_grafana_context() { - GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) - GRAFANA_RG=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f5) - GRAFANA_NAME=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f9) - ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') - EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" - - case "${KUSTO_INSTANCE}" in - "${EXPECTED_KUSTO_PREFIX}"*) - GEO_SHORT_ID=${KUSTO_INSTANCE#"${EXPECTED_KUSTO_PREFIX}"} - ;; - *) - log ERROR "Unexpected Kusto instance '${KUSTO_INSTANCE}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" - exit 1 - ;; - esac - - DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" -} - -delete_grafana_datasource() { - log STEP "Removing Grafana datasource" - if ! az resource wait \ - --custom "properties.provisioningState=='Succeeded'" \ - --ids "${GRAFANA_RESOURCE_ID}" \ - --api-version 2024-10-01 \ - --timeout 300; then - log ERROR "Failed waiting for Grafana resource '${GRAFANA_RESOURCE_ID}' to reach provisioningState 'Succeeded' within 300 seconds" - exit 1 - fi - - local existing_datasource - existing_datasource=$(az grafana data-source list \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --query "[?name=='${DATASOURCE_NAME}'].name | [0]" \ - --output tsv) - - if [[ -z "${existing_datasource}" ]]; then - log INFO "Grafana datasource '${DATASOURCE_NAME}' not found, nothing to delete" - return 0 +log STEP "Removing Kusto instance" +kusto_instance_id=$(get_kusto_instance_id) +if [[ -n "$kusto_instance_id" ]]; then + if has_locks "$kusto_instance_id"; then + log WARN "Skipping locked Kusto instance: $kusto_instance_id" + exit 0 fi if [[ "$DRY_RUN" == "true" ]]; then - log INFO "[DRY RUN] Would delete Grafana datasource: ${DATASOURCE_NAME}" - return 0 - fi - - log INFO "Deleting Grafana datasource: ${DATASOURCE_NAME}" - az grafana data-source delete \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" \ - --output none - log SUCCESS "Deleted Grafana datasource: ${DATASOURCE_NAME}" -} - -parse_grafana_context - -resource_group_exists=$(az group exists --name "$RESOURCE_GROUP" --output tsv 2>/dev/null || echo "false") - -# List all resources for debugging -if [[ "$DRY_RUN" == "true" && "$resource_group_exists" == "true" ]]; then - list_all_resources -fi - -if [[ "$resource_group_exists" == "true" ]]; then - log STEP "Removing Kusto instance" - kusto_instance_id=$(get_kusto_instance_id) - if [[ -n "$kusto_instance_id" ]]; then - if has_locks "$kusto_instance_id"; then - log WARN "Skipping locked Kusto instance: $kusto_instance_id" - exit 0 - fi - - if [[ "$DRY_RUN" == "true" ]]; then - log INFO "[DRY RUN] Would delete Kusto instance: $(basename "$kusto_instance_id")" + log INFO "[DRY RUN] Would delete Kusto instance: $(basename "$kusto_instance_id")" + else + log INFO "Deleting Kusto instance: $(basename "$kusto_instance_id")" + if az resource delete --ids "$kusto_instance_id" --output none 2>/dev/null; then + log SUCCESS "Deleted Kusto instance: $(basename "$kusto_instance_id")" else - log INFO "Deleting Kusto instance: $(basename "$kusto_instance_id")" - if az resource delete --ids "$kusto_instance_id" --output none 2>/dev/null; then - log SUCCESS "Deleted Kusto instance: $(basename "$kusto_instance_id")" - else - log ERROR "Failed to delete Kusto instance: $(basename "$kusto_instance_id")" - fi + log ERROR "Failed to delete Kusto instance: $(basename "$kusto_instance_id")" fi - else - log WARN "Kusto instance '$KUSTO_INSTANCE' not found in resource group '$RESOURCE_GROUP', continuing with datasource cleanup" fi else - log WARN "Resource group '$RESOURCE_GROUP' does not exist, skipping Kusto instance deletion" + log ERROR "Kusto instance '$KUSTO_INSTANCE' not found in resource group '$RESOURCE_GROUP'" + exit 1 fi - -delete_grafana_datasource diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 2925d7a3f2c..4c27f2dea5a 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -60,14 +60,70 @@ resourceGroups: - name: adx-datasource action: Shell omitFromServiceGroupCompletion: true - command: ./add-kusto-grafana-datasource.sh - workingDir: ./scripts + command: | + if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then + echo "ADX datasource provisioning disabled, skipping" + exit 0 + fi + + if [ -z "${ADX_CLUSTER_URL}" ]; then + echo "No Kusto cluster URI available (manageInstance is likely false), skipping" + exit 0 + fi + + ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') + EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" + + case "${KUSTO_NAME}" in + "${EXPECTED_KUSTO_PREFIX}"*) + GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} + ;; + *) + echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" + exit 1 + ;; + esac + + if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then + NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') + + if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then + echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" + exit 1 + fi + + case ",${NORMALIZED_GEOS}," in + *,"${NORMALIZED_GEO}",*) ;; + *) + echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" + exit 0 + ;; + esac + fi + + DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" + + az resource wait \ + --custom "properties.provisioningState=='Succeeded'" \ + --ids "${GRAFANA_RESOURCE_ID}" \ + --api-version 2024-10-01 + + go run ../tooling/grafanactl/main.go modify datasource reconcile-adx \ + --grafana-resource-id "${GRAFANA_RESOURCE_ID}" \ + --cluster-url "${ADX_CLUSTER_URL}" \ + --default-database "${ADX_DATABASE}" \ + --datasource-name "${DATASOURCE_NAME}" \ + --data-consistency strongconsistency automatedRetry: errorContainsAny: - - "is invalid as it is being provisioned with state" - - "503 Server Error: Service Unavailable for url" - - "(CreateOrUpdateConflict) Operation conflict occurred for workspace" - - "InvalidMonitorAccountIntegration" + - "grafana API POST /api/datasources request failed" + - "grafana API PUT /api/datasources/" + - "failed with status 429" + - "failed with status 500" + - "failed with status 502" + - "failed with status 503" + - "failed with status 504" maximumRetryCount: 5 durationBetweenRetries: 2m shellIdentity: @@ -87,8 +143,6 @@ resourceGroups: configRef: kusto.kustoName - name: GRAFANA_NAME configRef: monitoring.grafanaName - - name: GRAFANA_RG - configRef: global.rg - name: GRAFANA_RESOURCE_ID input: resourceGroup: global diff --git a/dev-infrastructure/modules/logs/kusto/main.bicep b/dev-infrastructure/modules/logs/kusto/main.bicep index b9ccaca6e1a..1d5e1652a84 100644 --- a/dev-infrastructure/modules/logs/kusto/main.bicep +++ b/dev-infrastructure/modules/logs/kusto/main.bicep @@ -213,7 +213,7 @@ module grafanaServiceLogsAccess 'grant-access.bicep' = if (hasGrafana) { params: { kustoName: kustoName databaseName: db.serviceLogs - readAccessPrincipalIds: [grafana.identity.principalId] + readAccessPrincipalIds: [grafana!.identity.principalId] } dependsOn: [serviceLogsTables] } diff --git a/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh b/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh deleted file mode 100755 index afecca67a2d..00000000000 --- a/dev-infrastructure/scripts/add-kusto-grafana-datasource.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then - echo "ADX datasource provisioning disabled, skipping" - exit 0 -fi - -if [ -z "${ADX_CLUSTER_URL}" ]; then - echo "No Kusto cluster URI available (manageInstance is likely false), skipping" - exit 0 -fi - -GRAFANA_SUBSCRIPTION_ID=$(printf '%s' "${GRAFANA_RESOURCE_ID}" | cut -d/ -f3) -ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') -EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" - -case "${KUSTO_NAME}" in - "${EXPECTED_KUSTO_PREFIX}"*) - GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} - ;; - *) - echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" - exit 1 - ;; -esac - -if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then - NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') - - if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then - echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" - exit 1 - fi - - case ",${NORMALIZED_GEOS}," in - *,"${NORMALIZED_GEO}",*) ;; - *) - echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" - exit 0 - ;; - esac -fi - -DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" - -az resource wait \ - --custom "properties.provisioningState=='Succeeded'" \ - --ids "${GRAFANA_RESOURCE_ID}" \ - --api-version 2024-10-01 - -DEFINITION=$(jq -nc \ - --arg name "${DATASOURCE_NAME}" \ - --arg clusterUrl "${ADX_CLUSTER_URL}" \ - --arg defaultDatabase "${ADX_DATABASE}" \ - '{ - name: $name, - type: "grafana-azure-data-explorer-datasource", - access: "proxy", - jsonData: { - clusterUrl: $clusterUrl, - defaultDatabase: $defaultDatabase, - dataConsistency: "strongconsistency", - azureCredentials: { - authType: "msi" - } - } - }') - -if az grafana data-source show \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" >/dev/null 2>&1; then - echo "Datasource ${DATASOURCE_NAME} exists, updating" - az grafana data-source update \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" \ - --definition "${DEFINITION}" -else - echo "Datasource ${DATASOURCE_NAME} not found, creating" - az grafana data-source create \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --definition "${DEFINITION}" -fi - -az grafana data-source show \ - --name "${GRAFANA_NAME}" \ - --resource-group "${GRAFANA_RG}" \ - --subscription "${GRAFANA_SUBSCRIPTION_ID}" \ - --data-source "${DATASOURCE_NAME}" \ - --query "{name:name, type:type, uid:uid}" \ - -o table diff --git a/dev-infrastructure/templates/kusto.bicep b/dev-infrastructure/templates/kusto.bicep index 47d18f6476e..a78dd75cb39 100644 --- a/dev-infrastructure/templates/kusto.bicep +++ b/dev-infrastructure/templates/kusto.bicep @@ -70,4 +70,4 @@ module kusto '../modules/logs/kusto/main.bicep' = if (manageInstance) { } } -output kustoUri string = manageInstance ? kusto.outputs.kustoUri : '' +output kustoUri string = manageInstance ? kusto!.outputs.kustoUri : '' diff --git a/docs/monitoring.md b/docs/monitoring.md index 3354dd46029..ecf17c43fb3 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -105,7 +105,7 @@ Provisioning is controlled by: - `monitoring.adxDatasourceEnabled` - `monitoring.adxDatasourceGeographies` -When `adxDatasourceEnabled` is `true`, the geography pipeline creates or updates the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. +When `adxDatasourceEnabled` is `true`, the geography pipeline uses `grafanactl` to create or update the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. During rollout, validate at least one datasource during the environment bake window with: @@ -118,11 +118,9 @@ During rollout, validate at least one datasource during the environment bake win `adxDatasourceEnabled: false` is a provisioning gate, not a deletion signal. Disabling it stops future create or update attempts, but it does **not** remove an existing datasource by itself. -If a geography's Kusto cluster is intentionally decommissioned through the `Microsoft.Azure.ARO.HCP.Kusto.Delete` cleanup rollout, that cleanup flow deletes the corresponding `kusto--` datasource from the shared Grafana workspace. - The regular geography pipeline still skips datasource changes when no `kustoUri` is available. That fail-closed behavior prevents accidental deletion caused by transient deployment or output issues during normal rollout or provisioning retries. -If a datasource needs to be removed outside the cleanup flow, delete it explicitly: +If a datasource needs to be removed, delete it explicitly: ```bash az grafana data-source delete \ @@ -131,7 +129,7 @@ az grafana data-source delete \ --data-source "kusto--" ``` -This manual delete is only needed for out-of-band cleanup. Normal Kusto decommission should use the cleanup rollout so the cluster teardown and datasource teardown stay aligned. +Datasource deletion is intentionally separate from normal Kusto cleanup so disabling or decommissioning a geography cannot remove shared Grafana configuration by accident. ### Regional Azure Monitor Workspace From b3d6d5f5a73e6ebf20cf3e165795bb2d7ff27e0c Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 28 Apr 2026 12:07:49 -0700 Subject: [PATCH 07/12] monitoring: constrain geography global output --- dev-infrastructure/geography-pipeline.yaml | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 4c27f2dea5a..a5e6ed8cc7a 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -5,6 +5,33 @@ resourceGroups: - name: global resourceGroup: '{{ .global.rg }}' subscription: '{{ .global.subscription.key }}' + executionConstraints: + - clouds: + - public + environments: + - int + regions: + - uksouth + - clouds: + - public + environments: + - stg + regions: + - uksouth + - clouds: + - public + environments: + - prod + regions: + - australiaeast + - brazilsouth + - canadacentral + - centralindia + - eastus2 + - eastus2euap + - westeurope + - switzerlandnorth + - uksouth steps: - name: output action: ARM From 0b702b5e58e63507b2cd0590e003abab310d25e1 Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 28 Apr 2026 12:51:56 -0700 Subject: [PATCH 08/12] monitoring: fix geography pipeline metadata --- dev-infrastructure/geography-pipeline.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index a5e6ed8cc7a..81a0db0ab91 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -2,7 +2,7 @@ $schema: pipeline.schema.v1 rolloutName: Geography Rollout serviceGroup: Microsoft.Azure.ARO.HCP.Geography resourceGroups: -- name: global +- name: geo-global resourceGroup: '{{ .global.rg }}' subscription: '{{ .global.subscription.key }}' executionConstraints: @@ -78,11 +78,11 @@ resourceGroups: variables: - name: grafanaResourceId input: - resourceGroup: global + resourceGroup: geo-global step: output name: grafanaResourceId dependsOn: - - resourceGroup: global + - resourceGroup: geo-global step: output - name: adx-datasource action: Shell @@ -142,6 +142,7 @@ resourceGroups: --default-database "${ADX_DATABASE}" \ --datasource-name "${DATASOURCE_NAME}" \ --data-consistency strongconsistency + workingDir: . automatedRetry: errorContainsAny: - "grafana API POST /api/datasources request failed" @@ -155,7 +156,7 @@ resourceGroups: durationBetweenRetries: 2m shellIdentity: input: - resourceGroup: global + resourceGroup: geo-global step: output name: globalMSIId variables: @@ -172,7 +173,7 @@ resourceGroups: configRef: monitoring.grafanaName - name: GRAFANA_RESOURCE_ID input: - resourceGroup: global + resourceGroup: geo-global step: output name: grafanaResourceId - name: ADX_DATASOURCE_ENABLED @@ -180,7 +181,7 @@ resourceGroups: - name: ADX_DATASOURCE_GEOGRAPHIES configRef: monitoring.adxDatasourceGeographies dependsOn: - - resourceGroup: global + - resourceGroup: geo-global step: output - resourceGroup: kusto-infra step: deploy From aa18f33072788922d510f4e0e0b19adb5f7801cc Mon Sep 17 00:00:00 2001 From: swiencki Date: Tue, 28 Apr 2026 12:59:39 -0700 Subject: [PATCH 09/12] monitoring: restore shared global metadata --- dev-infrastructure/geography-pipeline.yaml | 39 ++++------------------ 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index 81a0db0ab91..a055158c462 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -2,36 +2,9 @@ $schema: pipeline.schema.v1 rolloutName: Geography Rollout serviceGroup: Microsoft.Azure.ARO.HCP.Geography resourceGroups: -- name: geo-global +- name: global resourceGroup: '{{ .global.rg }}' subscription: '{{ .global.subscription.key }}' - executionConstraints: - - clouds: - - public - environments: - - int - regions: - - uksouth - - clouds: - - public - environments: - - stg - regions: - - uksouth - - clouds: - - public - environments: - - prod - regions: - - australiaeast - - brazilsouth - - canadacentral - - centralindia - - eastus2 - - eastus2euap - - westeurope - - switzerlandnorth - - uksouth steps: - name: output action: ARM @@ -78,11 +51,11 @@ resourceGroups: variables: - name: grafanaResourceId input: - resourceGroup: geo-global + resourceGroup: global step: output name: grafanaResourceId dependsOn: - - resourceGroup: geo-global + - resourceGroup: global step: output - name: adx-datasource action: Shell @@ -156,7 +129,7 @@ resourceGroups: durationBetweenRetries: 2m shellIdentity: input: - resourceGroup: geo-global + resourceGroup: global step: output name: globalMSIId variables: @@ -173,7 +146,7 @@ resourceGroups: configRef: monitoring.grafanaName - name: GRAFANA_RESOURCE_ID input: - resourceGroup: geo-global + resourceGroup: global step: output name: grafanaResourceId - name: ADX_DATASOURCE_ENABLED @@ -181,7 +154,7 @@ resourceGroups: - name: ADX_DATASOURCE_GEOGRAPHIES configRef: monitoring.adxDatasourceGeographies dependsOn: - - resourceGroup: geo-global + - resourceGroup: global step: output - resourceGroup: kusto-infra step: deploy From 297e8611d6e28bfca1fbd1b85975099bfd4e7cd3 Mon Sep 17 00:00:00 2001 From: swiencki Date: Thu, 30 Apr 2026 13:52:17 -0700 Subject: [PATCH 10/12] monitoring: use typed GrafanaDatasources for ADX --- config/config.schema.json | 4 +- dev-infrastructure/geography-pipeline.yaml | 115 +++------ .../modules/logs/kusto/grant-access.bicep | 4 +- docs/monitoring.md | 12 +- .../pkg/pipeline/grafana_datasources.go | 166 +++++++++++- .../pkg/pipeline/grafana_datasources_test.go | 239 ++++++++++++++++++ 6 files changed, 445 insertions(+), 95 deletions(-) create mode 100644 tooling/templatize/pkg/pipeline/grafana_datasources_test.go diff --git a/config/config.schema.json b/config/config.schema.json index 1782f92ec25..7f3fc62bb25 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -2792,8 +2792,8 @@ }, "adxDatasourceGeographies": { "type": "string", - "pattern": "^$|^[a-z0-9-]+(,[a-z0-9-]+)*$", - "description": "Comma-separated geography allowlist for optional targeted ADX datasource provisioning. Empty means all geographies." + "pattern": "^$|^\\s*[A-Za-z0-9-]+\\s*(,\\s*[A-Za-z0-9-]+\\s*)*$", + "description": "Comma-separated geography allowlist for optional targeted ADX datasource provisioning/removal. Empty means all geographies. Matching is case-insensitive and surrounding whitespace is ignored." }, "grafanaMajorVersion": { "type": "string" diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index a055158c462..ebc65ef01ef 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -58,68 +58,37 @@ resourceGroups: - resourceGroup: global step: output - name: adx-datasource - action: Shell + action: GrafanaDatasources omitFromServiceGroupCompletion: true - command: | - if [ "${ADX_DATASOURCE_ENABLED}" != "true" ]; then - echo "ADX datasource provisioning disabled, skipping" - exit 0 - fi - - if [ -z "${ADX_CLUSTER_URL}" ]; then - echo "No Kusto cluster URI available (manageInstance is likely false), skipping" - exit 0 - fi - - ENVIRONMENT_NAME=$(printf '%s' "${GRAFANA_NAME}" | sed 's/^arohcp-//') - EXPECTED_KUSTO_PREFIX="hcp-${ENVIRONMENT_NAME}-" - - case "${KUSTO_NAME}" in - "${EXPECTED_KUSTO_PREFIX}"*) - GEO_SHORT_ID=${KUSTO_NAME#"${EXPECTED_KUSTO_PREFIX}"} - ;; - *) - echo "ERROR: unexpected Kusto name '${KUSTO_NAME}', expected prefix '${EXPECTED_KUSTO_PREFIX}'" - exit 1 - ;; - esac - - if [ -n "${ADX_DATASOURCE_GEOGRAPHIES}" ]; then - NORMALIZED_GEOS=$(printf '%s' "${ADX_DATASOURCE_GEOGRAPHIES}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - NORMALIZED_GEO=$(printf '%s' "${GEO_SHORT_ID}" | tr '[:upper:]' '[:lower:]') - - if ! printf '%s' "${NORMALIZED_GEOS}" | grep -Eq '^[a-z0-9-]+(,[a-z0-9-]+)*$'; then - echo "ERROR: adxDatasourceGeographies has invalid format: '${ADX_DATASOURCE_GEOGRAPHIES}'" - exit 1 - fi - - case ",${NORMALIZED_GEOS}," in - *,"${NORMALIZED_GEO}",*) ;; - *) - echo "Geography ${GEO_SHORT_ID} not in allowlist (${NORMALIZED_GEOS}), skipping" - exit 0 - ;; - esac - fi - - DATASOURCE_NAME="kusto-${ENVIRONMENT_NAME}-${GEO_SHORT_ID}" - - az resource wait \ - --custom "properties.provisioningState=='Succeeded'" \ - --ids "${GRAFANA_RESOURCE_ID}" \ - --api-version 2024-10-01 - - go run ../tooling/grafanactl/main.go modify datasource reconcile-adx \ - --grafana-resource-id "${GRAFANA_RESOURCE_ID}" \ - --cluster-url "${ADX_CLUSTER_URL}" \ - --default-database "${ADX_DATABASE}" \ - --datasource-name "${DATASOURCE_NAME}" \ - --data-consistency strongconsistency - workingDir: . + grafanaName: "{{ .monitoring.grafanaName }}" + grafanaResourceId: + input: + resourceGroup: global + step: output + name: grafanaResourceId + azureMonitor: + enabled: false + adx: + enabled: + value: "{{ and .monitoring.adxDatasourceEnabled .kusto.manageInstance }}" + deleteWhenDisabled: true + clusterUrl: + input: + resourceGroup: kusto-infra + step: deploy + name: kustoUri + defaultDatabase: + configRef: kusto.serviceLogsDatabase + datasourceName: + value: "kusto-{{ .environmentName }}-{{ .azureGeoShortId }}" + geographies: + configRef: monitoring.adxDatasourceGeographies + dataConsistency: strongconsistency automatedRetry: errorContainsAny: - - "grafana API POST /api/datasources request failed" + - "grafana API POST /api/datasources" - "grafana API PUT /api/datasources/" + - "grafana API DELETE /api/datasources/name/" - "failed with status 429" - "failed with status 500" - "failed with status 502" @@ -127,32 +96,10 @@ resourceGroups: - "failed with status 504" maximumRetryCount: 5 durationBetweenRetries: 2m - shellIdentity: - input: - resourceGroup: global - step: output - name: globalMSIId - variables: - - name: ADX_CLUSTER_URL - input: - resourceGroup: kusto-infra - step: deploy - name: kustoUri - - name: ADX_DATABASE - configRef: kusto.serviceLogsDatabase - - name: KUSTO_NAME - configRef: kusto.kustoName - - name: GRAFANA_NAME - configRef: monitoring.grafanaName - - name: GRAFANA_RESOURCE_ID - input: - resourceGroup: global - step: output - name: grafanaResourceId - - name: ADX_DATASOURCE_ENABLED - configRef: monitoring.adxDatasourceEnabled - - name: ADX_DATASOURCE_GEOGRAPHIES - configRef: monitoring.adxDatasourceGeographies + identityFrom: + resourceGroup: global + step: output + name: globalMSIId dependsOn: - resourceGroup: global step: output diff --git a/dev-infrastructure/modules/logs/kusto/grant-access.bicep b/dev-infrastructure/modules/logs/kusto/grant-access.bicep index 0d3f701ea6a..d9ce736e881 100644 --- a/dev-infrastructure/modules/logs/kusto/grant-access.bicep +++ b/dev-infrastructure/modules/logs/kusto/grant-access.bicep @@ -14,7 +14,7 @@ resource database 'Microsoft.Kusto/clusters/databases@2024-04-13' existing = { resource grantIngest 'Microsoft.Kusto/clusters/databases/principalAssignments@2024-04-13' = [ for id in ingestAccessPrincipalIds: { parent: database - name: 'grant-${guid(id, databaseName)}' + name: 'grant-ingest-${guid(id, databaseName)}' properties: { principalId: id principalType: 'App' @@ -27,7 +27,7 @@ resource grantIngest 'Microsoft.Kusto/clusters/databases/principalAssignments@20 resource grantRead 'Microsoft.Kusto/clusters/databases/principalAssignments@2024-04-13' = [ for id in readAccessPrincipalIds: { parent: database - name: 'grant-${guid(id, databaseName)}' + name: 'grant-viewer-${guid(id, databaseName)}' properties: { principalId: id principalType: 'App' diff --git a/docs/monitoring.md b/docs/monitoring.md index ecf17c43fb3..4f15569840a 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -105,9 +105,9 @@ Provisioning is controlled by: - `monitoring.adxDatasourceEnabled` - `monitoring.adxDatasourceGeographies` -When `adxDatasourceEnabled` is `true`, the geography pipeline uses `grafanactl` to create or update the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. +When `adxDatasourceEnabled` is `true`, the geography pipeline uses the typed `GrafanaDatasources` action to create or update the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. Matching is case-insensitive and ignores surrounding whitespace around comma-separated entries. The same `grafanactl` reconcile path evaluates the allowlist during EV2 rollout and local templatize runs. -During rollout, validate at least one datasource during the environment bake window with: +After rollout, validate at least one datasource during the environment bake window with: 1. Grafana UI 2. Data sources @@ -116,11 +116,11 @@ During rollout, validate at least one datasource during the environment bake win ### Kusto datasource lifecycle and teardown -`adxDatasourceEnabled: false` is a provisioning gate, not a deletion signal. Disabling it stops future create or update attempts, but it does **not** remove an existing datasource by itself. +`adxDatasourceEnabled: false` disables the ADX desired state. The typed datasource reconcile step still runs and deletes the named datasource when deletion-on-disable is enabled, so disabling ADX or removing a geography from `adxDatasourceGeographies` cleans up stale Grafana datasource configuration. -The regular geography pipeline still skips datasource changes when no `kustoUri` is available. That fail-closed behavior prevents accidental deletion caused by transient deployment or output issues during normal rollout or provisioning retries. +The regular geography pipeline still fails closed when ADX is desired but no `kustoUri` is available. That prevents accidental deletion caused by transient deployment or output issues during normal rollout or provisioning retries. -If a datasource needs to be removed, delete it explicitly: +If a datasource needs to be removed outside the desired-state rollout path, delete it explicitly: ```bash az grafana data-source delete \ @@ -129,7 +129,7 @@ az grafana data-source delete \ --data-source "kusto--" ``` -Datasource deletion is intentionally separate from normal Kusto cleanup so disabling or decommissioning a geography cannot remove shared Grafana configuration by accident. +Datasource deletion is intentionally separate from normal Kusto cleanup so decommissioning Kusto infrastructure cannot remove shared Grafana configuration by accident. ### Regional Azure Monitor Workspace diff --git a/tooling/templatize/pkg/pipeline/grafana_datasources.go b/tooling/templatize/pkg/pipeline/grafana_datasources.go index 35df564ad9a..2427fee67db 100644 --- a/tooling/templatize/pkg/pipeline/grafana_datasources.go +++ b/tooling/templatize/pkg/pipeline/grafana_datasources.go @@ -17,18 +17,73 @@ package pipeline import ( "context" "fmt" + "strconv" + "strings" + configtypes "github.com/Azure/ARO-Tools/config/types" "github.com/Azure/ARO-Tools/pipelines/graph" "github.com/Azure/ARO-Tools/pipelines/types" "github.com/Azure/ARO-Tools/tools/grafanactl/cmd/modify" ) -func runGrafanaDatasourcesStep(_ graph.Identifier, step *types.GrafanaDatasourcesStep, ctx context.Context, options *StepRunOptions, executionTarget ExecutionTarget, _ *ExecutionState) error { +type resolvedGrafanaADXOptions struct { + Enabled bool + DeleteWhenDisabled bool + ClusterURL string + DefaultDatabase string + DatasourceName string + Geographies string + CurrentGeography string + DataConsistency string +} + +func runGrafanaDatasourcesStep(id graph.Identifier, step *types.GrafanaDatasourcesStep, ctx context.Context, options *StepRunOptions, executionTarget ExecutionTarget, state *ExecutionState) error { opts := modify.DefaultAddDatasourceOptions() opts.GrafanaName = step.GrafanaName opts.SubscriptionID = executionTarget.GetSubscriptionID() opts.ResourceGroup = executionTarget.GetResourceGroup() + if err := func() error { + state.RLock() + defer state.RUnlock() + + if step.GrafanaResourceID != nil { + grafanaResourceID, ok, err := resolveGrafanaDatasourceValue(id.ServiceGroup, "grafanaResourceId", *step.GrafanaResourceID, options.Configuration, state.Outputs) + if err != nil { + return fmt.Errorf("failed to resolve grafanaResourceId: %w", err) + } + if ok { + resolved, err := valueAsString("grafanaResourceId", grafanaResourceID) + if err != nil { + return err + } + opts.GrafanaResourceID = resolved + } + } + + if step.AzureMonitor != nil && step.AzureMonitor.Enabled != nil { + opts.AzureMonitorEnabled = *step.AzureMonitor.Enabled + } + + if step.ADX != nil { + adx, err := resolveGrafanaADXOptions(id.ServiceGroup, step.ADX, options.Configuration, state.Outputs) + if err != nil { + return err + } + opts.ADXEnabled = adx.Enabled + opts.ADXDeleteWhenDisabled = adx.DeleteWhenDisabled + opts.ADXClusterURL = adx.ClusterURL + opts.ADXDefaultDatabase = adx.DefaultDatabase + opts.ADXDatasourceName = adx.DatasourceName + opts.ADXGeographies = adx.Geographies + opts.ADXCurrentGeography = adx.CurrentGeography + opts.ADXDataConsistency = adx.DataConsistency + } + return nil + }(); err != nil { + return err + } + validated, err := opts.Validate(ctx) if err != nil { return fmt.Errorf("validation failed: %w", err) @@ -45,3 +100,112 @@ func runGrafanaDatasourcesStep(_ graph.Identifier, step *types.GrafanaDatasource return nil } + +func resolveGrafanaADXOptions(serviceGroup string, adx *types.GrafanaADXDatasource, cfg configtypes.Configuration, outputs Outputs) (*resolvedGrafanaADXOptions, error) { + enabled, err := resolveOptionalBool(serviceGroup, "adx.enabled", adx.Enabled, cfg, outputs) + if err != nil { + return nil, err + } + geographies, err := resolveOptionalString(serviceGroup, "adx.geographies", adx.Geographies, cfg, outputs) + if err != nil { + return nil, err + } + currentGeography, err := resolveCurrentGeography(cfg) + if err != nil { + return nil, err + } + + resolved := &resolvedGrafanaADXOptions{ + Enabled: enabled, + DeleteWhenDisabled: adx.DeleteWhenDisabled, + Geographies: geographies, + CurrentGeography: currentGeography, + DataConsistency: adx.DataConsistency, + } + + resolved.DatasourceName, err = resolveOptionalString(serviceGroup, "adx.datasourceName", adx.DatasourceName, cfg, outputs) + if err != nil { + return nil, err + } + resolved.ClusterURL, err = resolveOptionalString(serviceGroup, "adx.clusterUrl", adx.ClusterURL, cfg, outputs) + if err != nil { + return nil, err + } + + resolved.DefaultDatabase, err = resolveOptionalString(serviceGroup, "adx.defaultDatabase", adx.DefaultDatabase, cfg, outputs) + if err != nil { + return nil, err + } + + return resolved, nil +} + +func resolveOptionalBool(serviceGroup, name string, value types.Value, cfg configtypes.Configuration, outputs Outputs) (bool, error) { + raw, ok, err := resolveGrafanaDatasourceValue(serviceGroup, name, value, cfg, outputs) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + return valueAsBool(name, raw) +} + +func resolveOptionalString(serviceGroup, name string, value types.Value, cfg configtypes.Configuration, outputs Outputs) (string, error) { + raw, ok, err := resolveGrafanaDatasourceValue(serviceGroup, name, value, cfg, outputs) + if err != nil { + return "", err + } + if !ok { + return "", nil + } + return valueAsString(name, raw) +} + +func resolveGrafanaDatasourceValue(serviceGroup, name string, value types.Value, cfg configtypes.Configuration, outputs Outputs) (any, bool, error) { + if value.Input == nil && value.ConfigRef == "" && value.Value == nil { + return nil, false, nil + } + + values, err := getInputValues(serviceGroup, []types.Variable{{Name: name, Value: value}}, cfg, outputs) + if err != nil { + return nil, false, err + } + return values[name], true, nil +} + +func resolveCurrentGeography(cfg configtypes.Configuration) (string, error) { + currentRaw, err := cfg.GetByPath("azureGeoShortId") + if err != nil { + return "", fmt.Errorf("failed to lookup current geography short ID: %w", err) + } + currentGeography, err := valueAsString("azureGeoShortId", currentRaw) + if err != nil { + return "", err + } + return currentGeography, nil +} + +func valueAsBool(name string, value any) (bool, error) { + switch v := value.(type) { + case bool: + return v, nil + case string: + parsed, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil { + return false, fmt.Errorf("%s must resolve to a boolean, got %q", name, v) + } + return parsed, nil + default: + return false, fmt.Errorf("%s must resolve to a boolean, got %T", name, value) + } +} + +func valueAsString(name string, value any) (string, error) { + switch v := value.(type) { + case string: + return strings.TrimSpace(v), nil + default: + return "", fmt.Errorf("%s must resolve to a string, got %T", name, value) + } +} diff --git a/tooling/templatize/pkg/pipeline/grafana_datasources_test.go b/tooling/templatize/pkg/pipeline/grafana_datasources_test.go new file mode 100644 index 00000000000..f02e76bf458 --- /dev/null +++ b/tooling/templatize/pkg/pipeline/grafana_datasources_test.go @@ -0,0 +1,239 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipeline + +import ( + "strings" + "testing" + + configtypes "github.com/Azure/ARO-Tools/config/types" + "github.com/Azure/ARO-Tools/pipelines/types" + "github.com/Azure/ARO-Tools/tools/grafanactl/cmd/modify" +) + +func testGrafanaDatasourceConfig() configtypes.Configuration { + return configtypes.Configuration{ + "azureGeoShortId": "uks", + "kusto": map[string]any{ + "serviceLogsDatabase": "ServiceLogs", + }, + "monitoring": map[string]any{ + "adxDatasourceGeographies": " UKS, eus2 ", + }, + } +} + +func testGrafanaDatasourceOutputs(kustoURI string) Outputs { + return Outputs{ + "Microsoft.Azure.ARO.HCP.Geography": map[string]map[string]Output{ + "kusto-infra": { + "deploy": ArmOutput{ + "kustoUri": map[string]any{ + "type": "String", + "value": kustoURI, + }, + }, + }, + }, + } +} + +func TestResolveGrafanaADXOptionsEnabledAndAllowed(t *testing.T) { + adx := &types.GrafanaADXDatasource{ + Enabled: types.Value{Value: "true"}, + DeleteWhenDisabled: true, + ClusterURL: types.Value{Input: &types.Input{ + StepDependency: types.StepDependency{ResourceGroup: "kusto-infra", Step: "deploy"}, + Name: "kustoUri", + }}, + DefaultDatabase: types.Value{ConfigRef: "kusto.serviceLogsDatabase"}, + DatasourceName: types.Value{Value: "kusto-int-uks"}, + Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, + DataConsistency: "strongconsistency", + } + + resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, testGrafanaDatasourceConfig(), testGrafanaDatasourceOutputs("https://example.kusto.windows.net")) + if err != nil { + t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) + } + if !resolved.Enabled { + t.Fatal("expected ADX desired state enabled") + } + if !resolved.DeleteWhenDisabled { + t.Fatal("expected deleteWhenDisabled to be preserved") + } + if resolved.ClusterURL != "https://example.kusto.windows.net" { + t.Fatalf("expected cluster URL to resolve, got %q", resolved.ClusterURL) + } + if resolved.DefaultDatabase != "ServiceLogs" { + t.Fatalf("expected default database to resolve, got %q", resolved.DefaultDatabase) + } + if resolved.DatasourceName != "kusto-int-uks" { + t.Fatalf("expected datasource name to resolve, got %q", resolved.DatasourceName) + } + if resolved.Geographies != "UKS, eus2" { + t.Fatalf("expected geographies to resolve, got %q", resolved.Geographies) + } + if resolved.CurrentGeography != "uks" { + t.Fatalf("expected current geography to resolve, got %q", resolved.CurrentGeography) + } +} + +func TestResolveGrafanaADXOptionsDisabledWhenGeographyNotAllowed(t *testing.T) { + cfg := testGrafanaDatasourceConfig() + cfg["monitoring"] = map[string]any{ + "adxDatasourceGeographies": "eus2", + } + adx := &types.GrafanaADXDatasource{ + Enabled: types.Value{Value: "true"}, + DeleteWhenDisabled: true, + DatasourceName: types.Value{Value: "kusto-int-uks"}, + Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, + } + + resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, cfg, nil) + if err != nil { + t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) + } + if resolved.DatasourceName != "kusto-int-uks" { + t.Fatalf("expected datasource name to stay available for delete path, got %q", resolved.DatasourceName) + } + + opts := modify.DefaultAddDatasourceOptions() + opts.SubscriptionID = "subscription-id" + opts.ResourceGroup = "resource-group" + opts.GrafanaName = "grafana-name" + opts.AzureMonitorEnabled = false + opts.ADXEnabled = resolved.Enabled + opts.ADXDeleteWhenDisabled = resolved.DeleteWhenDisabled + opts.ADXDatasourceName = resolved.DatasourceName + opts.ADXGeographies = resolved.Geographies + opts.ADXCurrentGeography = resolved.CurrentGeography + + validated, err := opts.Validate(t.Context()) + if err != nil { + t.Fatalf("Validate returned error: %v", err) + } + if validated.ADXEnabled { + t.Fatal("expected ADX desired state disabled for disallowed geography") + } +} + +func TestResolveGrafanaADXOptionsDisabledByConfigRef(t *testing.T) { + cfg := testGrafanaDatasourceConfig() + cfg["monitoring"] = map[string]any{ + "adxDatasourceEnabled": false, + "adxDatasourceGeographies": "", + } + adx := &types.GrafanaADXDatasource{ + Enabled: types.Value{ConfigRef: "monitoring.adxDatasourceEnabled"}, + DeleteWhenDisabled: true, + DatasourceName: types.Value{Value: "kusto-int-uks"}, + Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, + } + + resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, cfg, nil) + if err != nil { + t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) + } + if resolved.Enabled { + t.Fatal("expected ADX desired state disabled by configRef") + } + + opts := modify.DefaultAddDatasourceOptions() + opts.SubscriptionID = "subscription-id" + opts.ResourceGroup = "resource-group" + opts.GrafanaName = "grafana-name" + opts.AzureMonitorEnabled = false + opts.ADXEnabled = resolved.Enabled + opts.ADXDeleteWhenDisabled = resolved.DeleteWhenDisabled + opts.ADXDatasourceName = resolved.DatasourceName + opts.ADXGeographies = resolved.Geographies + opts.ADXCurrentGeography = resolved.CurrentGeography + + if _, err := opts.Validate(t.Context()); err != nil { + t.Fatalf("Validate returned error: %v", err) + } +} + +func TestResolveGrafanaADXOptionsRejectsInvalidGeographyAllowlist(t *testing.T) { + cfg := testGrafanaDatasourceConfig() + cfg["monitoring"] = map[string]any{ + "adxDatasourceGeographies": "uks,!", + } + adx := &types.GrafanaADXDatasource{ + Enabled: types.Value{Value: "true"}, + Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, + } + + resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, cfg, nil) + if err != nil { + t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) + } + + opts := modify.DefaultAddDatasourceOptions() + opts.SubscriptionID = "subscription-id" + opts.ResourceGroup = "resource-group" + opts.GrafanaName = "grafana-name" + opts.AzureMonitorEnabled = false + opts.ADXEnabled = resolved.Enabled + opts.ADXGeographies = resolved.Geographies + opts.ADXCurrentGeography = resolved.CurrentGeography + + _, err = opts.Validate(t.Context()) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "invalid entry") { + t.Fatalf("expected invalid allowlist error, got %v", err) + } +} + +func TestResolveGrafanaADXOptionsFailsClosedOnMissingKustoURI(t *testing.T) { + adx := &types.GrafanaADXDatasource{ + Enabled: types.Value{Value: "true"}, + ClusterURL: types.Value{Input: &types.Input{ + StepDependency: types.StepDependency{ResourceGroup: "kusto-infra", Step: "deploy"}, + Name: "kustoUri", + }}, + DefaultDatabase: types.Value{Value: "ServiceLogs"}, + DatasourceName: types.Value{Value: "kusto-int-uks"}, + } + + resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, testGrafanaDatasourceConfig(), testGrafanaDatasourceOutputs("")) + if err != nil { + t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) + } + + opts := modify.DefaultAddDatasourceOptions() + opts.SubscriptionID = "subscription-id" + opts.ResourceGroup = "resource-group" + opts.GrafanaName = "grafana-name" + opts.AzureMonitorEnabled = false + opts.ADXEnabled = resolved.Enabled + opts.ADXClusterURL = resolved.ClusterURL + opts.ADXDefaultDatabase = resolved.DefaultDatabase + opts.ADXDatasourceName = resolved.DatasourceName + opts.ADXGeographies = resolved.Geographies + opts.ADXCurrentGeography = resolved.CurrentGeography + + _, err = opts.Validate(t.Context()) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "cluster URL is required") { + t.Fatalf("expected missing cluster URL error, got %v", err) + } +} From 1b32952731242ab15d003fe35eb70e0fae8b7208 Mon Sep 17 00:00:00 2001 From: swiencki Date: Thu, 30 Apr 2026 14:14:57 -0700 Subject: [PATCH 11/12] monitoring: use real geo short ID in ADX tests --- .../pkg/pipeline/grafana_datasources_test.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tooling/templatize/pkg/pipeline/grafana_datasources_test.go b/tooling/templatize/pkg/pipeline/grafana_datasources_test.go index f02e76bf458..f525913587f 100644 --- a/tooling/templatize/pkg/pipeline/grafana_datasources_test.go +++ b/tooling/templatize/pkg/pipeline/grafana_datasources_test.go @@ -25,12 +25,12 @@ import ( func testGrafanaDatasourceConfig() configtypes.Configuration { return configtypes.Configuration{ - "azureGeoShortId": "uks", + "azureGeoShortId": "uk", "kusto": map[string]any{ "serviceLogsDatabase": "ServiceLogs", }, "monitoring": map[string]any{ - "adxDatasourceGeographies": " UKS, eus2 ", + "adxDatasourceGeographies": " UK, eus2 ", }, } } @@ -59,7 +59,7 @@ func TestResolveGrafanaADXOptionsEnabledAndAllowed(t *testing.T) { Name: "kustoUri", }}, DefaultDatabase: types.Value{ConfigRef: "kusto.serviceLogsDatabase"}, - DatasourceName: types.Value{Value: "kusto-int-uks"}, + DatasourceName: types.Value{Value: "kusto-int-uk"}, Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, DataConsistency: "strongconsistency", } @@ -80,13 +80,13 @@ func TestResolveGrafanaADXOptionsEnabledAndAllowed(t *testing.T) { if resolved.DefaultDatabase != "ServiceLogs" { t.Fatalf("expected default database to resolve, got %q", resolved.DefaultDatabase) } - if resolved.DatasourceName != "kusto-int-uks" { + if resolved.DatasourceName != "kusto-int-uk" { t.Fatalf("expected datasource name to resolve, got %q", resolved.DatasourceName) } - if resolved.Geographies != "UKS, eus2" { + if resolved.Geographies != "UK, eus2" { t.Fatalf("expected geographies to resolve, got %q", resolved.Geographies) } - if resolved.CurrentGeography != "uks" { + if resolved.CurrentGeography != "uk" { t.Fatalf("expected current geography to resolve, got %q", resolved.CurrentGeography) } } @@ -99,7 +99,7 @@ func TestResolveGrafanaADXOptionsDisabledWhenGeographyNotAllowed(t *testing.T) { adx := &types.GrafanaADXDatasource{ Enabled: types.Value{Value: "true"}, DeleteWhenDisabled: true, - DatasourceName: types.Value{Value: "kusto-int-uks"}, + DatasourceName: types.Value{Value: "kusto-int-uk"}, Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, } @@ -107,7 +107,7 @@ func TestResolveGrafanaADXOptionsDisabledWhenGeographyNotAllowed(t *testing.T) { if err != nil { t.Fatalf("resolveGrafanaADXOptions returned error: %v", err) } - if resolved.DatasourceName != "kusto-int-uks" { + if resolved.DatasourceName != "kusto-int-uk" { t.Fatalf("expected datasource name to stay available for delete path, got %q", resolved.DatasourceName) } @@ -140,7 +140,7 @@ func TestResolveGrafanaADXOptionsDisabledByConfigRef(t *testing.T) { adx := &types.GrafanaADXDatasource{ Enabled: types.Value{ConfigRef: "monitoring.adxDatasourceEnabled"}, DeleteWhenDisabled: true, - DatasourceName: types.Value{Value: "kusto-int-uks"}, + DatasourceName: types.Value{Value: "kusto-int-uk"}, Geographies: types.Value{ConfigRef: "monitoring.adxDatasourceGeographies"}, } @@ -171,7 +171,7 @@ func TestResolveGrafanaADXOptionsDisabledByConfigRef(t *testing.T) { func TestResolveGrafanaADXOptionsRejectsInvalidGeographyAllowlist(t *testing.T) { cfg := testGrafanaDatasourceConfig() cfg["monitoring"] = map[string]any{ - "adxDatasourceGeographies": "uks,!", + "adxDatasourceGeographies": "uk,!", } adx := &types.GrafanaADXDatasource{ Enabled: types.Value{Value: "true"}, @@ -209,7 +209,7 @@ func TestResolveGrafanaADXOptionsFailsClosedOnMissingKustoURI(t *testing.T) { Name: "kustoUri", }}, DefaultDatabase: types.Value{Value: "ServiceLogs"}, - DatasourceName: types.Value{Value: "kusto-int-uks"}, + DatasourceName: types.Value{Value: "kusto-int-uk"}, } resolved, err := resolveGrafanaADXOptions("Microsoft.Azure.ARO.HCP.Geography", adx, testGrafanaDatasourceConfig(), testGrafanaDatasourceOutputs("")) From 4d471666345b126ab430c56230572f4563d27b19 Mon Sep 17 00:00:00 2001 From: swiencki Date: Fri, 8 May 2026 08:56:26 -0700 Subject: [PATCH 12/12] ARO-26057: consolidate Grafana datasource provisioning into region pipeline Per PR #4878 review. Drops the new GrafanaDatasources call from geography-pipeline.yaml and extends the existing add-grafana-datasource in region-pipeline.yaml with the ADX block. clusterUrl sources from regional kusto-lookup output. AzureMonitor reconcile preserved via the ARO-Tools default-true. Triple-AND gate keeps the three feature flags agreeing by construction. No Go/Bicep/config/test changes. --- dev-infrastructure/geography-pipeline.yaml | 48 -------------------- dev-infrastructure/region-pipeline.yaml | 51 ++++++++++++++++++++++ docs/monitoring.md | 4 +- 3 files changed, 53 insertions(+), 50 deletions(-) diff --git a/dev-infrastructure/geography-pipeline.yaml b/dev-infrastructure/geography-pipeline.yaml index ebc65ef01ef..8de2fac81b6 100644 --- a/dev-infrastructure/geography-pipeline.yaml +++ b/dev-infrastructure/geography-pipeline.yaml @@ -57,51 +57,3 @@ resourceGroups: dependsOn: - resourceGroup: global step: output - - name: adx-datasource - action: GrafanaDatasources - omitFromServiceGroupCompletion: true - grafanaName: "{{ .monitoring.grafanaName }}" - grafanaResourceId: - input: - resourceGroup: global - step: output - name: grafanaResourceId - azureMonitor: - enabled: false - adx: - enabled: - value: "{{ and .monitoring.adxDatasourceEnabled .kusto.manageInstance }}" - deleteWhenDisabled: true - clusterUrl: - input: - resourceGroup: kusto-infra - step: deploy - name: kustoUri - defaultDatabase: - configRef: kusto.serviceLogsDatabase - datasourceName: - value: "kusto-{{ .environmentName }}-{{ .azureGeoShortId }}" - geographies: - configRef: monitoring.adxDatasourceGeographies - dataConsistency: strongconsistency - automatedRetry: - errorContainsAny: - - "grafana API POST /api/datasources" - - "grafana API PUT /api/datasources/" - - "grafana API DELETE /api/datasources/name/" - - "failed with status 429" - - "failed with status 500" - - "failed with status 502" - - "failed with status 503" - - "failed with status 504" - maximumRetryCount: 5 - durationBetweenRetries: 2m - identityFrom: - resourceGroup: global - step: output - name: globalMSIId - dependsOn: - - resourceGroup: global - step: output - - resourceGroup: kusto-infra - step: deploy diff --git a/dev-infrastructure/region-pipeline.yaml b/dev-infrastructure/region-pipeline.yaml index 8ae3cc80870..a07bdea00d2 100644 --- a/dev-infrastructure/region-pipeline.yaml +++ b/dev-infrastructure/region-pipeline.yaml @@ -60,6 +60,53 @@ resourceGroups: maximumRetryCount: 3 durationBetweenRetries: 1m grafanaName: "{{ .monitoring.grafanaName }}" + grafanaResourceId: + input: + resourceGroup: global + step: output + name: grafanaResourceId + # ADX datasource reconciliation. The triple-AND gate is intentional: + # adxDatasourceEnabled is the feature flag, kusto.manageInstance is + # the per-region "this region owns the geo's Kusto cluster" marker + # (mirrors the per-region allowlist that the deleted geography-level + # step encoded as executionConstraints), and arobit.kusto.enabled is + # what kusto-lookup.bicep gates on for the kustoUri output. Today + # exactly one region per geoShortId has manageInstance=true; if that + # invariant ever changes, the geographies allowlist below would + # need to be region-aware to avoid two regions racing to reconcile + # the same global Grafana datasource. + adx: + enabled: + value: "{{ and .monitoring.adxDatasourceEnabled .kusto.manageInstance .arobit.kusto.enabled }}" + deleteWhenDisabled: true + clusterUrl: + input: + resourceGroup: kusto + step: output + name: kustoUri + defaultDatabase: + configRef: kusto.serviceLogsDatabase + datasourceName: + value: "kusto-{{ .environmentName }}-{{ .azureGeoShortId }}" + geographies: + configRef: monitoring.adxDatasourceGeographies + dataConsistency: strongconsistency + # Retry signature matches any Grafana datasource API mutation in this + # step (POST/PUT/DELETE on /api/datasources*), not only ADX. If + # AzureMonitor reconcile is ever explicitly toggled here, the same + # retry policy applies to those calls too. + automatedRetry: + errorContainsAny: + - "grafana API POST /api/datasources" + - "grafana API PUT /api/datasources/" + - "grafana API DELETE /api/datasources/name/" + - "failed with status 429" + - "failed with status 500" + - "failed with status 502" + - "failed with status 503" + - "failed with status 504" + maximumRetryCount: 5 + durationBetweenRetries: 2m identityFrom: resourceGroup: global step: output @@ -67,6 +114,10 @@ resourceGroups: dependsOn: - resourceGroup: regional step: output + - resourceGroup: global + step: output + - resourceGroup: kusto + step: output - name: kusto resourceGroup: '{{ .kusto.rg }}' subscription: '{{ .global.subscription.key }}' diff --git a/docs/monitoring.md b/docs/monitoring.md index 4f15569840a..5b0d4d25e9b 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -94,7 +94,7 @@ A single **Azure Managed Grafana** instance is deployed globally and configured Managed Grafana can also expose Azure Data Explorer datasources for ARO-HCP Kusto clusters. These datasources follow the same single-workspace-per-environment model as the Azure Monitor datasources above. -- **Provisioning scope**: Geography pipeline, once per managed Kusto cluster +- **Provisioning scope**: Region pipeline, gated per-region by `kusto.manageInstance` (one region per `geoShortId` owns the cluster) - **Datasource name**: `kusto--` (for example `kusto-int-uk`) - **Target database**: `ServiceLogs` - **Authentication**: Grafana workspace system-assigned managed identity @@ -105,7 +105,7 @@ Provisioning is controlled by: - `monitoring.adxDatasourceEnabled` - `monitoring.adxDatasourceGeographies` -When `adxDatasourceEnabled` is `true`, the geography pipeline uses the typed `GrafanaDatasources` action to create or update the datasource if that geography has a managed Kusto cluster (`kusto.manageInstance: true`). When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. Matching is case-insensitive and ignores surrounding whitespace around comma-separated entries. The same `grafanactl` reconcile path evaluates the allowlist during EV2 rollout and local templatize runs. +When `adxDatasourceEnabled` is `true`, the region pipeline's existing `add-grafana-datasource` step uses the typed `GrafanaDatasources` action to create or update the datasource. The step runs in every region the regional pipeline touches, but the runtime gate `adxDatasourceEnabled && kusto.manageInstance && arobit.kusto.enabled` limits actual reconciliation to the region that owns the geography's managed Kusto cluster. `clusterUrl` is sourced from the regional `kusto-lookup.bicep` output, which `existing`s the cluster the geography pipeline already created. When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. Matching is case-insensitive and ignores surrounding whitespace around comma-separated entries. The same `grafanactl` reconcile path evaluates the allowlist during EV2 rollout and local templatize runs. After rollout, validate at least one datasource during the environment bake window with: