ARO-26057: add ADX Grafana datasource provisioning

config/config.schema.json

@@ -2786,6 +2786,15 @@
         "grafanaName": {
           "type": "string"
         },
+        "adxDatasourceEnabled": {
+          "type": "boolean",
+          "description": "Whether to provision ADX datasources in Managed Grafana"
+        },
+        "adxDatasourceGeographies": {
+          "type": "string",
+          "pattern": "^$|^\\s*[A-Za-z0-9-]+\\s*(,\\s*[A-Za-z0-9-]+\\s*)*$",
+          "description": "Comma-separated geography allowlist for optional targeted ADX datasource provisioning/removal. Empty means all geographies. Matching is case-insensitive and surrounding whitespace is ignored."
+        },
         "grafanaMajorVersion": {
           "type": "string"
         },

config/config.yaml

@@ -345,6 +345,8 @@ defaults:
   # Monitoring
   monitoring:
     grafanaName: "arohcp-{{ .ctx.environment }}"
+    adxDatasourceEnabled: false
+    adxDatasourceGeographies: ""
     # Format:
     #   Multiline string using '>-' YAML block scalar
     #   One item per line, formatted as: UUID/PrincipalType/RoleName

dev-infrastructure/cleanup/delete.kusto.instance.sh

@@ -87,7 +87,7 @@ kusto_instance_id=$(get_kusto_instance_id)
 if [[ -n "$kusto_instance_id" ]]; then
     if has_locks "$kusto_instance_id"; then
         log WARN "Skipping locked Kusto instance: $kusto_instance_id"
-        continue
+        exit 0
     fi
 
     if [[ "$DRY_RUN" == "true" ]]; then

dev-infrastructure/configurations/kusto.tmpl.bicepparam

@@ -7,6 +7,8 @@ param tier = '{{ .kusto.tier }}'
 
 param kustoName = '{{ .kusto.kustoName }}'
 
+param grafanaResourceId = '__grafanaResourceId__'
+
 param manageInstance = {{ .kusto.manageInstance }}
 
 param serviceLogsDatabase = '{{ .kusto.serviceLogsDatabase }}'

dev-infrastructure/geography-pipeline.yaml

@@ -2,6 +2,16 @@ $schema: pipeline.schema.v1
 rolloutName: Geography Rollout
 serviceGroup: Microsoft.Azure.ARO.HCP.Geography
 resourceGroups:
+- name: global
+  resourceGroup: '{{ .global.rg }}'
+  subscription: '{{ .global.subscription.key }}'
+  steps:
+  - name: output
+    action: ARM
+    template: templates/output-global.bicep
+    parameters: configurations/output-global.tmpl.bicepparam
+    deploymentLevel: ResourceGroup
+    outputOnly: true
 - name: kusto-infra
   resourceGroup: '{{ .kusto.rg }}'
   subscription: '{{ .global.subscription.key }}'
@@ -38,3 +48,12 @@ resourceGroups:
     template: templates/kusto.bicep
     parameters: configurations/kusto.tmpl.bicepparam
     deploymentLevel: ResourceGroup
+    variables:
+    - name: grafanaResourceId
+      input:
+        resourceGroup: global
+        step: output
+        name: grafanaResourceId
+    dependsOn:
+    - resourceGroup: global
+      step: output

dev-infrastructure/modules/logs/kusto/cluster.bicep

@@ -120,3 +120,4 @@ resource kusto 'Microsoft.Kusto/clusters@2024-04-13' = {
 
 output id string = kusto.id
 output name string = kusto.name
+output uri string = kusto.properties.uri

dev-infrastructure/modules/logs/kusto/grant-access.bicep

@@ -14,7 +14,7 @@ resource database 'Microsoft.Kusto/clusters/databases@2024-04-13' existing = {
 resource grantIngest 'Microsoft.Kusto/clusters/databases/principalAssignments@2024-04-13' = [
   for id in ingestAccessPrincipalIds: {
     parent: database
-    name: 'grant-${guid(id, databaseName)}'
+    name: 'grant-ingest-${guid(id, databaseName)}'
     properties: {
       principalId: id
       principalType: 'App'
@@ -27,7 +27,7 @@ resource grantIngest 'Microsoft.Kusto/clusters/databases/principalAssignments@20
 resource grantRead 'Microsoft.Kusto/clusters/databases/principalAssignments@2024-04-13' = [
   for id in readAccessPrincipalIds: {
     parent: database
-    name: 'grant-${guid(id, databaseName)}'
+    name: 'grant-viewer-${guid(id, databaseName)}'
     properties: {
       principalId: id
       principalType: 'App'

dev-infrastructure/modules/logs/kusto/main.bicep

@@ -1,3 +1,5 @@
+import * as res from '../../resource.bicep'
+
 @description('Azure Region Location')
 param location string = resourceGroup().location
 
@@ -44,15 +46,26 @@ param crossClusterServiceLogsScript string = ''
 @description('Optional cross-cluster HostedControlPlaneLogs Kusto script content.')
 @secure()
 param crossClusterHostedControlPlaneLogsScript string = ''
+
+@description('Optional: Grafana resource ID for database-level Viewer access')
+param grafanaResourceId string = ''
+
 var db = {
   serviceLogs: serviceLogsDatabase
   hostedControlPlaneLogs: hostedControlPlaneLogsDatabase
 }
 
 var databases = [db.serviceLogs, db.hostedControlPlaneLogs]
+var hasGrafana = grafanaResourceId != ''
+var grafanaRef = hasGrafana ? res.grafanaRefFromId(grafanaResourceId) : null
 
 var dummyScript = '.create-or-alter function with (docstring = \'dummy function to run last and to remove permission\') dummyFunction() {print \'dummy\'}'
 
+resource grafana 'Microsoft.Dashboard/grafana@2024-10-01' existing = if (hasGrafana) {
+  name: grafanaRef!.name
+  scope: resourceGroup(grafanaRef!.resourceGroup.subscriptionId, grafanaRef!.resourceGroup.name)
+}
+
 var allServiceLogsTablesKQL = {
   backendLogs: loadTextContent('tables/backendLogs.kql')
   containerlogs: loadTextContent('tables/containerLogs.kql')
@@ -194,7 +207,18 @@ module crossClusterHostedControlPlaneLogsQueryScript 'script.bicep' = if (deploy
   ]
 }
 
-// 6. Remove the caller principal
+// 6. Grafana service logs access
+module grafanaServiceLogsAccess 'grant-access.bicep' = if (hasGrafana) {
+  name: 'grafana-serviceLogs-viewer'
+  params: {
+    kustoName: kustoName
+    databaseName: db.serviceLogs
+    readAccessPrincipalIds: [grafana!.identity.principalId]
+  }
+  dependsOn: [serviceLogsTables]
+}
+
+// 7. Remove the caller principal
 // THIS MUST BE THE LAST SCRIPT TO RUN
 module removePermission 'script.bicep' = [
   for (database, i) in databases: {
@@ -213,9 +237,11 @@ module removePermission 'script.bicep' = [
       hostedControlPlaneLogsTables
       crossClusterServiceLogsQueryScript
       crossClusterHostedControlPlaneLogsQueryScript
+      grafanaServiceLogsAccess
     ]
   }
 ]
 
 // Outputs mirror original contract
 output id string = cluster.outputs.id
+output kustoUri string = cluster.outputs.uri

dev-infrastructure/region-pipeline.yaml

@@ -60,13 +60,64 @@ resourceGroups:
       maximumRetryCount: 3
       durationBetweenRetries: 1m
     grafanaName: "{{ .monitoring.grafanaName }}"
+    grafanaResourceId:
+      input:
+        resourceGroup: global
+        step: output
+        name: grafanaResourceId
+    # ADX datasource reconciliation. The triple-AND gate is intentional:
+    # adxDatasourceEnabled is the feature flag, kusto.manageInstance is
+    # the per-region "this region owns the geo's Kusto cluster" marker
+    # (mirrors the per-region allowlist that the deleted geography-level
+    # step encoded as executionConstraints), and arobit.kusto.enabled is
+    # what kusto-lookup.bicep gates on for the kustoUri output. Today
+    # exactly one region per geoShortId has manageInstance=true; if that
+    # invariant ever changes, the geographies allowlist below would
+    # need to be region-aware to avoid two regions racing to reconcile
+    # the same global Grafana datasource.
+    adx:
+      enabled:
+        value: "{{ and .monitoring.adxDatasourceEnabled .kusto.manageInstance .arobit.kusto.enabled }}"
+      deleteWhenDisabled: true
+      clusterUrl:
+        input:
+          resourceGroup: kusto
+          step: output
+          name: kustoUri
+      defaultDatabase:
+        configRef: kusto.serviceLogsDatabase
+      datasourceName:
+        value: "kusto-{{ .environmentName }}-{{ .azureGeoShortId }}"
+      geographies:
+        configRef: monitoring.adxDatasourceGeographies
+      dataConsistency: strongconsistency
+    # Retry signature matches any Grafana datasource API mutation in this
+    # step (POST/PUT/DELETE on /api/datasources*), not only ADX. If
+    # AzureMonitor reconcile is ever explicitly toggled here, the same
+    # retry policy applies to those calls too.
+    automatedRetry:
+      errorContainsAny:
+      - "grafana API POST /api/datasources"
+      - "grafana API PUT /api/datasources/"
+      - "grafana API DELETE /api/datasources/name/"
+      - "failed with status 429"
+      - "failed with status 500"
+      - "failed with status 502"
+      - "failed with status 503"
+      - "failed with status 504"
+      maximumRetryCount: 5
+      durationBetweenRetries: 2m
     identityFrom:
       resourceGroup: global
       step: output
       name: globalMSIId
     dependsOn:
     - resourceGroup: regional
       step: output
+    - resourceGroup: global
+      step: output
+    - resourceGroup: kusto
+      step: output
 - name: kusto
   resourceGroup: '{{ .kusto.rg }}'
   subscription: '{{ .global.subscription.key }}'

dev-infrastructure/templates/kusto.bicep

@@ -44,6 +44,10 @@ param crossClusterServiceLogsScript string = ''
 @description('Optional cross-cluster HostedControlPlaneLogs Kusto script content.')
 @secure()
 param crossClusterHostedControlPlaneLogsScript string = ''
+
+@description('Optional: Grafana resource ID for database-level Viewer access')
+param grafanaResourceId string = ''
+
 module kusto '../modules/logs/kusto/main.bicep' = if (manageInstance) {
   name: 'kusto-${location}'
   params: {
@@ -62,5 +66,8 @@ module kusto '../modules/logs/kusto/main.bicep' = if (manageInstance) {
     enableAutoScale: enableAutoScale
     crossClusterServiceLogsScript: crossClusterServiceLogsScript
     crossClusterHostedControlPlaneLogsScript: crossClusterHostedControlPlaneLogsScript
+    grafanaResourceId: grafanaResourceId
   }
 }
+
+output kustoUri string = manageInstance ? kusto!.outputs.kustoUri : ''

docs/monitoring.md

@@ -90,6 +90,47 @@ A single **Azure Managed Grafana** instance is deployed globally and configured
 - Region-agnostic dashboard experience
 - Consolidated alerting and monitoring workflows
 
+### ADX / Kusto datasources
+
+Managed Grafana can also expose Azure Data Explorer datasources for ARO-HCP Kusto clusters. These datasources follow the same single-workspace-per-environment model as the Azure Monitor datasources above.
+
+- **Provisioning scope**: Region pipeline, gated per-region by `kusto.manageInstance` (one region per `geoShortId` owns the cluster)
+- **Datasource name**: `kusto-<env>-<geoShortId>` (for example `kusto-int-uk`)
+- **Target database**: `ServiceLogs`
+- **Authentication**: Grafana workspace system-assigned managed identity
+- **RBAC**: Database-level `Viewer` on `ServiceLogs` only
+
+Provisioning is controlled by:
+
+- `monitoring.adxDatasourceEnabled`
+- `monitoring.adxDatasourceGeographies`
+
+When `adxDatasourceEnabled` is `true`, the region pipeline's existing `add-grafana-datasource` step uses the typed `GrafanaDatasources` action to create or update the datasource. The step runs in every region the regional pipeline touches, but the runtime gate `adxDatasourceEnabled && kusto.manageInstance && arobit.kusto.enabled` limits actual reconciliation to the region that owns the geography's managed Kusto cluster. `clusterUrl` is sourced from the regional `kusto-lookup.bicep` output, which `existing`s the cluster the geography pipeline already created. When `adxDatasourceGeographies` is empty, all managed Kusto geographies in the environment are included. When it is set, only the listed geography short IDs are allowed. Matching is case-insensitive and ignores surrounding whitespace around comma-separated entries. The same `grafanactl` reconcile path evaluates the allowlist during EV2 rollout and local templatize runs.
+
+After rollout, validate at least one datasource during the environment bake window with:
+
+1. Grafana UI
+2. Data sources
+3. `kusto-<env>-<geoShortId>`
+4. **Save & Test**
+
+### Kusto datasource lifecycle and teardown
+
+`adxDatasourceEnabled: false` disables the ADX desired state. The typed datasource reconcile step still runs and deletes the named datasource when deletion-on-disable is enabled, so disabling ADX or removing a geography from `adxDatasourceGeographies` cleans up stale Grafana datasource configuration.
+
+The regular geography pipeline still fails closed when ADX is desired but no `kustoUri` is available. That prevents accidental deletion caused by transient deployment or output issues during normal rollout or provisioning retries.
+
+If a datasource needs to be removed outside the desired-state rollout path, delete it explicitly:
+
+```bash
+az grafana data-source delete \
+  --name "${GRAFANA_NAME}" \
+  --resource-group "${GRAFANA_RG}" \
+  --data-source "kusto-<env>-<geoShortId>"
+```
+
+Datasource deletion is intentionally separate from normal Kusto cleanup so decommissioning Kusto infrastructure cannot remove shared Grafana configuration by accident.
+
 ### Regional Azure Monitor Workspace
 
 Each region contains **two Azure Monitor Workspaces (AMW)**: