Taskfile.yaml

# Local DNS platform environment, built on the shared datum-cloud/test-infra
# tooling (same pattern milo-os/activity uses: a pinned remote Taskfile
# include). Requires:
#   export TASK_X_REMOTE_TASKFILES=1
#
# `task env:up` brings up two kind clusters - "control" (dns-operator +
# PowerDNS + RustFS) and "edge" (PowerDNS only) - and verifies that zone data
# written on the control cluster replicates to the edge cluster via PowerDNS
# Lightningstream over a shared RustFS (S3-compatible) bucket, the same way
# production replicates zone data to edge PoPs over GCS.
version: '3'

vars:
  # Bump to the merged tag once datum-cloud/test-infra#35 lands (it removes
  # the hardcoded NodePorts that block running two clusters concurrently).
  TEST_INFRA_REPO_REF: 'fix/remove-hardcoded-nodeports'
  # Upstream (control-plane) cluster runs the replicator; it pushes DNSZone/
  # DNSRecordSet resources to the control cluster, which plays the "downstream"
  # agent role for the replicator and the "control" (Lightningstream source)
  # role for edge replication - one cluster, two roles, mirroring production.
  UPSTREAM_CLUSTER_NAME: 'dns-upstream'
  CONTROL_CLUSTER_NAME: 'dns-control'
  EDGE_CLUSTER_NAME: 'dns-edge'
  IMG: 'ghcr.io/datum-cloud/dns-operator:latest'
  RUSTFS_BUCKET: 'lightningstream'
  RUSTFS_NODEPORT_FILE: 'dev/rustfs.nodeport'
  # In-cluster-rewritten kubeconfig for the control cluster, mounted into the
  # replicator (via a Secret) so it can reach control's API as its downstream.
  CONTROL_INCLUSTER_KUBECONFIG: 'dev/control.incluster.kubeconfig'
  # Admin kubeconfig for the Milo core control plane (milo-apiserver), minted
  # from the static test-admin-token. Used to install DNS CRDs and create the
  # Org + projects on the core CP. Server points at a local port-forward.
  MILO_ADMIN_KUBECONFIG: 'dev/milo.admin.kubeconfig'
  # dns-operator image used by the replicator on dns-upstream. Loaded into the
  # kind node; the overlays reference :latest with imagePullPolicy IfNotPresent.
  DNSOP_IMG: 'ghcr.io/datum-cloud/dns-operator:latest'
  # Cross-cluster kubeconfig the replicator uses to reach milo-apiserver on
  # dns-control, via the envoy gateway NodePort on the control node's IP.
  MILO_UPSTREAM_KUBECONFIG: 'dev/milo.upstream.kubeconfig'
  MILO_GATEWAY_NODEPORT: '32648'
  # Dedicated kubeconfig for this environment (not the user's default
  # ~/.kube/config). go-task's `env:` blocks are implemented as process-wide
  # os.Setenv calls that leak between sibling task/cmd invocations (a known
  # go-task quirk, not scoped per-task as the docs imply) - the included
  # test-infra Taskfile sets KUBECONFIG to its own `.test-infra/kubeconfig`
  # this way, and that value silently leaks into every cmd we run for the
  # rest of the process, clobbering the default kubeconfig assumption. Every
  # kubectl/kind/make invocation below explicitly sets KUBECONFIG to this
  # fixed path so behavior doesn't depend on call order or that leak.
  ENV_KUBECONFIG: 'dev/kubeconfig'

includes:
  test-infra:
    taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REPO_REF}}/Taskfile.yml
    vars:
      REPO_REF: "{{.TEST_INFRA_REPO_REF}}"

tasks:
  default:
    desc: "Show available commands"
    silent: true
    cmds:
      - echo "DNS platform local environment (upstream replicator + control + edge federation):"
      - echo "  task env:up          Bring up upstream + control + edge clusters, deploy a sample zone"
      - echo "  task env:stack-up    Bring up the full chain with no sample data (used by CI)"
      - echo "  task env:chainsaw    Run the full Chainsaw e2e suite against the environment"
      - echo "  task env:down        Tear down all clusters"
      - echo "  task env:dns-check   Query control + edge PowerDNS for the sample record"
      - echo ""
      - echo "Individual pieces:"
      - echo "  task env:control-up  Control cluster only (dns-operator + PowerDNS + RustFS)"
      - echo "  task env:edge-up     Edge cluster only (PowerDNS, replicates from control)"
      - echo "  task env:upstream-up Upstream cluster only (replicator, pushes to control)"
      - echo ""
      - echo "Underlying test-infra tasks are available as test-infra:<name>, e.g.:"
      - echo "  task test-infra:kubectl -- get pods --all-namespaces"

  env:rustfs-up:
    desc: "Deploy RustFS (S3-compatible object store) into the control cluster via Flux"
    silent: true
    cmds:
      - echo "➡️  Installing RustFS into kind-{{.CONTROL_CLUSTER_NAME}}..."
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} apply -k config/dependencies/rustfs
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n rustfs-system wait helmrelease/rustfs --for=condition=Ready --timeout=300s
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        mkdir -p dev
        SVC=$(kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} get svc -n rustfs-system -l app.kubernetes.io/instance=rustfs -o jsonpath='{.items[0].metadata.name}')
        kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n rustfs-system wait --for=jsonpath='{.spec.ports[0].nodePort}' svc/$SVC --timeout=60s
        PORT=$(kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} get svc -n rustfs-system $SVC -o jsonpath='{.spec.ports[0].nodePort}')
        echo "$PORT" > {{.RUSTFS_NODEPORT_FILE}}
        echo "✅ RustFS ready: svc/$SVC, NodePort=$PORT (recorded to {{.RUSTFS_NODEPORT_FILE}})"
      # RustFS (like any S3-compatible store) does not auto-create buckets,
      # and the pinned chart version predates its `setup.buckets` hook (see
      # config/dependencies/rustfs/helmrelease.yaml) - so create the bucket
      # Lightningstream syncs snapshots through ourselves, via a one-off pod.
      # Without this, lightningstream just polls forever logging
      # "list snapshots: Volume not found" and never actually syncs.
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n rustfs-system delete pod rustfs-mb --ignore-not-found
        kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n rustfs-system run rustfs-mb \
          --image=amazon/aws-cli:2.17.62 --restart=Never \
          --env=AWS_ACCESS_KEY_ID=dns-federation --env=AWS_SECRET_ACCESS_KEY=dns-federation-secret \
          --command -- \
          sh -c "aws --endpoint-url http://rustfs.rustfs-system.svc.cluster.local:9000 s3 mb s3://{{.RUSTFS_BUCKET}} --region us-east-1 || true"
        kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n rustfs-system wait --for=jsonpath='{.status.phase}'=Succeeded pod/rustfs-mb --timeout=60s
        echo "✅ RustFS bucket '{{.RUSTFS_BUCKET}}' ready"

  # internal: writes/updates the s3-credentials Secret dns-operator's agent
  # (see config/agent/manager.yaml) reads its Lightningstream S3 config from.
  env:s3-secret:
    internal: true
    silent: true
    cmds:
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        kubectl --context {{.CONTEXT}} create namespace dns-agent-system --dry-run=client -o yaml | kubectl --context {{.CONTEXT}} apply -f -
        kubectl --context {{.CONTEXT}} -n dns-agent-system create secret generic s3-credentials \
          --from-literal=accesskey=dns-federation \
          --from-literal=secretkey=dns-federation-secret \
          --from-literal=endpoint_url={{.ENDPOINT_URL}} \
          --from-literal=bucket={{.RUSTFS_BUCKET}} \
          --from-literal=region=us-east-1 \
          --from-literal=cluster_id={{.CLUSTER_ID}}- \
          --dry-run=client -o yaml | kubectl --context {{.CONTEXT}} apply -f -
        echo "✅ s3-credentials set in {{.CONTEXT}}: endpoint_url={{.ENDPOINT_URL}}"
        # The pdns-auth StatefulSet (see config/agent/manager.yaml) is applied
        # by kustomize-apply BEFORE this secret exists, so its lightningstream
        # container's secretKeyRef env vars (optional: true) start out empty.
        # Kubernetes does not live-update already-injected env vars when a
        # Secret is created/changed, so restart the pod to pick them up.
        kubectl --context {{.CONTEXT}} -n dns-agent-system rollout restart statefulset/pdns-auth
        kubectl --context {{.CONTEXT}} -n dns-agent-system rollout status statefulset/pdns-auth --timeout=120s

  # internal: brings up a test-infra-managed kind cluster (kind cluster +
  # Flux + cert-manager + Kyverno + Envoy Gateway) for CLUSTER_NAME.
  #
  # NOTE: this deliberately calls the individual test-infra sub-tasks
  # (ensure-repo/ensure-tools/create-kind/install-components) instead of the
  # composite `test-infra:cluster-up`/`up` task. As of test-infra's
  # fix/remove-hardcoded-nodeports branch (and go-task 3.49/3.50),
  # `cluster-up`'s trailing informational `echo "... # ..."` lines panic
  # go-task's command parser ("reached EOF without closing quote") because
  # go-task strips ` #...` as a shell comment even inside double quotes on
  # single-line (non `|` block) cmd entries - a go-task parsing quirk, not
  # a shell issue. Calling the sub-tasks directly gets the same cluster
  # setup while skipping the broken trailing echoes. If that upstream
  # Taskfile is fixed (or go-task fixes the quoting bug), this can go back
  # to a single `task: test-infra:cluster-up` call.
  env:test-infra-cluster-up:
    internal: true
    cmds:
      - task: test-infra:ensure-repo
        vars:
          CLUSTER_NAME: "{{.CLUSTER_NAME}}"
      - task: test-infra:ensure-tools
        vars:
          CLUSTER_NAME: "{{.CLUSTER_NAME}}"
      - task: test-infra:create-kind
        vars:
          CLUSTER_NAME: "{{.CLUSTER_NAME}}"
      - task: test-infra:install-components
        vars:
          CLUSTER_NAME: "{{.CLUSTER_NAME}}"
      # test-infra's create-kind writes the cluster's kubeconfig to its own
      # KUBECONFIG_FILE (e.g. .test-infra/kubeconfig when run from an
      # external repo like this one) via `kind create cluster --kubeconfig
      # <file>`, which does NOT merge a kind-<name> context into the
      # default kubeconfig. Every other task in this Taskfile (and `make`
      # targets it shells out to) assumes the standard `kubectl --context
      # kind-<name>` convention, so re-export/merge it into the default
      # kubeconfig here - into our own dedicated ENV_KUBECONFIG file (not the
      # default ~/.kube/config, which is subject to the same leaked-env risk
      # via kind's own default KUBECONFIG resolution). Every subsequent cmd
      # in this Taskfile explicitly sets KUBECONFIG to ENV_KUBECONFIG too.
      - kind export kubeconfig --name {{.CLUSTER_NAME}} --kubeconfig {{.ENV_KUBECONFIG}}

  env:control-up:
    desc: "Create the control kind cluster and deploy dns-operator + PowerDNS + RustFS"
    cmds:
      - task: env:test-infra-cluster-up
        vars:
          CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}"
      - task: env:rustfs-up
      - CLUSTER={{.CONTROL_CLUSTER_NAME}} IMG={{.IMG}} make kind-load-image
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}} make install-dns-operator-crds
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}} KUSTOMIZE_DIR=config/overlays/agent-powerdns-federated make kustomize-apply
      - task: env:s3-secret
        vars:
          CONTEXT: "kind-{{.CONTROL_CLUSTER_NAME}}"
          CLUSTER_ID: "{{.CONTROL_CLUSTER_NAME}}"
          ENDPOINT_URL:
            sh: |
              export KUBECONFIG={{.ENV_KUBECONFIG}}
              SVC=$(kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} get svc -n rustfs-system -l app.kubernetes.io/instance=rustfs -o jsonpath='{.items[0].metadata.name}')
              echo "http://$SVC.rustfs-system.svc.cluster.local:9000"
      - echo "✅ control cluster (kind-{{.CONTROL_CLUSTER_NAME}}) ready"

  env:edge-up:
    desc: "Create the edge kind cluster and deploy PowerDNS pointed at the control cluster's RustFS"
    cmds:
      - |
        if [ ! -f {{.RUSTFS_NODEPORT_FILE}} ]; then
          echo "❌ {{.RUSTFS_NODEPORT_FILE}} not found - run 'task env:control-up' first"
          exit 1
        fi
      - task: env:test-infra-cluster-up
        vars:
          CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}"
      - CLUSTER={{.EDGE_CLUSTER_NAME}} IMG={{.IMG}} make kind-load-image
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.EDGE_CLUSTER_NAME}} make install-dns-operator-crds
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.EDGE_CLUSTER_NAME}} KUSTOMIZE_DIR=config/overlays/agent-powerdns-federated make kustomize-apply
      - task: env:s3-secret
        vars:
          CONTEXT: "kind-{{.EDGE_CLUSTER_NAME}}"
          CLUSTER_ID: "{{.EDGE_CLUSTER_NAME}}"
          ENDPOINT_URL:
            sh: echo "http://{{.CONTROL_CLUSTER_NAME}}-control-plane:$(cat {{.RUSTFS_NODEPORT_FILE}})"
      - echo "✅ edge cluster (kind-{{.EDGE_CLUSTER_NAME}}) ready"

  env:upstream-up:
    desc: "Create the upstream kind cluster and deploy the replicator pointed at the control (downstream agent) cluster"
    cmds:
      - task: env:test-infra-cluster-up
        vars:
          CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}"
      - CLUSTER={{.UPSTREAM_CLUSTER_NAME}} IMG={{.IMG}} make kind-load-image
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} make install-dns-operator-crds
      # The replicator's manager watches networking-services types (Domain,
      # etc.) in addition to DNS resources; install those CRDs so its
      # controllers start cleanly (mirrors `make bootstrap-upstream`).
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} make install-networking-crds
      # The replicator reaches the control cluster's API (its "downstream")
      # through a kubeconfig Secret. Export control's kubeconfig rewritten to
      # the control-plane container's in-network address (host:6443, TLS-skip)
      # so it is reachable from the upstream cluster's pods over the shared
      # kind docker network - the same cross-cluster addressing env:edge-up
      # already uses for RustFS.
      - CLUSTER={{.CONTROL_CLUSTER_NAME}} OUT={{.CONTROL_INCLUSTER_KUBECONFIG}} KIND_KUBECONFIG_INSECURE=true make export-kind-kubeconfig
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} create namespace dns-replicator-system --dry-run=client -o yaml | kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} apply -f -
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} NAMESPACE=dns-replicator-system NAME=downstream-kubeconfig KEY=kubeconfig FILE={{.CONTROL_INCLUSTER_KUBECONFIG}} make secret-from-file
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} KUSTOMIZE_DIR=config/overlays/replicator make kustomize-apply
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} -n dns-replicator-system rollout status deployment/dns-operator-controller-manager --timeout=180s
      - echo "✅ upstream cluster (kind-{{.UPSTREAM_CLUSTER_NAME}}) ready"

  env:stack-up:
    desc: "Bring up all three clusters and the full DNS chain (no sample data) - used by CI"
    cmds:
      # Order matters: the replicator (upstream) needs the control cluster
      # reachable, and edge needs control's RustFS NodePort, so control comes
      # up first, then edge, then upstream.
      - task: env:control-up
      - task: env:edge-up
      - task: env:upstream-up

  env:up:
    desc: "Bring up the full local DNS federation environment (upstream + control + edge) with sample data"
    cmds:
      - task: env:stack-up
      # Dev convenience only: this applies the sample directly on the control
      # cluster to smoke-test the control->edge Lightningstream path. CI uses
      # env:stack-up instead and drives the full upstream->edge chain through
      # the Chainsaw suites (see env:chainsaw), so it never applies this - the
      # sample's example.com zone would otherwise collide with the
      # zones-and-records suite on the shared control PowerDNS.
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} apply -f config/samples/federation/dnszone-example.yaml
      - echo ""
      - echo "🎉 Environment up. upstream=kind-{{.UPSTREAM_CLUSTER_NAME}} control=kind-{{.CONTROL_CLUSTER_NAME}} edge=kind-{{.EDGE_CLUSTER_NAME}}"
      - echo "   Lightningstream syncs on an interval - give it a few seconds, then:"
      - echo "   task env:dns-check"

  env:dns-check:
    desc: "Query both control and edge PowerDNS for the sample www.example.com record"
    cmds:
      - echo "🔎 control (kind-{{.CONTROL_CLUSTER_NAME}}):"
      - KUBECONFIG={{.ENV_KUBECONFIG}} make dns-debug DNS_DEBUG_CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}} DNS_DEBUG_ZONE=example.com DNS_DEBUG_HOST=www
      - echo "🔎 edge (kind-{{.EDGE_CLUSTER_NAME}}) - should match control once Lightningstream syncs:"
      - KUBECONFIG={{.ENV_KUBECONFIG}} make dns-debug DNS_DEBUG_CONTEXT=kind-{{.EDGE_CLUSTER_NAME}} DNS_DEBUG_ZONE=example.com DNS_DEBUG_HOST=www

  env:chainsaw-prepare-kubeconfigs:
    desc: "Export upstream/control/edge kind kubeconfigs into test/e2e/ for the Chainsaw suites"
    cmds:
      - CLUSTER={{.UPSTREAM_CLUSTER_NAME}} OUT=test/e2e/kubeconfig-upstream make export-kind-kubeconfig-raw
      # The control cluster is addressed as "downstream" by the replicator
      # suites and as "control" by the federation/full-chain suites, so export
      # its kubeconfig under both names.
      - CLUSTER={{.CONTROL_CLUSTER_NAME}} OUT=test/e2e/kubeconfig-downstream make export-kind-kubeconfig-raw
      - CLUSTER={{.CONTROL_CLUSTER_NAME}} OUT=test/e2e/kubeconfig-control make export-kind-kubeconfig-raw
      - CLUSTER={{.EDGE_CLUSTER_NAME}} OUT=test/e2e/kubeconfig-edge make export-kind-kubeconfig-raw
      - echo "✅ wrote test/e2e/kubeconfig-{upstream,downstream,control,edge}"

  env:chainsaw:
    desc: "Run the full Chainsaw e2e suite (replicator + federation + full-chain) against the environment"
    deps:
      - env:chainsaw-prepare-kubeconfigs
    cmds:
      # Chainsaw needs *a* default KUBECONFIG to initialize its client even
      # though each suite addresses its clusters by name (spec.clusters) -
      # point it at the control kubeconfig we just exported. `.` runs every
      # suite under test/e2e (alias, display-annotations, zones-and-records,
      # federation, full-chain).
      #
      # The drift suites deliberately live outside this tree, in test/e2e-drift:
      # they need the Milo environment and the kubeconfigs that only
      # env:chainsaw-milo builds, and `.` would otherwise discover them here.
      - cd test/e2e && KUBECONFIG=kubeconfig-control chainsaw test .

  env:down:
    desc: "Tear down all clusters (upstream + control + edge)"
    cmds:
      - task: test-infra:delete-kind
        vars:
          CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}"
      # NOTE: calling the leaf `delete-kind` task directly (not the
      # `cluster-down`/`down` wrapper) - go-task does not forward
      # caller-supplied `vars:` through a wrapper task's own un-parameterized
      # `task:` calls, so `test-infra:cluster-down vars: {CLUSTER_NAME: ...}`
      # silently falls back to CLUSTER_NAME's taskfile-level default
      # ("test-infra") instead of deleting the cluster we actually created.
      # Same reasoning as env:test-infra-cluster-up above.
      - task: test-infra:delete-kind
        vars:
          CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}"
      - task: test-infra:delete-kind
        vars:
          CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}"
      - rm -f {{.RUSTFS_NODEPORT_FILE}} {{.ENV_KUBECONFIG}} {{.CONTROL_INCLUSTER_KUBECONFIG}}

  # ---- Milo control-plane (drift-detection / Phase B) --------------------
  # Installs the Milo apiserver + controller-manager into the control cluster
  # via Flux, so the replicator can run discovery.mode=milo and resource-metrics
  # can collect project + core control planes. See config/dependencies/milo and
  # docs/enhancements/controlplane-drift-detection.md.
  env:milo-up:
    desc: "Install Milo (apiserver + controller-manager) into the control cluster via Flux"
    silent: true
    cmds:
      - echo "➡️  Installing Milo into kind-{{.CONTROL_CLUSTER_NAME}}..."
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} apply -k config/dependencies/milo
      - echo "⏳ waiting for Flux OCIRepository 'milo' to reconcile..."
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait ocirepository/milo --for=condition=Ready --timeout=180s
      - echo "⏳ waiting for Flux Kustomization 'milo-infra-crds'..."
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait kustomization/milo-infra-crds --for=condition=Ready --timeout=300s
      - echo "⏳ waiting for Flux Kustomization 'milo' (cold start applies CRDs + deploys apiserver, up to 10m)..."
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait kustomization/milo --for=condition=Ready --timeout=600s
      - echo "✅ Milo installed. milo-system objects:"
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n milo-system get pods,svc

  env:derisk-control-milo:
    desc: "De-risk: bring up ONLY the control cluster + Milo (no agent/rustfs) to inspect the live milo-apiserver"
    cmds:
      - task: env:test-infra-cluster-up
        vars:
          CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}"
      - task: env:milo-up

  # internal: mint an admin kubeconfig for the Milo core CP via a transient
  # port-forward, then run CMD (a kubectl invocation with KUBECONFIG=$mk).
  # milo-apiserver's NodePort is not host-mapped, and on macOS the host cannot
  # reach the kind docker-network IP, so a port-forward is the portable way to
  # drive the core CP from the host (macOS dev + Linux CI).
  env:with-milo-admin:
    internal: true
    silent: true
    cmds:
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n milo-system port-forward svc/milo-apiserver 16443:6443 >/tmp/milo-pf.log 2>&1 &
        pf=$!; trap 'kill $pf 2>/dev/null || true' EXIT
        for i in $(seq 1 30); do
          curl -sk -o /dev/null -m 2 https://127.0.0.1:16443/livez && break || sleep 1
        done
        mk={{.MILO_ADMIN_KUBECONFIG}}
        kubectl config --kubeconfig="$mk" set-cluster milo --server=https://127.0.0.1:16443 --insecure-skip-tls-verify=true >/dev/null
        kubectl config --kubeconfig="$mk" set-credentials admin --token=test-admin-token >/dev/null
        kubectl config --kubeconfig="$mk" set-context milo --cluster=milo --user=admin >/dev/null
        kubectl config --kubeconfig="$mk" use-context milo >/dev/null
        KUBECONFIG="$mk" {{.CMD}}

  env:observability-up:
    desc: "Install the observability stack (Victoria Metrics + OTel + Prometheus CRDs) into the control cluster"
    cmds:
      - KUBECONFIG={{.ENV_KUBECONFIG}} TASK_X_REMOTE_TASKFILES=1 task --yes test-infra:install-observability

  env:metrics-up:
    desc: "Deploy both resource-metrics collectors (upstream mode:milo, downstream mode:single), their policies, and the drift rules"
    vars:
      KCTL: KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}}
    cmds:
      # Two collectors, because the two replication seams are two API servers:
      #   controller/  mode:milo   — engages every project CP, reads its policy
      #                              from the Milo core CP  -> upstream series
      #   downstream/  mode:single — collects dns-control's own apiserver, where
      #                              the replicator writes the shadow objects,
      #                              and reads its policy from there
      #                                                     -> downstream series
      # All Flux objects live on the LOCAL (dns-control) cluster; only the core
      # CP Kustomization targets a different API server, via kubeConfig. Every
      # manifest comes from resource-metrics' published bundle (no vendored
      # manifests, no local image build — the pin ships a multi-arch image).
      - "{{.KCTL}} apply -k config/dependencies/resource-metrics/controller"
      - "{{.KCTL}} apply -k config/dependencies/resource-metrics/core-control-plane/crd"
      - "{{.KCTL}} apply -k config/dependencies/resource-metrics/downstream"
      - "{{.KCTL}} -n flux-system wait ocirepository/resource-metrics --for=condition=Ready --timeout=120s"
      # Installs the ResourceMetricsPolicy CRD onto each cluster from the bundle.
      - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics-crd --for=condition=Ready --timeout=180s"
      - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics-downstream-crd --for=condition=Ready --timeout=180s"
      # Each policy goes to the cluster whose collector reads it, once that
      # cluster's CRD is Established. dns-metrics is the same policy infra ships
      # to production; dns-downstream-metrics is drift-detection specific.
      # Applied as a plain file, not with `apply -k` on
      # config/dependencies/resource-metrics/core-control-plane/policy: that
      # kustomization references the policy outside its own root, which needs
      # --load-restrictor LoadRestrictionsNone, and `kubectl apply -k` has no way
      # to pass it. The kustomization stays as the reference for how infra wires
      # this; here we apply what it points at.
      - task: env:with-milo-admin
        vars:
          CMD: kubectl apply -f config/milo/resource-metrics/policies/dns-metrics.yaml
      - "{{.KCTL}} apply -k config/observability/cluster-policy"
      - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics --for=condition=Ready --timeout=300s"
      - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics-downstream --for=condition=Ready --timeout=300s"
      # Same deployment name in both cases — the two collectors are told apart by
      # namespace, not by name.
      - "{{.KCTL}} -n resource-metrics-system rollout status deploy/resource-metrics-controller-manager --timeout=150s"
      - "{{.KCTL}} -n resource-metrics-downstream-system rollout status deploy/resource-metrics-controller-manager --timeout=150s"
      - "{{.KCTL}} -n telemetry-system apply -k config/observability/rules"
      - echo "✅ upstream + downstream resource-metrics, both policies, and the drift rules deployed"

  env:milo-all-up:
    desc: "Full clean bring-up: control+Milo, CRDs+projects, observability, resource-metrics, upstream replicator"
    cmds:
      - task: env:derisk-control-milo
      - task: env:milo-bootstrap
      - task: env:observability-up
      - task: env:metrics-up
      - task: env:upstream-milo-up

  env:chainsaw-milo:
    desc: "Run the control-plane drift Chainsaw suite (happy-path/orphan/missing) against the running Milo env"
    cmds:
      - |
        set -euo pipefail
        root="$(pwd)"
        cd test/e2e-drift
        # infra + downstream are both the dns-control kind API: it hosts VM/OTel
        # and it is also where the replicator writes the shadow objects. They
        # get separate kubeconfigs so each scenario reads as what it is doing.
        # replicator = dns-upstream kind. All kind APIs are host-reachable.
        kind get kubeconfig --name {{.CONTROL_CLUSTER_NAME}} > kubeconfig-infra
        cp kubeconfig-infra kubeconfig-downstream
        kind get kubeconfig --name {{.UPSTREAM_CLUSTER_NAME}} > kubeconfig-replicator
        # alpha (project CP): reach milo-apiserver via a localhost port-forward —
        # portable across macOS + Linux, since the gateway NodePort is not
        # host-mapped and macOS can't reach the kind docker-network IP directly.
        KUBECONFIG="$root/{{.ENV_KUBECONFIG}}" kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} \
          -n milo-system port-forward svc/milo-apiserver 16443:6443 >/tmp/milo-pf-chainsaw.log 2>&1 &
        pf=$!; trap 'kill $pf 2>/dev/null || true' EXIT
        for i in $(seq 1 30); do curl -sk -o /dev/null -m2 https://127.0.0.1:16443/livez && break || sleep 1; done
        kubectl config --kubeconfig=kubeconfig-alpha set-credentials a --token=test-admin-token >/dev/null
        kubectl config --kubeconfig=kubeconfig-alpha set-context alpha --cluster=alpha --user=a >/dev/null
        kubectl config --kubeconfig=kubeconfig-alpha use-context alpha >/dev/null
        kubectl config --kubeconfig=kubeconfig-alpha set-cluster alpha \
          --server=https://127.0.0.1:16443/apis/resourcemanager.miloapis.com/v1alpha1/projects/alpha/control-plane \
          --insecure-skip-tls-verify=true >/dev/null
        KUBECONFIG=kubeconfig-infra chainsaw test --config .chainsaw.yaml {{.CHAINSAW_SUITE | default "."}}

  env:upstream-milo-up:
    desc: "Create dns-upstream and deploy the replicator (discovery.mode=milo, 2 replicas): discovery via Milo, shadow objects to dns-control's apiserver"
    cmds:
      - task: env:test-infra-cluster-up
        vars:
          CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}"
      - kind load docker-image {{.DNSOP_IMG}} --name {{.UPSTREAM_CLUSTER_NAME}}
      # Mint the cross-cluster milo kubeconfig (control node IP + gateway
      # NodePort) and stash it as the milo-kubeconfig Secret the replicator-milo
      # overlay mounts at /milo. NODE_IP via python to avoid go-task/docker
      # template brace conflicts.
      - |
        set -euo pipefail
        export KUBECONFIG={{.ENV_KUBECONFIG}}
        NODE_IP=$(docker inspect {{.CONTROL_CLUSTER_NAME}}-control-plane | python3 -c "import json,sys; n=json.load(sys.stdin)[0]['NetworkSettings']['Networks']; print(next(iter(n.values()))['IPAddress'])")
        # Derive the envoy gateway NodePort dynamically — test-infra's
        # fix/remove-hardcoded-nodeports branch assigns it per-cluster, so it is
        # NOT stable across rebuilds. Look up the nodePort mapped to gateway
        # port 8443 (the HTTPS listener that fronts milo-apiserver).
        NODEPORT=$(kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n envoy-gateway-system get svc \
          -o jsonpath='{range .items[*]}{range .spec.ports[?(@.port==8443)]}{.nodePort}{end}{end}')
        test -n "$NODEPORT" || { echo "could not resolve envoy gateway 8443 NodePort" >&2; exit 1; }
        echo "➡️  milo gateway endpoint for replicator: https://${NODE_IP}:${NODEPORT}"
        mk={{.MILO_UPSTREAM_KUBECONFIG}}
        kubectl config --kubeconfig="$mk" set-cluster milo --server="https://${NODE_IP}:${NODEPORT}" --insecure-skip-tls-verify=true >/dev/null
        kubectl config --kubeconfig="$mk" set-credentials admin --token=test-admin-token >/dev/null
        kubectl config --kubeconfig="$mk" set-context milo --cluster=milo --user=admin >/dev/null
        kubectl config --kubeconfig="$mk" use-context milo >/dev/null
        kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} create ns dns-replicator-system --dry-run=client -o yaml | kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} apply -f -
        kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} -n dns-replicator-system create secret generic milo-kubeconfig --from-file=kubeconfig="$mk" --dry-run=client -o yaml | kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} apply -f -
      # Milo is only half the replicator's wiring: the shadow objects go to
      # dns-control's own apiserver, a different API server than milo-apiserver.
      # Export control's kubeconfig rewritten to the control-plane container's
      # in-network address so pods on dns-upstream can reach it over the shared
      # kind docker network — the same mechanism env:upstream-up uses.
      - CLUSTER={{.CONTROL_CLUSTER_NAME}} OUT={{.CONTROL_INCLUSTER_KUBECONFIG}} KIND_KUBECONFIG_INSECURE=true make export-kind-kubeconfig
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} NAMESPACE=dns-replicator-system NAME=downstream-kubeconfig KEY=kubeconfig FILE={{.CONTROL_INCLUSTER_KUBECONFIG}} make secret-from-file
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} KUSTOMIZE_DIR=config/overlays/replicator-milo make kustomize-apply
      - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} -n dns-replicator-system rollout status deploy/dns-operator-controller-manager --timeout=180s
      - echo "✅ replicator up on kind-{{.UPSTREAM_CLUSTER_NAME}} (mode=milo, 2 replicas)"

  env:milo-bootstrap:
    desc: "Install DNS + networking CRDs into the Milo core CP and the downstream cluster, and create the drift Org + projects (alpha, beta)"
    cmds:
      - task: env:with-milo-admin
        vars:
          CMD: kubectl apply -k config/crd
      # The DNS CRDs are needed on BOTH seams: the core CP shares them with every
      # project CP (upstream), and dns-control's own apiserver stores the shadow
      # objects the replicator writes (downstream). Production installs the
      # downstream copy through the agent bundle, which pulls in config/crd.
      - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}} make install-dns-operator-crds
      # The replicator (dnszone/dnszonediscovery controllers) indexes Domain
      # (networking.datumapis.com); without its CRD, project-CP engagement fails
      # on cache-index setup. Generate the networking CRDs and install them into
      # the core CP (shared to all project CPs). The make target also applies to
      # the local kind API (harmless) and writes dev/crds/network-services.
      - KUBECONFIG={{.ENV_KUBECONFIG}} make install-networking-crds CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}}
      - task: env:with-milo-admin
        vars:
          CMD: kubectl apply -f dev/crds/network-services
      - task: env:with-milo-admin
        vars:
          CMD: kubectl apply -f test/e2e-drift/fixtures/milo-projects.yaml
      - task: env:with-milo-admin
        vars:
          CMD: kubectl wait --for=condition=Ready project/alpha project/beta --timeout=120s
      - echo "✅ Milo core CP has DNS + networking CRDs; projects alpha, beta are Ready"