diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4050aad0..5da4ea4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,8 @@ jobs: - name: Run GoReleaser (publish) uses: goreleaser/goreleaser-action@v7 with: - version: latest + # renovate: datasource=github-releases depName=goreleaser/goreleaser + version: "v2.14.3" args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 93183e66..664b83d5 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,37 +2,37 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "6.36.0" + version = "6.37.0" constraints = ">= 4.56.0" hashes = [ - "h1:4sdEWcVrQOP3xZJz1Ld7mJsRUghMD5/u3z/h2DryFzA=", - "h1:547WNLU+9TkcXoGBzH3G3A8dF+bTpCMhoBuMq7bjGxE=", - "h1:AiIqpFaYiDAO68q7WYQrQ2zGpRVojIzLG5bb9VFi4CY=", - "h1:I36O/YXrM2U+wQd+ncqAoPM/LwODXAHelhN2alctC94=", - "h1:I3tZF008rcWRN19En34I04cLrapWjCinPLLv6sB1zlE=", - "h1:MOGEhZW49aW+2Js3b1XVGwm7dyhcuH9k59q5buyalQM=", - "h1:NVoHUXXTNemFijAA9VsKxVzWKlIv66Hu8NBKzWHXXKY=", - "h1:NsGSj9/Rbd/NhsYR2hgDCaIhvcbUDu7f960fRvcMa9g=", - "h1:ObcjqX/3XDBVq7hMisa46jbhE/ku3bIPLlffoVlUooQ=", - "h1:SJhaTTZS+/tUPuQMPi9CFFP8M58FQm0Cvml4r1TC1Yw=", - "h1:UYt6mrz0d3PfTJRbJAxe+fLcJt7voXJCLLdwfHrApGk=", - "h1:gaxpIeOTszPUGCIiwbbAL9n64e4NB0Y/Tslw8e1fwHw=", - "h1:ivdqYb0xTWGul8CjUp2SRJjJtSUJbrZi+zPLIDkDTIw=", - "h1:r9icn1WEZVvEXiy6ZKexLzAPnXkkt+22jJ9WQYPfKB0=", - "zh:0eb4481315564aaeec4905a804fd0df22c40f509ad2af63615eeaa90abacf81c", - "zh:12c3cddc461a8dbaa04387fe83420b64c4c05cb5479d181674168ca7daefcc38", - "zh:1b55a09661e80acf6826faa38dd8fbff24c2ef620d2a0a16918491a222c55370", - "zh:269cb1a406d0cac762bce82119247395a0bbf0d4ad2492fb2ea5653b4f44bc05", - "zh:3bfb78e3345f0c3846e76578952a09fb5dda05d2d73e19473fb0af0000469a66", - "zh:3ead4f4388c7dd78ed198082a981746324da0d7a51460c9b455fd884d86fc82c", - "zh:44906654199991b3f1a21c6a984bc5f9f556ff4baa4e5f77e168968e941c2725", - "zh:4803d050d581b05b0fd0ae5cce95ec1784d66e2bc9da4b1f7663df0ce7914609", - "zh:4cf9fe8fae58b62e83c0672a9c66e0963b7289aaf768a250e9bc44570d82cbd5", - "zh:5bfd7a1fb3116164b411777115dd4b272a68984fa949c687e41a3041318c82f1", - "zh:77cbcf2db512617f10b81e11c20d40fa534ef07163171cbe35214fa8f74b4e85", - "zh:8201cabed01f1434bf9ea7fbcf2a95612a87a0398b870b2643bd1a5119793d2d", - "zh:9aaded4cf36ec2abbe35086733a4510e08819698180b21a9387ba4112aee02e0", + "h1:2OzkE8hCk+rVC0Ljxw6oCwNHpJfHHiU/i6sBIGg1ZZQ=", + "h1:4sToAMU3GsSC0Orc3JqcgOMRdMxHbAtbVRSav4fbvMk=", + "h1:IvmQvCUpUSL2T1xtxxPhP/MXl9efLD4NmXDBAG1YTNE=", + "h1:KMbPmTWY92+hoTEIYCJvmh5F7eJCESoswyOirL23Tws=", + "h1:OdQC/z+3ReUAPhlXCJIJYUZlSQ3b96TWfFK3lCT5zKU=", + "h1:Q6oMyOOO0SgKhXvDODv2nBWlO7m8/uXaGF7XALj1tuQ=", + "h1:Um4kBHMX/BW9k3wYGoyxcpNmd/5UdxHhSlyNXp6tVCU=", + "h1:VKlndKPfI8jFbdNAHdlzglmOt/XLynNc14wizVQY5z0=", + "h1:Z9l6B2KTCXI81ljNyriWU2vUd4gZlOs6uceBowDcOrQ=", + "h1:l6dLejTvvF+0HnvW7DsjY/jVKjgTo627Pc4C6nYGNHM=", + "h1:mcQsALya7Snbo3yxJ4Kb/ZJzYd6xFw7BX/ejn0qydCU=", + "h1:uUrMuvVNnDE13pIUuwNN/yyZcPB83cQzsc5RRHQQ14E=", + "h1:w3z/TApcKD3b/aMZoZZKSxOld4xw+gEtQ1ka6C1UN+4=", + "h1:z+wSA7CUTunt2Kb+O4TKNDT4pmWOovNoUxijZLn3n9w=", + "zh:0427fadb719ed5a32feb09f047539d2348e659056f3b8a8589d34d8f0a95be7a", + "zh:3891c670674aba2125a7ac6d4348cde43646b1b46ce6f829e6f4724091bc0dcd", + "zh:632cb24b7b5790b730b33bcbe9f1a7b75f2644fb52f9d6aaafb0249c9e7601d2", + "zh:6e96ed1f824c2efa9de5b7c22ab3715624ba34c28564a06e9a15e71bc3d3a30b", + "zh:7b8fd86907b659bc45f4a3f42c3c0ccc66925a74e265b01e9e66242c0b2cafef", + "zh:81f9a587deddef4dfcc2101c54ec28a3a554056837f68ebb920c83fe8327b16f", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:f594ef2683a0d23d3a6f0ad6c84a55ed79368c158ee08c2f3b7c41ec446a701f", + "zh:a9a38a67cb98d690fec951ec3e133b6836279629db2ed3a0ebf97a5bea58674f", + "zh:b18f60d62e4bd4d466077e09c39259d1a85355f0f00b801fe8aedbc50193d357", + "zh:b7a51bc0faf60d17043b4df1d1b7bb55129eaa4bdeb65ff55f5b00b9b8fee9f7", + "zh:c28c42f91ca3a6b65b3fd3ed6e891fc0fc28d0cb5ab65dea65eda8eec5cea5f3", + "zh:d895ddc04280ed26b6ca64ca05b78caaa7b72c8e167af4093545efbc608d5482", + "zh:f4a56f5157009ef160fbd79105078fe675df479cb73c1b7e1fea2741403a0b67", + "zh:f547d6ca371b96fec97b972fc0c93bcfc23d58e34a9da215b94e9d2aa170fb77", + "zh:f7b0a3cd4adadd3f4b9609a54e651ed5eafa22c196ab229042fc1d0aa0ab8f3a", ] } diff --git a/aws-source/module/provider/.github/workflows/release.yml b/aws-source/module/provider/.github/workflows/release.yml index ae76b5a4..465023d4 100644 --- a/aws-source/module/provider/.github/workflows/release.yml +++ b/aws-source/module/provider/.github/workflows/release.yml @@ -45,7 +45,8 @@ jobs: - name: Run GoReleaser uses: goreleaser/goreleaser-action@v7 with: - version: latest + # renovate: datasource=github-releases depName=goreleaser/goreleaser + version: "v2.14.3" args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs.overmind.tech/docs/sources/gcp/Types/gcp-dataflow-job.md b/docs.overmind.tech/docs/sources/gcp/Types/gcp-dataflow-job.md new file mode 100644 index 00000000..5f2860eb --- /dev/null +++ b/docs.overmind.tech/docs/sources/gcp/Types/gcp-dataflow-job.md @@ -0,0 +1,59 @@ +--- +title: GCP Dataflow Job +sidebar_label: gcp-dataflow-job +--- + +A **Google Cloud Dataflow Job** is a managed Apache Beam pipeline that processes streaming or batch data at scale. Dataflow handles resource provisioning, autoscaling, and fault tolerance, allowing you to run data processing workloads without managing the underlying infrastructure. Jobs can read from and write to Pub/Sub, BigQuery, Spanner, Bigtable, and other GCP services. See the official documentation for full details: https://cloud.google.com/dataflow/docs. + +**Terraform Mappings:** + +- `google_dataflow_job.job_id` +- `google_dataflow_flex_template_job.job_id` + +## Supported Methods + +- `GET`: Get a gcp-dataflow-job by its "locations|jobs" +- ~~`LIST`~~ +- `SEARCH`: Search for gcp-dataflow-job by location + +## Possible Links + +### [`gcp-big-query-dataset`](/sources/gcp/Types/gcp-big-query-dataset) + +Dataflow jobs that read from or write to BigQuery reference the dataset containing the tables they use. If the dataset is deleted or misconfigured, the job may fail to access data. + +### [`gcp-big-query-table`](/sources/gcp/Types/gcp-big-query-table) + +Dataflow jobs can read from or write to specific BigQuery tables. If a table is deleted or its schema changes, the job may fail. + +### [`gcp-big-table-admin-instance`](/sources/gcp/Types/gcp-big-table-admin-instance) + +Dataflow jobs that use Bigtable as a source or sink reference the Bigtable instance. If the instance is deleted or misconfigured, the job may fail. + +### [`gcp-cloud-kms-crypto-key`](/sources/gcp/Types/gcp-cloud-kms-crypto-key) + +When customer-managed encryption keys (CMEK) are enabled for the Dataflow job environment, the job references the Cloud KMS Crypto Key used for encryption. + +### [`gcp-compute-network`](/sources/gcp/Types/gcp-compute-network) + +Dataflow worker VMs are attached to a VPC network. If the network is deleted or misconfigured, workers may lose connectivity or fail to start. + +### [`gcp-compute-subnetwork`](/sources/gcp/Types/gcp-compute-subnetwork) + +Dataflow workers run in a specific subnetwork. If the subnetwork is deleted or misconfigured, workers may lose connectivity or fail to start. + +### [`gcp-iam-service-account`](/sources/gcp/Types/gcp-iam-service-account) + +Dataflow workers run under a service account that grants them permissions to access other GCP services. If the service account is deleted or its permissions change, the job may fail. + +### [`gcp-pub-sub-subscription`](/sources/gcp/Types/gcp-pub-sub-subscription) + +Dataflow jobs that consume messages from Pub/Sub reference the subscription. If the subscription is deleted or misconfigured, the job may fail to consume messages. + +### [`gcp-pub-sub-topic`](/sources/gcp/Types/gcp-pub-sub-topic) + +Dataflow jobs that publish to or consume from Pub/Sub reference the topic. If the topic is deleted or misconfigured, the job may fail to read or write messages. + +### [`gcp-spanner-instance`](/sources/gcp/Types/gcp-spanner-instance) + +Dataflow jobs that use Spanner reference the Spanner instance. If the instance is deleted or misconfigured, the job may fail. diff --git a/docs.overmind.tech/docs/sources/gcp/configuration.md b/docs.overmind.tech/docs/sources/gcp/configuration.md index 4d56efcc..f89dfb9c 100644 --- a/docs.overmind.tech/docs/sources/gcp/configuration.md +++ b/docs.overmind.tech/docs/sources/gcp/configuration.md @@ -360,7 +360,7 @@ Overmind requires read-only access to discover and map your GCP infrastructure. **Read-only viewer roles** for GCP services including: -- Compute Engine, GKE, Cloud Run, Cloud Functions +- Compute Engine, GKE, Cloud Run, Cloud Functions, Dataflow - Cloud SQL, BigQuery, Spanner, Cloud Storage - IAM, networking, monitoring, and logging resources - And other GCP services @@ -408,6 +408,7 @@ Here are all the predefined GCP roles that Overmind requires, plus the custom ro | `roles/dataform.viewer` | Dataform resource discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dataform#dataform.viewer) | | `roles/dataplex.catalogViewer` | Dataplex catalog resource discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dataplex#dataplex.catalogViewer) | | `roles/dataplex.viewer` | Dataplex resource discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dataplex#dataplex.viewer) | +| `roles/dataflow.viewer` | Dataflow job discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dataflow#dataflow.viewer) | | `roles/dataproc.viewer` | Dataproc cluster discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dataproc#dataproc.viewer) | | `roles/dns.reader` | Cloud DNS resource discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/dns#dns.reader) | | `roles/essentialcontacts.viewer` | Essential Contacts discovery [GCP Docs](https://cloud.google.com/iam/docs/roles-permissions/essentialcontacts#essentialcontacts.viewer) | diff --git a/docs.overmind.tech/docs/sources/gcp/data/gcp-dataflow-job.json b/docs.overmind.tech/docs/sources/gcp/data/gcp-dataflow-job.json new file mode 100644 index 00000000..932854e9 --- /dev/null +++ b/docs.overmind.tech/docs/sources/gcp/data/gcp-dataflow-job.json @@ -0,0 +1,31 @@ +{ + "type": "gcp-dataflow-job", + "category": 7, + "potentialLinks": [ + "gcp-big-query-dataset", + "gcp-big-query-table", + "gcp-big-table-admin-instance", + "gcp-cloud-kms-crypto-key", + "gcp-compute-network", + "gcp-compute-subnetwork", + "gcp-iam-service-account", + "gcp-pub-sub-subscription", + "gcp-pub-sub-topic", + "gcp-spanner-instance" + ], + "descriptiveName": "GCP Dataflow Job", + "supportedQueryMethods": { + "get": true, + "getDescription": "Get a gcp-dataflow-job by its \"locations|jobs\"", + "search": true, + "searchDescription": "Search for gcp-dataflow-job by location" + }, + "terraformMappings": [ + { + "terraformQueryMap": "google_dataflow_job.job_id" + }, + { + "terraformQueryMap": "google_dataflow_flex_template_job.job_id" + } + ] +} diff --git a/go.mod b/go.mod index c46a5e38..c2d13176 100644 --- a/go.mod +++ b/go.mod @@ -17,13 +17,14 @@ require ( cloud.google.com/go/auth v0.18.2 cloud.google.com/go/auth/oauth2adapt v0.2.8 cloud.google.com/go/bigquery v1.74.0 - cloud.google.com/go/bigtable v1.42.0 + cloud.google.com/go/bigtable v1.43.0 cloud.google.com/go/certificatemanager v1.9.6 cloud.google.com/go/compute v1.57.0 cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/container v1.46.0 cloud.google.com/go/dataplex v1.28.0 cloud.google.com/go/dataproc/v2 v2.16.0 + cloud.google.com/go/eventarc v1.18.0 cloud.google.com/go/filestore v1.10.3 cloud.google.com/go/functions v1.19.7 cloud.google.com/go/iam v1.5.3 @@ -60,34 +61,34 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/MrAlias/otel-schema-utils v0.4.0-alpha github.com/auth0/go-jwt-middleware/v3 v3.0.0 - github.com/aws/aws-sdk-go-v2 v1.41.3 - github.com/aws/aws-sdk-go-v2/config v1.32.11 - github.com/aws/aws-sdk-go-v2/credentials v1.19.11 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 - github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6 - github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.2 - github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.2 - github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.1 - github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.13 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.1 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0 - github.com/aws/aws-sdk-go-v2/service/ecs v1.73.1 - github.com/aws/aws-sdk-go-v2/service/efs v1.41.12 - github.com/aws/aws-sdk-go-v2/service/eks v1.81.0 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.21 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.8 - github.com/aws/aws-sdk-go-v2/service/iam v1.53.5 - github.com/aws/aws-sdk-go-v2/service/kms v1.50.2 - github.com/aws/aws-sdk-go-v2/service/lambda v1.88.2 - github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.5 - github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.6 - github.com/aws/aws-sdk-go-v2/service/rds v1.116.2 - github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 - github.com/aws/aws-sdk-go-v2/service/sns v1.39.13 - github.com/aws/aws-sdk-go-v2/service/sqs v1.42.23 - github.com/aws/aws-sdk-go-v2/service/ssm v1.68.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 + github.com/aws/aws-sdk-go-v2/service/apigateway v1.39.0 + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.3 + github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.3 + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.2 + github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.14 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.2 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 + github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0 + github.com/aws/aws-sdk-go-v2/service/efs v1.41.13 + github.com/aws/aws-sdk-go-v2/service/eks v1.81.1 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.22 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.9 + github.com/aws/aws-sdk-go-v2/service/iam v1.53.6 + github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 + github.com/aws/aws-sdk-go-v2/service/lambda v1.88.3 + github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.6 + github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.7 + github.com/aws/aws-sdk-go-v2/service/rds v1.116.3 + github.com/aws/aws-sdk-go-v2/service/route53 v1.62.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 + github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 + github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 + github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 github.com/aws/smithy-go v1.24.2 github.com/cenkalti/backoff/v5 v5.0.3 github.com/charmbracelet/glamour v0.10.0 @@ -97,7 +98,7 @@ require ( github.com/go-jose/go-jose/v4 v4.1.3 github.com/google/btree v1.1.3 github.com/google/uuid v1.6.0 - github.com/googleapis/gax-go/v2 v2.18.0 + github.com/googleapis/gax-go/v2 v2.19.0 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/hashicorp/go-retryablehttp v0.7.8 github.com/hashicorp/hcl/v2 v2.24.0 @@ -111,7 +112,7 @@ require ( github.com/miekg/dns v1.1.72 github.com/mitchellh/go-homedir v1.1.0 github.com/muesli/reflow v0.3.0 - github.com/nats-io/jwt/v2 v2.8.0 + github.com/nats-io/jwt/v2 v2.8.1 github.com/nats-io/nats-server/v2 v2.12.5 github.com/nats-io/nats.go v1.49.0 github.com/nats-io/nkeys v0.4.15 @@ -147,15 +148,15 @@ require ( golang.org/x/sync v0.20.0 golang.org/x/text v0.35.0 gonum.org/v1/gonum v0.17.0 - google.golang.org/api v0.271.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c - google.golang.org/grpc v1.79.2 + google.golang.org/api v0.272.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 gopkg.in/ini.v1 v1.67.1 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.35.2 - k8s.io/apimachinery v0.35.2 - k8s.io/client-go v0.35.2 + k8s.io/api v0.35.3 + k8s.io/apimachinery v0.35.3 + k8s.io/client-go v0.35.3 sigs.k8s.io/kind v0.31.0 sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) @@ -184,19 +185,19 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -340,8 +341,8 @@ require ( golang.org/x/tools v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect diff --git a/go.sum b/go.sum index a03a71f8..2a972151 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.74.0 h1:Q6bAMv+eyvufOpIrfrYxhM46qq1D3ZQTdgUDQqKS+n8= cloud.google.com/go/bigquery v1.74.0/go.mod h1:iViO7Cx3A/cRKcHNRsHB3yqGAMInFBswrE9Pxazsc90= -cloud.google.com/go/bigtable v1.42.0 h1:SREvT4jLhJQZXUjsLmFs/1SMQJ+rKEj1cJuPE9liQs8= -cloud.google.com/go/bigtable v1.42.0/go.mod h1:oZ30nofVB6/UYGg7lBwGLWSea7NZUvw/WvBBgLY07xU= +cloud.google.com/go/bigtable v1.43.0 h1:ysbiG+AZElMELOKDrTkHF2N9xPzj5Dl7tmPr92/FUiQ= +cloud.google.com/go/bigtable v1.43.0/go.mod h1:uH4AQlKpsGMZvicLU+RJs263UiTQhQKzXFmANPjOSUk= cloud.google.com/go/certificatemanager v1.9.6 h1:v5X8X+THKrS9OFZb6k0GRDP1WQxLXTdMko7OInBliw4= cloud.google.com/go/certificatemanager v1.9.6/go.mod h1:vWogV874jKZkSRDFCMM3r7wqybv8WXs3XhyNff6o/Zo= cloud.google.com/go/compute v1.57.0 h1:uACoYJCUftJxxoI7si8u1S9szRDalftrWSjo1Dizfx4= @@ -42,6 +42,8 @@ cloud.google.com/go/dataplex v1.28.0 h1:rROI3iqMVI9nXT701ULoFRETQVAOAPC3mPSWFDxX cloud.google.com/go/dataplex v1.28.0/go.mod h1:VB+xlYJiJ5kreonXsa2cHPj0A3CfPh/mgiHG4JFhbUA= cloud.google.com/go/dataproc/v2 v2.16.0 h1:0g2hnjlQ8SQTnNeu+Bqqa61QPssfSZF3t+9ldRmx+VQ= cloud.google.com/go/dataproc/v2 v2.16.0/go.mod h1:HlzFg8k1SK+bJN3Zsy2z5g6OZS1D4DYiDUgJtF0gJnE= +cloud.google.com/go/eventarc v1.18.0 h1:8WWG1/ogInYur1NQjML6EMHQ0ZBzAdMDGlUVpLD56cI= +cloud.google.com/go/eventarc v1.18.0/go.mod h1:/6SDoqh5+9QNUqCX4/oQcJVK16fG/snHBSXu7lrJtO8= cloud.google.com/go/filestore v1.10.3 h1:3KZifUVTqGhNNv6MLeONYth1HjlVM4vDhaH+xrdPljU= cloud.google.com/go/filestore v1.10.3/go.mod h1:94ZGyLTx9j+aWKozPQ6Wbq1DuImie/L/HIdGMshtwac= cloud.google.com/go/functions v1.19.7 h1:7LcOD18euIVGRUPaeCmgO6vfWSLNIsi6STWRQcdANG8= @@ -183,88 +185,88 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmms github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/auth0/go-jwt-middleware/v3 v3.0.0 h1:+rvUPCT+VbAuK4tpS13fWfZrMyqTwLopt3VoY0Y7kvA= github.com/auth0/go-jwt-middleware/v3 v3.0.0/go.mod h1:iU42jqjRyeKbf9YYSnRnolr836gk6Ty/jnUNuVq2b0o= -github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= -github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= -github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= -github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6 h1:dzd86UudvxJ1c6z/o+hHh7ZhkoBrh81XYz/M11zwQYI= -github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6/go.mod h1:jWmyEnBPJdt+RaHSRzZDKp3HyyzjOofGp4+xXY503Do= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.2 h1:pzFtdV2DArJul6aM3+WiWjUQ63IzrSnSbvBr8FAokt4= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.2/go.mod h1:8xQlcle6cf4R66HrXbiahORXakWpLlvJXoiGae5BlIc= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.2 h1:+5lijyTp+IoU5oh6rL3374yEkaPeFnaes+b4WWUQC2I= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.2/go.mod h1:Ndq7ECdcXc8jmE4WPhl409BdAAWW6jrirMFgliMxMtU= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.1 h1:s+ZS2lmYFeCISy20RkSerTmfMIzxlevj4LyWNuE3cfY= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.1/go.mod h1:xXUsqpyas4oCIPxrKoCeqvyvFBLEYSohybRVV0bHq9A= -github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.13 h1:nUrVaHNZ82u7H3012w+gqscrbFjVLfSFWvbgeP7+J90= -github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.13/go.mod h1:DTuhYIuDsUBwdSHh6Dg8NNRq7CCeVPI8w0D/ZXQiE40= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.1 h1:EkW4NqA2mwCkL7YCDYh6OpA/bCMhKYbZgpRHt2FD2Ow= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.1/go.mod h1:OQp5333OH1IjmJmJpTU4IwoaOoCMnDrThg0zIx169rE= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0 h1:776KnBqePBBR6zEDi0bUIHXzUBOISa2WgAKEgckUF8M= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0/go.mod h1:rB577GvkmJADVOFGY8/j9sPv/ewcsEtQNsd9Lrn7Zx0= -github.com/aws/aws-sdk-go-v2/service/ecs v1.73.1 h1:TSmcWx+RzhGJrPNoFkuqANafJQ7xY3W2UBg6ShN3ae8= -github.com/aws/aws-sdk-go-v2/service/ecs v1.73.1/go.mod h1:KWILGx+bRowcGyJU/va2Ift48c658blP5e1qvldnIRE= -github.com/aws/aws-sdk-go-v2/service/efs v1.41.12 h1:YZXW11dESIf6CNhMG2ICZonCkzKBaGLuFamSJTYV5g0= -github.com/aws/aws-sdk-go-v2/service/efs v1.41.12/go.mod h1:+rjniKD0YQAmjiDNJvLodKXn1vXWwMpctrr/M4zm1V4= -github.com/aws/aws-sdk-go-v2/service/eks v1.81.0 h1:sI0DVFTT+ILkk4QDNKU5a+arj+kduG2qJZoI3vnDKcc= -github.com/aws/aws-sdk-go-v2/service/eks v1.81.0/go.mod h1:nx52u/3RVDWkOcrAchYgt7CXkrd03A6Gvzi0trtMFjQ= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.21 h1:VriOdPKF8YrkMpnT76ZwA2LXk5aBInOfuzN14QGTOJc= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.21/go.mod h1:sp4Mz5YUnYCvIkGNEcdEPp+DuHqquEZYXyIuKXuHzig= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.8 h1:xUwbqWhKASQsigeQfeBjhbm6dAP1EeTulHnNSYv5Xfc= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.8/go.mod h1:sQoz/dTooY3kCkNNGxVLTS7EacLA0qXUaK4BkpMjGOc= -github.com/aws/aws-sdk-go-v2/service/iam v1.53.5 h1:J8qtztl/SJ6lhk/Rke/F6PgpZ7V6UYq0my0Zc8hdLuc= -github.com/aws/aws-sdk-go-v2/service/iam v1.53.5/go.mod h1:seDE466zJ4haVuAVcRk+yIH4DWb3s6cqt3Od8GxnGAA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.19 h1:jdCj9vbCXwzTcIJX+MVd2UdssFhRJFTrWlPZwZB8Hpk= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.19/go.mod h1:Dgg2d5WGRr7YB8JJsELskBxLUhgwWppXPwlvmuQKhbc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.2 h1:UOHOXigIzDRaEU03CBQcZ5uW7FNC7E+vwfhsQWXl5RQ= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.2/go.mod h1:nAa5gmcmAmjXN3tGuhPSHLXFeWv+7nzKhjZzh8F7MH0= -github.com/aws/aws-sdk-go-v2/service/lambda v1.88.2 h1:j+IFEtr7aykD6jJRE86kv/+TgN1UK90LudBuz2bjjYw= -github.com/aws/aws-sdk-go-v2/service/lambda v1.88.2/go.mod h1:IDvS3hFp41ZJTByY7BO8PNgQkPNeQDjJfU/0cHJ2V4o= -github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.5 h1:atVRUNiG3hrpntduj0OExYB31F59zr+eavoAecVNMhQ= -github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.5/go.mod h1:Lr/sslNngRPyPo2FeWkEo02t9f/CjkzSIeR0MqRh8ao= -github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.6 h1:G6+LJP+mxaLuM65jEwpgOcef2fmGSyr92U0zjE988A0= -github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.6/go.mod h1:GIy6ofSymO4ZAPKlYhb6Na4sXqsJrmPZPP/NyheE0rk= -github.com/aws/aws-sdk-go-v2/service/rds v1.116.2 h1:KQLPCn9BWXW0Y8DyzEokbTF9HOiOQoR77Eu9GKcjBWU= -github.com/aws/aws-sdk-go-v2/service/rds v1.116.2/go.mod h1:aPw0arz1e+cZUbF4LU7ZMYB1ZSYsJKi/tsAq9wADfeE= -github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3 h1:JRPXnIr0WwFsSHBmuCvT/uh0Vgys+crvwkOghbJEqi8= -github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3/go.mod h1:DHddp7OO4bY467WVCqWBzk5+aEWn7vqYkap7UigJzGk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.13 h1:8xP94tDzFpgwIOsusGiEFHPaqrpckDojoErk/ZFZTio= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.13/go.mod h1:RwF6Xnba8PlINxJUQq1IAWeon6IglvqsnhNqV8QsQjk= -github.com/aws/aws-sdk-go-v2/service/sqs v1.42.23 h1:Rw3+8VaLH0jozccNR52bSvCPYtkiQeNn576l7HCHvL0= -github.com/aws/aws-sdk-go-v2/service/sqs v1.42.23/go.mod h1:MdjRkQEd2EUOiifYnkg/6f1NGtZSN3dFOLNByzufXok= -github.com/aws/aws-sdk-go-v2/service/ssm v1.68.2 h1:idKv7B7NjmTDd05YHQYMMEFNeD0rWxs/kVX4lsjEiDo= -github.com/aws/aws-sdk-go-v2/service/ssm v1.68.2/go.mod h1:1NiL45h4A60CO/hu/UdNyG5AD3VEsdpaQx1l5KtpurA= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.39.0 h1:FQ0FLNsNkhwHcpv6rkAZaR+Royay19A+M88mtOOSg7w= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.39.0/go.mod h1:xnkbxhrdrHvrz8qrNVvMAlARU/6suQoKtIjlaciWr3I= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.3 h1:YBBu7ZhnMkHjBCFVa50NO+AQo1I/8SK8yFIXxQSR/Do= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.3/go.mod h1:wRRzL5slhAzg/F8SwmTgIl3XXFB4V+X1t5qmuJf+O6k= +github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.3 h1:RB1bsGTqfbLymRdVDqHoomyZ6XfPcXtBP041qHLtlv4= +github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.3/go.mod h1:hF/cipBvnkoCnN0v+lw1ZnKTj6LyDfSX7yZW2d9buJQ= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.2 h1:mleWBVIxwceEzyItUVoqMFiv6TmOP6ECPoN6WB/VWXc= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.55.2/go.mod h1:cMApt548kNgu87UsBTNWVv+fpzjbUTFRSFjD1688SBs= +github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.14 h1:ikGt5kTzRE0+ehjewNLHjEO9pgAjdLv2IiwUuIdWagw= +github.com/aws/aws-sdk-go-v2/service/directconnect v1.38.14/go.mod h1:WnsYjhq0txJK9bw4UeVijF1zG0Iuz6FdzwxNihVGcxI= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.2 h1:xi/ECwajy2mixviBD7bKAlGGSwzEaFKX2wIhrZt9NGw= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.56.2/go.mod h1:dLREOeW66eVaaGIOi2ZlLHDgkR3nuJ02rd00j0YSlBE= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 h1:98Miqj16un1WLNyM1RjVDhXYumhqZrQfAeG8i4jPG6o= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0/go.mod h1:T6ndRfdhnXLIY5oKBHjYZDVj706los2zGdpThppquvA= +github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0 h1:YS5TXaEvzDb+sV+wdQFUtuCAk0GeFR9Ai6HFdxpz6q8= +github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0/go.mod h1:10kBgdaNJz0FO/+JWDUH+0rtSjkn5yafgavDDmmhFzs= +github.com/aws/aws-sdk-go-v2/service/efs v1.41.13 h1:C11r13KfnzxlLuILWOjBNdSJDRsJ2HDqc8kTNGc6VcM= +github.com/aws/aws-sdk-go-v2/service/efs v1.41.13/go.mod h1:YP65UYTCBf/NQKrZH+jfX/EHD5zFWLwioLpNoioIscU= +github.com/aws/aws-sdk-go-v2/service/eks v1.81.1 h1:wMMZ6vc0xljHGxZB4Hz3kVX9wSLTUau8RiaZAZtszCA= +github.com/aws/aws-sdk-go-v2/service/eks v1.81.1/go.mod h1:F8fvMS/6YtJPi40rwXWnuWPn6SYIGXPmLY5k87S3Td4= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.22 h1:YyV5ec8Hl6zezAzEQdetqYORGXHtNexaHWLw4TDfuBw= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.22/go.mod h1:480LHdOg5a74xgOBvXwI/yQuaK7SCHUVuujGyOkCw3c= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.9 h1:F7t1rvo++Bv9mTsFbd/0gThSx8vZqdHmIAURQ4dc8Jc= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.9/go.mod h1:1ethHYerpOsRYxSkV8mFNNDmDWPqCdLcrUmdd7aUYN4= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.6 h1:GPQvvxy8+FDnD9xKYzGKJMjIm5xkVM5pd3bFgRldNSo= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.6/go.mod h1:RJNVc52A0K41fCDJOnsCLeWJf8mwa0q30fM3CfE9U18= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.20 h1:ru+seMuylHiNZlvgZei83eD8h37hRjm1XIMOEmcV0BU= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.20/go.mod h1:ihZMtPTKoX/ugQRHbui6zNdSgVYN1KY2Dgwb2d3hXlc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 h1:s/zDSG/a/Su9aX+v0Ld9cimUCdkr5FWPmBV8owaEbZY= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3/go.mod h1:/iSgiUor15ZuxFGQSTf3lA2FmKxFsQoc2tADOarQBSw= +github.com/aws/aws-sdk-go-v2/service/lambda v1.88.3 h1:VlSZQKfbHSjeKJaTpBfp3WVxPH7qa2SbneFtjT9vft8= +github.com/aws/aws-sdk-go-v2/service/lambda v1.88.3/go.mod h1:/C3/ZU9bR0pjMwIYivZVpdcj4HjvOfk+OTPiiXKoTSE= +github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.6 h1:++XPZP+nXaKtZ755Yt8sfqu6lzyyIOu66CaUKA7o8eE= +github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.59.6/go.mod h1:geeH6hXRfXvEXn5tnVljRTl7PyDA3N6fadTnp4z5s8c= +github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.7 h1:NKDyxMTFdm1C/+a2mt4QqmAk2GEfC1iETCsyw9qCEow= +github.com/aws/aws-sdk-go-v2/service/networkmanager v1.41.7/go.mod h1:SGskKh/tt+FOs3//n2K6rNvdsfHQ91hPe7XRtRejOEg= +github.com/aws/aws-sdk-go-v2/service/rds v1.116.3 h1:H/ZYZ6QR4EXJAYElI5xkIM/yCz+A4uHIvWpzl+IfJks= +github.com/aws/aws-sdk-go-v2/service/rds v1.116.3/go.mod h1:QbXW4coAMakHQhf1qhE0eVVCen9gwB/Kvn+HHHKhpGY= +github.com/aws/aws-sdk-go-v2/service/route53 v1.62.4 h1:64aYPyHg3RjLvnMMSYQSg7aP+r1WRCPIS9SP9KfHjWg= +github.com/aws/aws-sdk-go-v2/service/route53 v1.62.4/go.mod h1:bPSPzWTn9LSX6e0KPp4LlPoaspouZdKAlIdSMdhBBrs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 h1:p8WdWDh5AwSZdp19Haa3XMyPCICi9Z375a/Nu3IIEZY= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.14/go.mod h1:NKVY7DER6VXHkt2I/ycmHakALNboi3Rqwt4eEf/1Cnk= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 h1:JP2wjWGmUp8lTCZb13Dv0Eciyc1jbO8pd0HZVMHFlrc= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24/go.mod h1:Ql9ziDutk8ERAN9HMaYANCW3lop451ppebkxEJMLCTM= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 h1:bBoWhx8lsFLTXintRX64ZBXcmFZbGqUmaPUrjXECqIc= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -419,8 +421,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.18.0 h1:jxP5Uuo3bxm3M6gGtV94P4lliVetoCB4Wk2x8QA86LI= -github.com/googleapis/gax-go/v2 v2.18.0/go.mod h1:uSzZN4a356eRG985CzJ3WfbFSpqkLTjsnhWGJR6EwrE= +github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE= +github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -586,8 +588,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g= -github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.12.5 h1:EOHLbsLJgUHUwzkj9gBTOlubkX+dmSs0EYWMdBiHivU= github.com/nats-io/nats-server/v2 v2.12.5/go.mod h1:JQDAKcwdXs0NRhvYO31dzsXkzCyDkOBS7SKU3Nozu14= github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= @@ -856,19 +858,19 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.271.0 h1:cIPN4qcUc61jlh7oXu6pwOQqbJW2GqYh5PS6rB2C/JY= -google.golang.org/api v0.271.0/go.mod h1:CGT29bhwkbF+i11qkRUJb2KMKqcJ1hdFceEIRd9u64Q= +google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA= +google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d h1:vsOm753cOAMkt76efriTCDKjpCbK18XGHMJHo0JUKhc= -google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:0oz9d7g9QLSdv9/lgbIjowW1JoxMbxmBVNe8i6tORJI= -google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= -google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c h1:xgCzyF2LFIO/0X2UAoVRiXKU5Xg6VjToG4i2/ecSswk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -892,12 +894,12 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.35.2 h1:tW7mWc2RpxW7HS4CoRXhtYHSzme1PN1UjGHJ1bdrtdw= -k8s.io/api v0.35.2/go.mod h1:7AJfqGoAZcwSFhOjcGM7WV05QxMMgUaChNfLTXDRE60= -k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= -k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= -k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= -k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g= +k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= +k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg= +k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= diff --git a/go/sdp-go/changes.go b/go/sdp-go/changes.go index b338f2b1..b55d29da 100644 --- a/go/sdp-go/changes.go +++ b/go/sdp-go/changes.go @@ -147,17 +147,17 @@ func (r *Reference) ToMap() map[string]any { // ToMap converts a Risk to a map for serialization, including related items. func (r *Risk) ToMap() map[string]any { - relatedItems := make([]map[string]any, len(r.GetRelatedItems())) - for i, ri := range r.GetRelatedItems() { + relatedItems := make([]map[string]any, len(r.GetRelatedItemRefs())) + for i, ri := range r.GetRelatedItemRefs() { relatedItems[i] = ri.ToMap() } return map[string]any{ - "uuid": stringFromUuidBytes(r.GetUUID()), - "title": r.GetTitle(), - "severity": r.GetSeverity().String(), - "description": r.GetDescription(), - "relatedItems": relatedItems, + "uuid": stringFromUuidBytes(r.GetUUID()), + "title": r.GetTitle(), + "severity": r.GetSeverity().String(), + "description": r.GetDescription(), + "relatedItemRefs": relatedItems, } } diff --git a/go/sdp-go/changes.pb.go b/go/sdp-go/changes.pb.go index 789cb46c..6cd2b4be 100644 --- a/go/sdp-go/changes.pb.go +++ b/go/sdp-go/changes.pb.go @@ -3739,7 +3739,7 @@ func (x *PopulateChangeFiltersResponse) GetAuthors() []string { type ItemDiffSummary struct { state protoimpl.MessageState `protogen:"open.v1"` // A reference to the item that this diff is related to - Item *Reference `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + ItemRef *Reference `protobuf:"bytes,1,opt,name=itemRef,proto3" json:"itemRef,omitempty"` // The status of the item Status ItemDiffStatus `protobuf:"varint,4,opt,name=status,proto3,enum=changes.ItemDiffStatus" json:"status,omitempty"` // The health of the item currently (as opposed to before the change) @@ -3778,9 +3778,9 @@ func (*ItemDiffSummary) Descriptor() ([]byte, []int) { return file_changes_proto_rawDescGZIP(), []int{51} } -func (x *ItemDiffSummary) GetItem() *Reference { +func (x *ItemDiffSummary) GetItemRef() *Reference { if x != nil { - return x.Item + return x.ItemRef } return nil } @@ -6423,14 +6423,14 @@ func (x *EndChangeSimpleResponse) GetQueuedAfterStart() bool { } type Risk struct { - state protoimpl.MessageState `protogen:"open.v1"` - UUID []byte `protobuf:"bytes,5,opt,name=UUID,proto3" json:"UUID,omitempty"` - Title string `protobuf:"bytes,1,opt,name=title,proto3" json:"title,omitempty"` - Severity Risk_Severity `protobuf:"varint,2,opt,name=severity,proto3,enum=changes.Risk_Severity" json:"severity,omitempty"` - Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` - RelatedItems []*Reference `protobuf:"bytes,4,rep,name=relatedItems,proto3" json:"relatedItems,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + UUID []byte `protobuf:"bytes,5,opt,name=UUID,proto3" json:"UUID,omitempty"` + Title string `protobuf:"bytes,1,opt,name=title,proto3" json:"title,omitempty"` + Severity Risk_Severity `protobuf:"varint,2,opt,name=severity,proto3,enum=changes.Risk_Severity" json:"severity,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + RelatedItemRefs []*Reference `protobuf:"bytes,4,rep,name=relatedItemRefs,proto3" json:"relatedItemRefs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Risk) Reset() { @@ -6491,9 +6491,9 @@ func (x *Risk) GetDescription() string { return "" } -func (x *Risk) GetRelatedItems() []*Reference { +func (x *Risk) GetRelatedItemRefs() []*Reference { if x != nil { - return x.RelatedItems + return x.RelatedItemRefs } return nil } @@ -7034,10 +7034,10 @@ const file_changes_proto_rawDesc = "" + "\x1cPopulateChangeFiltersRequest\"O\n" + "\x1dPopulateChangeFiltersResponse\x12\x14\n" + "\x05repos\x18\x01 \x03(\tR\x05repos\x12\x18\n" + - "\aauthors\x18\x02 \x03(\tR\aauthors\"\x8d\x01\n" + - "\x0fItemDiffSummary\x12\x1e\n" + - "\x04item\x18\x01 \x01(\v2\n" + - ".ReferenceR\x04item\x12/\n" + + "\aauthors\x18\x02 \x03(\tR\aauthors\"\x93\x01\n" + + "\x0fItemDiffSummary\x12$\n" + + "\aitemRef\x18\x01 \x01(\v2\n" + + ".ReferenceR\aitemRef\x12/\n" + "\x06status\x18\x04 \x01(\x0e2\x17.changes.ItemDiffStatusR\x06status\x12)\n" + "\vhealthAfter\x18\x05 \x01(\x0e2\a.HealthR\vhealthAfter\"\xa0\x02\n" + "\bItemDiff\x12#\n" + @@ -7258,14 +7258,14 @@ const file_changes_proto_rawDesc = "" + "\x19StartChangeSimpleResponse\"_\n" + "\x17EndChangeSimpleResponse\x12\x16\n" + "\x06queued\x18\x01 \x01(\bR\x06queued\x12,\n" + - "\x12queued_after_start\x18\x02 \x01(\bR\x10queuedAfterStart\"\x96\x02\n" + + "\x12queued_after_start\x18\x02 \x01(\bR\x10queuedAfterStart\"\x9c\x02\n" + "\x04Risk\x12\x12\n" + "\x04UUID\x18\x05 \x01(\fR\x04UUID\x12\x14\n" + "\x05title\x18\x01 \x01(\tR\x05title\x122\n" + "\bseverity\x18\x02 \x01(\x0e2\x16.changes.Risk.SeverityR\bseverity\x12 \n" + - "\vdescription\x18\x03 \x01(\tR\vdescription\x12.\n" + - "\frelatedItems\x18\x04 \x03(\v2\n" + - ".ReferenceR\frelatedItems\"^\n" + + "\vdescription\x18\x03 \x01(\tR\vdescription\x124\n" + + "\x0frelatedItemRefs\x18\x04 \x03(\v2\n" + + ".ReferenceR\x0frelatedItemRefs\"^\n" + "\bSeverity\x12\x18\n" + "\x14SEVERITY_UNSPECIFIED\x10\x00\x12\x10\n" + "\fSEVERITY_LOW\x10\x01\x12\x13\n" + @@ -7586,7 +7586,7 @@ var file_changes_proto_depIdxs = []int32{ 125, // 58: changes.ChangeFiltersRequest.sortOrder:type_name -> SortOrder 71, // 59: changes.ListHomeChangesResponse.changes:type_name -> changes.ChangeSummary 126, // 60: changes.ListHomeChangesResponse.pagination:type_name -> PaginationResponse - 127, // 61: changes.ItemDiffSummary.item:type_name -> Reference + 127, // 61: changes.ItemDiffSummary.itemRef:type_name -> Reference 4, // 62: changes.ItemDiffSummary.status:type_name -> changes.ItemDiffStatus 128, // 63: changes.ItemDiffSummary.healthAfter:type_name -> Health 127, // 64: changes.ItemDiff.item:type_name -> Reference @@ -7638,7 +7638,7 @@ var file_changes_proto_depIdxs = []int32{ 9, // 110: changes.StartChangeResponse.state:type_name -> changes.StartChangeResponse.State 10, // 111: changes.EndChangeResponse.state:type_name -> changes.EndChangeResponse.State 11, // 112: changes.Risk.severity:type_name -> changes.Risk.Severity - 127, // 113: changes.Risk.relatedItems:type_name -> Reference + 127, // 113: changes.Risk.relatedItemRefs:type_name -> Reference 12, // 114: changes.ChangeAnalysisStatus.status:type_name -> changes.ChangeAnalysisStatus.Status 8, // 115: changes.SubmitRiskFeedbackRequest.sentiment:type_name -> changes.RiskFeedbackSentiment 116, // 116: changes.SubmitRiskFeedbackRequest.metadata:type_name -> changes.SubmitRiskFeedbackRequest.MetadataEntry diff --git a/go/sdp-go/config.pb.go b/go/sdp-go/config.pb.go index c20030ae..412962bd 100644 --- a/go/sdp-go/config.pb.go +++ b/go/sdp-go/config.pb.go @@ -1334,8 +1334,13 @@ type GithubAppInformation struct { BotAutomationPercentage int64 `protobuf:"varint,7,opt,name=botAutomationPercentage,proto3" json:"botAutomationPercentage,omitempty"` AverageMergeTime string `protobuf:"bytes,8,opt,name=averageMergeTime,proto3" json:"averageMergeTime,omitempty"` AverageCommitFrequency string `protobuf:"bytes,9,opt,name=averageCommitFrequency,proto3" json:"averageCommitFrequency,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Pending installation request fields (populated when a non-admin user + // has requested the app but admin approval is still pending) + RequestedOrgName *string `protobuf:"bytes,10,opt,name=requestedOrgName,proto3,oneof" json:"requestedOrgName,omitempty"` + RequestedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=requestedAt,proto3,oneof" json:"requestedAt,omitempty"` + RequestedBy *string `protobuf:"bytes,12,opt,name=requestedBy,proto3,oneof" json:"requestedBy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GithubAppInformation) Reset() { @@ -1431,6 +1436,27 @@ func (x *GithubAppInformation) GetAverageCommitFrequency() string { return "" } +func (x *GithubAppInformation) GetRequestedOrgName() string { + if x != nil && x.RequestedOrgName != nil { + return *x.RequestedOrgName + } + return "" +} + +func (x *GithubAppInformation) GetRequestedAt() *timestamppb.Timestamp { + if x != nil { + return x.RequestedAt + } + return nil +} + +func (x *GithubAppInformation) GetRequestedBy() string { + if x != nil && x.RequestedBy != nil { + return *x.RequestedBy + } + return "" +} + // this is all the information required to display the github app information type GetGithubAppInformationResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1685,6 +1711,92 @@ func (*DeleteGithubAppProfileAndGithubInstallationIDResponse) Descriptor() ([]by return file_config_proto_rawDescGZIP(), []int{29} } +// No parameters required — the account is determined from the caller's auth context. +type CreateGithubInstallURLRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateGithubInstallURLRequest) Reset() { + *x = CreateGithubInstallURLRequest{} + mi := &file_config_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateGithubInstallURLRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateGithubInstallURLRequest) ProtoMessage() {} + +func (x *CreateGithubInstallURLRequest) ProtoReflect() protoreflect.Message { + mi := &file_config_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateGithubInstallURLRequest.ProtoReflect.Descriptor instead. +func (*CreateGithubInstallURLRequest) Descriptor() ([]byte, []int) { + return file_config_proto_rawDescGZIP(), []int{30} +} + +type CreateGithubInstallURLResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The full GitHub App install URL including the state query parameter. + // The URL is built from the server-configured GitHub App slug (which GitHub + // restricts to [a-z0-9-]) and is NOT additionally URL-encoded. Consumers + // (especially the frontend) should use this URL as-is for redirection and + // must not assume it is pre-escaped. + InstallUrl string `protobuf:"bytes,1,opt,name=install_url,json=installUrl,proto3" json:"install_url,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateGithubInstallURLResponse) Reset() { + *x = CreateGithubInstallURLResponse{} + mi := &file_config_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateGithubInstallURLResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateGithubInstallURLResponse) ProtoMessage() {} + +func (x *CreateGithubInstallURLResponse) ProtoReflect() protoreflect.Message { + mi := &file_config_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateGithubInstallURLResponse.ProtoReflect.Descriptor instead. +func (*CreateGithubInstallURLResponse) Descriptor() ([]byte, []int) { + return file_config_proto_rawDescGZIP(), []int{31} +} + +func (x *CreateGithubInstallURLResponse) GetInstallUrl() string { + if x != nil { + return x.InstallUrl + } + return "" +} + var File_config_proto protoreflect.FileDescriptor const file_config_proto_rawDesc = "" + @@ -1762,7 +1874,7 @@ const file_config_proto_rawDesc = "" + "\x05WEEKS\x10\x01\x12\n" + "\n" + "\x06MONTHS\x10\x02\" \n" + - "\x1eGetGithubAppInformationRequest\"\xca\x03\n" + + "\x1eGetGithubAppInformationRequest\"\x9a\x05\n" + "\x14GithubAppInformation\x12&\n" + "\x0einstallationID\x18\x01 \x01(\x03R\x0einstallationID\x12 \n" + "\vinstalledBy\x18\x02 \x01(\tR\vinstalledBy\x12<\n" + @@ -1772,7 +1884,14 @@ const file_config_proto_rawDesc = "" + "\x10contributorCount\x18\x06 \x01(\x03R\x10contributorCount\x128\n" + "\x17botAutomationPercentage\x18\a \x01(\x03R\x17botAutomationPercentage\x12*\n" + "\x10averageMergeTime\x18\b \x01(\tR\x10averageMergeTime\x126\n" + - "\x16averageCommitFrequency\x18\t \x01(\tR\x16averageCommitFrequency\"s\n" + + "\x16averageCommitFrequency\x18\t \x01(\tR\x16averageCommitFrequency\x12/\n" + + "\x10requestedOrgName\x18\n" + + " \x01(\tH\x00R\x10requestedOrgName\x88\x01\x01\x12A\n" + + "\vrequestedAt\x18\v \x01(\v2\x1a.google.protobuf.TimestampH\x01R\vrequestedAt\x88\x01\x01\x12%\n" + + "\vrequestedBy\x18\f \x01(\tH\x02R\vrequestedBy\x88\x01\x01B\x13\n" + + "\x11_requestedOrgNameB\x0e\n" + + "\f_requestedAtB\x0e\n" + + "\f_requestedBy\"s\n" + "\x1fGetGithubAppInformationResponse\x12P\n" + "\x14githubAppInformation\x18\x01 \x01(\v2\x1c.config.GithubAppInformationR\x14githubAppInformation\"#\n" + "!RegenerateGithubAppProfileRequest\"\x8f\x01\n" + @@ -1782,7 +1901,11 @@ const file_config_proto_rawDesc = "" + "\"RegenerateGithubAppProfileResponse\x12_\n" + "\x19githubOrganisationProfile\x18\x01 \x01(\v2!.config.GithubOrganisationProfileR\x19githubOrganisationProfile\"6\n" + "4DeleteGithubAppProfileAndGithubInstallationIDRequest\"7\n" + - "5DeleteGithubAppProfileAndGithubInstallationIDResponse2\xd8\b\n" + + "5DeleteGithubAppProfileAndGithubInstallationIDResponse\"\x1f\n" + + "\x1dCreateGithubInstallURLRequest\"A\n" + + "\x1eCreateGithubInstallURLResponse\x12\x1f\n" + + "\vinstall_url\x18\x01 \x01(\tR\n" + + "installUrl2\xc1\t\n" + "\x14ConfigurationService\x12U\n" + "\x10GetAccountConfig\x12\x1f.config.GetAccountConfigRequest\x1a .config.GetAccountConfigResponse\x12^\n" + "\x13UpdateAccountConfig\x12\".config.UpdateAccountConfigRequest\x1a#.config.UpdateAccountConfigResponse\x12R\n" + @@ -1794,7 +1917,8 @@ const file_config_proto_rawDesc = "" + "\x12UpdateSignalConfig\x12!.config.UpdateSignalConfigRequest\x1a\".config.UpdateSignalConfigResponse\x12j\n" + "\x17GetGithubAppInformation\x12&.config.GetGithubAppInformationRequest\x1a'.config.GetGithubAppInformationResponse\x12s\n" + "\x1aRegenerateGithubAppProfile\x12).config.RegenerateGithubAppProfileRequest\x1a*.config.RegenerateGithubAppProfileResponse\x12\xac\x01\n" + - "-DeleteGithubAppProfileAndGithubInstallationID\x12<.config.DeleteGithubAppProfileAndGithubInstallationIDRequest\x1a=.config.DeleteGithubAppProfileAndGithubInstallationIDResponseB1Z/github.com/overmindtech/workspace/go/sdp-go;sdpb\x06proto3" + "-DeleteGithubAppProfileAndGithubInstallationID\x12<.config.DeleteGithubAppProfileAndGithubInstallationIDRequest\x1a=.config.DeleteGithubAppProfileAndGithubInstallationIDResponse\x12g\n" + + "\x16CreateGithubInstallURL\x12%.config.CreateGithubInstallURLRequest\x1a&.config.CreateGithubInstallURLResponseB1Z/github.com/overmindtech/workspace/go/sdp-go;sdpb\x06proto3" var ( file_config_proto_rawDescOnce sync.Once @@ -1809,7 +1933,7 @@ func file_config_proto_rawDescGZIP() []byte { } var file_config_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_config_proto_msgTypes = make([]protoimpl.MessageInfo, 30) +var file_config_proto_msgTypes = make([]protoimpl.MessageInfo, 32) var file_config_proto_goTypes = []any{ (AccountConfig_BlastRadiusPreset)(0), // 0: config.AccountConfig.BlastRadiusPreset (GetHcpConfigResponse_Status)(0), // 1: config.GetHcpConfigResponse.Status @@ -1844,23 +1968,25 @@ var file_config_proto_goTypes = []any{ (*RegenerateGithubAppProfileResponse)(nil), // 30: config.RegenerateGithubAppProfileResponse (*DeleteGithubAppProfileAndGithubInstallationIDRequest)(nil), // 31: config.DeleteGithubAppProfileAndGithubInstallationIDRequest (*DeleteGithubAppProfileAndGithubInstallationIDResponse)(nil), // 32: config.DeleteGithubAppProfileAndGithubInstallationIDResponse - (*durationpb.Duration)(nil), // 33: google.protobuf.Duration - (*CreateAPIKeyResponse)(nil), // 34: apikeys.CreateAPIKeyResponse - (*timestamppb.Timestamp)(nil), // 35: google.protobuf.Timestamp + (*CreateGithubInstallURLRequest)(nil), // 33: config.CreateGithubInstallURLRequest + (*CreateGithubInstallURLResponse)(nil), // 34: config.CreateGithubInstallURLResponse + (*durationpb.Duration)(nil), // 35: google.protobuf.Duration + (*CreateAPIKeyResponse)(nil), // 36: apikeys.CreateAPIKeyResponse + (*timestamppb.Timestamp)(nil), // 37: google.protobuf.Timestamp } var file_config_proto_depIdxs = []int32{ - 33, // 0: config.BlastRadiusConfig.changeAnalysisTargetDuration:type_name -> google.protobuf.Duration + 35, // 0: config.BlastRadiusConfig.changeAnalysisTargetDuration:type_name -> google.protobuf.Duration 0, // 1: config.AccountConfig.blastRadiusPreset:type_name -> config.AccountConfig.BlastRadiusPreset 3, // 2: config.AccountConfig.blastRadius:type_name -> config.BlastRadiusConfig 4, // 3: config.GetAccountConfigResponse.config:type_name -> config.AccountConfig 4, // 4: config.UpdateAccountConfigRequest.config:type_name -> config.AccountConfig 4, // 5: config.UpdateAccountConfigResponse.config:type_name -> config.AccountConfig 11, // 6: config.CreateHcpConfigResponse.config:type_name -> config.HcpConfig - 34, // 7: config.CreateHcpConfigResponse.apiKey:type_name -> apikeys.CreateAPIKeyResponse + 36, // 7: config.CreateHcpConfigResponse.apiKey:type_name -> apikeys.CreateAPIKeyResponse 11, // 8: config.GetHcpConfigResponse.config:type_name -> config.HcpConfig 1, // 9: config.GetHcpConfigResponse.status:type_name -> config.GetHcpConfigResponse.Status 11, // 10: config.ReplaceHcpApiKeyResponse.config:type_name -> config.HcpConfig - 34, // 11: config.ReplaceHcpApiKeyResponse.apiKey:type_name -> apikeys.CreateAPIKeyResponse + 36, // 11: config.ReplaceHcpApiKeyResponse.apiKey:type_name -> apikeys.CreateAPIKeyResponse 22, // 12: config.GetSignalConfigResponse.config:type_name -> config.SignalConfig 22, // 13: config.UpdateSignalConfigRequest.config:type_name -> config.SignalConfig 22, // 14: config.UpdateSignalConfigResponse.config:type_name -> config.SignalConfig @@ -1869,36 +1995,39 @@ var file_config_proto_depIdxs = []int32{ 29, // 17: config.SignalConfig.githubOrganisationProfile:type_name -> config.GithubOrganisationProfile 2, // 18: config.RoutineChangesConfig.eventsPerUnit:type_name -> config.RoutineChangesConfig.DurationUnit 2, // 19: config.RoutineChangesConfig.durationUnit:type_name -> config.RoutineChangesConfig.DurationUnit - 35, // 20: config.GithubAppInformation.installedAt:type_name -> google.protobuf.Timestamp - 26, // 21: config.GetGithubAppInformationResponse.githubAppInformation:type_name -> config.GithubAppInformation - 29, // 22: config.RegenerateGithubAppProfileResponse.githubOrganisationProfile:type_name -> config.GithubOrganisationProfile - 5, // 23: config.ConfigurationService.GetAccountConfig:input_type -> config.GetAccountConfigRequest - 7, // 24: config.ConfigurationService.UpdateAccountConfig:input_type -> config.UpdateAccountConfigRequest - 9, // 25: config.ConfigurationService.CreateHcpConfig:input_type -> config.CreateHcpConfigRequest - 12, // 26: config.ConfigurationService.GetHcpConfig:input_type -> config.GetHcpConfigRequest - 14, // 27: config.ConfigurationService.DeleteHcpConfig:input_type -> config.DeleteHcpConfigRequest - 16, // 28: config.ConfigurationService.ReplaceHcpApiKey:input_type -> config.ReplaceHcpApiKeyRequest - 18, // 29: config.ConfigurationService.GetSignalConfig:input_type -> config.GetSignalConfigRequest - 20, // 30: config.ConfigurationService.UpdateSignalConfig:input_type -> config.UpdateSignalConfigRequest - 25, // 31: config.ConfigurationService.GetGithubAppInformation:input_type -> config.GetGithubAppInformationRequest - 28, // 32: config.ConfigurationService.RegenerateGithubAppProfile:input_type -> config.RegenerateGithubAppProfileRequest - 31, // 33: config.ConfigurationService.DeleteGithubAppProfileAndGithubInstallationID:input_type -> config.DeleteGithubAppProfileAndGithubInstallationIDRequest - 6, // 34: config.ConfigurationService.GetAccountConfig:output_type -> config.GetAccountConfigResponse - 8, // 35: config.ConfigurationService.UpdateAccountConfig:output_type -> config.UpdateAccountConfigResponse - 10, // 36: config.ConfigurationService.CreateHcpConfig:output_type -> config.CreateHcpConfigResponse - 13, // 37: config.ConfigurationService.GetHcpConfig:output_type -> config.GetHcpConfigResponse - 15, // 38: config.ConfigurationService.DeleteHcpConfig:output_type -> config.DeleteHcpConfigResponse - 17, // 39: config.ConfigurationService.ReplaceHcpApiKey:output_type -> config.ReplaceHcpApiKeyResponse - 19, // 40: config.ConfigurationService.GetSignalConfig:output_type -> config.GetSignalConfigResponse - 21, // 41: config.ConfigurationService.UpdateSignalConfig:output_type -> config.UpdateSignalConfigResponse - 27, // 42: config.ConfigurationService.GetGithubAppInformation:output_type -> config.GetGithubAppInformationResponse - 30, // 43: config.ConfigurationService.RegenerateGithubAppProfile:output_type -> config.RegenerateGithubAppProfileResponse - 32, // 44: config.ConfigurationService.DeleteGithubAppProfileAndGithubInstallationID:output_type -> config.DeleteGithubAppProfileAndGithubInstallationIDResponse - 34, // [34:45] is the sub-list for method output_type - 23, // [23:34] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 37, // 20: config.GithubAppInformation.installedAt:type_name -> google.protobuf.Timestamp + 37, // 21: config.GithubAppInformation.requestedAt:type_name -> google.protobuf.Timestamp + 26, // 22: config.GetGithubAppInformationResponse.githubAppInformation:type_name -> config.GithubAppInformation + 29, // 23: config.RegenerateGithubAppProfileResponse.githubOrganisationProfile:type_name -> config.GithubOrganisationProfile + 5, // 24: config.ConfigurationService.GetAccountConfig:input_type -> config.GetAccountConfigRequest + 7, // 25: config.ConfigurationService.UpdateAccountConfig:input_type -> config.UpdateAccountConfigRequest + 9, // 26: config.ConfigurationService.CreateHcpConfig:input_type -> config.CreateHcpConfigRequest + 12, // 27: config.ConfigurationService.GetHcpConfig:input_type -> config.GetHcpConfigRequest + 14, // 28: config.ConfigurationService.DeleteHcpConfig:input_type -> config.DeleteHcpConfigRequest + 16, // 29: config.ConfigurationService.ReplaceHcpApiKey:input_type -> config.ReplaceHcpApiKeyRequest + 18, // 30: config.ConfigurationService.GetSignalConfig:input_type -> config.GetSignalConfigRequest + 20, // 31: config.ConfigurationService.UpdateSignalConfig:input_type -> config.UpdateSignalConfigRequest + 25, // 32: config.ConfigurationService.GetGithubAppInformation:input_type -> config.GetGithubAppInformationRequest + 28, // 33: config.ConfigurationService.RegenerateGithubAppProfile:input_type -> config.RegenerateGithubAppProfileRequest + 31, // 34: config.ConfigurationService.DeleteGithubAppProfileAndGithubInstallationID:input_type -> config.DeleteGithubAppProfileAndGithubInstallationIDRequest + 33, // 35: config.ConfigurationService.CreateGithubInstallURL:input_type -> config.CreateGithubInstallURLRequest + 6, // 36: config.ConfigurationService.GetAccountConfig:output_type -> config.GetAccountConfigResponse + 8, // 37: config.ConfigurationService.UpdateAccountConfig:output_type -> config.UpdateAccountConfigResponse + 10, // 38: config.ConfigurationService.CreateHcpConfig:output_type -> config.CreateHcpConfigResponse + 13, // 39: config.ConfigurationService.GetHcpConfig:output_type -> config.GetHcpConfigResponse + 15, // 40: config.ConfigurationService.DeleteHcpConfig:output_type -> config.DeleteHcpConfigResponse + 17, // 41: config.ConfigurationService.ReplaceHcpApiKey:output_type -> config.ReplaceHcpApiKeyResponse + 19, // 42: config.ConfigurationService.GetSignalConfig:output_type -> config.GetSignalConfigResponse + 21, // 43: config.ConfigurationService.UpdateSignalConfig:output_type -> config.UpdateSignalConfigResponse + 27, // 44: config.ConfigurationService.GetGithubAppInformation:output_type -> config.GetGithubAppInformationResponse + 30, // 45: config.ConfigurationService.RegenerateGithubAppProfile:output_type -> config.RegenerateGithubAppProfileResponse + 32, // 46: config.ConfigurationService.DeleteGithubAppProfileAndGithubInstallationID:output_type -> config.DeleteGithubAppProfileAndGithubInstallationIDResponse + 34, // 47: config.ConfigurationService.CreateGithubInstallURL:output_type -> config.CreateGithubInstallURLResponse + 36, // [36:48] is the sub-list for method output_type + 24, // [24:36] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_config_proto_init() } @@ -1910,13 +2039,14 @@ func file_config_proto_init() { file_config_proto_msgTypes[0].OneofWrappers = []any{} file_config_proto_msgTypes[1].OneofWrappers = []any{} file_config_proto_msgTypes[19].OneofWrappers = []any{} + file_config_proto_msgTypes[23].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_config_proto_rawDesc), len(file_config_proto_rawDesc)), NumEnums: 3, - NumMessages: 30, + NumMessages: 32, NumExtensions: 0, NumServices: 1, }, diff --git a/go/sdp-go/gateway.pb.go b/go/sdp-go/gateway.pb.go index 3728f320..54179d78 100644 --- a/go/sdp-go/gateway.pb.go +++ b/go/sdp-go/gateway.pb.go @@ -233,7 +233,7 @@ type GatewayResponse struct { // *GatewayResponse_Status // *GatewayResponse_Error // *GatewayResponse_QueryError - // *GatewayResponse_DeleteItem + // *GatewayResponse_DeleteItemRef // *GatewayResponse_DeleteEdge // *GatewayResponse_UpdateItem // *GatewayResponse_SnapshotStoreResult @@ -331,10 +331,10 @@ func (x *GatewayResponse) GetQueryError() *QueryError { return nil } -func (x *GatewayResponse) GetDeleteItem() *Reference { +func (x *GatewayResponse) GetDeleteItemRef() *Reference { if x != nil { - if x, ok := x.ResponseType.(*GatewayResponse_DeleteItem); ok { - return x.DeleteItem + if x, ok := x.ResponseType.(*GatewayResponse_DeleteItemRef); ok { + return x.DeleteItemRef } } return nil @@ -454,8 +454,8 @@ type GatewayResponse_QueryError struct { QueryError *QueryError `protobuf:"bytes,6,opt,name=queryError,proto3,oneof"` // A new error that was encountered as part of a query } -type GatewayResponse_DeleteItem struct { - DeleteItem *Reference `protobuf:"bytes,7,opt,name=deleteItem,proto3,oneof"` // An item that should be deleted from local state +type GatewayResponse_DeleteItemRef struct { + DeleteItemRef *Reference `protobuf:"bytes,7,opt,name=deleteItemRef,proto3,oneof"` // An item that should be deleted from local state } type GatewayResponse_DeleteEdge struct { @@ -508,7 +508,7 @@ func (*GatewayResponse_Error) isGatewayResponse_ResponseType() {} func (*GatewayResponse_QueryError) isGatewayResponse_ResponseType() {} -func (*GatewayResponse_DeleteItem) isGatewayResponse_ResponseType() {} +func (*GatewayResponse_DeleteItemRef) isGatewayResponse_ResponseType() {} func (*GatewayResponse_DeleteEdge) isGatewayResponse_ResponseType() {} @@ -2037,7 +2037,7 @@ const file_gateway_proto_rawDesc = "" + "\vchatMessage\x18\x10 \x01(\v2\x14.gateway.ChatMessageH\x00R\vchatMessage\x12L\n" + "\x11minStatusInterval\x18\x02 \x01(\v2\x19.google.protobuf.DurationH\x01R\x11minStatusInterval\x88\x01\x01B\x0e\n" + "\frequest_typeB\x14\n" + - "\x12_minStatusInterval\"\x84\a\n" + + "\x12_minStatusInterval\"\x8a\a\n" + "\x0fGatewayResponse\x12!\n" + "\anewItem\x18\x02 \x01(\v2\x05.ItemH\x00R\anewItem\x12!\n" + "\anewEdge\x18\x03 \x01(\v2\x05.EdgeH\x00R\anewEdge\x127\n" + @@ -2045,11 +2045,9 @@ const file_gateway_proto_rawDesc = "" + "\x05error\x18\x05 \x01(\tH\x00R\x05error\x12-\n" + "\n" + "queryError\x18\x06 \x01(\v2\v.QueryErrorH\x00R\n" + - "queryError\x12,\n" + - "\n" + - "deleteItem\x18\a \x01(\v2\n" + - ".ReferenceH\x00R\n" + - "deleteItem\x12'\n" + + "queryError\x122\n" + + "\rdeleteItemRef\x18\a \x01(\v2\n" + + ".ReferenceH\x00R\rdeleteItemRef\x12'\n" + "\n" + "deleteEdge\x18\b \x01(\v2\x05.EdgeH\x00R\n" + "deleteEdge\x12'\n" + @@ -2234,7 +2232,7 @@ var file_gateway_proto_depIdxs = []int32{ 29, // 10: gateway.GatewayResponse.newEdge:type_name -> Edge 2, // 11: gateway.GatewayResponse.status:type_name -> gateway.GatewayRequestStatus 30, // 12: gateway.GatewayResponse.queryError:type_name -> QueryError - 31, // 13: gateway.GatewayResponse.deleteItem:type_name -> Reference + 31, // 13: gateway.GatewayResponse.deleteItemRef:type_name -> Reference 29, // 14: gateway.GatewayResponse.deleteEdge:type_name -> Edge 28, // 15: gateway.GatewayResponse.updateItem:type_name -> Item 8, // 16: gateway.GatewayResponse.snapshotStoreResult:type_name -> gateway.SnapshotStoreResult @@ -2290,7 +2288,7 @@ func file_gateway_proto_init() { (*GatewayResponse_Status)(nil), (*GatewayResponse_Error)(nil), (*GatewayResponse_QueryError)(nil), - (*GatewayResponse_DeleteItem)(nil), + (*GatewayResponse_DeleteItemRef)(nil), (*GatewayResponse_DeleteEdge)(nil), (*GatewayResponse_UpdateItem)(nil), (*GatewayResponse_SnapshotStoreResult)(nil), diff --git a/go/sdp-go/items.pb.go b/go/sdp-go/items.pb.go index f05460e2..628e1240 100644 --- a/go/sdp-go/items.pb.go +++ b/go/sdp-go/items.pb.go @@ -1264,7 +1264,7 @@ func (x *CancelQuery) GetUUID() []byte { type Expand struct { state protoimpl.MessageState `protogen:"open.v1"` // The item that should be expanded - Item *Reference `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + ItemRef *Reference `protobuf:"bytes,1,opt,name=itemRef,proto3" json:"itemRef,omitempty"` // How many levels of expansion should be run LinkDepth uint32 `protobuf:"varint,2,opt,name=linkDepth,proto3" json:"linkDepth,omitempty"` // A UUID to uniquely identify the request. This should be stored by the @@ -1308,9 +1308,9 @@ func (*Expand) Descriptor() ([]byte, []int) { return file_items_proto_rawDescGZIP(), []int{13} } -func (x *Expand) GetItem() *Reference { +func (x *Expand) GetItemRef() *Reference { if x != nil { - return x.Item + return x.ItemRef } return nil } @@ -1624,10 +1624,10 @@ const file_items_proto_rawDesc = "" + "\aNOSCOPE\x10\x02\x12\v\n" + "\aTIMEOUT\x10\x03\"!\n" + "\vCancelQuery\x12\x12\n" + - "\x04UUID\x18\x01 \x01(\fR\x04UUID\"\x92\x01\n" + - "\x06Expand\x12\x1e\n" + - "\x04item\x18\x01 \x01(\v2\n" + - ".ReferenceR\x04item\x12\x1c\n" + + "\x04UUID\x18\x01 \x01(\fR\x04UUID\"\x98\x01\n" + + "\x06Expand\x12$\n" + + "\aitemRef\x18\x01 \x01(\v2\n" + + ".ReferenceR\aitemRef\x12\x1c\n" + "\tlinkDepth\x18\x02 \x01(\rR\tlinkDepth\x12\x12\n" + "\x04UUID\x18\x03 \x01(\fR\x04UUID\x126\n" + "\bdeadline\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\bdeadline\"\xbf\x01\n" + @@ -1722,7 +1722,7 @@ var file_items_proto_depIdxs = []int32{ 19, // 21: QueryResponse.edge:type_name -> Edge 2, // 22: QueryStatus.status:type_name -> QueryStatus.Status 3, // 23: QueryError.errorType:type_name -> QueryError.ErrorType - 18, // 24: Expand.item:type_name -> Reference + 18, // 24: Expand.itemRef:type_name -> Reference 23, // 25: Expand.deadline:type_name -> google.protobuf.Timestamp 1, // 26: Reference.method:type_name -> QueryMethod 18, // 27: Edge.from:type_name -> Reference diff --git a/go/sdp-go/revlink.pb.go b/go/sdp-go/revlink.pb.go index 6ede8fd4..2b43e25c 100644 --- a/go/sdp-go/revlink.pb.go +++ b/go/sdp-go/revlink.pb.go @@ -27,7 +27,7 @@ type GetReverseEdgesRequest struct { // The account that the item belongs to Account string `protobuf:"bytes,1,opt,name=account,proto3" json:"account,omitempty"` // The item that you would like to find reverse edges for - Item *Reference `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + ItemRef *Reference `protobuf:"bytes,2,opt,name=itemRef,proto3" json:"itemRef,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -69,9 +69,9 @@ func (x *GetReverseEdgesRequest) GetAccount() string { return "" } -func (x *GetReverseEdgesRequest) GetItem() *Reference { +func (x *GetReverseEdgesRequest) GetItemRef() *Reference { if x != nil { - return x.Item + return x.ItemRef } return nil } @@ -342,11 +342,11 @@ var File_revlink_proto protoreflect.FileDescriptor const file_revlink_proto_rawDesc = "" + "\n" + - "\rrevlink.proto\x12\arevlink\x1a\vitems.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"t\n" + + "\rrevlink.proto\x12\arevlink\x1a\vitems.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"z\n" + "\x16GetReverseEdgesRequest\x12\x18\n" + - "\aaccount\x18\x01 \x01(\tR\aaccount\x12\x1e\n" + - "\x04item\x18\x02 \x01(\v2\n" + - ".ReferenceR\x04itemJ\x04\b\x03\x10\x04R\x1afollowOnlyBlastPropagation\"6\n" + + "\aaccount\x18\x01 \x01(\tR\aaccount\x12$\n" + + "\aitemRef\x18\x02 \x01(\v2\n" + + ".ReferenceR\aitemRefJ\x04\b\x03\x10\x04R\x1afollowOnlyBlastPropagation\"6\n" + "\x17GetReverseEdgesResponse\x12\x1b\n" + "\x05edges\x18\x01 \x03(\v2\x05.EdgeR\x05edges\"\x8f\x01\n" + "\x1cIngestGatewayResponseRequest\x12\x18\n" + @@ -390,7 +390,7 @@ var file_revlink_proto_goTypes = []any{ (*Item)(nil), // 8: Item } var file_revlink_proto_depIdxs = []int32{ - 6, // 0: revlink.GetReverseEdgesRequest.item:type_name -> Reference + 6, // 0: revlink.GetReverseEdgesRequest.itemRef:type_name -> Reference 7, // 1: revlink.GetReverseEdgesResponse.edges:type_name -> Edge 8, // 2: revlink.IngestGatewayResponseRequest.newItem:type_name -> Item 7, // 3: revlink.IngestGatewayResponseRequest.newEdge:type_name -> Edge diff --git a/go/sdp-go/sdpconnect/config.connect.go b/go/sdp-go/sdpconnect/config.connect.go index 86befb73..bb758bc5 100644 --- a/go/sdp-go/sdpconnect/config.connect.go +++ b/go/sdp-go/sdpconnect/config.connect.go @@ -66,6 +66,9 @@ const ( // ConfigurationServiceDeleteGithubAppProfileAndGithubInstallationIDProcedure is the fully-qualified // name of the ConfigurationService's DeleteGithubAppProfileAndGithubInstallationID RPC. ConfigurationServiceDeleteGithubAppProfileAndGithubInstallationIDProcedure = "/config.ConfigurationService/DeleteGithubAppProfileAndGithubInstallationID" + // ConfigurationServiceCreateGithubInstallURLProcedure is the fully-qualified name of the + // ConfigurationService's CreateGithubInstallURL RPC. + ConfigurationServiceCreateGithubInstallURLProcedure = "/config.ConfigurationService/CreateGithubInstallURL" ) // ConfigurationServiceClient is a client for the config.ConfigurationService service. @@ -98,6 +101,11 @@ type ConfigurationServiceClient interface { // remove the github app installation id and github organisation profile from the signal config // this will not uninstall the app from github, that must be done manually by the user DeleteGithubAppProfileAndGithubInstallationID(context.Context, *connect.Request[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDRequest]) (*connect.Response[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDResponse], error) + // Create a GitHub App install URL with a DB-backed state parameter for CSRF + // protection. The frontend calls this RPC, then redirects the user to the + // returned URL. GitHub will redirect back with the state UUID, which the + // callback handler consumes to identify the Overmind account. + CreateGithubInstallURL(context.Context, *connect.Request[sdp_go.CreateGithubInstallURLRequest]) (*connect.Response[sdp_go.CreateGithubInstallURLResponse], error) } // NewConfigurationServiceClient constructs a client for the config.ConfigurationService service. By @@ -177,6 +185,12 @@ func NewConfigurationServiceClient(httpClient connect.HTTPClient, baseURL string connect.WithSchema(configurationServiceMethods.ByName("DeleteGithubAppProfileAndGithubInstallationID")), connect.WithClientOptions(opts...), ), + createGithubInstallURL: connect.NewClient[sdp_go.CreateGithubInstallURLRequest, sdp_go.CreateGithubInstallURLResponse]( + httpClient, + baseURL+ConfigurationServiceCreateGithubInstallURLProcedure, + connect.WithSchema(configurationServiceMethods.ByName("CreateGithubInstallURL")), + connect.WithClientOptions(opts...), + ), } } @@ -193,6 +207,7 @@ type configurationServiceClient struct { getGithubAppInformation *connect.Client[sdp_go.GetGithubAppInformationRequest, sdp_go.GetGithubAppInformationResponse] regenerateGithubAppProfile *connect.Client[sdp_go.RegenerateGithubAppProfileRequest, sdp_go.RegenerateGithubAppProfileResponse] deleteGithubAppProfileAndGithubInstallationID *connect.Client[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDRequest, sdp_go.DeleteGithubAppProfileAndGithubInstallationIDResponse] + createGithubInstallURL *connect.Client[sdp_go.CreateGithubInstallURLRequest, sdp_go.CreateGithubInstallURLResponse] } // GetAccountConfig calls config.ConfigurationService.GetAccountConfig. @@ -251,6 +266,11 @@ func (c *configurationServiceClient) DeleteGithubAppProfileAndGithubInstallation return c.deleteGithubAppProfileAndGithubInstallationID.CallUnary(ctx, req) } +// CreateGithubInstallURL calls config.ConfigurationService.CreateGithubInstallURL. +func (c *configurationServiceClient) CreateGithubInstallURL(ctx context.Context, req *connect.Request[sdp_go.CreateGithubInstallURLRequest]) (*connect.Response[sdp_go.CreateGithubInstallURLResponse], error) { + return c.createGithubInstallURL.CallUnary(ctx, req) +} + // ConfigurationServiceHandler is an implementation of the config.ConfigurationService service. type ConfigurationServiceHandler interface { // Get the account config for the user's account @@ -281,6 +301,11 @@ type ConfigurationServiceHandler interface { // remove the github app installation id and github organisation profile from the signal config // this will not uninstall the app from github, that must be done manually by the user DeleteGithubAppProfileAndGithubInstallationID(context.Context, *connect.Request[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDRequest]) (*connect.Response[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDResponse], error) + // Create a GitHub App install URL with a DB-backed state parameter for CSRF + // protection. The frontend calls this RPC, then redirects the user to the + // returned URL. GitHub will redirect back with the state UUID, which the + // callback handler consumes to identify the Overmind account. + CreateGithubInstallURL(context.Context, *connect.Request[sdp_go.CreateGithubInstallURLRequest]) (*connect.Response[sdp_go.CreateGithubInstallURLResponse], error) } // NewConfigurationServiceHandler builds an HTTP handler from the service implementation. It returns @@ -356,6 +381,12 @@ func NewConfigurationServiceHandler(svc ConfigurationServiceHandler, opts ...con connect.WithSchema(configurationServiceMethods.ByName("DeleteGithubAppProfileAndGithubInstallationID")), connect.WithHandlerOptions(opts...), ) + configurationServiceCreateGithubInstallURLHandler := connect.NewUnaryHandler( + ConfigurationServiceCreateGithubInstallURLProcedure, + svc.CreateGithubInstallURL, + connect.WithSchema(configurationServiceMethods.ByName("CreateGithubInstallURL")), + connect.WithHandlerOptions(opts...), + ) return "/config.ConfigurationService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case ConfigurationServiceGetAccountConfigProcedure: @@ -380,6 +411,8 @@ func NewConfigurationServiceHandler(svc ConfigurationServiceHandler, opts ...con configurationServiceRegenerateGithubAppProfileHandler.ServeHTTP(w, r) case ConfigurationServiceDeleteGithubAppProfileAndGithubInstallationIDProcedure: configurationServiceDeleteGithubAppProfileAndGithubInstallationIDHandler.ServeHTTP(w, r) + case ConfigurationServiceCreateGithubInstallURLProcedure: + configurationServiceCreateGithubInstallURLHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -432,3 +465,7 @@ func (UnimplementedConfigurationServiceHandler) RegenerateGithubAppProfile(conte func (UnimplementedConfigurationServiceHandler) DeleteGithubAppProfileAndGithubInstallationID(context.Context, *connect.Request[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDRequest]) (*connect.Response[sdp_go.DeleteGithubAppProfileAndGithubInstallationIDResponse], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("config.ConfigurationService.DeleteGithubAppProfileAndGithubInstallationID is not implemented")) } + +func (UnimplementedConfigurationServiceHandler) CreateGithubInstallURL(context.Context, *connect.Request[sdp_go.CreateGithubInstallURLRequest]) (*connect.Response[sdp_go.CreateGithubInstallURLResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("config.ConfigurationService.CreateGithubInstallURL is not implemented")) +} diff --git a/go/sdp-go/sdpws/client.go b/go/sdp-go/sdpws/client.go index f862824a..efb59ea2 100644 --- a/go/sdp-go/sdpws/client.go +++ b/go/sdp-go/sdpws/client.go @@ -232,8 +232,8 @@ func (c *Client) receive(ctx context.Context) { c.postRequestChan(u, msg) } - case *sdp.GatewayResponse_DeleteItem: - item := msg.GetDeleteItem() + case *sdp.GatewayResponse_DeleteItemRef: + item := msg.GetDeleteItemRef() if c.handler != nil { c.handler.DeleteItem(ctx, item) } diff --git a/go/sdp-go/signal.pb.go b/go/sdp-go/signal.pb.go index aaa0e88b..9e1b7c12 100644 --- a/go/sdp-go/signal.pb.go +++ b/go/sdp-go/signal.pb.go @@ -725,7 +725,7 @@ type GetItemSignalDetailsRequest struct { // The item for which we want to get the details of the signals. // it is the reference of the terraform item before/after. // NB it is not the lookup item from resolve mapping queries. - Item *Reference `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + ItemRef *Reference `protobuf:"bytes,1,opt,name=itemRef,proto3" json:"itemRef,omitempty"` // The UUID of the change this item is associated with. ChangeUUID []byte `protobuf:"bytes,2,opt,name=changeUUID,proto3" json:"changeUUID,omitempty"` // The category of the signals we want to get. This is used to filter the signals by category. @@ -764,9 +764,9 @@ func (*GetItemSignalDetailsRequest) Descriptor() ([]byte, []int) { return file_signal_proto_rawDescGZIP(), []int{14} } -func (x *GetItemSignalDetailsRequest) GetItem() *Reference { +func (x *GetItemSignalDetailsRequest) GetItemRef() *Reference { if x != nil { - return x.Item + return x.ItemRef } return nil } @@ -1133,10 +1133,10 @@ const file_signal_proto_rawDesc = "" + "changeUUID\x12\x1a\n" + "\bcategory\x18\x02 \x01(\tR\bcategory\"N\n" + "\"GetCustomSignalsByCategoryResponse\x12(\n" + - "\asignals\x18\x01 \x03(\v2\x0e.signal.SignalR\asignals\"y\n" + - "\x1bGetItemSignalDetailsRequest\x12\x1e\n" + - "\x04item\x18\x01 \x01(\v2\n" + - ".ReferenceR\x04item\x12\x1e\n" + + "\asignals\x18\x01 \x03(\v2\x0e.signal.SignalR\asignals\"\x7f\n" + + "\x1bGetItemSignalDetailsRequest\x12$\n" + + "\aitemRef\x18\x01 \x01(\v2\n" + + ".ReferenceR\aitemRef\x12\x1e\n" + "\n" + "changeUUID\x18\x02 \x01(\fR\n" + "changeUUID\x12\x1a\n" + @@ -1227,7 +1227,7 @@ var file_signal_proto_depIdxs = []int32{ 22, // 9: signal.ItemAggregationV2.status:type_name -> changes.ItemDiffStatus 10, // 10: signal.GetItemSignalsResponseV2.itemAggregations:type_name -> signal.ItemAggregationV2 18, // 11: signal.GetCustomSignalsByCategoryResponse.signals:type_name -> signal.Signal - 21, // 12: signal.GetItemSignalDetailsRequest.item:type_name -> Reference + 21, // 12: signal.GetItemSignalDetailsRequest.itemRef:type_name -> Reference 18, // 13: signal.GetItemSignalDetailsResponse.signals:type_name -> signal.Signal 21, // 14: signal.SignalProperties.item:type_name -> Reference 16, // 15: signal.Signal.metadata:type_name -> signal.SignalMetadata diff --git a/go/sdpcache/bolt.go b/go/sdpcache/bolt.go new file mode 100644 index 00000000..ec5f1082 --- /dev/null +++ b/go/sdpcache/bolt.go @@ -0,0 +1,118 @@ +package sdpcache + +import ( + "context" + "time" + + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// BoltCache wraps boltStore and delegates Lookup control-flow to +// lookupCoordinator. Purge scheduling lives here; boltStore only handles +// storage and purge execution. +type BoltCache struct { + purger + + *boltStore + pending *pendingWork + lookup *lookupCoordinator +} + +// assert interface +var _ Cache = (*BoltCache)(nil) + +// NewBoltCache creates a new BoltCache at the specified path. +// If a cache file already exists at the path, it will be opened and used. +// The existing file will be automatically handled by the purge process, +// which removes expired items. No explicit cleanup is needed on startup. +func NewBoltCache(path string, opts ...BoltCacheOption) (*BoltCache, error) { + store, err := newBoltCacheStore(path, opts...) + if err != nil { + return nil, err + } + + pending := newPendingWork() + c := &BoltCache{ + boltStore: store, + pending: pending, + lookup: newLookupCoordinator(pending), + } + c.purgeFunc = c.boltStore.Purge + return c, nil +} + +// Lookup performs a cache lookup for the given query parameters. +func (c *BoltCache) Lookup(ctx context.Context, srcName string, method sdp.QueryMethod, scope string, typ string, query string, ignoreCache bool) (bool, CacheKey, []*sdp.Item, *sdp.QueryError, func()) { + ctx, span := tracing.Tracer().Start(ctx, "BoltCache.Lookup", + trace.WithAttributes( + attribute.String("ovm.cache.sourceName", srcName), + attribute.String("ovm.cache.method", method.String()), + attribute.String("ovm.cache.scope", scope), + attribute.String("ovm.cache.type", typ), + attribute.String("ovm.cache.query", query), + attribute.Bool("ovm.cache.ignoreCache", ignoreCache), + ), + ) + defer span.End() + + ck := CacheKeyFromParts(srcName, method, scope, typ, query) + + if c == nil || c.boltStore == nil { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache not initialised"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, ck, nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "cache has not been initialised", + Scope: scope, + SourceName: srcName, + ItemType: typ, + }, noopDone + } + + // Set disk usage metrics + c.setDiskUsageAttributes(span) + + if ignoreCache { + span.SetAttributes( + attribute.String("ovm.cache.result", "ignore cache"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, ck, nil, nil, noopDone + } + + lookup := c.lookup + if lookup == nil { + lookup = newLookupCoordinator(c.pending) + } + + hit, items, qErr, done := lookup.Lookup( + ctx, + c, + ck, + method, + ) + return hit, ck, items, qErr, done +} + +// StoreItem delegates to boltStore and pokes the purge timer. +func (c *BoltCache) StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) { + if item == nil { + return + } + c.boltStore.StoreItem(ctx, item, duration, ck) + c.setNextPurgeIfEarlier(time.Now().Add(duration)) +} + +// StoreUnavailableItem delegates to boltStore and pokes the purge timer. +func (c *BoltCache) StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, ck CacheKey) { + if err == nil { + return + } + c.boltStore.StoreUnavailableItem(ctx, err, duration, ck) + c.setNextPurgeIfEarlier(time.Now().Add(duration)) +} diff --git a/go/sdpcache/bolt_cache.go b/go/sdpcache/boltstore.go similarity index 72% rename from go/sdpcache/bolt_cache.go rename to go/sdpcache/boltstore.go index b3a97948..96219ff4 100644 --- a/go/sdpcache/bolt_cache.go +++ b/go/sdpcache/boltstore.go @@ -158,7 +158,7 @@ func parseExpiryKey(key []byte) (time.Time, SSTHash, []byte, error) { } expiryNanoUint := binary.BigEndian.Uint64(key[0:8]) - expiryNano := int64(expiryNanoUint) //nolint:gosec // G115 (overflow): guarded by underflow check on lines 153-155 that clamps to zero + expiryNano := int64(expiryNanoUint) //nolint:gosec // G115 (overflow): guarded by underflow check that clamps to zero // Check for overflow when converting uint64 to int64 if expiryNano < 0 && expiryNanoUint > 0 { expiryNano = 0 @@ -178,27 +178,16 @@ func parseExpiryKey(key []byte) (time.Time, SSTHash, []byte, error) { return expiry, sstHash, entryKey, nil } -// BoltCache implements the Cache interface using bbolt for persistent storage -type BoltCache struct { +// boltStore holds the bbolt-backed storage implementation reused by both +// BoltCache and ShardedCache. It handles storage and purge execution only; +// purge scheduling (timer, goroutine) is owned by the Cache-level wrapper. +type boltStore struct { db *bbolt.DB path string - // Minimum amount of time to wait between cache purges - MinWaitTime time.Duration - // CompactThreshold is the number of deleted bytes before triggering compaction CompactThreshold int64 - // The timer that is used to trigger the next purge - purgeTimer *time.Timer - - // The time that the purger will run next - nextPurge time.Time - - // Ensures that purge stats like `purgeTimer` and `nextPurge` aren't being - // modified concurrently - purgeMutex sync.Mutex - // Track deleted bytes for compaction deletedBytes int64 deletedMu sync.Mutex @@ -206,37 +195,19 @@ type BoltCache struct { // Ensures that compaction operations aren't running concurrently // Read operations use RLock, write operations and compaction use Lock compactMutex sync.RWMutex - - // Tracks in-flight lookups to prevent duplicate work when multiple - // goroutines request the same cache key simultaneously - pending *pendingWork } -// assert interface -var _ Cache = (*BoltCache)(nil) - -// BoltCacheOption is a functional option for configuring BoltCache -type BoltCacheOption func(*BoltCache) - -// WithMinWaitTime sets the minimum wait time between purges -func WithMinWaitTime(d time.Duration) BoltCacheOption { - return func(c *BoltCache) { - c.MinWaitTime = d - } -} +// BoltCacheOption is a functional option for configuring bolt-backed storage. +type BoltCacheOption func(*boltStore) // WithCompactThreshold sets the threshold for triggering compaction func WithCompactThreshold(bytes int64) BoltCacheOption { - return func(c *BoltCache) { + return func(c *boltStore) { c.CompactThreshold = bytes } } -// NewBoltCache creates a new BoltCache at the specified path. -// If a cache file already exists at the path, it will be opened and used. -// The existing file will be automatically handled by the purge process, -// which removes expired items. No explicit cleanup is needed on startup. -func NewBoltCache(path string, opts ...BoltCacheOption) (*BoltCache, error) { +func newBoltCacheStore(path string, opts ...BoltCacheOption) (*boltStore, error) { // Ensure the directory exists dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o755); err != nil { @@ -249,11 +220,10 @@ func NewBoltCache(path string, opts ...BoltCacheOption) (*BoltCache, error) { return nil, fmt.Errorf("failed to open bolt database: %w", err) } - c := &BoltCache{ + c := &boltStore{ db: db, path: path, CompactThreshold: DefaultCompactThreshold, - pending: newPendingWork(), } for _, opt := range opts { @@ -276,7 +246,7 @@ func NewBoltCache(path string, opts ...BoltCacheOption) (*BoltCache, error) { } // initBuckets creates the required buckets if they don't exist -func (c *BoltCache) initBuckets() error { +func (c *boltStore) initBuckets() error { return c.db.Update(func(tx *bbolt.Tx) error { if _, err := tx.CreateBucketIfNotExists(itemsBucketName); err != nil { return fmt.Errorf("failed to create items bucket: %w", err) @@ -292,7 +262,7 @@ func (c *BoltCache) initBuckets() error { } // loadDeletedBytes loads the deleted bytes counter from the meta bucket -func (c *BoltCache) loadDeletedBytes() error { +func (c *boltStore) loadDeletedBytes() error { return c.db.View(func(tx *bbolt.Tx) error { meta := tx.Bucket(metaBucketName) if meta == nil { @@ -302,7 +272,7 @@ func (c *BoltCache) loadDeletedBytes() error { data := meta.Get(deletedBytesKey) if len(data) == 8 { deletedBytesUint := binary.BigEndian.Uint64(data) - deletedBytes := int64(deletedBytesUint) //nolint:gosec // G115 (overflow): guarded by underflow check on lines 299-301 that clamps to zero + deletedBytes := int64(deletedBytesUint) //nolint:gosec // G115 (overflow): guarded by underflow check that clamps to zero // Check for overflow when converting uint64 to int64 if deletedBytes < 0 && deletedBytesUint > 0 { deletedBytes = 0 @@ -314,7 +284,7 @@ func (c *BoltCache) loadDeletedBytes() error { } // saveDeletedBytes saves the deleted bytes counter to the meta bucket -func (c *BoltCache) saveDeletedBytes(tx *bbolt.Tx) error { +func (c *boltStore) saveDeletedBytes(tx *bbolt.Tx) error { meta := tx.Bucket(metaBucketName) if meta == nil { return errors.New("meta bucket not found") @@ -333,28 +303,28 @@ func (c *BoltCache) saveDeletedBytes(tx *bbolt.Tx) error { } // addDeletedBytes adds to the deleted bytes counter (thread-safe) -func (c *BoltCache) addDeletedBytes(n int64) { +func (c *boltStore) addDeletedBytes(n int64) { c.deletedMu.Lock() c.deletedBytes += n c.deletedMu.Unlock() } // getDeletedBytes returns the current deleted bytes count (thread-safe) -func (c *BoltCache) getDeletedBytes() int64 { +func (c *boltStore) getDeletedBytes() int64 { c.deletedMu.Lock() defer c.deletedMu.Unlock() return c.deletedBytes } // resetDeletedBytes resets the deleted bytes counter (thread-safe) -func (c *BoltCache) resetDeletedBytes() { +func (c *boltStore) resetDeletedBytes() { c.deletedMu.Lock() c.deletedBytes = 0 c.deletedMu.Unlock() } // getFileSize returns the size of the BoltDB file, logging any errors -func (c *BoltCache) getFileSize() int64 { +func (c *boltStore) getFileSize() int64 { if c == nil || c.path == "" { return 0 } @@ -372,25 +342,14 @@ func (c *BoltCache) getFileSize() int64 { return stat.Size() } -// getDiskUsageMetrics returns disk usage metrics for the BoltDB file -func (c *BoltCache) getDiskUsageMetrics() (fileSize int64, deletedBytes int64) { - if c == nil || c.path == "" { - return 0, 0 - } - - fileSize = c.getFileSize() - deletedBytes = c.getDeletedBytes() - - return fileSize, deletedBytes -} - // setDiskUsageAttributes sets disk usage attributes on a span -func (c *BoltCache) setDiskUsageAttributes(span trace.Span) { +func (c *boltStore) setDiskUsageAttributes(span trace.Span) { if c == nil { return } - fileSize, deletedBytes := c.getDiskUsageMetrics() + fileSize := c.getFileSize() + deletedBytes := c.getDeletedBytes() span.SetAttributes( attribute.Int64("ovm.boltdb.fileSizeBytes", fileSize), attribute.Int64("ovm.boltdb.deletedBytes", deletedBytes), @@ -400,7 +359,7 @@ func (c *BoltCache) setDiskUsageAttributes(span trace.Span) { // CloseAndDestroy closes the database and deletes the cache file. // This method makes the destructive behavior explicit. -func (c *BoltCache) CloseAndDestroy() error { +func (c *boltStore) CloseAndDestroy() error { if c == nil { return nil } @@ -423,7 +382,7 @@ func (c *BoltCache) CloseAndDestroy() error { // deleteCacheFile removes the cache file entirely. This is used as a last resort // when the disk is full and cleanup doesn't help. It closes the database, // removes the file, and resets internal state. -func (c *BoltCache) deleteCacheFile(ctx context.Context) error { +func (c *boltStore) deleteCacheFile(ctx context.Context) error { if c == nil { return nil } @@ -442,7 +401,7 @@ func (c *BoltCache) deleteCacheFile(ctx context.Context) error { } // deleteCacheFileLocked is the internal version that assumes the caller already holds compactMutex.Lock() -func (c *BoltCache) deleteCacheFileLocked(ctx context.Context, span trace.Span) error { +func (c *boltStore) deleteCacheFileLocked(ctx context.Context, span trace.Span) error { // Close the database if it's open if err := c.db.Close(); err != nil { span.RecordError(err) @@ -483,214 +442,10 @@ func (c *BoltCache) deleteCacheFileLocked(ctx context.Context, span trace.Span) return nil } -// createDoneFunc returns a done function that calls pending.Complete for the given cache key. -// The done function releases resources and unblocks waiting goroutines. -// The done function is safe to call multiple times (idempotent via sync.Once). -func (c *BoltCache) createDoneFunc(ck CacheKey) func() { - if c == nil || c.pending == nil { - return noopDone - } - key := ck.String() - var once sync.Once - return func() { - once.Do(func() { - c.pending.Complete(key) - }) - } -} - -// Lookup performs a cache lookup for the given query parameters. -func (c *BoltCache) Lookup(ctx context.Context, srcName string, method sdp.QueryMethod, scope string, typ string, query string, ignoreCache bool) (bool, CacheKey, []*sdp.Item, *sdp.QueryError, func()) { - ctx, span := tracing.Tracer().Start(ctx, "BoltCache.Lookup", - trace.WithAttributes( - attribute.String("ovm.cache.sourceName", srcName), - attribute.String("ovm.cache.method", method.String()), - attribute.String("ovm.cache.scope", scope), - attribute.String("ovm.cache.type", typ), - attribute.String("ovm.cache.query", query), - attribute.Bool("ovm.cache.ignoreCache", ignoreCache), - ), - ) - defer span.End() - - ck := CacheKeyFromParts(srcName, method, scope, typ, query) - - // Set disk usage metrics - c.setDiskUsageAttributes(span) - - if c == nil { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache not initialised"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, &sdp.QueryError{ - ErrorType: sdp.QueryError_OTHER, - ErrorString: "cache has not been initialised", - Scope: scope, - SourceName: srcName, - ItemType: typ, - }, noopDone - } - - if ignoreCache { - span.SetAttributes( - attribute.String("ovm.cache.result", "ignore cache"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - // Search already has RLock, so we don't need to add another one here - initialSearchStart := time.Now() - items, err := c.search(ctx, ck) - initialSearchDuration := time.Since(initialSearchStart) - span.SetAttributes( - attribute.Float64("ovm.cache.initialSearchDuration_ms", float64(initialSearchDuration.Milliseconds())), - ) - - if err != nil { - var qErr *sdp.QueryError - if errors.Is(err, ErrCacheNotFound) { - // Cache miss - check if another goroutine is already fetching this data - shouldWork, entry := c.pending.StartWork(ck.String()) - if shouldWork { - // We're the first caller, return miss so caller does the work - span.SetAttributes( - attribute.String("ovm.cache.result", "cache miss"), - attribute.Bool("ovm.cache.hit", false), - attribute.Bool("ovm.cache.workPending", false), - ) - return false, ck, nil, nil, c.createDoneFunc(ck) - } - - // Another goroutine is fetching this data, wait for it to complete - pendingWaitStart := time.Now() - ok := c.pending.Wait(ctx, entry) - pendingWaitDuration := time.Since(pendingWaitStart) - - span.SetAttributes( - attribute.Float64("ovm.cache.pendingWaitDuration_ms", float64(pendingWaitDuration.Milliseconds())), - attribute.Bool("ovm.cache.pendingWaitSuccess", ok), - ) - - if !ok { - // Context was cancelled or work was cancelled, return miss - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work cancelled or timeout"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - // Work is complete, re-check the cache for results - recheckSearchStart := time.Now() - items, recheckErr := c.search(ctx, ck) - recheckSearchDuration := time.Since(recheckSearchStart) - span.SetAttributes( - attribute.Float64("ovm.cache.recheckSearchDuration_ms", float64(recheckSearchDuration.Milliseconds())), - ) - if recheckErr != nil { - if errors.Is(recheckErr, ErrCacheNotFound) { - // Cache still empty after pending work completed - // This is valid - worker may have found nothing or cancelled - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work completed but cache still empty"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - var recheckQErr *sdp.QueryError - if errors.As(recheckErr, &recheckQErr) { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work: error"), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, nil, recheckQErr, noopDone - } - // Truly unexpected error - return miss - span.SetAttributes( - attribute.String("ovm.cache.result", "unexpected error on re-check"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } else if errors.As(err, &qErr) { - if qErr.GetErrorType() == sdp.QueryError_NOTFOUND { - span.SetAttributes(attribute.String("ovm.cache.result", "cache hit: item not found")) - } else { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: QueryError"), - attribute.String("ovm.cache.error", err.Error()), - ) - } - - span.SetAttributes(attribute.Bool("ovm.cache.hit", true)) - return true, ck, nil, qErr, noopDone - } else { - qErr = &sdp.QueryError{ - ErrorType: sdp.QueryError_OTHER, - ErrorString: err.Error(), - Scope: scope, - SourceName: srcName, - ItemType: typ, - } - - span.SetAttributes( - attribute.String("ovm.cache.error", err.Error()), - attribute.String("ovm.cache.result", "cache hit: unknown QueryError"), - attribute.Bool("ovm.cache.hit", true), - ) - - return true, ck, nil, qErr, noopDone - } - } - - if method == sdp.QueryMethod_GET { - if len(items) < 2 { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: 1 item"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } else { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache returned >1 value, purging and continuing"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", false), - ) - // Delete already has Lock(), so we can call it directly - c.Delete(ck) - return false, ck, nil, nil, noopDone - } - } - - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: multiple items"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - - // RLock already released above - return true, ck, items, nil, noopDone -} - // Search performs a lower-level search using a CacheKey, bypassing pendingWork -// deduplication. This is used by ShardedCache to do raw reads on individual shards. -func (c *BoltCache) Search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { - return c.search(ctx, ck) -} - -// search performs a lower-level search using a CacheKey. +// deduplication. This is used by ShardedCache and lookupCoordinator. // If ctx contains a span, detailed timing metrics will be added as span attributes. -func (c *BoltCache) search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { +func (c *boltStore) Search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { if c == nil { return nil, nil } @@ -790,7 +545,7 @@ func (c *BoltCache) search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error } // StoreItem stores an item in the cache with the specified duration. -func (c *BoltCache) StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) { +func (c *boltStore) StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) { if item == nil || c == nil { return } @@ -820,13 +575,6 @@ func (c *BoltCache) StoreItem(ctx context.Context, item *sdp.Item, duration time // Set disk usage metrics c.setDiskUsageAttributes(span) - // Ensure minimum duration to avoid items expiring immediately - // This handles cases where time.Until() returns 0 or negative due to timing - // Use 100ms to account for race detector overhead and slow CI environments - if duration <= 100*time.Millisecond { - duration = 100 * time.Millisecond - } - res := CachedResult{ Item: item, Error: nil, @@ -849,7 +597,7 @@ func (c *BoltCache) StoreItem(ctx context.Context, item *sdp.Item, duration time } // StoreUnavailableItem stores an error in the cache with the specified duration. -func (c *BoltCache) StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, ck CacheKey) { +func (c *boltStore) StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, ck CacheKey) { if c == nil || err == nil { return } @@ -878,12 +626,6 @@ func (c *BoltCache) StoreUnavailableItem(ctx context.Context, err error, duratio // Set disk usage metrics c.setDiskUsageAttributes(span) - // Ensure minimum duration to avoid items expiring immediately - // Use 100ms to account for race detector overhead and slow CI environments - if duration <= 100*time.Millisecond { - duration = 100 * time.Millisecond - } - res := CachedResult{ Item: nil, Error: err, @@ -895,7 +637,7 @@ func (c *BoltCache) StoreUnavailableItem(ctx context.Context, err error, duratio } // storeResult stores a CachedResult in the database -func (c *BoltCache) storeResult(ctx context.Context, res CachedResult) { +func (c *boltStore) storeResult(ctx context.Context, res CachedResult) { span := trace.SpanFromContext(ctx) entry, err := fromCachedResult(&res) @@ -1046,13 +788,10 @@ func (c *BoltCache) storeResult(ctx context.Context, res CachedResult) { attribute.Int64("ovm.boltdb.entrySizeBytes", entrySize), ) c.setDiskUsageAttributes(span) - - // Update the purge time if required - c.setNextPurgeIfEarlier(res.Expiry) } // Delete removes all entries matching the given cache key. -func (c *BoltCache) Delete(ck CacheKey) { +func (c *boltStore) Delete(ck CacheKey) { if c == nil { return } @@ -1119,7 +858,7 @@ func (c *BoltCache) Delete(ck CacheKey) { } // Clear removes all entries from the cache. -func (c *BoltCache) Clear() { +func (c *boltStore) Clear() { if c == nil { return } @@ -1150,7 +889,7 @@ func (c *BoltCache) Clear() { } // Purge removes all expired items from the cache. -func (c *BoltCache) Purge(ctx context.Context, before time.Time) PurgeStats { +func (c *boltStore) Purge(ctx context.Context, before time.Time) PurgeStats { if c == nil { return PurgeStats{} } @@ -1194,7 +933,7 @@ func (c *BoltCache) Purge(ctx context.Context, before time.Time) PurgeStats { // purgeLocked is the internal version that assumes the caller already holds compactMutex.Lock() // It performs the actual purging work and returns the stats, but does not handle compaction. -func (c *BoltCache) purgeLocked(ctx context.Context, before time.Time) PurgeStats { +func (c *boltStore) purgeLocked(ctx context.Context, before time.Time) PurgeStats { span := trace.SpanFromContext(ctx) // Set initial disk usage metrics @@ -1312,7 +1051,7 @@ func (c *BoltCache) purgeLocked(ctx context.Context, before time.Time) PurgeStat } // compact performs database compaction to reclaim disk space -func (c *BoltCache) compact(ctx context.Context) error { +func (c *boltStore) compact(ctx context.Context) error { // Acquire global lock to prevent any concurrent bbolt operations c.compactMutex.Lock() defer c.compactMutex.Unlock() @@ -1435,85 +1174,3 @@ func (c *BoltCache) compact(ctx context.Context) error { return nil } - -// GetMinWaitTime returns the minimum time between purge operations -func (c *BoltCache) GetMinWaitTime() time.Duration { - if c == nil { - return 0 - } - - if c.MinWaitTime == 0 { - return MinWaitDefault - } - - return c.MinWaitTime -} - -// StartPurger starts a background goroutine that automatically purges expired items. -func (c *BoltCache) StartPurger(ctx context.Context) { - if c == nil { - return - } - - c.purgeMutex.Lock() - if c.purgeTimer == nil { - c.purgeTimer = time.NewTimer(0) - c.purgeMutex.Unlock() - } else { - c.purgeMutex.Unlock() - log.WithContext(ctx).Info("Purger already running") - return // the purger is already running, so we don't need to start it again - } - - go func(ctx context.Context) { - for { - select { - case <-c.purgeTimer.C: - stats := c.Purge(ctx, time.Now()) - c.setNextPurgeFromStats(stats) - case <-ctx.Done(): - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - c.purgeTimer.Stop() - c.purgeTimer = nil - return - } - } - }(ctx) -} - -// setNextPurgeFromStats sets when the next purge should run based on the stats -func (c *BoltCache) setNextPurgeFromStats(stats PurgeStats) { - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - if stats.NextExpiry == nil { - c.purgeTimer.Reset(1000 * time.Hour) - c.nextPurge = time.Now().Add(1000 * time.Hour) - } else { - if time.Until(*stats.NextExpiry) < c.GetMinWaitTime() { - c.purgeTimer.Reset(c.GetMinWaitTime()) - c.nextPurge = time.Now().Add(c.GetMinWaitTime()) - } else { - c.purgeTimer.Reset(time.Until(*stats.NextExpiry)) - c.nextPurge = *stats.NextExpiry - } - } -} - -// setNextPurgeIfEarlier sets the next purge time if the provided time is sooner -func (c *BoltCache) setNextPurgeIfEarlier(t time.Time) { - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - if t.Before(c.nextPurge) { - if c.purgeTimer == nil { - return - } - - c.purgeTimer.Stop() - c.nextPurge = t - c.purgeTimer.Reset(time.Until(t)) - } -} diff --git a/go/sdpcache/boltstore_test.go b/go/sdpcache/boltstore_test.go new file mode 100644 index 00000000..73beddba --- /dev/null +++ b/go/sdpcache/boltstore_test.go @@ -0,0 +1,717 @@ +package sdpcache + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/overmindtech/cli/go/sdp-go" +) + +// TestBoltStoreCloseAndDestroy verifies that CloseAndDestroy() correctly +// closes the database and deletes the cache file. +func TestBoltStoreCloseAndDestroy(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + // Create a cache and store some data + ctx := t.Context() + cache1, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + + // Store an item + item1 := GenerateRandomItem() + ck1 := CacheKeyFromQuery(item1.GetMetadata().GetSourceQuery(), item1.GetMetadata().GetSourceName()) + cache1.StoreItem(ctx, item1, 10*time.Second, ck1) + + // Store another item with a short TTL (will expire) + item2 := GenerateRandomItem() + ck2 := CacheKeyFromQuery(item2.GetMetadata().GetSourceQuery(), item2.GetMetadata().GetSourceName()) + cache1.StoreItem(ctx, item2, 100*time.Millisecond, ck2) + + // Verify both items are in the cache + items, err := testSearch(t.Context(), cache1, ck1) + if err != nil { + t.Errorf("failed to search for item1: %v", err) + } + if len(items) != 1 { + t.Errorf("expected 1 item for ck1, got %d", len(items)) + } + + // Verify the cache file exists + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + t.Fatal("cache file should exist before CloseAndDestroy") + } + + // Close and destroy the cache + if err := cache1.CloseAndDestroy(); err != nil { + t.Fatalf("failed to close and destroy cache1: %v", err) + } + + // Verify the cache file is deleted + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Error("cache file should be deleted after CloseAndDestroy") + } + + // Create a new cache at the same path - should create a fresh, empty cache + cache2, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create new BoltCache: %v", err) + } + defer func() { + _ = cache2.CloseAndDestroy() + }() + + // Verify the old item is NOT accessible (cache was destroyed) + items, err = testSearch(ctx, cache2, ck1) + if !errors.Is(err, ErrCacheNotFound) { + t.Errorf("expected cache miss for item1 in new cache, got: err=%v, items=%d", err, len(items)) + } + + // Verify we can store new items in the fresh cache + item3 := GenerateRandomItem() + ck3 := CacheKeyFromQuery(item3.GetMetadata().GetSourceQuery(), item3.GetMetadata().GetSourceName()) + cache2.StoreItem(ctx, item3, 10*time.Second, ck3) + + items, err = testSearch(ctx, cache2, ck3) + if err != nil { + t.Errorf("failed to search for newly stored item3: %v", err) + } + if len(items) != 1 { + t.Errorf("expected 1 item for ck3, got %d", len(items)) + } +} + +// TestBoltStoreOperationsAfterCloseAndDestroy verifies that operations after +// CloseAndDestroy() return proper errors instead of panicking. +func TestBoltStoreOperationsAfterCloseAndDestroy(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + ctx := t.Context() + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + + // Store an item before closing + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + // Close and destroy the cache + if err := cache.CloseAndDestroy(); err != nil { + t.Fatalf("failed to close and destroy cache: %v", err) + } + + // Now try various operations after the cache is closed and destroyed + // These should return errors, not panic + + t.Run("Search after CloseAndDestroy", func(t *testing.T) { + // This should error because the database is closed + _, err := testSearch(ctx, cache, ck) + if err == nil { + t.Error("expected error when searching after CloseAndDestroy, got nil") + } + t.Logf("Search returned expected error: %v", err) + }) + + t.Run("StoreItem after CloseAndDestroy", func(t *testing.T) { + // This should not panic - it might silently fail or error + // The key is that it doesn't panic + newItem := GenerateRandomItem() + newCk := CacheKeyFromQuery(newItem.GetMetadata().GetSourceQuery(), newItem.GetMetadata().GetSourceName()) + + // This should either complete without panic or handle the closed DB gracefully + cache.StoreItem(ctx, newItem, 10*time.Second, newCk) + t.Log("StoreItem completed without panic (may have failed internally)") + }) + + t.Run("Delete after CloseAndDestroy", func(t *testing.T) { + // This should not panic + cache.Delete(ck) + t.Log("Delete completed without panic (may have failed internally)") + }) + + t.Run("Purge after CloseAndDestroy", func(t *testing.T) { + // This should not panic + stats := cache.Purge(ctx, time.Now()) + t.Logf("Purge completed without panic, purged %d items", stats.NumPurged) + }) +} + +// TestBoltStoreConcurrentCloseAndDestroy verifies that CloseAndDestroy() +// properly synchronizes with concurrent operations using the compaction lock. +func TestBoltStoreConcurrentCloseAndDestroy(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + ctx := t.Context() + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + + // Store some items + for range 10 { + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + } + + // Start some concurrent operations + var wg sync.WaitGroup + numOperations := 50 + + // Launch concurrent read/write operations + for range numOperations { + wg.Go(func() { + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + }) + } + + // Wait a bit to let operations start + time.Sleep(10 * time.Millisecond) + + // Close and destroy while operations are in flight + // The compaction lock should serialize this properly + wg.Go(func() { + err := cache.CloseAndDestroy() + if err != nil { + t.Logf("CloseAndDestroy returned error: %v", err) + } + }) + + // Wait for all operations to complete + wg.Wait() + + // Verify the file is deleted + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Error("cache file should be deleted after CloseAndDestroy") + } + + t.Log("Concurrent operations with CloseAndDestroy completed without data races") +} + +// TestIsDiskFullError tests the isDiskFullError helper function. +func TestIsDiskFullError(t *testing.T) { + // Test that non-disk-full errors are not detected. + regularErr := errors.New("some other error") + if isDiskFullError(regularErr) { + t.Error("isDiskFullError should return false for regular errors") + } + + // Test nil error. + if isDiskFullError(nil) { + t.Error("isDiskFullError should return false for nil") + } +} + +// TestBoltStoreDeleteCacheFile recreates the DB file and clears data by exercising +// deleteCacheFile(). This is the behavior relied upon in disk-full recovery paths. +func TestBoltStoreDeleteCacheFile(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + // Create a cache and store some data + ctx := t.Context() + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + + // Store an item + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + // Verify the cache file exists + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + t.Fatal("cache file should exist") + } + + // Verify item is in cache + items, err := testSearch(t.Context(), cache, ck) + if err != nil { + t.Errorf("failed to search: %v", err) + } + if len(items) != 1 { + t.Errorf("expected 1 item, got %d", len(items)) + } + + // Delete the cache file (cache is already *BoltCache) + if err := cache.deleteCacheFile(ctx); err != nil { + t.Fatalf("failed to delete cache file: %v", err) + } + + // Verify the cache file is gone + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + t.Error("cache file should be recreated") + } + + // Verify the database is closed (can't search anymore) + _, _ = testSearch(t.Context(), cache, ck) + // The search might fail or return empty, but the important thing is the file is gone + // and we can't use the cache anymore +} + +// TestBoltStoreCompactThresholdTriggeredByPurge verifies that purge-triggered +// compaction keeps the store usable afterwards. +func TestBoltStoreCompactThresholdTriggeredByPurge(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath, WithCompactThreshold(1024)) // Small threshold to trigger compaction + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { + _ = cache.CloseAndDestroy() + }() + + ctx := t.Context() + + // Store enough items to trigger compaction + // We'll store items and then delete them to accumulate deleted bytes + for range 10 { + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + } + + // Manually set deleted bytes to trigger compaction + cache.addDeletedBytes(cache.CompactThreshold) + + // Trigger purge which should trigger compaction + stats := cache.Purge(ctx, time.Now().Add(-1*time.Hour)) // Purge items from an hour ago (none should exist) + _ = stats // Use stats to avoid unused variable + + // Verify cache still works after compaction attempt + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + items, err := testSearch(t.Context(), cache, ck) + if err != nil { + t.Errorf("failed to search after compaction: %v", err) + } + if len(items) != 1 { + t.Errorf("expected 1 item after compaction, got %d", len(items)) + } +} + +// TestBoltCacheLookupDeduplicatesConcurrentMisses verifies that multiple concurrent +// Lookup calls for the same cache key result in only one caller doing the work. +func TestBoltCacheLookupDeduplicatesConcurrentMisses(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { _ = cache.CloseAndDestroy() }() + + ctx := t.Context() + + // Create a cache key for the test - use LIST method to avoid UniqueAttributeValue matching issues + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_LIST + + // Track how many goroutines actually do work (get cache miss as first caller) + var workCount int32 + var mu sync.Mutex + var wg sync.WaitGroup + + numGoroutines := 10 + results := make([]struct { + hit bool + items []*sdp.Item + err *sdp.QueryError + }, numGoroutines) + + // Start barrier to ensure all goroutines start at roughly the same time + startBarrier := make(chan struct{}) + + for i := range numGoroutines { + wg.Go(func() { + // Wait for the start signal + <-startBarrier + + // Lookup the cache - all should get miss initially + hit, ck, items, qErr, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) + defer done() + + if !hit { + // This goroutine is doing the work + mu.Lock() + workCount++ + mu.Unlock() + + // Simulate some work + time.Sleep(50 * time.Millisecond) + + // Create and store the item + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + + cache.StoreItem(ctx, item, 10*time.Second, ck) + + // Re-lookup to get the stored item for our result + hit, _, items, qErr, done = cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) + defer done() + } + + results[i] = struct { + hit bool + items []*sdp.Item + err *sdp.QueryError + }{hit, items, qErr} + }) + } + + // Release all goroutines at once + close(startBarrier) + + // Wait for all goroutines to complete + wg.Wait() + + // Verify that only one goroutine did the work + if workCount != 1 { + t.Errorf("expected exactly 1 goroutine to do work, got %d", workCount) + } + + // Verify all goroutines got results + for i, r := range results { + if !r.hit { + t.Errorf("goroutine %d: expected cache hit after dedup, got miss", i) + } + if len(r.items) != 1 { + t.Errorf("goroutine %d: expected 1 item, got %d", i, len(r.items)) + } + } +} + +// TestBoltCacheLookupDeduplicationRespectsWaiterTimeout verifies that waiter +// lookups return when their context deadline is exceeded. +func TestBoltCacheLookupDeduplicationRespectsWaiterTimeout(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { _ = cache.CloseAndDestroy() }() + + ctx := t.Context() + + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_GET + query := "timeout-test" + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + // First goroutine: does the work but takes a long time + wg.Go(func() { + <-startBarrier + + hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + if hit { + t.Error("first goroutine: expected cache miss") + return + } + + // Simulate slow work + time.Sleep(500 * time.Millisecond) + + // Store the item + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + cache.StoreItem(ctx, item, 10*time.Second, ck) + }) + + // Second goroutine: should timeout waiting + var secondHit bool + wg.Go(func() { + <-startBarrier + + // Small delay to ensure first goroutine starts first + time.Sleep(10 * time.Millisecond) + + // Use a short timeout context + shortCtx, done := context.WithTimeout(ctx, 50*time.Millisecond) + defer done() + + hit, _, _, _, done := cache.Lookup(shortCtx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + secondHit = hit + }) + + // Release all goroutines + close(startBarrier) + wg.Wait() + + // Second goroutine should have timed out and returned miss + if secondHit { + t.Error("second goroutine should have timed out and returned miss") + } +} + +// TestBoltCacheLookupDeduplicationPropagatesStoredError verifies that waiters +// receive the error stored by the first caller. +func TestBoltCacheLookupDeduplicationPropagatesStoredError(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { _ = cache.CloseAndDestroy() }() + + ctx := t.Context() + + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_GET + query := "error-test" + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + expectedError := &sdp.QueryError{ + ErrorType: sdp.QueryError_NOTFOUND, + ErrorString: "item not found", + Scope: sst.Scope, + SourceName: sst.SourceName, + ItemType: sst.Type, + } + + // Track results from waiters + var waiterErrors []*sdp.QueryError + var waiterMu sync.Mutex + + numWaiters := 5 + + // First goroutine: does the work and stores an error + wg.Go(func() { + <-startBarrier + + hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + if hit { + t.Error("first goroutine: expected cache miss") + return + } + + // Simulate work that results in an error + time.Sleep(50 * time.Millisecond) + + // Store the error + cache.StoreUnavailableItem(ctx, expectedError, 10*time.Second, ck) + }) + + // Waiter goroutines: should receive the error + for range numWaiters { + wg.Go(func() { + <-startBarrier + + // Small delay to ensure first goroutine starts first + time.Sleep(10 * time.Millisecond) + + hit, _, _, qErr, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + + waiterMu.Lock() + if hit && qErr != nil { + waiterErrors = append(waiterErrors, qErr) + } + waiterMu.Unlock() + }) + } + + // Release all goroutines + close(startBarrier) + wg.Wait() + + // All waiters should have received the error + if len(waiterErrors) != numWaiters { + t.Errorf("expected %d waiters to receive error, got %d", numWaiters, len(waiterErrors)) + } + + // Verify the error content + for i, qErr := range waiterErrors { + if qErr.GetErrorType() != expectedError.GetErrorType() { + t.Errorf("waiter %d: expected error type %v, got %v", i, expectedError.GetErrorType(), qErr.GetErrorType()) + } + } +} + +// TestBoltCacheLookupDeduplicationReturnsMissAfterCancel verifies that waiters +// return misses when the in-flight work is cancelled. +func TestBoltCacheLookupDeduplicationReturnsMissAfterCancel(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { _ = cache.CloseAndDestroy() }() + + ctx := t.Context() + + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_GET + query := "done-test" + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + // Track results + var waiterHits []bool + var waiterMu sync.Mutex + + numWaiters := 3 + + // First goroutine: starts work but then calls done() without storing anything + wg.Go(func() { + <-startBarrier + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + if hit { + t.Error("first goroutine: expected cache miss") + done() + return + } + + // Simulate work that fails - done the pending work + time.Sleep(50 * time.Millisecond) + done() + }) + + // Waiter goroutines + for range numWaiters { + wg.Go(func() { + <-startBarrier + + // Small delay to ensure first goroutine starts first + time.Sleep(10 * time.Millisecond) + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + + waiterMu.Lock() + waiterHits = append(waiterHits, hit) + waiterMu.Unlock() + }) + } + + // Release all goroutines + close(startBarrier) + wg.Wait() + + // When work is cancelled, waiters receive ok=false from Wait + // (because entry.cancelled is true) and return a cache miss without re-checking. + // This is the correct behavior - waiters don't hang forever and can retry. + if len(waiterHits) != numWaiters { + t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) + } +} + +// TestBoltCacheLookupDeduplicationReturnsMissWhenCompletedWithoutStore verifies +// waiter behavior when the first caller completes without storing data. +func TestBoltCacheLookupDeduplicationReturnsMissWhenCompletedWithoutStore(t *testing.T) { + tempDir := t.TempDir() + cachePath := filepath.Join(tempDir, "cache.db") + + cache, err := NewBoltCache(cachePath) + if err != nil { + t.Fatalf("failed to create BoltCache: %v", err) + } + defer func() { _ = cache.CloseAndDestroy() }() + + ctx := t.Context() + + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_LIST + query := "complete-without-store-test" + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + // Track results + var waiterHits []bool + var waiterMu sync.Mutex + + numWaiters := 3 + + // First goroutine: starts work and completes without storing anything + // This simulates a LIST query that returns 0 items + wg.Go(func() { + <-startBarrier + + hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + if hit { + t.Error("first goroutine: expected cache miss") + return + } + + // Simulate work that completes successfully but returns nothing + time.Sleep(50 * time.Millisecond) + + // Complete without storing anything - no items, no error + // This triggers the ErrCacheNotFound path in waiters' re-check + cache.pending.Complete(ck.String()) + }) + + // Waiter goroutines + for range numWaiters { + wg.Go(func() { + <-startBarrier + + // Small delay to ensure first goroutine starts first + time.Sleep(10 * time.Millisecond) + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + + waiterMu.Lock() + waiterHits = append(waiterHits, hit) + waiterMu.Unlock() + }) + } + + // Release all goroutines + close(startBarrier) + wg.Wait() + + // When Complete is called without storing anything: + // 1. Waiters' Wait returns ok=true (not cancelled) + // 2. Waiters re-check the cache and get ErrCacheNotFound + // 3. Waiters return hit=false (cache miss) + if len(waiterHits) != numWaiters { + t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) + } + + // All waiters should get a cache miss since nothing was stored + for i, hit := range waiterHits { + if hit { + t.Errorf("waiter %d: expected cache miss (hit=false), got hit=true", i) + } + } +} diff --git a/go/sdpcache/cache.go b/go/sdpcache/cache.go index f84bb423..47b4ce06 100644 --- a/go/sdpcache/cache.go +++ b/go/sdpcache/cache.go @@ -6,15 +6,9 @@ import ( "errors" "fmt" "strings" - "sync" "time" - "github.com/google/btree" "github.com/overmindtech/cli/go/sdp-go" - log "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "google.golang.org/protobuf/proto" ) // noopDone is a reusable no-op done function returned when no cleanup is needed @@ -177,51 +171,92 @@ type CachedResult struct { // SSTHash Represents the hash of `SourceName`, `Scope` and `Type` type SSTHash string -// Cache provides operations for caching SDP items and errors +// Cache provides operations for caching SDP query results (items and errors). +// +// # Lookup state matrix +// +// Lookup returns (hit bool, ck CacheKey, items []*sdp.Item, qErr *sdp.QueryError, done func()). +// The return values follow one of three states: +// +// - Miss: hit=false, items=nil, qErr=nil — no cached data +// - Item hit: hit=true, len(items)>0, qErr=nil — cached items found +// - Error hit: hit=true, items=nil, qErr!=nil — cached error found +// +// # done() contract +// +// On a cache miss the returned done function MUST be called after storing +// results (or deciding to store nothing). It releases the pending-work slot +// so that waiting goroutines can proceed. The done function is idempotent +// (safe to call multiple times). On a cache hit or for goroutines that were +// waiting, done is a no-op. +// +// # ignoreCache +// +// When ignoreCache=true, Lookup always returns a miss without checking the +// cache or registering pending work. The returned done is a no-op. +// +// # GET cardinality +// +// If a GET lookup finds more than one cached item for the same key, the +// cache treats the data as inconsistent, purges the key, and returns a miss. +// +// # Item ordering +// +// The order of items returned from Lookup or any fan-out search is +// implementation-defined and must not be relied upon by callers. +// +// # Error precedence +// +// If both items and an error are cached under the same CacheKey, the error +// takes precedence: Lookup returns an error hit with nil items. +// +// # TTL handling +// +// There is no minimum TTL floor. A zero or negative duration stores the +// entry with an expiry at (or before) the current time. The entry will +// not survive a Purge(ctx, time.Now()) call and will be skipped by +// subsequent searches once the clock advances past the stored expiry. +// +// # Copy semantics +// +// Stored items are copied; mutating an item after StoreItem will not alter +// the cached copy. type Cache interface { // Lookup performs a cache lookup for the given query parameters. - // Returns: (cache hit, cache key, items, query error, done function) - // - // If hit=false, you MUST call the returned done function when finished, after storing all - // items/errors. The done function releases resources and unblocks waiting goroutines. - // You should defer the done function immediately after calling Lookup to ensure it's called. - // - // Example usage for cache miss: - // hit, ck, _, _, done := cache.Lookup(...) - // defer done() // MUST be called when finished - // if !hit { - // // Store all items first - // for _, item := range items { - // cache.StoreItem(ctx, item, duration, ck) - // } - // // done() is called via defer, releasing waiters - // } - // - // The done function is safe to call multiple times (idempotent). - // For cache hits or waiting goroutines, the done function is a no-op. + // See the Cache-level doc for the state matrix, done() obligations, + // ignoreCache semantics, and GET cardinality rules. Lookup(ctx context.Context, srcName string, method sdp.QueryMethod, scope string, typ string, query string, ignoreCache bool) (bool, CacheKey, []*sdp.Item, *sdp.QueryError, func()) - // StoreItem stores an item in the cache with the specified duration. + // StoreItem stores an item in the cache with the specified TTL. + // The item is deep-copied before storage; the caller retains ownership + // of the original. Storing under the same CacheKey overwrites any + // previous entry with matching IndexValues. StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) - // StoreUnavailableItem stores an error in the cache with the specified duration. + // StoreUnavailableItem stores an error in the cache with the specified TTL. + // A subsequent Lookup for the same key returns an error hit. StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, ck CacheKey) - // Delete removes all entries matching the given cache key. + // Delete removes all entries whose IndexValues match the supplied + // CacheKey. Because CacheKey fields are optional, omitting Method or + // UniqueAttributeValue acts as a wildcard across those dimensions. Delete(ck CacheKey) - // Clear removes all entries from the cache. + // Clear removes every entry from the cache. Clear() - // Purge removes all expired items from the cache. - // Returns statistics about the purge operation. + // Purge removes entries that expired before the given time. + // Returns PurgeStats with the count of purged entries and the next + // expiry time (nil when the cache is empty after purging). Purge(ctx context.Context, before time.Time) PurgeStats - // GetMinWaitTime returns the minimum time between purge operations + // GetMinWaitTime returns the minimum interval between automatic purge + // cycles. Stateful implementations return a positive duration; + // NoOpCache returns 0. GetMinWaitTime() time.Duration - // StartPurger starts a background goroutine that automatically purges expired items. - // The purger will stop when the context is cancelled. + // StartPurger starts a background goroutine that periodically calls + // Purge. The goroutine exits when ctx is cancelled. StartPurger(ctx context.Context) } @@ -229,6 +264,8 @@ type Cache interface { // It can be used in tests or when caching is not desired, avoiding nil checks. type NoOpCache struct{} +var _ Cache = (*NoOpCache)(nil) + // NewNoOpCache creates a new no-op cache that implements the Cache interface // but performs no operations. Useful for testing or when caching is disabled. func NewNoOpCache() Cache { @@ -275,706 +312,9 @@ func (n *NoOpCache) GetMinWaitTime() time.Duration { func (n *NoOpCache) StartPurger(ctx context.Context) { } -type MemoryCache struct { - // Minimum amount of time to wait between cache purges - MinWaitTime time.Duration - - // The timer that is used to trigger the next purge - purgeTimer *time.Timer - - // The time that the purger will run next - nextPurge time.Time - - indexes map[SSTHash]*indexSet - - // This index is used to track item expiries, since items can have different - // expiry durations we need to use a btree here rather than just appending - // to a slice or something. The purge process uses this to determine what - // needs deleting, then calls into each specific index to delete as required - expiryIndex *btree.BTreeG[*CachedResult] - - // Mutex for reading caches - indexMutex sync.RWMutex - - // Ensures that purge stats like `purgeTimer` and `nextPurge` aren't being - // modified concurrently - purgeMutex sync.Mutex - - // Tracks in-flight lookups to prevent duplicate work when multiple - // goroutines request the same cache key simultaneously - pending *pendingWork -} - -// NewMemoryCache creates a new in-memory cache implementation -func NewMemoryCache() *MemoryCache { - return &MemoryCache{ - indexes: make(map[SSTHash]*indexSet), - expiryIndex: newExpiryIndex(), - pending: newPendingWork(), - } -} - // NewCache creates a new cache. This function returns a Cache interface backed // by a ShardedCache (N independent BoltDB files) for write concurrency. // The passed context will be used to start the purger. func NewCache(ctx context.Context) Cache { return newShardedCacheForProduction(ctx) } - -func newExpiryIndex() *btree.BTreeG[*CachedResult] { - return btree.NewG(2, func(a, b *CachedResult) bool { - return a.Expiry.Before(b.Expiry) - }) -} - -type indexSet struct { - uniqueAttributeValueIndex *btree.BTreeG[*CachedResult] - methodIndex *btree.BTreeG[*CachedResult] - queryIndex *btree.BTreeG[*CachedResult] -} - -func newIndexSet() *indexSet { - return &indexSet{ - uniqueAttributeValueIndex: btree.NewG(2, func(a, b *CachedResult) bool { - return sortString(a.IndexValues.UniqueAttributeValue, a.Item) < sortString(b.IndexValues.UniqueAttributeValue, b.Item) - }), - methodIndex: btree.NewG(2, func(a, b *CachedResult) bool { - return sortString(a.IndexValues.Method.String(), a.Item) < sortString(b.IndexValues.Method.String(), b.Item) - }), - queryIndex: btree.NewG(2, func(a, b *CachedResult) bool { - return sortString(a.IndexValues.Query, a.Item) < sortString(b.IndexValues.Query, b.Item) - }), - } -} - -// createDoneFunc returns a done function that calls pending.Complete for the given cache key. -// The done function releases resources and unblocks waiting goroutines. -// The done function is safe to call multiple times (idempotent via sync.Once). -func (c *MemoryCache) createDoneFunc(ck CacheKey) func() { - if c == nil || c.pending == nil { - return noopDone - } - key := ck.String() - var once sync.Once - return func() { - once.Do(func() { - c.pending.Complete(key) - }) - } -} - -// Lookup returns true/false whether or not the cache has a result for the given -// query. If there are results, they will be returned as slice of `sdp.Item`s or -// an `*sdp.QueryError`. -// The CacheKey is always returned, even if the lookup otherwise fails or errors -func (c *MemoryCache) Lookup(ctx context.Context, srcName string, method sdp.QueryMethod, scope string, typ string, query string, ignoreCache bool) (bool, CacheKey, []*sdp.Item, *sdp.QueryError, func()) { - span := trace.SpanFromContext(ctx) - ck := CacheKeyFromParts(srcName, method, scope, typ, query) - - if c == nil { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache not initialised"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, &sdp.QueryError{ - ErrorType: sdp.QueryError_OTHER, - ErrorString: "cache has not been initialised", - Scope: scope, - SourceName: srcName, - ItemType: typ, - }, noopDone - } - - if ignoreCache { - span.SetAttributes( - attribute.String("ovm.cache.result", "ignore cache"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - items, err := c.search(ctx, ck) - if err != nil { - var qErr *sdp.QueryError - if errors.Is(err, ErrCacheNotFound) { - // Cache miss - check if another goroutine is already fetching this data - shouldWork, entry := c.pending.StartWork(ck.String()) - if shouldWork { - // We're the first caller, return miss so caller does the work - span.SetAttributes( - attribute.String("ovm.cache.result", "cache miss"), - attribute.Bool("ovm.cache.hit", false), - attribute.Bool("ovm.cache.workPending", false), - ) - return false, ck, nil, nil, c.createDoneFunc(ck) - } - - // Another goroutine is fetching this data, wait for it to complete - ok := c.pending.Wait(ctx, entry) - if !ok { - // Context was cancelled or work was cancelled, return miss - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work cancelled or timeout"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - // Work is complete, re-check the cache for results - items, recheckErr := c.search(ctx, ck) - if recheckErr != nil { - if errors.Is(recheckErr, ErrCacheNotFound) { - // Cache still empty after pending work completed - // This is valid - worker may have found nothing or cancelled - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work completed but cache still empty"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - var recheckQErr *sdp.QueryError - if errors.As(recheckErr, &recheckQErr) { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work: error"), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, nil, recheckQErr, noopDone - } - // Truly unexpected error - return miss - span.SetAttributes( - attribute.String("ovm.cache.result", "unexpected error on re-check"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } else if errors.As(err, &qErr) { - if qErr.GetErrorType() == sdp.QueryError_NOTFOUND { - span.SetAttributes(attribute.String("ovm.cache.result", "cache hit: item not found")) - } else { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: QueryError"), - attribute.String("ovm.cache.error", err.Error()), - ) - } - - span.SetAttributes(attribute.Bool("ovm.cache.hit", true)) - return true, ck, nil, qErr, noopDone - } else { - // If it's an unknown error, convert it to SDP and skip this source - qErr = &sdp.QueryError{ - ErrorType: sdp.QueryError_OTHER, - ErrorString: err.Error(), - Scope: scope, - SourceName: srcName, - ItemType: typ, - } - - span.SetAttributes( - attribute.String("ovm.cache.error", err.Error()), - attribute.String("ovm.cache.result", "cache hit: unknown QueryError"), - attribute.Bool("ovm.cache.hit", true), - ) - - return true, ck, nil, qErr, noopDone - } - } - - if method == sdp.QueryMethod_GET { - // If the method was Get we should validate that we have - // only pulled one thing from the cache - - if len(items) < 2 { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: 1 item"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } else { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache returned >1 value, purging and continuing"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", false), - ) - c.Delete(ck) - return false, ck, nil, nil, noopDone - } - } - - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: multiple items"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - - return true, ck, items, nil, noopDone -} - -// Search Runs a given query against the cache. If a cached error is found it -// will be returned immediately, if nothing is found a ErrCacheNotFound will -// be returned. Otherwise this will return items that match ALL of the given -// query parameters. Context is accepted for tracing but not currently used -// by MemoryCache (no I/O operations). -func (c *MemoryCache) search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { - if c == nil { - return nil, nil - } - - items := make([]*sdp.Item, 0) - - results := c.getResults(ck) - - if len(results) == 0 { - return nil, ErrCacheNotFound - } - - now := time.Now() - - // If there is an error we want to return that, so we need to range over the - // results and separate items and errors. This is computationally less - // efficient than extracting errors inside of `getResults()` but logically - // it's a lot less complicated since `Delete()` uses the same method but - // applies different logic - for _, res := range results { - // Check if the cached result has expired - if res.Expiry.Before(now) { - // Skip expired results - continue - } - - if res.Error != nil { - return nil, res.Error - } - - // Return a copy of the item so the user can do whatever they want with - // it - itemCopy := proto.Clone(res.Item).(*sdp.Item) - - items = append(items, itemCopy) - } - - // If all results were expired, return cache not found - if len(items) == 0 { - return nil, ErrCacheNotFound - } - - return items, nil -} - -// Delete Deletes anything that matches the given cache query -func (c *MemoryCache) Delete(ck CacheKey) { - if c == nil { - return - } - - c.deleteResults(c.getResults(ck)) -} - -// getResults Searches indexes for cached results, doing no other logic. If -// nothing is found an empty slice will be returned. -func (c *MemoryCache) getResults(ck CacheKey) []*CachedResult { - c.indexMutex.RLock() - defer c.indexMutex.RUnlock() - - results := make([]*CachedResult, 0) - - // Get the relevant set of indexes based on the SST Hash - sstHash := ck.SST.Hash() - indexes, exists := c.indexes[sstHash] - pivot := CachedResult{ - IndexValues: IndexValues{ - SSTHash: sstHash, - }, - } - - if !exists { - // If we don't have a set of indexes then it definitely doesn't exist - return results - } - - // Start with the most specific index and fall back to the least specific. - // Checking all matching items and returning. These is no need to check all - // indexes since they all have the same content - if ck.UniqueAttributeValue != nil { - pivot.IndexValues.UniqueAttributeValue = *ck.UniqueAttributeValue - - indexes.uniqueAttributeValueIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { - if *ck.UniqueAttributeValue == result.IndexValues.UniqueAttributeValue { - if ck.Matches(result.IndexValues) { - results = append(results, result) - } - - // Always return true so that we continue to iterate - return true - } - - return false - }) - - return results - } - - if ck.Query != nil { - pivot.IndexValues.Query = *ck.Query - - indexes.queryIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { - if *ck.Query == result.IndexValues.Query { - if ck.Matches(result.IndexValues) { - results = append(results, result) - } - - // Always return true so that we continue to iterate - return true - } - - return false - }) - - return results - } - - if ck.Method != nil { - pivot.IndexValues.Method = *ck.Method - - indexes.methodIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { - if *ck.Method == result.IndexValues.Method { - // If the methods match, check the rest - if ck.Matches(result.IndexValues) { - results = append(results, result) - } - - // Always return true so that we continue to iterate - return true - } - - return false - }) - - return results - } - - // If nothing other than SST has been set then return everything - indexes.methodIndex.Ascend(func(result *CachedResult) bool { - results = append(results, result) - - return true - }) - - return results -} - -// StoreItem Stores an item in the cache. Note that this item must be fully -// populated (including metadata) for indexing to work correctly -func (c *MemoryCache) StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) { - if item == nil || c == nil { - return - } - - itemCopy := proto.Clone(item).(*sdp.Item) - - res := CachedResult{ - Item: itemCopy, - Error: nil, - Expiry: time.Now().Add(duration), - IndexValues: IndexValues{ - UniqueAttributeValue: itemCopy.UniqueAttributeValue(), - }, - } - - if ck.Method != nil { - res.IndexValues.Method = *ck.Method - } - if ck.Query != nil { - res.IndexValues.Query = *ck.Query - } - - res.IndexValues.SSTHash = ck.SST.Hash() - - c.storeResult(ctx, res) -} - -// StoreUnavailableItem Stores an error for the given duration. -func (c *MemoryCache) StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, cacheQuery CacheKey) { - if c == nil || err == nil { - return - } - - res := CachedResult{ - Item: nil, - Error: err, - Expiry: time.Now().Add(duration), - IndexValues: cacheQuery.ToIndexValues(), - } - - c.storeResult(ctx, res) -} - -// Clear Delete all data in cache -func (c *MemoryCache) Clear() { - if c == nil { - return - } - - c.indexMutex.Lock() - defer c.indexMutex.Unlock() - - c.indexes = make(map[SSTHash]*indexSet) - c.expiryIndex = newExpiryIndex() -} - -func (c *MemoryCache) storeResult(ctx context.Context, res CachedResult) { - c.indexMutex.Lock() - defer c.indexMutex.Unlock() - - // Create the index if it doesn't exist - indexes, ok := c.indexes[res.IndexValues.SSTHash] - - if !ok { - indexes = newIndexSet() - c.indexes[res.IndexValues.SSTHash] = indexes - } - - // Add the item to the indexes and check if we're overwriting an unexpired entry - // We only need to check one index since they all reference the same CachedResult - oldResult, replaced := indexes.methodIndex.ReplaceOrInsert(&res) - indexes.queryIndex.ReplaceOrInsert(&res) - indexes.uniqueAttributeValueIndex.ReplaceOrInsert(&res) - - // Get the current span to add attributes - span := trace.SpanFromContext(ctx) - - // Check if we overwrote an entry that hasn't expired yet - // This indicates potential thundering-herd issues where multiple identical - // queries are executed concurrently instead of waiting for the first result - overwritten := false - if replaced && oldResult != nil { - now := time.Now() - if oldResult.Expiry.After(now) { - overwritten = true - timeUntilExpiry := oldResult.Expiry.Sub(now) - - // Build attributes for the overwrite event - attrs := []attribute.KeyValue{ - attribute.Bool("ovm.cache.unexpired_overwrite", true), - attribute.String("ovm.cache.time_until_expiry", timeUntilExpiry.String()), - attribute.String("ovm.cache.sst_hash", string(res.IndexValues.SSTHash)), - attribute.String("ovm.cache.query_method", res.IndexValues.Method.String()), - } - - if res.Item != nil { - attrs = append(attrs, - attribute.String("ovm.cache.item_type", res.Item.GetType()), - attribute.String("ovm.cache.item_scope", res.Item.GetScope()), - ) - } - - if res.IndexValues.Query != "" { - attrs = append(attrs, attribute.String("ovm.cache.query", res.IndexValues.Query)) - } - - if res.IndexValues.UniqueAttributeValue != "" { - attrs = append(attrs, attribute.String("ovm.cache.unique_attribute", res.IndexValues.UniqueAttributeValue)) - } - - span.SetAttributes(attrs...) - } - } - - // Always set the overwrite attribute, even if false, for consistent tracking - if !overwritten { - span.SetAttributes(attribute.Bool("ovm.cache.unexpired_overwrite", false)) - } - - // Add the item to the expiry index - c.expiryIndex.ReplaceOrInsert(&res) - - // Update the purge time if required - c.setNextPurgeIfEarlier(res.Expiry) -} - -// sortString Returns the string that the cached result should be sorted on. -// This has a prefix of the index value and suffix of the GloballyUniqueName if -// relevant -func sortString(indexValue string, item *sdp.Item) string { - if item == nil { - return indexValue - } else { - return indexValue + item.GloballyUniqueName() - } -} - -// PurgeStats Stats about the Purge -type PurgeStats struct { - // How many items were timed out of the cache - NumPurged int - // How long purging took overall - TimeTaken time.Duration - // The expiry time of the next item to expire. If there are no more items in - // the cache, this will be nil - NextExpiry *time.Time -} - -// deleteResults Deletes many cached results at once -func (c *MemoryCache) deleteResults(results []*CachedResult) { - c.indexMutex.Lock() - defer c.indexMutex.Unlock() - - for _, res := range results { - if indexSet, ok := c.indexes[res.IndexValues.SSTHash]; ok { - // For each expired item, delete it from all of the indexes that it will be in - if indexSet.methodIndex != nil { - indexSet.methodIndex.Delete(res) - } - if indexSet.queryIndex != nil { - indexSet.queryIndex.Delete(res) - } - if indexSet.uniqueAttributeValueIndex != nil { - indexSet.uniqueAttributeValueIndex.Delete(res) - } - } - - c.expiryIndex.Delete(res) - } -} - -// Purge Purges all expired items from the cache. The user must pass in the -// `before` time. All items that expired before this will be purged. Usually -// this would be just `time.Now()` however it could be overridden for testing -func (c *MemoryCache) Purge(ctx context.Context, before time.Time) PurgeStats { - if c == nil { - return PurgeStats{} - } - - // Store the current time rather than calling it a million times - start := time.Now() - - var nextExpiry *time.Time - - expired := make([]*CachedResult, 0) - - // Look through the expiry cache and work out what has expired - c.indexMutex.RLock() - c.expiryIndex.Ascend(func(res *CachedResult) bool { - if res.Expiry.Before(before) { - expired = append(expired, res) - - return true - } - - // Take note of the next expiry so we can schedule the next run - nextExpiry = &res.Expiry - - // As soon as hit this we'll stop ascending - return false - }) - c.indexMutex.RUnlock() - - c.deleteResults(expired) - - return PurgeStats{ - NumPurged: len(expired), - TimeTaken: time.Since(start), - NextExpiry: nextExpiry, - } -} - -// MinWaitDefault The default minimum wait time -const MinWaitDefault = (5 * time.Second) - -// GetMinWaitTime Returns the minimum wait time or the default if not set -func (c *MemoryCache) GetMinWaitTime() time.Duration { - if c == nil { - return 0 - } - - if c.MinWaitTime == 0 { - return MinWaitDefault - } - - return c.MinWaitTime -} - -// StartPurger Starts the purge process in the background, it will be cancelled -// when the context is cancelled. The cache will be purged initially, at which -// point the process will sleep until the next time an item expires -func (c *MemoryCache) StartPurger(ctx context.Context) { - if c == nil { - return - } - - c.purgeMutex.Lock() - if c.purgeTimer == nil { - c.purgeTimer = time.NewTimer(0) - c.purgeMutex.Unlock() - } else { - c.purgeMutex.Unlock() - log.WithContext(ctx).Info("Purger already running") - return // the purger is already running, so we don't need to start it again - } - - go func(ctx context.Context) { - for { - select { - case <-c.purgeTimer.C: - stats := c.Purge(ctx, time.Now()) - - c.setNextPurgeFromStats(stats) - case <-ctx.Done(): - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - c.purgeTimer.Stop() - c.purgeTimer = nil - return - } - } - }(ctx) -} - -// setNextPurgeFromStats Sets when the next purge should run based on the stats of the -// previous purge -func (c *MemoryCache) setNextPurgeFromStats(stats PurgeStats) { - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - if stats.NextExpiry == nil { - // If there is nothing else in the cache, wait basically - // forever - c.purgeTimer.Reset(1000 * time.Hour) - c.nextPurge = time.Now().Add(1000 * time.Hour) - } else { - if time.Until(*stats.NextExpiry) < c.GetMinWaitTime() { - c.purgeTimer.Reset(c.GetMinWaitTime()) - c.nextPurge = time.Now().Add(c.GetMinWaitTime()) - } else { - c.purgeTimer.Reset(time.Until(*stats.NextExpiry)) - c.nextPurge = *stats.NextExpiry - } - } -} - -// setNextPurgeIfEarlier Sets the next time the purger will run, if the provided -// time is sooner than the current scheduled purge time. While the purger is -// active this will be constantly updated, however if the purger is sleeping and -// new items are added this method ensures that the purger is woken up -func (c *MemoryCache) setNextPurgeIfEarlier(t time.Time) { - c.purgeMutex.Lock() - defer c.purgeMutex.Unlock() - - if t.Before(c.nextPurge) { - if c.purgeTimer == nil { - return - } - - c.purgeTimer.Stop() - c.nextPurge = t - c.purgeTimer.Reset(time.Until(t)) - } -} diff --git a/go/sdpcache/cache_benchmark_test.go b/go/sdpcache/cache_benchmark_test.go index 893fda93..4255332c 100644 --- a/go/sdpcache/cache_benchmark_test.go +++ b/go/sdpcache/cache_benchmark_test.go @@ -81,11 +81,11 @@ func NewPopulatedCacheWithMultipleBuckets(cache Cache, itemsPerBucket, numBucket for i := range itemsPerBucket { item := GenerateRandomItem() - item.Scope = sst.Scope - item.Type = sst.Type - item.Metadata.SourceName = sst.SourceName - uniqueValue := fmt.Sprintf("bucket-%d-item-%d", bucketIdx, i) - item.GetAttributes().Set("name", uniqueValue) + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + uniqueValue := fmt.Sprintf("bucket-%d-item-%d", bucketIdx, i) + item.GetAttributes().Set("name", uniqueValue) cache.StoreItem(context.Background(), item, CacheDuration, keys[bucketIdx]) } @@ -104,7 +104,6 @@ func BenchmarkCache1SingleItem(b *testing.B) { for range b.N { // Search for a single item _, err = testSearch(context.Background(), c, query) - if err != nil { b.Fatal(err) } @@ -121,7 +120,6 @@ func BenchmarkCache10SingleItem(b *testing.B) { for range b.N { // Search for a single item _, err = testSearch(context.Background(), c, query) - if err != nil { b.Fatal(err) } @@ -138,7 +136,6 @@ func BenchmarkCache100SingleItem(b *testing.B) { for range b.N { // Search for a single item _, err = testSearch(context.Background(), c, query) - if err != nil { b.Fatal(err) } @@ -155,7 +152,6 @@ func BenchmarkCache1000SingleItem(b *testing.B) { for range b.N { // Search for a single item _, err = testSearch(context.Background(), c, query) - if err != nil { b.Fatal(err) } @@ -172,7 +168,6 @@ func BenchmarkCache10_000SingleItem(b *testing.B) { for range b.N { // Search for a single item _, err = testSearch(context.Background(), c, query) - if err != nil { b.Fatal(err) } @@ -202,28 +197,28 @@ func BenchmarkListQueryLookup(b *testing.B) { b.ResetTimer() b.ReportAllocs() - // Benchmark - for range b.N { - hit, _, items, qErr, done := cache.Lookup( - b.Context(), - sst.SourceName, - sdp.QueryMethod_LIST, - sst.Scope, - sst.Type, - "", - false, // ignoreCache - ) - done() // Clean up immediately - if qErr != nil { - b.Fatalf("unexpected query error: %v", qErr) - } - if !hit { - b.Fatal("expected cache hit, got miss") - } - if len(items) != size { - b.Fatalf("expected %d items, got %d", size, len(items)) + // Benchmark + for range b.N { + hit, _, items, qErr, done := cache.Lookup( + b.Context(), + sst.SourceName, + sdp.QueryMethod_LIST, + sst.Scope, + sst.Type, + "", + false, // ignoreCache + ) + done() // Clean up immediately + if qErr != nil { + b.Fatalf("unexpected query error: %v", qErr) + } + if !hit { + b.Fatal("expected cache hit, got miss") + } + if len(items) != size { + b.Fatalf("expected %d items, got %d", size, len(items)) + } } - } }) } }) @@ -260,17 +255,17 @@ func BenchmarkListQueryConcurrent(b *testing.B) { bucketIdx := rand.Intn(numBuckets) ck := cacheKeys[bucketIdx] - // Use Lookup() to match production behavior - hit, _, items, qErr, done := cache.Lookup( - b.Context(), - ck.SST.SourceName, - sdp.QueryMethod_LIST, - ck.SST.Scope, - ck.SST.Type, - "", - false, // ignoreCache - ) - done() // Clean up immediately + // Use Lookup() to match production behavior + hit, _, items, qErr, done := cache.Lookup( + b.Context(), + ck.SST.SourceName, + sdp.QueryMethod_LIST, + ck.SST.Scope, + ck.SST.Type, + "", + false, // ignoreCache + ) + done() // Clean up immediately if qErr != nil { b.Errorf("unexpected query error: %v", qErr) return @@ -320,19 +315,19 @@ func BenchmarkListQueryConcurrentSameKey(b *testing.B) { // Benchmark: All goroutines hit the same key b.RunParallel(func(pb *testing.PB) { for pb.Next() { - // Use Lookup() to match production behavior - hit, _, items, qErr, done := cache.Lookup( - b.Context(), - sst.SourceName, - sdp.QueryMethod_LIST, - sst.Scope, - sst.Type, - "", - false, // ignoreCache - ) - done() // Clean up immediately - if qErr != nil { - b.Errorf("unexpected query error: %v", qErr) + // Use Lookup() to match production behavior + hit, _, items, qErr, done := cache.Lookup( + b.Context(), + sst.SourceName, + sdp.QueryMethod_LIST, + sst.Scope, + sst.Type, + "", + false, // ignoreCache + ) + done() // Clean up immediately + if qErr != nil { + b.Errorf("unexpected query error: %v", qErr) return } if !hit { @@ -416,7 +411,7 @@ func benchmarkPendingWorkContentionScenario( // Track timing metrics across all goroutines var ( - firstStartTime time.Time + firstStartTime time.Time firstCompleteTime time.Time lastCompleteTime time.Time timingMutex sync.Mutex @@ -441,87 +436,84 @@ func benchmarkPendingWorkContentionScenario( lastCompleteTime = time.Time{} var wg sync.WaitGroup - wg.Add(concurrency) // Spawn all goroutines for range concurrency { - go func() { - defer wg.Done() - + wg.Go(func() { // Wait for start signal to ensure simultaneous execution <-startBarrier startTime := time.Now() - // Call Lookup - this is where the contention happens - hit, _, items, qErr, done := cache.Lookup( - b.Context(), - sst.SourceName, - sdp.QueryMethod_LIST, - sst.Scope, - sst.Type, - "", - false, // ignoreCache - ) - - endTime := time.Now() + // Call Lookup - this is where the contention happens + hit, _, items, qErr, done := cache.Lookup( + b.Context(), + sst.SourceName, + sdp.QueryMethod_LIST, + sst.Scope, + sst.Type, + "", + false, // ignoreCache + ) + + endTime := time.Now() + + // Check if this goroutine was the first one (the worker) + isFirst := firstGoroutine.CompareAndSwap(false, true) + + if isFirst { + // This goroutine got the cache miss and needs to do the work + if hit { + b.Errorf("First goroutine should get cache miss, got hit") + done() + return + } - // Check if this goroutine was the first one (the worker) - isFirst := firstGoroutine.CompareAndSwap(false, true) + // Record when work started + timingMutex.Lock() + firstStartTime = startTime + timingMutex.Unlock() - if isFirst { - // This goroutine got the cache miss and needs to do the work - if hit { - b.Errorf("First goroutine should get cache miss, got hit") - done() - return - } + // Simulate slow fetch operation (like aggregatedList) + time.Sleep(fetchDuration) - // Record when work started - timingMutex.Lock() - firstStartTime = startTime - timingMutex.Unlock() + // Store items in cache (simulating results from aggregatedList) + for itemIdx := range resultSize { + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + item.GetAttributes().Set("name", fmt.Sprintf("item-%d", itemIdx)) - // Simulate slow fetch operation (like aggregatedList) - time.Sleep(fetchDuration) + cache.StoreItem(b.Context(), item, CacheDuration, sharedCacheKey) + } - // Store items in cache (simulating results from aggregatedList) - for itemIdx := range resultSize { - item := GenerateRandomItem() - item.Scope = sst.Scope - item.Type = sst.Type - item.Metadata.SourceName = sst.SourceName - item.GetAttributes().Set("name", fmt.Sprintf("item-%d", itemIdx)) + // Record when work completed + timingMutex.Lock() + firstCompleteTime = time.Now() + timingMutex.Unlock() - cache.StoreItem(b.Context(), item, CacheDuration, sharedCacheKey) - } - - // Record when work completed - timingMutex.Lock() - firstCompleteTime = time.Now() - timingMutex.Unlock() - - // Call done() to complete pending work and release waiting goroutines - done() - } else { - // This goroutine should have waited in pending.Wait() and then got a cache hit - // Note: It might get partial results if it wakes up while the first goroutine - // is still storing items (released when the first goroutine calls done()) - done() // No-op for waiters, but good practice - if !hit { - b.Errorf("Waiting goroutine should get cache hit after pending work completes, got miss") - return - } - if qErr != nil { - b.Errorf("Waiting goroutine got error: %v", qErr) - return - } - if len(items) == 0 { - b.Errorf("Waiting goroutine got cache hit but no items") - return + // Call done() to complete pending work and release waiting goroutines + done() + } else { + // This goroutine should have waited in pending.Wait() and then got a cache hit + // Note: It might get partial results if it wakes up while the first goroutine + // is still storing items (released when the first goroutine calls done()) + done() // No-op for waiters, but good practice + if !hit { + b.Errorf("Waiting goroutine should get cache hit after pending work completes, got miss") + return + } + if qErr != nil { + b.Errorf("Waiting goroutine got error: %v", qErr) + return + } + if len(items) == 0 { + b.Errorf("Waiting goroutine got cache hit but no items") + return + } + // Don't check exact count - waiters may get partial results } - // Don't check exact count - waiters may get partial results - } // Track when each goroutine completes timingMutex.Lock() @@ -529,7 +521,7 @@ func benchmarkPendingWorkContentionScenario( lastCompleteTime = endTime } timingMutex.Unlock() - }() + }) } // Release all goroutines simultaneously @@ -633,10 +625,10 @@ func benchmarkConcurrentMultiKeyWritesScenario( // Track timing metrics var ( - goroutineStartTimes []time.Time - goroutineEndTimes []time.Time - timesMutex sync.Mutex - totalStoreItemCalls atomic.Int64 + goroutineStartTimes []time.Time + goroutineEndTimes []time.Time + timesMutex sync.Mutex + totalStoreItemCalls atomic.Int64 ) // Use a start barrier to ensure all goroutines begin simultaneously @@ -672,53 +664,53 @@ func benchmarkConcurrentMultiKeyWritesScenario( goroutineStartTimes = append(goroutineStartTimes, startTime) timesMutex.Unlock() - // Call Lookup with unique cache key - should be a cache miss - myCacheKey := cacheKeys[goroutineIdx] - hit, _, _, qErr, done := cache.Lookup( - b.Context(), - myCacheKey.SST.SourceName, - sdp.QueryMethod_LIST, - myCacheKey.SST.Scope, - myCacheKey.SST.Type, - "", - false, // ignoreCache - ) - - if hit { - b.Errorf("Expected cache miss for goroutine %d, got hit", goroutineIdx) - done() - return - } - if qErr != nil { - b.Errorf("Unexpected error for goroutine %d: %v", goroutineIdx, qErr) - done() - return - } + // Call Lookup with unique cache key - should be a cache miss + myCacheKey := cacheKeys[goroutineIdx] + hit, _, _, qErr, done := cache.Lookup( + b.Context(), + myCacheKey.SST.SourceName, + sdp.QueryMethod_LIST, + myCacheKey.SST.Scope, + myCacheKey.SST.Type, + "", + false, // ignoreCache + ) - // Simulate slow fetch operation (like aggregatedList API call) - time.Sleep(fetchDuration) + if hit { + b.Errorf("Expected cache miss for goroutine %d, got hit", goroutineIdx) + done() + return + } + if qErr != nil { + b.Errorf("Unexpected error for goroutine %d: %v", goroutineIdx, qErr) + done() + return + } - // Store multiple items (simulating API results) - for itemIdx := range itemsPerGoroutine { - item := GenerateRandomItem() - item.Scope = myCacheKey.SST.Scope - item.Type = myCacheKey.SST.Type - item.Metadata.SourceName = myCacheKey.SST.SourceName - item.GetAttributes().Set("name", fmt.Sprintf("goroutine-%d-item-%d", goroutineIdx, itemIdx)) + // Simulate slow fetch operation (like aggregatedList API call) + time.Sleep(fetchDuration) - cache.StoreItem(b.Context(), item, CacheDuration, myCacheKey) - totalStoreItemCalls.Add(1) - } + // Store multiple items (simulating API results) + for itemIdx := range itemsPerGoroutine { + item := GenerateRandomItem() + item.Scope = myCacheKey.SST.Scope + item.Type = myCacheKey.SST.Type + item.Metadata.SourceName = myCacheKey.SST.SourceName + item.GetAttributes().Set("name", fmt.Sprintf("goroutine-%d-item-%d", goroutineIdx, itemIdx)) - // Call done() to complete pending work - done() + cache.StoreItem(b.Context(), item, CacheDuration, myCacheKey) + totalStoreItemCalls.Add(1) + } + + // Call done() to complete pending work + done() - endTime := time.Now() + endTime := time.Now() - // Track end time - timesMutex.Lock() - goroutineEndTimes = append(goroutineEndTimes, endTime) - timesMutex.Unlock() + // Track end time + timesMutex.Lock() + goroutineEndTimes = append(goroutineEndTimes, endTime) + timesMutex.Unlock() }() } diff --git a/go/sdpcache/cache_contract_test.go b/go/sdpcache/cache_contract_test.go new file mode 100644 index 00000000..0f594d14 --- /dev/null +++ b/go/sdpcache/cache_contract_test.go @@ -0,0 +1,910 @@ +package sdpcache + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/overmindtech/cli/go/sdp-go" +) + +// ────────────────────────────────────────────────────────────────────── +// Contract tests for the Cache interface. +// +// Every test in this file exercises only the public Cache methods and +// asserts guarantees documented on the Cache interface in cache.go. +// Implementation internals (Search, pending, shardFor, …) are tested +// in the backend-specific test files. +// +// NoOpCache is intentionally excluded; its dedicated no-op semantics +// are validated in noop_cache_test.go. +// ────────────────────────────────────────────────────────────────────── + +// --- Lookup: miss / item-hit / error-hit ------------------------------------ + +func TestCacheContract_LookupMiss(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + + hit, ck, items, qErr, done := cache.Lookup( + t.Context(), "src", sdp.QueryMethod_GET, "scope", "type", "query", false, + ) + defer done() + + if hit { + t.Fatal("expected miss on empty cache") + } + if len(items) != 0 { + t.Fatalf("expected no items, got %d", len(items)) + } + if qErr != nil { + t.Fatalf("expected nil error, got %v", qErr) + } + if ck.SST.SourceName != "src" || ck.SST.Scope != "scope" || ck.SST.Type != "type" { + t.Fatalf("returned CacheKey SST mismatch: %v", ck) + } + }) + } +} + +func TestCacheContract_LookupItemHit(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + hit, _, items, qErr, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if !hit { + t.Fatal("expected item hit") + } + if qErr != nil { + t.Fatalf("expected nil error, got %v", qErr) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].GetType() != item.GetType() { + t.Errorf("type mismatch: got %q, want %q", items[0].GetType(), item.GetType()) + } + }) + } +} + +func TestCacheContract_LookupErrorHit(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + ck := CacheKey{SST: sst, Method: new(sdp.QueryMethod_GET), UniqueAttributeValue: new("q")} + + qErr := &sdp.QueryError{ + ErrorType: sdp.QueryError_NOTFOUND, + ErrorString: "not found", + Scope: sst.Scope, + SourceName: sst.SourceName, + ItemType: sst.Type, + } + cache.StoreUnavailableItem(ctx, qErr, 10*time.Second, ck) + + hit, _, items, retErr, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_GET, sst.Scope, sst.Type, "q", false) + defer done() + + if !hit { + t.Fatal("expected error hit") + } + if items != nil { + t.Fatalf("expected nil items on error hit, got %d", len(items)) + } + if retErr == nil { + t.Fatal("expected non-nil QueryError") + } + if retErr.GetErrorType() != sdp.QueryError_NOTFOUND { + t.Errorf("error type: got %v, want NOTFOUND", retErr.GetErrorType()) + } + }) + } +} + +// --- ignoreCache ----------------------------------------------------------- + +func TestCacheContract_IgnoreCache(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + hit, _, items, qErr, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + true, // ignoreCache + ) + defer done() + + if hit { + t.Fatal("expected miss with ignoreCache=true") + } + if len(items) != 0 { + t.Errorf("expected no items, got %d", len(items)) + } + if qErr != nil { + t.Errorf("expected nil error, got %v", qErr) + } + }) + } +} + +// --- done() idempotency ---------------------------------------------------- + +func TestCacheContract_DoneIdempotent(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + + _, _, _, _, done := cache.Lookup( + t.Context(), "src", sdp.QueryMethod_GET, "scope", "type", "q", false, + ) + done() + done() // must not panic + }) + } +} + +func TestCacheContract_DoneIdempotentOnHit(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + _, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + done() + done() // must not panic + }) + } +} + +// --- GET cardinality ------------------------------------------------------- + +func TestCacheContract_GETMultipleItemsPurgesAndMisses(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + listMethod := sdp.QueryMethod_LIST + ck := CacheKey{SST: sst, Method: &listMethod} + + // Store two distinct entries (different GUN via scope) that both share + // the same unique attribute value used by GET lookup. + for i := range 2 { + item := GenerateRandomItem() + item.Scope = fmt.Sprintf("%s-%d", sst.Scope, i) + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + item.GetAttributes().Set("name", "shared-uav") + cache.StoreItem(ctx, item, 10*time.Second, ck) + } + + // Precondition: both entries are retrievable. + hit, _, items, qErr, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done() + if !hit { + t.Fatal("expected LIST hit before GET cardinality purge") + } + if qErr != nil { + t.Fatalf("expected nil error for LIST precondition, got %v", qErr) + } + if len(items) != 2 { + t.Fatalf("expected 2 LIST items before purge, got %d", len(items)) + } + + hit, _, _, _, done2 := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_GET, sst.Scope, sst.Type, "shared-uav", false) + defer done2() + if hit { + t.Fatal("expected miss when GET finds >1 item (cardinality purge)") + } + + // The purge should have removed all entries that matched the GET key. + hit, _, _, _, done3 := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done3() + if hit { + t.Fatal("expected LIST miss after GET cardinality purge") + } + }) + } +} + +// --- Copy semantics -------------------------------------------------------- + +func TestCacheContract_StoreItemCopies(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + original := item.GetType() + item.Type = "mutated-after-store" + + hit, _, items, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + original, + item.UniqueAttributeValue(), + false, + ) + defer done() + + if !hit || len(items) == 0 { + t.Fatal("expected hit after StoreItem") + } + if items[0].GetType() == "mutated-after-store" { + t.Error("cached item was mutated through original pointer") + } + }) + } +} + +// --- StoreItem + Lookup round-trip for LIST & SEARCH ----------------------- + +func TestCacheContract_LISTReturnsMultipleItems(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + ck := CacheKey{SST: sst, Method: new(sdp.QueryMethod_LIST)} + + for i := range 3 { + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + item.GetAttributes().Set("name", fmt.Sprintf("item-%d", i)) + cache.StoreItem(ctx, item, 10*time.Second, ck) + } + + hit, _, items, qErr, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done() + + if qErr != nil { + t.Fatalf("unexpected error: %v", qErr) + } + if !hit { + t.Fatal("expected hit") + } + if len(items) != 3 { + t.Errorf("expected 3 items, got %d", len(items)) + } + }) + } +} + +func TestCacheContract_SEARCHIsolatesByQuery(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + + ck1 := CacheKey{SST: sst, Method: new(sdp.QueryMethod_SEARCH), Query: new("alpha")} + item1 := GenerateRandomItem() + item1.Scope = sst.Scope + item1.Type = sst.Type + item1.Metadata.SourceName = sst.SourceName + cache.StoreItem(ctx, item1, 10*time.Second, ck1) + + ck2 := CacheKey{SST: sst, Method: new(sdp.QueryMethod_SEARCH), Query: new("beta")} + item2 := GenerateRandomItem() + item2.Scope = sst.Scope + item2.Type = sst.Type + item2.Metadata.SourceName = sst.SourceName + cache.StoreItem(ctx, item2, 10*time.Second, ck2) + + // Lookup alpha + hit, _, items, _, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_SEARCH, sst.Scope, sst.Type, "alpha", false) + defer done() + if !hit || len(items) != 1 { + t.Errorf("alpha: hit=%v, items=%d", hit, len(items)) + } + + // Lookup beta + hit, _, items, _, done2 := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_SEARCH, sst.Scope, sst.Type, "beta", false) + defer done2() + if !hit || len(items) != 1 { + t.Errorf("beta: hit=%v, items=%d", hit, len(items)) + } + }) + } +} + +// --- SEARCH items retrievable via GET (cross-method hit) ------------------- + +func TestCacheContract_SEARCHItemRetrievableViaGET(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + item.Metadata.SourceQuery.Method = sdp.QueryMethod_SEARCH + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + hit, _, items, qErr, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if qErr != nil { + t.Fatalf("unexpected error: %v", qErr) + } + if !hit { + t.Fatal("expected GET hit for SEARCH-stored item") + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + }) + } +} + +// --- Delete ---------------------------------------------------------------- + +func TestCacheContract_DeleteRemovesEntry(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + cache.Delete(ck) + + hit, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if hit { + t.Fatal("expected miss after Delete") + } + }) + } +} + +func TestCacheContract_DeleteWildcard(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + + for i := range 3 { + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + item.GetAttributes().Set("name", fmt.Sprintf("wc-%d", i)) + ck := CacheKey{SST: sst, Method: new(sdp.QueryMethod_LIST)} + cache.StoreItem(ctx, item, 10*time.Second, ck) + } + + // Delete with SST-only (wildcard on method/uav) + cache.Delete(CacheKey{SST: sst}) + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done() + + if hit { + t.Fatal("expected miss after wildcard Delete") + } + }) + } +} + +func TestCacheContract_DeleteOnEmptyCacheIsIdempotent(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + cache.Delete(CacheKey{SST: SST{SourceName: "x", Scope: "y", Type: "z"}}) + }) + } +} + +// --- Clear ----------------------------------------------------------------- + +func TestCacheContract_Clear(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + cache.Clear() + + hit, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if hit { + t.Fatal("expected miss after Clear") + } + }) + } +} + +func TestCacheContract_ClearThenStoreWorks(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + cache.Clear() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 10*time.Second, ck) + + hit, _, items, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if !hit || len(items) != 1 { + t.Fatalf("expected hit with 1 item after Clear+Store, got hit=%v items=%d", hit, len(items)) + } + }) + } +} + +func TestCacheContract_ClearOnEmptyCacheIsIdempotent(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + cache.Clear() + cache.Clear() + }) + } +} + +// --- Purge ----------------------------------------------------------------- + +func TestCacheContract_PurgeRemovesExpired(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 50*time.Millisecond, ck) + + stats := cache.Purge(ctx, time.Now().Add(100*time.Millisecond)) + + if stats.NumPurged != 1 { + t.Errorf("expected 1 purged, got %d", stats.NumPurged) + } + + hit, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if hit { + t.Fatal("expected miss after purge") + } + }) + } +} + +func TestCacheContract_PurgeStatsNextExpiry(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item1 := GenerateRandomItem() + ck1 := CacheKeyFromQuery(item1.GetMetadata().GetSourceQuery(), item1.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item1, 50*time.Millisecond, ck1) + + item2 := GenerateRandomItem() + ck2 := CacheKeyFromQuery(item2.GetMetadata().GetSourceQuery(), item2.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item2, 5*time.Second, ck2) + + stats := cache.Purge(ctx, time.Now().Add(100*time.Millisecond)) + + if stats.NumPurged != 1 { + t.Errorf("expected 1 purged, got %d", stats.NumPurged) + } + if stats.NextExpiry == nil { + t.Fatal("expected non-nil NextExpiry (second item still cached)") + } + }) + } +} + +func TestCacheContract_PurgeEmptyCache(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + stats := cache.Purge(t.Context(), time.Now()) + + if stats.NumPurged != 0 { + t.Errorf("expected 0 purged on empty cache, got %d", stats.NumPurged) + } + if stats.NextExpiry != nil { + t.Errorf("expected nil NextExpiry on empty cache, got %v", stats.NextExpiry) + } + }) + } +} + +// --- GetMinWaitTime -------------------------------------------------------- + +func TestCacheContract_GetMinWaitTimePositive(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + if d := cache.GetMinWaitTime(); d <= 0 { + t.Errorf("stateful cache should return positive min wait time, got %v", d) + } + }) + } +} + +// --- StartPurger ----------------------------------------------------------- + +func TestCacheContract_StartPurgerPurgesExpired(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 50*time.Millisecond, ck) + + cache.StartPurger(ctx) + + // Wait long enough for at least one purge cycle. + time.Sleep(cache.GetMinWaitTime() + 200*time.Millisecond) + + hit, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if hit { + t.Error("expected miss after purger ran (item expired)") + } + }) + } +} + +// --- Thundering herd / deduplication (documented contract) ---------------- + +func TestCacheContract_LookupDeduplication(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + ctx := t.Context() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + + var workCount int + var mu sync.Mutex + var wg sync.WaitGroup + + numGoroutines := 10 + results := make([]bool, numGoroutines) + startBarrier := make(chan struct{}) + + for idx := range numGoroutines { + wg.Go(func() { + <-startBarrier + + hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done() + + if !hit { + mu.Lock() + workCount++ + mu.Unlock() + + time.Sleep(50 * time.Millisecond) + + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + cache.StoreItem(ctx, item, 10*time.Second, ck) + + hit, _, _, _, done2 := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) + defer done2() + results[idx] = hit + } else { + results[idx] = true + } + }) + } + + close(startBarrier) + wg.Wait() + + if workCount != 1 { + t.Fatalf("expected 1 worker, got %d", workCount) + } + for i, hit := range results { + if !hit { + t.Errorf("goroutine %d: expected hit after dedup, got miss", i) + } + } + }) + } +} + +func TestCacheContract_WaitersGetMissWhenWorkerStoresNothing(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + cache := impl.factory() + ctx := t.Context() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + numWaiters := 3 + waiterHits := make([]bool, 0, numWaiters) + var waiterMu sync.Mutex + + // Worker: gets miss, completes without storing. + wg.Go(func() { + <-startBarrier + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "no-store", false) + if hit { + t.Error("worker: expected miss") + } + time.Sleep(50 * time.Millisecond) + done() + }) + + for range numWaiters { + wg.Go(func() { + <-startBarrier + time.Sleep(10 * time.Millisecond) + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "no-store", false) + defer done() + + waiterMu.Lock() + waiterHits = append(waiterHits, hit) + waiterMu.Unlock() + }) + } + + close(startBarrier) + wg.Wait() + + for i, hit := range waiterHits { + if hit { + t.Errorf("waiter %d: expected miss when worker stored nothing", i) + } + } + }) + } +} + +// --- Error precedence over items ------------------------------------------- + +func TestCacheContract_ErrorTakesPrecedenceOverItems(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + sst := SST{SourceName: "src", Scope: "scope", Type: "type"} + ck := CacheKey{SST: sst, Method: new(sdp.QueryMethod_GET), UniqueAttributeValue: new("prec")} + + // Store an item first. + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + item.GetAttributes().Set("name", "prec") + cache.StoreItem(ctx, item, 10*time.Second, ck) + + // Then store an error under the same key. + cache.StoreUnavailableItem(ctx, &sdp.QueryError{ + ErrorType: sdp.QueryError_NOTFOUND, + ErrorString: "gone", + Scope: sst.Scope, + SourceName: sst.SourceName, + ItemType: sst.Type, + }, 10*time.Second, ck) + + hit, _, items, qErr, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_GET, sst.Scope, sst.Type, "prec", false) + defer done() + + if !hit { + t.Fatal("expected hit") + } + if qErr == nil { + t.Fatal("expected error hit (error should take precedence over items)") + } + if items != nil { + t.Errorf("expected nil items when error takes precedence, got %d", len(items)) + } + }) + } +} + +// --- Zero/negative TTL ----------------------------------------------------- + +func TestCacheContract_ZeroTTLPurgedImmediately(t *testing.T) { + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, item, 0, ck) + + // A zero-TTL item sets expiry to ~time.Now(). It may survive a + // Search in the same nanosecond (strict Before check) but must + // not survive a Purge with a future cutoff. + stats := cache.Purge(ctx, time.Now().Add(time.Second)) + if stats.NumPurged != 1 { + t.Errorf("expected 1 purged, got %d", stats.NumPurged) + } + + hit, _, _, _, done := cache.Lookup(ctx, + item.GetMetadata().GetSourceName(), + sdp.QueryMethod_GET, + item.GetScope(), + item.GetType(), + item.UniqueAttributeValue(), + false, + ) + defer done() + + if hit { + t.Error("expected miss after purging zero-TTL item") + } + }) + } +} + +// --- Multiple error types -------------------------------------------------- + +func TestCacheContract_StoreUnavailableItemTypes(t *testing.T) { + errorTypes := []sdp.QueryError_ErrorType{ + sdp.QueryError_NOTFOUND, + sdp.QueryError_NOSCOPE, + sdp.QueryError_TIMEOUT, + sdp.QueryError_OTHER, + } + + for _, impl := range cacheImplementations(t) { + t.Run(impl.name, func(t *testing.T) { + ctx := t.Context() + cache := impl.factory() + + for i, et := range errorTypes { + t.Run(et.String(), func(t *testing.T) { + sst := SST{ + SourceName: fmt.Sprintf("src-%d", i), + Scope: "scope", + Type: "type", + } + ck := CacheKey{SST: sst, Method: new(sdp.QueryMethod_GET), UniqueAttributeValue: new("q")} + + qErr := &sdp.QueryError{ + ErrorType: et, + ErrorString: fmt.Sprintf("err %s", et), + Scope: sst.Scope, + SourceName: sst.SourceName, + ItemType: sst.Type, + } + cache.StoreUnavailableItem(ctx, qErr, 10*time.Second, ck) + + hit, _, items, retErr, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_GET, sst.Scope, sst.Type, "q", false) + defer done() + + if !hit { + t.Fatal("expected hit for cached error") + } + if items != nil { + t.Errorf("expected nil items, got %d", len(items)) + } + if retErr == nil || retErr.GetErrorType() != et { + t.Errorf("error type: got %v, want %v", retErr.GetErrorType(), et) + } + }) + } + }) + } +} diff --git a/go/sdpcache/cache_stuck_test.go b/go/sdpcache/cache_stuck_test.go index b8fe1956..b046868f 100644 --- a/go/sdpcache/cache_stuck_test.go +++ b/go/sdpcache/cache_stuck_test.go @@ -1,5 +1,14 @@ package sdpcache +// ────────────────────────────────────────────────────────────────────── +// "Stuck" scenario tests for the done() / pending-work lifecycle. +// +// These tests verify that proper use of done() and StoreUnavailableItem prevents +// goroutines from blocking indefinitely. They exercise the public Cache +// API and complement the contract suite with real-world error-recovery +// patterns. +// ────────────────────────────────────────────────────────────────────── + import ( "context" "sync" diff --git a/go/sdpcache/cache_test.go b/go/sdpcache/cache_test.go index 073816bb..8f02695d 100644 --- a/go/sdpcache/cache_test.go +++ b/go/sdpcache/cache_test.go @@ -1,10 +1,18 @@ package sdpcache +// ────────────────────────────────────────────────────────────────────── +// Implementation-detail tests for stateful cache backends. +// +// These tests exercise the internal Search method and storage internals +// that are NOT part of the public Cache contract. Contract-level tests +// (using only the public Cache interface) live in cache_contract_test.go. +// NoOpCache-specific tests live in noop_cache_test.go. +// ────────────────────────────────────────────────────────────────────── + import ( "context" "errors" "fmt" - "os" "path/filepath" "sync" "testing" @@ -13,23 +21,23 @@ import ( "github.com/overmindtech/cli/go/sdp-go" ) -// testSearch is a helper function that calls the internal search method -// on either MemoryCache, BoltCache, or ShardedCache implementations for testing purposes +type searchableCache interface { + Search(context.Context, CacheKey) ([]*sdp.Item, error) +} + +// testSearch is a helper function that calls the lower-level Search method on +// cache implementations for testing purposes. func testSearch(ctx context.Context, cache Cache, ck CacheKey) ([]*sdp.Item, error) { - switch c := cache.(type) { - case *MemoryCache: - return c.search(ctx, ck) - case *BoltCache: - return c.search(ctx, ck) - case *ShardedCache: - return c.searchByKey(ctx, ck) - default: - return nil, fmt.Errorf("unsupported cache type for search: %T", cache) + if c, ok := cache.(searchableCache); ok { + return c.Search(ctx, ck) } + + return nil, fmt.Errorf("unsupported cache type for search: %T", cache) } -// cacheImplementations returns the list of cache implementations to test -// Accepts testing.TB so it can be used by both tests and benchmarks +// cacheImplementations returns stateful cache implementations used by shared +// behavior tests. NoOpCache is intentionally excluded and tested separately. +// Accepts testing.TB so it can be used by both tests and benchmarks. func cacheImplementations(tb testing.TB) []struct { name string factory func() Cache @@ -349,236 +357,6 @@ func TestDelete(t *testing.T) { } } -func TestPointers(t *testing.T) { - implementations := cacheImplementations(t) - - for _, impl := range implementations { - t.Run(impl.name, func(t *testing.T) { - cache := impl.factory() - - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(t.Context(), item, time.Minute, ck) - - item.Type = "bad" - - items, err := testSearch(t.Context(), cache, ck) - if err != nil { - t.Error(err) - } - - if len(items) != 1 { - t.Errorf("expected 1 item, got %v", len(items)) - } - - if items[0].GetType() == "bad" { - t.Error("item was changed in cache") - } - }) - } -} - -func TestCacheClear(t *testing.T) { - implementations := cacheImplementations(t) - - for _, impl := range implementations { - t.Run(impl.name, func(t *testing.T) { - ctx := t.Context() - cache := impl.factory() - - cache.Clear() - - // Populate the cache - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(ctx, item, 500*time.Millisecond, ck) - - // Start purging just to make sure it doesn't break - ctx, done := context.WithCancel(ctx) - defer done() - cache.StartPurger(ctx) - - // Make sure the cache is populated - _, err := testSearch(t.Context(), cache, ck) - if err != nil { - t.Error(err) - } - - // Clear the cache - cache.Clear() - - // Make sure the cache is empty - _, err = testSearch(t.Context(), cache, ck) - - if err == nil { - t.Error("expected error, cache not cleared") - } - - // Make sure we can populate it again - cache.StoreItem(ctx, item, 500*time.Millisecond, ck) - _, err = testSearch(t.Context(), cache, ck) - if err != nil { - t.Error(err) - } - }) - } -} - -func TestLookup(t *testing.T) { - implementations := cacheImplementations(t) - - for _, impl := range implementations { - t.Run(impl.name, func(t *testing.T) { - ctx := t.Context() - cache := impl.factory() - - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(ctx, item, 10*time.Second, ck) - - // ignore the cache - cacheHit, _, cachedItems, err, done := cache.Lookup(ctx, item.GetMetadata().GetSourceName(), sdp.QueryMethod_GET, item.GetScope(), item.GetType(), item.UniqueAttributeValue(), true) - defer done() - if err != nil { - t.Fatal(err) - } - if cacheHit { - t.Error("expected cache miss, got hit") - } - if cachedItems != nil { - t.Errorf("expected nil items, got %v", cachedItems) - } - - // Lookup the item - cacheHit, _, cachedItems, err, done = cache.Lookup(ctx, item.GetMetadata().GetSourceName(), sdp.QueryMethod_GET, item.GetScope(), item.GetType(), item.UniqueAttributeValue(), false) - defer done() - - if err != nil { - t.Fatal(err) - } - if !cacheHit { - t.Fatal("expected cache hit, got miss") - } - if len(cachedItems) != 1 { - t.Fatalf("expected 1 item, got %v", len(cachedItems)) - } - - if cachedItems[0].GetType() != item.GetType() { - t.Errorf("expected type %v, got %v", item.GetType(), cachedItems[0].GetType()) - } - - if cachedItems[0].Health == nil { - t.Error("expected health to be set") - } - - if len(cachedItems[0].GetTags()) != len(item.GetTags()) { - t.Error("expected tags to be set") - } - - stats := cache.Purge(ctx, time.Now().Add(1*time.Hour)) - if stats.NumPurged != 1 { - t.Errorf("expected 1 item purged, got %v", stats.NumPurged) - } - - // Lookup the item - cacheHit, _, cachedItems, err, done = cache.Lookup(ctx, item.GetMetadata().GetSourceName(), sdp.QueryMethod_GET, item.GetScope(), item.GetType(), item.UniqueAttributeValue(), false) - defer done() - - if err != nil { - t.Fatal(err) - } - if cacheHit { - t.Fatal("expected cache miss, got hit") - } - if len(cachedItems) != 0 { - t.Fatalf("expected 0 item, got %v", len(cachedItems)) - } - }) - } -} - -func TestStoreSearch(t *testing.T) { - implementations := cacheImplementations(t) - - for _, impl := range implementations { - t.Run(impl.name, func(t *testing.T) { - ctx := t.Context() - cache := impl.factory() - - item := GenerateRandomItem() - item.Metadata.SourceQuery.Method = sdp.QueryMethod_SEARCH - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(ctx, item, 10*time.Second, ck) - - // Lookup the item as GET request - cacheHit, _, cachedItems, err, done := cache.Lookup(ctx, item.GetMetadata().GetSourceName(), sdp.QueryMethod_GET, item.GetScope(), item.GetType(), item.UniqueAttributeValue(), false) - defer done() - if err != nil { - t.Fatal(err) - } - - if !cacheHit { - t.Fatal("expected cache hit, got miss") - } - - if len(cachedItems) != 1 { - t.Fatalf("expected 1 item, got %v", len(cachedItems)) - } - - if cachedItems[0].GetType() != item.GetType() { - t.Errorf("expected type %v, got %v", item.GetType(), cachedItems[0].GetType()) - } - }) - } -} - -func TestLookupWithListMethod(t *testing.T) { - implementations := cacheImplementations(t) - - for _, impl := range implementations { - t.Run(impl.name, func(t *testing.T) { - ctx := t.Context() - cache := impl.factory() - - // Store multiple items with same SST - sst := SST{SourceName: "test", Scope: "scope", Type: "type"} - listMethod := sdp.QueryMethod_LIST - - item1 := GenerateRandomItem() - item1.Scope = sst.Scope - item1.Type = sst.Type - item1.Metadata.SourceName = sst.SourceName - ck1 := CacheKey{SST: sst, Method: &listMethod} - cache.StoreItem(ctx, item1, 10*time.Second, ck1) - - item2 := GenerateRandomItem() - item2.Scope = sst.Scope - item2.Type = sst.Type - item2.Metadata.SourceName = sst.SourceName - ck2 := CacheKey{SST: sst, Method: &listMethod} - cache.StoreItem(ctx, item2, 10*time.Second, ck2) - - // Lookup with LIST should return both items - cacheHit, _, cachedItems, err, done := cache.Lookup(ctx, sst.SourceName, sdp.QueryMethod_LIST, sst.Scope, sst.Type, "", false) - defer done() - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !cacheHit { - t.Fatal("expected cache hit, got miss") - } - if len(cachedItems) != 2 { - t.Errorf("expected 2 items, got %v", len(cachedItems)) - } - }) - } -} - func TestSearchWithListMethod(t *testing.T) { implementations := cacheImplementations(t) @@ -917,293 +695,6 @@ func TestMultipleItemsSameSST(t *testing.T) { } } -// Implementation-specific tests for MemoryCache - -// TestMemoryCacheStartPurge tests the memory cache implementation's purger -func TestMemoryCacheStartPurge(t *testing.T) { - ctx := t.Context() - cache := NewMemoryCache() - cache.MinWaitTime = 100 * time.Millisecond - - cachedItems := []struct { - Item *sdp.Item - Expiry time.Time - }{ - { - Item: GenerateRandomItem(), - Expiry: time.Now().Add(0), - }, - { - Item: GenerateRandomItem(), - Expiry: time.Now().Add(100 * time.Millisecond), - }, - } - - for _, i := range cachedItems { - ck := CacheKeyFromQuery(i.Item.GetMetadata().GetSourceQuery(), i.Item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, i.Item, time.Until(i.Expiry), ck) - } - - ctx, done := context.WithCancel(ctx) - defer done() - - cache.StartPurger(ctx) - - // Wait for everything to be purged - time.Sleep(200 * time.Millisecond) - - // At this point everything should be been cleaned, and the purger should be - // sleeping forever - items, err := testSearch(t.Context(), cache, CacheKeyFromQuery( - cachedItems[1].Item.GetMetadata().GetSourceQuery(), - cachedItems[1].Item.GetMetadata().GetSourceName(), - )) - - if !errors.Is(err, ErrCacheNotFound) { - t.Errorf("unexpected error: %v", err) - t.Errorf("unexpected items: %v", len(items)) - } - - cache.purgeMutex.Lock() - if cache.nextPurge.Before(time.Now().Add(time.Hour)) { - // If the next purge is within the next hour that's an error, it should - // be really, really for in the future - t.Errorf("Expected next purge to be in 1000 years, got %v", cache.nextPurge.String()) - } - cache.purgeMutex.Unlock() - - // Adding a new item should kick off the purging again - for _, i := range cachedItems { - ck := CacheKeyFromQuery(i.Item.GetMetadata().GetSourceQuery(), i.Item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, i.Item, 100*time.Millisecond, ck) - } - - time.Sleep(200 * time.Millisecond) - - // It should be empty again - items, err = testSearch(t.Context(), cache, CacheKeyFromQuery( - cachedItems[1].Item.GetMetadata().GetSourceQuery(), - cachedItems[1].Item.GetMetadata().GetSourceName(), - )) - - if !errors.Is(err, ErrCacheNotFound) { - t.Errorf("unexpected error: %v", err) - t.Errorf("unexpected items: %v: %v", len(items), items) - } -} - -// TestMemoryCacheStopPurge tests the memory cache implementation's purger stop functionality -func TestMemoryCacheStopPurge(t *testing.T) { - cache := NewMemoryCache() - cache.MinWaitTime = 1 * time.Millisecond - - ctx, done := context.WithCancel(t.Context()) - - cache.StartPurger(ctx) - - // Stop the purger - done() - - // Insert an item - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(ctx, item, 1*time.Second, ck) - sst := SST{ - SourceName: item.GetMetadata().GetSourceName(), - Scope: item.GetScope(), - Type: item.GetType(), - } - - // Make sure it's not purged - time.Sleep(100 * time.Millisecond) - items, err := testSearch(t.Context(), cache, CacheKey{ - SST: sst, - }) - if err != nil { - t.Error(err) - } - - if len(items) != 1 { - t.Errorf("Expected 1 item, got %v", len(items)) - } -} - -// TestMemoryCacheConcurrent tests the memory cache implementation for data races. -// This test is designed to be run with -race to ensure that there aren't any -// data races -func TestMemoryCacheConcurrent(t *testing.T) { - cache := NewMemoryCache() - // Run the purger super fast to generate a worst-case scenario - cache.MinWaitTime = 1 * time.Millisecond - - ctx, done := context.WithCancel(t.Context()) - defer done() - cache.StartPurger(ctx) - var wg sync.WaitGroup - - numParallel := 1_000 - - for range numParallel { - wg.Go(func() { - // Store the item - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - - cache.StoreItem(ctx, item, 100*time.Millisecond, ck) - - // Create a goroutine to also delete in parallel - wg.Go(func() { - cache.Delete(ck) - }) - }) - } - - wg.Wait() -} - -// TestMemoryCacheLookupDeduplication tests that multiple concurrent Lookup calls -// for the same cache key in MemoryCache result in only one caller doing work. -func TestMemoryCacheLookupDeduplication(t *testing.T) { - cache := NewMemoryCache() - ctx := t.Context() - - // Create a cache key for the test - use LIST method to avoid UniqueAttributeValue matching issues - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_LIST - - // Track how many goroutines actually do work - var workCount int32 - var mu sync.Mutex - var wg sync.WaitGroup - - numGoroutines := 10 - results := make([]struct { - hit bool - items []*sdp.Item - }, numGoroutines) - - startBarrier := make(chan struct{}) - - for i := range numGoroutines { - wg.Add(1) - go func(idx int) { - defer wg.Done() - <-startBarrier - - hit, ck, items, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) - defer done() - - if !hit { - mu.Lock() - workCount++ - mu.Unlock() - - time.Sleep(50 * time.Millisecond) - - item := GenerateRandomItem() - item.Scope = sst.Scope - item.Type = sst.Type - item.Metadata.SourceName = sst.SourceName - - cache.StoreItem(ctx, item, 10*time.Second, ck) - hit, _, items, _, done = cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) - defer done() - } - - results[idx] = struct { - hit bool - items []*sdp.Item - }{hit, items} - }(i) - } - - close(startBarrier) - wg.Wait() - - if workCount != 1 { - t.Errorf("expected exactly 1 goroutine to do work, got %d", workCount) - } - - for i, r := range results { - if !r.hit { - t.Errorf("goroutine %d: expected cache hit after dedup, got miss", i) - } - if len(r.items) != 1 { - t.Errorf("goroutine %d: expected 1 item, got %d", i, len(r.items)) - } - } -} - -// TestMemoryCacheLookupDeduplicationCompleteWithoutStore tests the scenario where -// Complete is called but nothing was stored in the cache. This tests the explicit -// ErrCacheNotFound check in the re-check logic. -func TestMemoryCacheLookupDeduplicationCompleteWithoutStore(t *testing.T) { - cache := NewMemoryCache() - ctx := t.Context() - - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_LIST - query := "complete-without-store-test" - - var wg sync.WaitGroup - startBarrier := make(chan struct{}) - - // Track results - var waiterHits []bool - var waiterMu sync.Mutex - - numWaiters := 3 - - // First goroutine: starts work and completes without storing anything - wg.Go(func() { - <-startBarrier - - hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - if hit { - t.Error("first goroutine: expected cache miss") - return - } - - // Simulate work that completes successfully but returns nothing - time.Sleep(50 * time.Millisecond) - - // Complete without storing anything - triggers ErrCacheNotFound on re-check - cache.pending.Complete(ck.String()) - }) - - // Waiter goroutines - for range numWaiters { - wg.Go(func() { - <-startBarrier - - time.Sleep(10 * time.Millisecond) - - hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - - waiterMu.Lock() - waiterHits = append(waiterHits, hit) - waiterMu.Unlock() - }) - } - - close(startBarrier) - wg.Wait() - - if len(waiterHits) != numWaiters { - t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) - } - - // All waiters should get a cache miss since nothing was stored - for i, hit := range waiterHits { - if hit { - t.Errorf("waiter %d: expected cache miss (hit=false), got hit=true", i) - } - } -} - func TestToIndexValues(t *testing.T) { ck := CacheKey{ SST: SST{ @@ -1332,722 +823,6 @@ func TestUnexpiredOverwriteLogging(t *testing.T) { }) } -// TestBoltCacheCloseAndDestroy verifies that CloseAndDestroy() correctly -// closes the database and deletes the cache file. -func TestBoltCacheCloseAndDestroy(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - // Create a cache and store some data - ctx := t.Context() - cache1, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - - // Store an item - item1 := GenerateRandomItem() - ck1 := CacheKeyFromQuery(item1.GetMetadata().GetSourceQuery(), item1.GetMetadata().GetSourceName()) - cache1.StoreItem(ctx, item1, 10*time.Second, ck1) - - // Store another item with a short TTL (will expire) - item2 := GenerateRandomItem() - ck2 := CacheKeyFromQuery(item2.GetMetadata().GetSourceQuery(), item2.GetMetadata().GetSourceName()) - cache1.StoreItem(ctx, item2, 100*time.Millisecond, ck2) - - // Verify both items are in the cache - items, err := testSearch(t.Context(), cache1, ck1) - if err != nil { - t.Errorf("failed to search for item1: %v", err) - } - if len(items) != 1 { - t.Errorf("expected 1 item for ck1, got %d", len(items)) - } - - // Verify the cache file exists - if _, err := os.Stat(cachePath); os.IsNotExist(err) { - t.Fatal("cache file should exist before CloseAndDestroy") - } - - // Close and destroy the cache - if err := cache1.CloseAndDestroy(); err != nil { - t.Fatalf("failed to close and destroy cache1: %v", err) - } - - // Verify the cache file is deleted - if _, err := os.Stat(cachePath); !os.IsNotExist(err) { - t.Error("cache file should be deleted after CloseAndDestroy") - } - - // Create a new cache at the same path - should create a fresh, empty cache - cache2, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create new BoltCache: %v", err) - } - defer func() { - _ = cache2.CloseAndDestroy() - }() - - // Verify the old item is NOT accessible (cache was destroyed) - items, err = testSearch(ctx, cache2, ck1) - if !errors.Is(err, ErrCacheNotFound) { - t.Errorf("expected cache miss for item1 in new cache, got: err=%v, items=%d", err, len(items)) - } - - // Verify we can store new items in the fresh cache - item3 := GenerateRandomItem() - ck3 := CacheKeyFromQuery(item3.GetMetadata().GetSourceQuery(), item3.GetMetadata().GetSourceName()) - cache2.StoreItem(ctx, item3, 10*time.Second, ck3) - - items, err = testSearch(ctx, cache2, ck3) - if err != nil { - t.Errorf("failed to search for newly stored item3: %v", err) - } - if len(items) != 1 { - t.Errorf("expected 1 item for ck3, got %d", len(items)) - } -} - -// TestBoltCacheOperationsAfterCloseAndDestroy verifies that operations after -// CloseAndDestroy() return proper errors instead of panicking. -func TestBoltCacheOperationsAfterCloseAndDestroy(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - ctx := t.Context() - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - - // Store an item before closing - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - - // Close and destroy the cache - if err := cache.CloseAndDestroy(); err != nil { - t.Fatalf("failed to close and destroy cache: %v", err) - } - - // Now try various operations after the cache is closed and destroyed - // These should return errors, not panic - - t.Run("Search after CloseAndDestroy", func(t *testing.T) { - // This should error because the database is closed - _, err := testSearch(ctx, cache, ck) - if err == nil { - t.Error("expected error when searching after CloseAndDestroy, got nil") - } - t.Logf("Search returned expected error: %v", err) - }) - - t.Run("StoreItem after CloseAndDestroy", func(t *testing.T) { - // This should not panic - it might silently fail or error - // The key is that it doesn't panic - newItem := GenerateRandomItem() - newCk := CacheKeyFromQuery(newItem.GetMetadata().GetSourceQuery(), newItem.GetMetadata().GetSourceName()) - - // This should either complete without panic or handle the closed DB gracefully - cache.StoreItem(ctx, newItem, 10*time.Second, newCk) - t.Log("StoreItem completed without panic (may have failed internally)") - }) - - t.Run("Delete after CloseAndDestroy", func(t *testing.T) { - // This should not panic - cache.Delete(ck) - t.Log("Delete completed without panic (may have failed internally)") - }) - - t.Run("Purge after CloseAndDestroy", func(t *testing.T) { - // This should not panic - stats := cache.Purge(ctx, time.Now()) - t.Logf("Purge completed without panic, purged %d items", stats.NumPurged) - }) -} - -// TestBoltCacheConcurrentCloseAndDestroy verifies that CloseAndDestroy() -// properly synchronizes with concurrent operations using the compaction lock. -func TestBoltCacheConcurrentCloseAndDestroy(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - ctx := t.Context() - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - - // Store some items - for range 10 { - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - } - - // Start some concurrent operations - var wg sync.WaitGroup - numOperations := 50 - - // Launch concurrent read/write operations - for range numOperations { - wg.Go(func() { - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - }) - } - - // Wait a bit to let operations start - time.Sleep(10 * time.Millisecond) - - // Close and destroy while operations are in flight - // The compaction lock should serialize this properly - wg.Go(func() { - err := cache.CloseAndDestroy() - if err != nil { - t.Logf("CloseAndDestroy returned error: %v", err) - } - }) - - // Wait for all operations to complete - wg.Wait() - - // Verify the file is deleted - if _, err := os.Stat(cachePath); !os.IsNotExist(err) { - t.Error("cache file should be deleted after CloseAndDestroy") - } - - t.Log("Concurrent operations with CloseAndDestroy completed without data races") -} - -// TestBoltCacheDiskFullErrorDetection tests the isDiskFullError helper function -func TestBoltCacheDiskFullErrorDetection(t *testing.T) { - // This test verifies that isDiskFullError correctly identifies disk full errors - // We can't easily simulate actual disk full in tests, but we can test the detection logic - - // Note: We can't directly test syscall.ENOSPC without actually filling the disk, - // but we can verify the function exists and works with the error types it's designed for. - // In a real scenario, BoltDB would return syscall.ENOSPC when the disk is full. - - // Test that non-disk-full errors are not detected - regularErr := errors.New("some other error") - if isDiskFullError(regularErr) { - t.Error("isDiskFullError should return false for regular errors") - } - - // Test nil error - if isDiskFullError(nil) { - t.Error("isDiskFullError should return false for nil") - } -} - -// TestBoltCacheDeleteOnDiskFull tests that the cache is deleted when disk is full -// and cleanup doesn't help. Since we can't easily simulate disk full in unit tests, -// this test verifies the deleteCacheFile method works correctly. -func TestBoltCacheDeleteOnDiskFull(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - // Create a cache and store some data - ctx := t.Context() - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - - // Store an item - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - - // Verify the cache file exists - if _, err := os.Stat(cachePath); os.IsNotExist(err) { - t.Fatal("cache file should exist") - } - - // Verify item is in cache - items, err := testSearch(t.Context(), cache, ck) - if err != nil { - t.Errorf("failed to search: %v", err) - } - if len(items) != 1 { - t.Errorf("expected 1 item, got %d", len(items)) - } - - // Delete the cache file (cache is already *BoltCache) - if err := cache.deleteCacheFile(ctx); err != nil { - t.Fatalf("failed to delete cache file: %v", err) - } - - // Verify the cache file is gone - if _, err := os.Stat(cachePath); os.IsNotExist(err) { - t.Error("cache file should be recreated") - } - - // Verify the database is closed (can't search anymore) - _, _ = testSearch(t.Context(), cache, ck) - // The search might fail or return empty, but the important thing is the file is gone - // and we can't use the cache anymore -} - -// TestBoltCacheDiskFullDuringCompact tests error handling during compaction. -// Since we can't easily simulate disk full, this test verifies the compaction -// process works normally and that the error handling paths exist. -func TestBoltCacheDiskFullDuringCompact(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath, WithCompactThreshold(1024)) // Small threshold to trigger compaction - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { - _ = cache.CloseAndDestroy() - }() - - ctx := t.Context() - - // Store enough items to trigger compaction - // We'll store items and then delete them to accumulate deleted bytes - for range 10 { - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - } - - // Manually set deleted bytes to trigger compaction - cache.addDeletedBytes(cache.CompactThreshold) - - // Trigger purge which should trigger compaction - stats := cache.Purge(ctx, time.Now().Add(-1*time.Hour)) // Purge items from an hour ago (none should exist) - _ = stats // Use stats to avoid unused variable - - // Verify cache still works after compaction attempt - item := GenerateRandomItem() - ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) - cache.StoreItem(ctx, item, 10*time.Second, ck) - - items, err := testSearch(t.Context(), cache, ck) - if err != nil { - t.Errorf("failed to search after compaction: %v", err) - } - if len(items) != 1 { - t.Errorf("expected 1 item after compaction, got %d", len(items)) - } -} - -// TestBoltCacheLookupDeduplication tests that multiple concurrent Lookup calls -// for the same cache key result in only one caller doing the actual work. -func TestBoltCacheLookupDeduplication(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { _ = cache.CloseAndDestroy() }() - - ctx := t.Context() - - // Create a cache key for the test - use LIST method to avoid UniqueAttributeValue matching issues - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_LIST - - // Track how many goroutines actually do work (get cache miss as first caller) - var workCount int32 - var mu sync.Mutex - var wg sync.WaitGroup - - numGoroutines := 10 - results := make([]struct { - hit bool - items []*sdp.Item - err *sdp.QueryError - }, numGoroutines) - - // Start barrier to ensure all goroutines start at roughly the same time - startBarrier := make(chan struct{}) - - for i := range numGoroutines { - wg.Add(1) - go func(idx int) { - defer wg.Done() - - // Wait for the start signal - <-startBarrier - - // Lookup the cache - all should get miss initially - hit, ck, items, qErr, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) - defer done() - - if !hit { - // This goroutine is doing the work - mu.Lock() - workCount++ - mu.Unlock() - - // Simulate some work - time.Sleep(50 * time.Millisecond) - - // Create and store the item - item := GenerateRandomItem() - item.Scope = sst.Scope - item.Type = sst.Type - item.Metadata.SourceName = sst.SourceName - - cache.StoreItem(ctx, item, 10*time.Second, ck) - - // Re-lookup to get the stored item for our result - hit, _, items, qErr, done = cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) - defer done() - } - - results[idx] = struct { - hit bool - items []*sdp.Item - err *sdp.QueryError - }{hit, items, qErr} - }(i) - } - - // Release all goroutines at once - close(startBarrier) - - // Wait for all goroutines to complete - wg.Wait() - - // Verify that only one goroutine did the work - if workCount != 1 { - t.Errorf("expected exactly 1 goroutine to do work, got %d", workCount) - } - - // Verify all goroutines got results - for i, r := range results { - if !r.hit { - t.Errorf("goroutine %d: expected cache hit after dedup, got miss", i) - } - if len(r.items) != 1 { - t.Errorf("goroutine %d: expected 1 item, got %d", i, len(r.items)) - } - } -} - -// TestBoltCacheLookupDeduplicationTimeout tests that waiters properly timeout -// when the context is cancelled. -func TestBoltCacheLookupDeduplicationTimeout(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { _ = cache.CloseAndDestroy() }() - - ctx := t.Context() - - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_GET - query := "timeout-test" - - var wg sync.WaitGroup - startBarrier := make(chan struct{}) - - // First goroutine: does the work but takes a long time - wg.Go(func() { - <-startBarrier - - hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - if hit { - t.Error("first goroutine: expected cache miss") - return - } - - // Simulate slow work - time.Sleep(500 * time.Millisecond) - - // Store the item - item := GenerateRandomItem() - item.Scope = sst.Scope - item.Type = sst.Type - cache.StoreItem(ctx, item, 10*time.Second, ck) - }) - - // Second goroutine: should timeout waiting - var secondHit bool - wg.Go(func() { - <-startBarrier - - // Small delay to ensure first goroutine starts first - time.Sleep(10 * time.Millisecond) - - // Use a short timeout context - shortCtx, done := context.WithTimeout(ctx, 50*time.Millisecond) - defer done() - - hit, _, _, _, done := cache.Lookup(shortCtx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - secondHit = hit - }) - - // Release all goroutines - close(startBarrier) - wg.Wait() - - // Second goroutine should have timed out and returned miss - if secondHit { - t.Error("second goroutine should have timed out and returned miss") - } -} - -// TestBoltCacheLookupDeduplicationError tests that waiters receive the error -// when the first caller stores an error. -func TestBoltCacheLookupDeduplicationError(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { _ = cache.CloseAndDestroy() }() - - ctx := t.Context() - - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_GET - query := "error-test" - - var wg sync.WaitGroup - startBarrier := make(chan struct{}) - - expectedError := &sdp.QueryError{ - ErrorType: sdp.QueryError_NOTFOUND, - ErrorString: "item not found", - Scope: sst.Scope, - SourceName: sst.SourceName, - ItemType: sst.Type, - } - - // Track results from waiters - var waiterErrors []*sdp.QueryError - var waiterMu sync.Mutex - - numWaiters := 5 - - // First goroutine: does the work and stores an error - wg.Go(func() { - <-startBarrier - - hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - if hit { - t.Error("first goroutine: expected cache miss") - return - } - - // Simulate work that results in an error - time.Sleep(50 * time.Millisecond) - - // Store the error - cache.StoreUnavailableItem(ctx, expectedError, 10*time.Second, ck) - }) - - // Waiter goroutines: should receive the error - for range numWaiters { - wg.Go(func() { - <-startBarrier - - // Small delay to ensure first goroutine starts first - time.Sleep(10 * time.Millisecond) - - hit, _, _, qErr, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - - waiterMu.Lock() - if hit && qErr != nil { - waiterErrors = append(waiterErrors, qErr) - } - waiterMu.Unlock() - }) - } - - // Release all goroutines - close(startBarrier) - wg.Wait() - - // All waiters should have received the error - if len(waiterErrors) != numWaiters { - t.Errorf("expected %d waiters to receive error, got %d", numWaiters, len(waiterErrors)) - } - - // Verify the error content - for i, qErr := range waiterErrors { - if qErr.GetErrorType() != expectedError.GetErrorType() { - t.Errorf("waiter %d: expected error type %v, got %v", i, expectedError.GetErrorType(), qErr.GetErrorType()) - } - } -} - -// TestBoltCacheLookupDeduplicationCancel tests the Cancel() path for error recovery. -func TestBoltCacheLookupDeduplicationCancel(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { _ = cache.CloseAndDestroy() }() - - ctx := t.Context() - - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_GET - query := "done-test" - - var wg sync.WaitGroup - startBarrier := make(chan struct{}) - - // Track results - var waiterHits []bool - var waiterMu sync.Mutex - - numWaiters := 3 - - // First goroutine: starts work but then calls done() without storing anything - wg.Go(func() { - <-startBarrier - - hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - if hit { - t.Error("first goroutine: expected cache miss") - done() - return - } - - // Simulate work that fails - done the pending work - time.Sleep(50 * time.Millisecond) - done() - }) - - // Waiter goroutines - for range numWaiters { - wg.Go(func() { - <-startBarrier - - // Small delay to ensure first goroutine starts first - time.Sleep(10 * time.Millisecond) - - hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - - waiterMu.Lock() - waiterHits = append(waiterHits, hit) - waiterMu.Unlock() - }) - } - - // Release all goroutines - close(startBarrier) - wg.Wait() - - // When work is cancelled, waiters receive ok=false from Wait - // (because entry.cancelled is true) and return a cache miss without re-checking. - // This is the correct behavior - waiters don't hang forever and can retry. - if len(waiterHits) != numWaiters { - t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) - } -} - -// TestBoltCacheLookupDeduplicationCompleteWithoutStore tests the scenario where -// Complete is called but nothing was stored in the cache. This tests the explicit -// ErrCacheNotFound check in the re-check logic. -func TestBoltCacheLookupDeduplicationCompleteWithoutStore(t *testing.T) { - tempDir := t.TempDir() - cachePath := filepath.Join(tempDir, "cache.db") - - cache, err := NewBoltCache(cachePath) - if err != nil { - t.Fatalf("failed to create BoltCache: %v", err) - } - defer func() { _ = cache.CloseAndDestroy() }() - - ctx := t.Context() - - sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} - method := sdp.QueryMethod_LIST - query := "complete-without-store-test" - - var wg sync.WaitGroup - startBarrier := make(chan struct{}) - - // Track results - var waiterHits []bool - var waiterMu sync.Mutex - - numWaiters := 3 - - // First goroutine: starts work and completes without storing anything - // This simulates a LIST query that returns 0 items - wg.Go(func() { - <-startBarrier - - hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - if hit { - t.Error("first goroutine: expected cache miss") - return - } - - // Simulate work that completes successfully but returns nothing - time.Sleep(50 * time.Millisecond) - - // Complete without storing anything - no items, no error - // This triggers the ErrCacheNotFound path in waiters' re-check - cache.pending.Complete(ck.String()) - }) - - // Waiter goroutines - for range numWaiters { - wg.Go(func() { - <-startBarrier - - // Small delay to ensure first goroutine starts first - time.Sleep(10 * time.Millisecond) - - hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) - defer done() - - waiterMu.Lock() - waiterHits = append(waiterHits, hit) - waiterMu.Unlock() - }) - } - - // Release all goroutines - close(startBarrier) - wg.Wait() - - // When Complete is called without storing anything: - // 1. Waiters' Wait returns ok=true (not cancelled) - // 2. Waiters re-check the cache and get ErrCacheNotFound - // 3. Waiters return hit=false (cache miss) - if len(waiterHits) != numWaiters { - t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) - } - - // All waiters should get a cache miss since nothing was stored - for i, hit := range waiterHits { - if hit { - t.Errorf("waiter %d: expected cache miss (hit=false), got hit=true", i) - } - } -} - // TestPendingWorkUnit tests the pendingWork component in isolation. func TestPendingWorkUnit(t *testing.T) { t.Run("StartWork first caller", func(t *testing.T) { diff --git a/go/sdpcache/item_generator_test.go b/go/sdpcache/item_generator_test.go index 2729a5cd..13a84e82 100644 --- a/go/sdpcache/item_generator_test.go +++ b/go/sdpcache/item_generator_test.go @@ -28,12 +28,14 @@ var Types = []string{ "lemur", } -const MaxAttributes = 30 -const MaxTags = 10 -const MaxTagKeyLength = 10 -const MaxTagValueLength = 10 -const MaxAttributeKeyLength = 20 -const MaxAttributeValueLength = 50 +const ( + MaxAttributes = 30 + MaxTags = 10 + MaxTagKeyLength = 10 + MaxTagValueLength = 10 + MaxAttributeKeyLength = 20 + MaxAttributeValueLength = 50 +) // TODO(LIQs): rewrite this to `MaxEdges` const MaxLinkedItems = 10 diff --git a/go/sdpcache/lookup_coordinator.go b/go/sdpcache/lookup_coordinator.go new file mode 100644 index 00000000..c898e1f4 --- /dev/null +++ b/go/sdpcache/lookup_coordinator.go @@ -0,0 +1,187 @@ +package sdpcache + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/overmindtech/cli/go/sdp-go" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// lookupBackend is the storage-facing interface used by lookupCoordinator. +// Implementations should focus on cache I/O while lookupCoordinator owns +// pending-work deduplication and shared branching behavior. +type lookupBackend interface { + Search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) + Delete(ck CacheKey) +} + +// lookupCoordinator centralizes shared Lookup control flow: +// cache miss deduplication, wait/re-check behavior, error classification, +// and GET cardinality validation. +type lookupCoordinator struct { + pending *pendingWork +} + +func newLookupCoordinator(pending *pendingWork) *lookupCoordinator { + if pending == nil { + pending = newPendingWork() + } + + return &lookupCoordinator{ + pending: pending, + } +} + +func (lc *lookupCoordinator) doneForMiss(ck CacheKey) func() { + if lc == nil || lc.pending == nil { + return noopDone + } + + key := ck.String() + var once sync.Once + + return func() { + once.Do(func() { + lc.pending.Complete(key) + }) + } +} + +func (lc *lookupCoordinator) Lookup( + ctx context.Context, + backend lookupBackend, + ck CacheKey, + requestedMethod sdp.QueryMethod, +) (bool, []*sdp.Item, *sdp.QueryError, func()) { + span := trace.SpanFromContext(ctx) + + initialSearchStart := time.Now() + items, err := backend.Search(ctx, ck) + span.SetAttributes(attribute.Float64("ovm.cache.initialSearchDurationMs", float64(time.Since(initialSearchStart).Milliseconds()))) + + if err != nil { + var qErr *sdp.QueryError + + if errors.Is(err, ErrCacheNotFound) { + shouldWork, entry := lc.pending.StartWork(ck.String()) + if shouldWork { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache miss"), + attribute.Bool("ovm.cache.hit", false), + attribute.Bool("ovm.cache.workPending", false), + ) + return false, nil, nil, lc.doneForMiss(ck) + } + + pendingWaitStart := time.Now() + ok := lc.pending.Wait(ctx, entry) + pendingWaitDuration := time.Since(pendingWaitStart) + span.SetAttributes( + attribute.Float64("ovm.cache.pendingWaitDurationMs", float64(pendingWaitDuration.Milliseconds())), + attribute.Bool("ovm.cache.pendingWaitSuccess", ok), + ) + + if !ok { + span.SetAttributes( + attribute.String("ovm.cache.result", "pending work cancelled or timeout"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, nil, nil, noopDone + } + + recheckStart := time.Now() + items, recheckErr := backend.Search(ctx, ck) + span.SetAttributes(attribute.Float64("ovm.cache.recheckSearchDurationMs", float64(time.Since(recheckStart).Milliseconds()))) + if recheckErr != nil { + if errors.Is(recheckErr, ErrCacheNotFound) { + span.SetAttributes( + attribute.String("ovm.cache.result", "pending work completed but cache still empty"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, nil, nil, noopDone + } + + var recheckQErr *sdp.QueryError + if errors.As(recheckErr, &recheckQErr) { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache hit from pending work: error"), + attribute.Bool("ovm.cache.hit", true), + ) + return true, nil, recheckQErr, noopDone + } + + span.SetAttributes( + attribute.String("ovm.cache.result", "unexpected error on re-check"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, nil, nil, noopDone + } + + span.SetAttributes( + attribute.String("ovm.cache.result", "cache hit from pending work"), + attribute.Int("ovm.cache.numItems", len(items)), + attribute.Bool("ovm.cache.hit", true), + ) + return true, items, nil, noopDone + } + + if errors.As(err, &qErr) { + if qErr.GetErrorType() == sdp.QueryError_NOTFOUND { + span.SetAttributes(attribute.String("ovm.cache.result", "cache hit: item not found")) + } else { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache hit: QueryError"), + attribute.String("ovm.cache.error", err.Error()), + ) + } + + span.SetAttributes(attribute.Bool("ovm.cache.hit", true)) + return true, nil, qErr, noopDone + } + + qErr = &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: err.Error(), + Scope: ck.SST.Scope, + SourceName: ck.SST.SourceName, + ItemType: ck.SST.Type, + } + + span.SetAttributes( + attribute.String("ovm.cache.error", err.Error()), + attribute.String("ovm.cache.result", "cache hit: unknown QueryError"), + attribute.Bool("ovm.cache.hit", true), + ) + return true, nil, qErr, noopDone + } + + if requestedMethod == sdp.QueryMethod_GET { + if len(items) < 2 { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache hit: 1 item"), + attribute.Int("ovm.cache.numItems", len(items)), + attribute.Bool("ovm.cache.hit", true), + ) + return true, items, nil, noopDone + } + + span.SetAttributes( + attribute.String("ovm.cache.result", "cache returned >1 value, purging and continuing"), + attribute.Int("ovm.cache.numItems", len(items)), + attribute.Bool("ovm.cache.hit", false), + ) + backend.Delete(ck) + return false, nil, nil, noopDone + } + + span.SetAttributes( + attribute.String("ovm.cache.result", "cache hit: multiple items"), + attribute.Int("ovm.cache.numItems", len(items)), + attribute.Bool("ovm.cache.hit", true), + ) + return true, items, nil, noopDone +} diff --git a/go/sdpcache/memory.go b/go/sdpcache/memory.go new file mode 100644 index 00000000..43277a36 --- /dev/null +++ b/go/sdpcache/memory.go @@ -0,0 +1,476 @@ +package sdpcache + +import ( + "context" + "sync" + "time" + + "github.com/google/btree" + "github.com/overmindtech/cli/go/sdp-go" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "google.golang.org/protobuf/proto" +) + +type MemoryCache struct { + purger + + indexes map[SSTHash]*indexSet + + // This index is used to track item expiries, since items can have different + // expiry durations we need to use a btree here rather than just appending + // to a slice or something. The purge process uses this to determine what + // needs deleting, then calls into each specific index to delete as required + expiryIndex *btree.BTreeG[*CachedResult] + + // Mutex for reading caches + indexMutex sync.RWMutex + + // Tracks in-flight lookups to prevent duplicate work when multiple + // goroutines request the same cache key simultaneously + pending *pendingWork + + lookup *lookupCoordinator +} + +var _ Cache = (*MemoryCache)(nil) + +// NewMemoryCache creates a new in-memory cache implementation. +func NewMemoryCache() *MemoryCache { + pending := newPendingWork() + c := &MemoryCache{ + indexes: make(map[SSTHash]*indexSet), + expiryIndex: newExpiryIndex(), + pending: pending, + lookup: newLookupCoordinator(pending), + } + c.purgeFunc = c.Purge + return c +} + +func newExpiryIndex() *btree.BTreeG[*CachedResult] { + return btree.NewG(2, func(a, b *CachedResult) bool { + return a.Expiry.Before(b.Expiry) + }) +} + +type indexSet struct { + uniqueAttributeValueIndex *btree.BTreeG[*CachedResult] + methodIndex *btree.BTreeG[*CachedResult] + queryIndex *btree.BTreeG[*CachedResult] +} + +func newIndexSet() *indexSet { + return &indexSet{ + uniqueAttributeValueIndex: btree.NewG(2, func(a, b *CachedResult) bool { + return sortString(a.IndexValues.UniqueAttributeValue, a.Item) < sortString(b.IndexValues.UniqueAttributeValue, b.Item) + }), + methodIndex: btree.NewG(2, func(a, b *CachedResult) bool { + return sortString(a.IndexValues.Method.String(), a.Item) < sortString(b.IndexValues.Method.String(), b.Item) + }), + queryIndex: btree.NewG(2, func(a, b *CachedResult) bool { + return sortString(a.IndexValues.Query, a.Item) < sortString(b.IndexValues.Query, b.Item) + }), + } +} + +// Lookup returns true/false whether or not the cache has a result for the given +// query. If there are results, they will be returned as slice of `sdp.Item`s or +// an `*sdp.QueryError`. +// The CacheKey is always returned, even if the lookup otherwise fails or errors. +func (c *MemoryCache) Lookup(ctx context.Context, srcName string, method sdp.QueryMethod, scope string, typ string, query string, ignoreCache bool) (bool, CacheKey, []*sdp.Item, *sdp.QueryError, func()) { + span := trace.SpanFromContext(ctx) + ck := CacheKeyFromParts(srcName, method, scope, typ, query) + + if c == nil { + span.SetAttributes( + attribute.String("ovm.cache.result", "cache not initialised"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, ck, nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "cache has not been initialised", + Scope: scope, + SourceName: srcName, + ItemType: typ, + }, noopDone + } + + if ignoreCache { + span.SetAttributes( + attribute.String("ovm.cache.result", "ignore cache"), + attribute.Bool("ovm.cache.hit", false), + ) + return false, ck, nil, nil, noopDone + } + + lookup := c.lookup + if lookup == nil { + lookup = newLookupCoordinator(c.pending) + } + + hit, items, qErr, done := lookup.Lookup( + ctx, + c, + ck, + method, + ) + return hit, ck, items, qErr, done +} + +// Search performs a lower-level search using a CacheKey. +// This bypasses pending-work deduplication and is used by lookupCoordinator. +func (c *MemoryCache) Search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { + return c.search(ctx, ck) +} + +// search performs a lower-level search using a CacheKey. +func (c *MemoryCache) search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { + if c == nil { + return nil, nil + } + + items := make([]*sdp.Item, 0) + + results := c.getResults(ck) + + if len(results) == 0 { + return nil, ErrCacheNotFound + } + + now := time.Now() + + // If there is an error we want to return that, so we need to range over the + // results and separate items and errors. This is computationally less + // efficient than extracting errors inside of `getResults()` but logically + // it's a lot less complicated since `Delete()` uses the same method but + // applies different logic + for _, res := range results { + // Check if the cached result has expired + if res.Expiry.Before(now) { + // Skip expired results + continue + } + + if res.Error != nil { + return nil, res.Error + } + + // Return a copy of the item so the user can do whatever they want with it + itemCopy := proto.Clone(res.Item).(*sdp.Item) + + items = append(items, itemCopy) + } + + // If all results were expired, return cache not found + if len(items) == 0 { + return nil, ErrCacheNotFound + } + + return items, nil +} + +// Delete deletes anything that matches the given cache query. +func (c *MemoryCache) Delete(ck CacheKey) { + if c == nil { + return + } + + c.deleteResults(c.getResults(ck)) +} + +// getResults searches indexes for cached results, doing no other logic. If +// nothing is found an empty slice will be returned. +func (c *MemoryCache) getResults(ck CacheKey) []*CachedResult { + c.indexMutex.RLock() + defer c.indexMutex.RUnlock() + + results := make([]*CachedResult, 0) + + // Get the relevant set of indexes based on the SST Hash + sstHash := ck.SST.Hash() + indexes, exists := c.indexes[sstHash] + pivot := CachedResult{ + IndexValues: IndexValues{ + SSTHash: sstHash, + }, + } + + if !exists { + // If we don't have a set of indexes then it definitely doesn't exist + return results + } + + // Start with the most specific index and fall back to the least specific. + // Checking all matching items and returning. These is no need to check all + // indexes since they all have the same content + if ck.UniqueAttributeValue != nil { + pivot.IndexValues.UniqueAttributeValue = *ck.UniqueAttributeValue + + indexes.uniqueAttributeValueIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { + if *ck.UniqueAttributeValue == result.IndexValues.UniqueAttributeValue { + if ck.Matches(result.IndexValues) { + results = append(results, result) + } + + // Always return true so that we continue to iterate + return true + } + + return false + }) + + return results + } + + if ck.Query != nil { + pivot.IndexValues.Query = *ck.Query + + indexes.queryIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { + if *ck.Query == result.IndexValues.Query { + if ck.Matches(result.IndexValues) { + results = append(results, result) + } + + // Always return true so that we continue to iterate + return true + } + + return false + }) + + return results + } + + if ck.Method != nil { + pivot.IndexValues.Method = *ck.Method + + indexes.methodIndex.AscendGreaterOrEqual(&pivot, func(result *CachedResult) bool { + if *ck.Method == result.IndexValues.Method { + // If the methods match, check the rest + if ck.Matches(result.IndexValues) { + results = append(results, result) + } + + // Always return true so that we continue to iterate + return true + } + + return false + }) + + return results + } + + // If nothing other than SST has been set then return everything + indexes.methodIndex.Ascend(func(result *CachedResult) bool { + results = append(results, result) + + return true + }) + + return results +} + +// StoreItem stores an item in the cache. Note that this item must be fully +// populated (including metadata) for indexing to work correctly. +func (c *MemoryCache) StoreItem(ctx context.Context, item *sdp.Item, duration time.Duration, ck CacheKey) { + if item == nil || c == nil { + return + } + + itemCopy := proto.Clone(item).(*sdp.Item) + + res := CachedResult{ + Item: itemCopy, + Error: nil, + Expiry: time.Now().Add(duration), + IndexValues: IndexValues{ + UniqueAttributeValue: itemCopy.UniqueAttributeValue(), + }, + } + + if ck.Method != nil { + res.IndexValues.Method = *ck.Method + } + if ck.Query != nil { + res.IndexValues.Query = *ck.Query + } + + res.IndexValues.SSTHash = ck.SST.Hash() + + c.storeResult(ctx, res) +} + +// StoreUnavailableItem stores an error for the given duration. +func (c *MemoryCache) StoreUnavailableItem(ctx context.Context, err error, duration time.Duration, cacheQuery CacheKey) { + if c == nil || err == nil { + return + } + + res := CachedResult{ + Item: nil, + Error: err, + Expiry: time.Now().Add(duration), + IndexValues: cacheQuery.ToIndexValues(), + } + + c.storeResult(ctx, res) +} + +// Clear deletes all data in cache. +func (c *MemoryCache) Clear() { + if c == nil { + return + } + + c.indexMutex.Lock() + defer c.indexMutex.Unlock() + + c.indexes = make(map[SSTHash]*indexSet) + c.expiryIndex = newExpiryIndex() +} + +func (c *MemoryCache) storeResult(ctx context.Context, res CachedResult) { + c.indexMutex.Lock() + defer c.indexMutex.Unlock() + + // Create the index if it doesn't exist + indexes, ok := c.indexes[res.IndexValues.SSTHash] + + if !ok { + indexes = newIndexSet() + c.indexes[res.IndexValues.SSTHash] = indexes + } + + // Add the item to the indexes and check if we're overwriting an unexpired entry + // We only need to check one index since they all reference the same CachedResult + oldResult, replaced := indexes.methodIndex.ReplaceOrInsert(&res) + indexes.queryIndex.ReplaceOrInsert(&res) + indexes.uniqueAttributeValueIndex.ReplaceOrInsert(&res) + + // Get the current span to add attributes + span := trace.SpanFromContext(ctx) + + // Check if we overwrote an entry that hasn't expired yet + // This indicates potential thundering-herd issues where multiple identical + // queries are executed concurrently instead of waiting for the first result + overwritten := false + if replaced && oldResult != nil { + now := time.Now() + if oldResult.Expiry.After(now) { + overwritten = true + timeUntilExpiry := oldResult.Expiry.Sub(now) + + // Build attributes for the overwrite event + attrs := []attribute.KeyValue{ + attribute.Bool("ovm.cache.unexpired_overwrite", true), + attribute.String("ovm.cache.time_until_expiry", timeUntilExpiry.String()), + attribute.String("ovm.cache.sst_hash", string(res.IndexValues.SSTHash)), + attribute.String("ovm.cache.query_method", res.IndexValues.Method.String()), + } + + if res.Item != nil { + attrs = append(attrs, + attribute.String("ovm.cache.item_type", res.Item.GetType()), + attribute.String("ovm.cache.item_scope", res.Item.GetScope()), + ) + } + + if res.IndexValues.Query != "" { + attrs = append(attrs, attribute.String("ovm.cache.query", res.IndexValues.Query)) + } + + if res.IndexValues.UniqueAttributeValue != "" { + attrs = append(attrs, attribute.String("ovm.cache.unique_attribute", res.IndexValues.UniqueAttributeValue)) + } + + span.SetAttributes(attrs...) + } + } + + // Always set the overwrite attribute, even if false, for consistent tracking + if !overwritten { + span.SetAttributes(attribute.Bool("ovm.cache.unexpired_overwrite", false)) + } + + // Add the item to the expiry index + c.expiryIndex.ReplaceOrInsert(&res) + + // Update the purge time if required + c.setNextPurgeIfEarlier(res.Expiry) +} + +// sortString returns the string that the cached result should be sorted on. +// This has a prefix of the index value and suffix of the GloballyUniqueName if +// relevant. +func sortString(indexValue string, item *sdp.Item) string { + if item == nil { + return indexValue + } + return indexValue + item.GloballyUniqueName() +} + +// deleteResults deletes many cached results at once. +func (c *MemoryCache) deleteResults(results []*CachedResult) { + c.indexMutex.Lock() + defer c.indexMutex.Unlock() + + for _, res := range results { + if indexSet, ok := c.indexes[res.IndexValues.SSTHash]; ok { + // For each expired item, delete it from all of the indexes that it will be in + if indexSet.methodIndex != nil { + indexSet.methodIndex.Delete(res) + } + if indexSet.queryIndex != nil { + indexSet.queryIndex.Delete(res) + } + if indexSet.uniqueAttributeValueIndex != nil { + indexSet.uniqueAttributeValueIndex.Delete(res) + } + } + + c.expiryIndex.Delete(res) + } +} + +// Purge purges all expired items from the cache. The user must pass in the +// `before` time. All items that expired before this will be purged. Usually +// this would be just `time.Now()` however it could be overridden for testing. +func (c *MemoryCache) Purge(ctx context.Context, before time.Time) PurgeStats { + if c == nil { + return PurgeStats{} + } + + // Store the current time rather than calling it a million times + start := time.Now() + + var nextExpiry *time.Time + + expired := make([]*CachedResult, 0) + + // Look through the expiry cache and work out what has expired + c.indexMutex.RLock() + c.expiryIndex.Ascend(func(res *CachedResult) bool { + if res.Expiry.Before(before) { + expired = append(expired, res) + + return true + } + + // Take note of the next expiry so we can schedule the next run + nextExpiry = &res.Expiry + + // As soon as hit this we'll stop ascending + return false + }) + c.indexMutex.RUnlock() + + c.deleteResults(expired) + + return PurgeStats{ + NumPurged: len(expired), + TimeTaken: time.Since(start), + NextExpiry: nextExpiry, + } +} diff --git a/go/sdpcache/memory_test.go b/go/sdpcache/memory_test.go new file mode 100644 index 00000000..478176e5 --- /dev/null +++ b/go/sdpcache/memory_test.go @@ -0,0 +1,294 @@ +package sdpcache + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/overmindtech/cli/go/sdp-go" +) + +// TestMemoryCacheStartPurge tests the memory cache implementation's purger. +func TestMemoryCacheStartPurge(t *testing.T) { + ctx := t.Context() + cache := NewMemoryCache() + cache.minWaitTime = 100 * time.Millisecond + + cachedItems := []struct { + Item *sdp.Item + Expiry time.Time + }{ + { + Item: GenerateRandomItem(), + Expiry: time.Now().Add(0), + }, + { + Item: GenerateRandomItem(), + Expiry: time.Now().Add(100 * time.Millisecond), + }, + } + + for _, i := range cachedItems { + ck := CacheKeyFromQuery(i.Item.GetMetadata().GetSourceQuery(), i.Item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, i.Item, time.Until(i.Expiry), ck) + } + + ctx, done := context.WithCancel(ctx) + defer done() + + cache.StartPurger(ctx) + + // Wait for everything to be purged + time.Sleep(200 * time.Millisecond) + + // At this point everything should be been cleaned, and the purger should be + // sleeping forever + items, err := testSearch(t.Context(), cache, CacheKeyFromQuery( + cachedItems[1].Item.GetMetadata().GetSourceQuery(), + cachedItems[1].Item.GetMetadata().GetSourceName(), + )) + + if !errors.Is(err, ErrCacheNotFound) { + t.Errorf("unexpected error: %v", err) + t.Errorf("unexpected items: %v", len(items)) + } + + cache.purgeMutex.Lock() + if cache.nextPurge.Before(time.Now().Add(time.Hour)) { + // If the next purge is within the next hour that's an error, it should + // be really, really for in the future + t.Errorf("Expected next purge to be in 1000 years, got %v", cache.nextPurge.String()) + } + cache.purgeMutex.Unlock() + + // Adding a new item should kick off the purging again + for _, i := range cachedItems { + ck := CacheKeyFromQuery(i.Item.GetMetadata().GetSourceQuery(), i.Item.GetMetadata().GetSourceName()) + cache.StoreItem(ctx, i.Item, 100*time.Millisecond, ck) + } + + time.Sleep(200 * time.Millisecond) + + // It should be empty again + items, err = testSearch(t.Context(), cache, CacheKeyFromQuery( + cachedItems[1].Item.GetMetadata().GetSourceQuery(), + cachedItems[1].Item.GetMetadata().GetSourceName(), + )) + + if !errors.Is(err, ErrCacheNotFound) { + t.Errorf("unexpected error: %v", err) + t.Errorf("unexpected items: %v: %v", len(items), items) + } +} + +// TestMemoryCacheStopPurge tests the memory cache implementation's purger stop functionality. +func TestMemoryCacheStopPurge(t *testing.T) { + cache := NewMemoryCache() + cache.minWaitTime = 1 * time.Millisecond + + ctx, done := context.WithCancel(t.Context()) + + cache.StartPurger(ctx) + + // Stop the purger + done() + + // Insert an item + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + + cache.StoreItem(ctx, item, 1*time.Second, ck) + sst := SST{ + SourceName: item.GetMetadata().GetSourceName(), + Scope: item.GetScope(), + Type: item.GetType(), + } + + // Make sure it's not purged + time.Sleep(100 * time.Millisecond) + items, err := testSearch(t.Context(), cache, CacheKey{ + SST: sst, + }) + if err != nil { + t.Error(err) + } + + if len(items) != 1 { + t.Errorf("Expected 1 item, got %v", len(items)) + } +} + +// TestMemoryCacheConcurrent tests the memory cache implementation for data races. +// This test is designed to be run with -race to ensure that there aren't any +// data races. +func TestMemoryCacheConcurrent(t *testing.T) { + cache := NewMemoryCache() + // Run the purger super fast to generate a worst-case scenario + cache.minWaitTime = 1 * time.Millisecond + + ctx, done := context.WithCancel(t.Context()) + defer done() + cache.StartPurger(ctx) + var wg sync.WaitGroup + + numParallel := 1_000 + + for range numParallel { + wg.Go(func() { + // Store the item + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + + cache.StoreItem(ctx, item, 100*time.Millisecond, ck) + + // Create a goroutine to also delete in parallel + wg.Go(func() { + cache.Delete(ck) + }) + }) + } + + wg.Wait() +} + +// TestMemoryCacheLookupDeduplication tests that multiple concurrent Lookup calls +// for the same cache key in MemoryCache result in only one caller doing work. +func TestMemoryCacheLookupDeduplication(t *testing.T) { + cache := NewMemoryCache() + ctx := t.Context() + + // Create a cache key for the test - use LIST method to avoid UniqueAttributeValue matching issues + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_LIST + + // Track how many goroutines actually do work + var workCount int32 + var mu sync.Mutex + var wg sync.WaitGroup + + numGoroutines := 10 + results := make([]struct { + hit bool + items []*sdp.Item + }, numGoroutines) + + startBarrier := make(chan struct{}) + + for idx := range numGoroutines { + wg.Go(func() { + <-startBarrier + + hit, ck, items, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) + defer done() + + if !hit { + mu.Lock() + workCount++ + mu.Unlock() + + time.Sleep(50 * time.Millisecond) + + item := GenerateRandomItem() + item.Scope = sst.Scope + item.Type = sst.Type + item.Metadata.SourceName = sst.SourceName + + cache.StoreItem(ctx, item, 10*time.Second, ck) + hit, _, items, _, done = cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) + defer done() + } + + results[idx] = struct { + hit bool + items []*sdp.Item + }{hit, items} + }) + } + + close(startBarrier) + wg.Wait() + + if workCount != 1 { + t.Errorf("expected exactly 1 goroutine to do work, got %d", workCount) + } + + for i, r := range results { + if !r.hit { + t.Errorf("goroutine %d: expected cache hit after dedup, got miss", i) + } + if len(r.items) != 1 { + t.Errorf("goroutine %d: expected 1 item, got %d", i, len(r.items)) + } + } +} + +// TestMemoryCacheLookupDeduplicationCompleteWithoutStore tests the scenario where +// Complete is called but nothing was stored in the cache. This tests the explicit +// ErrCacheNotFound check in the re-check logic. +func TestMemoryCacheLookupDeduplicationCompleteWithoutStore(t *testing.T) { + cache := NewMemoryCache() + ctx := t.Context() + + sst := SST{SourceName: "test-source", Scope: "test-scope", Type: "test-type"} + method := sdp.QueryMethod_LIST + query := "complete-without-store-test" + + var wg sync.WaitGroup + startBarrier := make(chan struct{}) + + // Track results + var waiterHits []bool + var waiterMu sync.Mutex + + numWaiters := 3 + + // First goroutine: starts work and completes without storing anything + wg.Go(func() { + <-startBarrier + + hit, ck, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + if hit { + t.Error("first goroutine: expected cache miss") + return + } + + // Simulate work that completes successfully but returns nothing + time.Sleep(50 * time.Millisecond) + + // Complete without storing anything - triggers ErrCacheNotFound on re-check + cache.pending.Complete(ck.String()) + }) + + // Waiter goroutines + for range numWaiters { + wg.Go(func() { + <-startBarrier + + time.Sleep(10 * time.Millisecond) + + hit, _, _, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, query, false) + defer done() + + waiterMu.Lock() + waiterHits = append(waiterHits, hit) + waiterMu.Unlock() + }) + } + + close(startBarrier) + wg.Wait() + + if len(waiterHits) != numWaiters { + t.Errorf("expected %d waiter results, got %d", numWaiters, len(waiterHits)) + } + + // All waiters should get a cache miss since nothing was stored + for i, hit := range waiterHits { + if hit { + t.Errorf("waiter %d: expected cache miss (hit=false), got hit=true", i) + } + } +} diff --git a/go/sdpcache/noop_cache_test.go b/go/sdpcache/noop_cache_test.go new file mode 100644 index 00000000..d737a690 --- /dev/null +++ b/go/sdpcache/noop_cache_test.go @@ -0,0 +1,95 @@ +package sdpcache + +import ( + "testing" + "time" + + "github.com/overmindtech/cli/go/sdp-go" +) + +func TestNoOpCacheLookupAlwaysMiss(t *testing.T) { + cache := NewNoOpCache() + + hit, ck, items, qErr, done := cache.Lookup( + t.Context(), + "test-source", + sdp.QueryMethod_GET, + "test-scope", + "test-type", + "test-query", + false, + ) + + if hit { + t.Fatal("expected miss, got hit") + } + if qErr != nil { + t.Fatalf("expected nil error, got %v", qErr) + } + if len(items) != 0 { + t.Fatalf("expected no items, got %d", len(items)) + } + + expected := CacheKeyFromParts("test-source", sdp.QueryMethod_GET, "test-scope", "test-type", "test-query") + if ck.String() != expected.String() { + t.Fatalf("expected cache key %q, got %q", expected.String(), ck.String()) + } + + // done() should be a no-op and idempotent. + done() + done() +} + +func TestNoOpCacheIgnoresAllMutations(t *testing.T) { + cache := NewNoOpCache() + ctx := t.Context() + + item := GenerateRandomItem() + ck := CacheKeyFromQuery(item.GetMetadata().GetSourceQuery(), item.GetMetadata().GetSourceName()) + + cache.StoreItem(ctx, item, time.Second, ck) + cache.StoreUnavailableItem(ctx, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "noop", + }, time.Second, ck) + cache.Delete(ck) + cache.Clear() + cache.StartPurger(ctx) + + hit, _, items, qErr, done := cache.Lookup( + ctx, + item.GetMetadata().GetSourceName(), + item.GetMetadata().GetSourceQuery().GetMethod(), + item.GetMetadata().GetSourceQuery().GetScope(), + item.GetMetadata().GetSourceQuery().GetType(), + item.GetMetadata().GetSourceQuery().GetQuery(), + false, + ) + defer done() + + if hit { + t.Fatal("expected miss after no-op mutations, got hit") + } + if qErr != nil { + t.Fatalf("expected nil error after no-op mutations, got %v", qErr) + } + if len(items) != 0 { + t.Fatalf("expected no items after no-op mutations, got %d", len(items)) + } +} + +func TestNoOpCachePurgeAndMinWaitDefaults(t *testing.T) { + cache := NewNoOpCache() + + if got := cache.GetMinWaitTime(); got != 0 { + t.Fatalf("expected min wait time 0, got %v", got) + } + + stats := cache.Purge(t.Context(), time.Now()) + if stats.NumPurged != 0 { + t.Fatalf("expected NumPurged=0, got %d", stats.NumPurged) + } + if stats.NextExpiry != nil { + t.Fatalf("expected NextExpiry=nil, got %v", stats.NextExpiry) + } +} diff --git a/go/sdpcache/pending.go b/go/sdpcache/pending.go index dc72864b..7f46086b 100644 --- a/go/sdpcache/pending.go +++ b/go/sdpcache/pending.go @@ -60,16 +60,16 @@ func (p *pendingWork) Wait(ctx context.Context, entry *workEntry) (ok bool) { // Calculate safety timeout based on when work started deadline := entry.startTime.Add(maxPendingWorkAge) timeUntilDeadline := time.Until(deadline) - + // If we're already past the deadline, return immediately if timeUntilDeadline <= 0 { return false } - + // Create a timer for the safety timeout timer := time.NewTimer(timeUntilDeadline) defer timer.Stop() - + select { case <-entry.done: // Work completed normally diff --git a/go/sdpcache/purger.go b/go/sdpcache/purger.go new file mode 100644 index 00000000..a39523d2 --- /dev/null +++ b/go/sdpcache/purger.go @@ -0,0 +1,113 @@ +package sdpcache + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// MinWaitDefault is the default minimum wait time between purge cycles. +const MinWaitDefault = 5 * time.Second + +// PurgeStats holds statistics from a single purge run. +type PurgeStats struct { + // How many items were timed out of the cache + NumPurged int + // How long purging took overall + TimeTaken time.Duration + // The expiry time of the next item to expire. If there are no more items in + // the cache, this will be nil + NextExpiry *time.Time +} + +// purger manages timer-based scheduling for periodic cache purging. +// MemoryCache and boltStore embed this struct to share the scheduling logic; +// storage-specific purge work is injected via the purgeFunc callback. +type purger struct { + purgeFunc func(context.Context, time.Time) PurgeStats + minWaitTime time.Duration + purgeTimer *time.Timer + nextPurge time.Time + purgeMutex sync.Mutex +} + +// GetMinWaitTime returns the minimum wait time or the default if not set. +func (p *purger) GetMinWaitTime() time.Duration { + if p.minWaitTime == 0 { + return MinWaitDefault + } + return p.minWaitTime +} + +// StartPurger starts the purge process in the background, it will be cancelled +// when the context is cancelled. The cache will be purged initially, at which +// point the process will sleep until the next time an item expires. +func (p *purger) StartPurger(ctx context.Context) { + p.purgeMutex.Lock() + if p.purgeTimer == nil { + p.purgeTimer = time.NewTimer(0) + p.purgeMutex.Unlock() + } else { + p.purgeMutex.Unlock() + log.WithContext(ctx).Info("Purger already running") + return + } + + go func(ctx context.Context) { + for { + select { + case <-p.purgeTimer.C: + stats := p.purgeFunc(ctx, time.Now()) + p.setNextPurgeFromStats(stats) + case <-ctx.Done(): + p.purgeMutex.Lock() + defer p.purgeMutex.Unlock() + + p.purgeTimer.Stop() + p.purgeTimer = nil + return + } + } + }(ctx) +} + +// setNextPurgeFromStats sets when the next purge should run based on the stats +// of the previous purge. +func (p *purger) setNextPurgeFromStats(stats PurgeStats) { + p.purgeMutex.Lock() + defer p.purgeMutex.Unlock() + + if stats.NextExpiry == nil { + p.purgeTimer.Reset(1000 * time.Hour) + p.nextPurge = time.Now().Add(1000 * time.Hour) + } else { + if time.Until(*stats.NextExpiry) < p.GetMinWaitTime() { + p.purgeTimer.Reset(p.GetMinWaitTime()) + p.nextPurge = time.Now().Add(p.GetMinWaitTime()) + } else { + p.purgeTimer.Reset(time.Until(*stats.NextExpiry)) + p.nextPurge = *stats.NextExpiry + } + } +} + +// setNextPurgeIfEarlier sets the next time the purger will run, if the provided +// time is sooner than the current scheduled purge time. While the purger is +// active this will be constantly updated, however if the purger is sleeping and +// new items are added this method ensures that the purger is woken up. +func (p *purger) setNextPurgeIfEarlier(t time.Time) { + p.purgeMutex.Lock() + defer p.purgeMutex.Unlock() + + if t.Before(p.nextPurge) { + if p.purgeTimer == nil { + return + } + + p.purgeTimer.Stop() + p.nextPurge = t + p.purgeTimer.Reset(time.Until(t)) + } +} diff --git a/go/sdpcache/sharded_cache.go b/go/sdpcache/sharded.go similarity index 61% rename from go/sdpcache/sharded_cache.go rename to go/sdpcache/sharded.go index 42423eb2..93252d3d 100644 --- a/go/sdpcache/sharded_cache.go +++ b/go/sdpcache/sharded.go @@ -32,12 +32,15 @@ const DefaultShardCount = 17 // shards in parallel and merge results. pendingWork deduplication lives at the // ShardedCache level to prevent duplicate API calls across the fan-out. type ShardedCache struct { - shards []*BoltCache + purger + + shards []*boltStore dir string // pendingWork lives at the ShardedCache level so that deduplication spans // the entire cache, not individual shards. pending *pendingWork + lookup *lookupCoordinator } var _ Cache = (*ShardedCache)(nil) @@ -53,22 +56,20 @@ func NewShardedCache(dir string, shardCount int, opts ...BoltCacheOption) (*Shar return nil, fmt.Errorf("failed to create shard directory: %w", err) } - shards := make([]*BoltCache, shardCount) + shards := make([]*boltStore, shardCount) errs := make([]error, shardCount) var wg sync.WaitGroup for i := range shardCount { - wg.Add(1) - go func(idx int) { - defer wg.Done() - path := filepath.Join(dir, fmt.Sprintf("shard-%02d.db", idx)) - c, err := NewBoltCache(path, opts...) + wg.Go(func() { + path := filepath.Join(dir, fmt.Sprintf("shard-%02d.db", i)) + c, err := newBoltCacheStore(path, opts...) if err != nil { - errs[idx] = fmt.Errorf("shard %d: %w", idx, err) + errs[i] = fmt.Errorf("shard %d: %w", i, err) return } - shards[idx] = c - }(i) + shards[i] = c + }) } wg.Wait() @@ -84,11 +85,15 @@ func NewShardedCache(dir string, shardCount int, opts ...BoltCacheOption) (*Shar } } - return &ShardedCache{ + pending := newPendingWork() + sc := &ShardedCache{ shards: shards, dir: dir, - pending: newPendingWork(), - }, nil + pending: pending, + lookup: newLookupCoordinator(pending), + } + sc.purgeFunc = sc.Purge + return sc, nil } // shardFor returns the shard index for a given item identity. @@ -125,124 +130,23 @@ func (sc *ShardedCache) Lookup(ctx context.Context, srcName string, method sdp.Q return false, ck, nil, nil, noopDone } - items, err := sc.searchByKey(ctx, ck) - - if err != nil { - var qErr *sdp.QueryError - if errors.Is(err, ErrCacheNotFound) { - shouldWork, entry := sc.pending.StartWork(ck.String()) - if shouldWork { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache miss"), - attribute.Bool("ovm.cache.hit", false), - attribute.Bool("ovm.cache.workPending", false), - ) - return false, ck, nil, nil, sc.createDoneFunc(ck) - } - - pendingWaitStart := time.Now() - ok := sc.pending.Wait(ctx, entry) - pendingWaitDuration := time.Since(pendingWaitStart) - span.SetAttributes( - attribute.Float64("ovm.cache.pendingWaitDuration_ms", float64(pendingWaitDuration.Milliseconds())), - attribute.Bool("ovm.cache.pendingWaitSuccess", ok), - ) - - if !ok { - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work cancelled or timeout"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - items, recheckErr := sc.searchByKey(ctx, ck) - if recheckErr != nil { - if errors.Is(recheckErr, ErrCacheNotFound) { - span.SetAttributes( - attribute.String("ovm.cache.result", "pending work completed but cache still empty"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - var recheckQErr *sdp.QueryError - if errors.As(recheckErr, &recheckQErr) { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work: error"), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, nil, recheckQErr, noopDone - } - span.SetAttributes( - attribute.String("ovm.cache.result", "unexpected error on re-check"), - attribute.Bool("ovm.cache.hit", false), - ) - return false, ck, nil, nil, noopDone - } - - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit from pending work"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } else if errors.As(err, &qErr) { - if qErr.GetErrorType() == sdp.QueryError_NOTFOUND { - span.SetAttributes(attribute.String("ovm.cache.result", "cache hit: item not found")) - } else { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: QueryError"), - attribute.String("ovm.cache.error", err.Error()), - ) - } - span.SetAttributes(attribute.Bool("ovm.cache.hit", true)) - return true, ck, nil, qErr, noopDone - } else { - qErr = &sdp.QueryError{ - ErrorType: sdp.QueryError_OTHER, - ErrorString: err.Error(), - Scope: scope, - SourceName: srcName, - ItemType: typ, - } - span.SetAttributes( - attribute.String("ovm.cache.error", err.Error()), - attribute.String("ovm.cache.result", "cache hit: unknown QueryError"), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, nil, qErr, noopDone - } - } - - if method == sdp.QueryMethod_GET { - if len(items) < 2 { - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: 1 item"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), - ) - return true, ck, items, nil, noopDone - } - span.SetAttributes( - attribute.String("ovm.cache.result", "cache returned >1 value, purging and continuing"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", false), - ) - sc.Delete(ck) - return false, ck, nil, nil, noopDone + lookup := sc.lookup + if lookup == nil { + lookup = newLookupCoordinator(sc.pending) } - span.SetAttributes( - attribute.String("ovm.cache.result", "cache hit: multiple items"), - attribute.Int("ovm.cache.numItems", len(items)), - attribute.Bool("ovm.cache.hit", true), + hit, items, qErr, done := lookup.Lookup( + ctx, + sc, + ck, + method, ) - return true, ck, items, nil, noopDone + return hit, ck, items, qErr, done } -// searchByKey routes GET queries to a single shard and LIST/SEARCH/unspecified -// queries to all shards via fan-out. -func (sc *ShardedCache) searchByKey(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { +// Search performs a lower-level search using a CacheKey. +// This bypasses pending-work deduplication and is used by lookupCoordinator. +func (sc *ShardedCache) Search(ctx context.Context, ck CacheKey) ([]*sdp.Item, error) { span := trace.SpanFromContext(ctx) if ck.UniqueAttributeValue != nil { @@ -271,13 +175,11 @@ func (sc *ShardedCache) searchAll(ctx context.Context, ck CacheKey) ([]*sdp.Item var wg sync.WaitGroup for i, shard := range sc.shards { - wg.Add(1) - go func(i int, shard *BoltCache) { - defer wg.Done() + wg.Go(func() { start := time.Now() items, err := shard.Search(ctx, ck) results[i] = result{items: items, err: err, dur: time.Since(start)} - }(i, shard) + }) } wg.Wait() @@ -340,6 +242,7 @@ func (sc *ShardedCache) StoreItem(ctx context.Context, item *sdp.Item, duration span.SetAttributes(attribute.Int("ovm.cache.shardIndex", idx)) sc.shards[idx].StoreItem(ctx, item, duration, ck) + sc.setNextPurgeIfEarlier(time.Now().Add(duration)) } // StoreUnavailableItem routes the error based on the CacheKey: @@ -359,17 +262,16 @@ func (sc *ShardedCache) StoreUnavailableItem(ctx context.Context, err error, dur span.SetAttributes(attribute.Int("ovm.cache.shardIndex", idx)) sc.shards[idx].StoreUnavailableItem(ctx, err, duration, ck) + sc.setNextPurgeIfEarlier(time.Now().Add(duration)) } // Delete fans out to all shards. func (sc *ShardedCache) Delete(ck CacheKey) { var wg sync.WaitGroup - for _, shard := range sc.shards { - wg.Add(1) - go func(s *BoltCache) { - defer wg.Done() + for _, s := range sc.shards { + wg.Go(func() { s.Delete(ck) - }(shard) + }) } wg.Wait() } @@ -377,12 +279,10 @@ func (sc *ShardedCache) Delete(ck CacheKey) { // Clear fans out to all shards. func (sc *ShardedCache) Clear() { var wg sync.WaitGroup - for _, shard := range sc.shards { - wg.Add(1) - go func(s *BoltCache) { - defer wg.Done() + for _, s := range sc.shards { + wg.Go(func() { s.Clear() - }(shard) + }) } wg.Wait() } @@ -391,6 +291,13 @@ func (sc *ShardedCache) Clear() { // TimeTaken reflects wall-clock time of the parallel fan-out, not the sum of // per-shard durations. func (sc *ShardedCache) Purge(ctx context.Context, before time.Time) PurgeStats { + ctx, span := tracing.Tracer().Start(ctx, "ShardedCache.Purge", + trace.WithAttributes( + attribute.Int("ovm.cache.shardCount", len(sc.shards)), + ), + ) + defer span.End() + type result struct { stats PurgeStats } @@ -399,12 +306,10 @@ func (sc *ShardedCache) Purge(ctx context.Context, before time.Time) PurgeStats start := time.Now() var wg sync.WaitGroup - for i, shard := range sc.shards { - wg.Add(1) - go func(i int, s *BoltCache) { - defer wg.Done() + for i, s := range sc.shards { + wg.Go(func() { results[i] = result{stats: s.Purge(ctx, before)} - }(i, shard) + }) } wg.Wait() @@ -419,22 +324,13 @@ func (sc *ShardedCache) Purge(ctx context.Context, before time.Time) PurgeStats } } } - return combined -} -// GetMinWaitTime returns the minimum wait time from the first shard. -func (sc *ShardedCache) GetMinWaitTime() time.Duration { - if len(sc.shards) == 0 { - return 0 - } - return sc.shards[0].GetMinWaitTime() -} + span.SetAttributes( + attribute.Int("ovm.cache.numPurged", combined.NumPurged), + attribute.Float64("ovm.cache.purgeDurationMs", float64(combined.TimeTaken.Milliseconds())), + ) -// StartPurger starts a purger on each shard independently. -func (sc *ShardedCache) StartPurger(ctx context.Context) { - for _, shard := range sc.shards { - shard.StartPurger(ctx) - } + return combined } // CloseAndDestroy closes and destroys all shard files in parallel, then removes @@ -443,12 +339,10 @@ func (sc *ShardedCache) CloseAndDestroy() error { errs := make([]error, len(sc.shards)) var wg sync.WaitGroup - for i, shard := range sc.shards { - wg.Add(1) - go func(i int, s *BoltCache) { - defer wg.Done() + for i, s := range sc.shards { + wg.Go(func() { errs[i] = s.CloseAndDestroy() - }(i, shard) + }) } wg.Wait() @@ -461,21 +355,6 @@ func (sc *ShardedCache) CloseAndDestroy() error { return os.RemoveAll(sc.dir) } -// createDoneFunc returns a done function that calls pending.Complete for the -// given cache key. Safe to call multiple times (idempotent via sync.Once). -func (sc *ShardedCache) createDoneFunc(ck CacheKey) func() { - if sc == nil || sc.pending == nil { - return noopDone - } - key := ck.String() - var once sync.Once - return func() { - once.Do(func() { - sc.pending.Complete(key) - }) - } -} - // newShardedCacheForProduction is used by NewCache to create a production // ShardedCache with appropriate defaults. It logs and falls back to MemoryCache // on failure. @@ -494,7 +373,6 @@ func newShardedCacheForProduction(ctx context.Context) Cache { cache, err := NewShardedCache( dir, DefaultShardCount, - WithMinWaitTime(30*time.Second), WithCompactThreshold(perShardThreshold), ) if err != nil { @@ -506,6 +384,7 @@ func newShardedCacheForProduction(ctx context.Context) Cache { return memCache } + cache.minWaitTime = 30 * time.Second cache.StartPurger(ctx) return cache } diff --git a/go/sdpcache/sharded_cache_test.go b/go/sdpcache/sharded_test.go similarity index 99% rename from go/sdpcache/sharded_cache_test.go rename to go/sdpcache/sharded_test.go index 32a6f153..56333fb6 100644 --- a/go/sdpcache/sharded_cache_test.go +++ b/go/sdpcache/sharded_test.go @@ -250,10 +250,8 @@ func TestShardedCachePendingWorkDeduplication(t *testing.T) { startBarrier := make(chan struct{}) - for i := range numGoroutines { - wg.Add(1) - go func(idx int) { - defer wg.Done() + for idx := range numGoroutines { + wg.Go(func() { <-startBarrier hit, ck, items, _, done := cache.Lookup(ctx, sst.SourceName, method, sst.Scope, sst.Type, "", false) @@ -280,7 +278,7 @@ func TestShardedCachePendingWorkDeduplication(t *testing.T) { hit bool items []*sdp.Item }{hit, items} - }(i) + }) } close(startBarrier) @@ -564,8 +562,7 @@ func TestShardedCacheConcurrentWriteThroughput(t *testing.T) { var wg sync.WaitGroup numParallel := 100 - for i := range numParallel { - idx := i + for idx := range numParallel { wg.Go(func() { item := GenerateRandomItem() item.Scope = sst.Scope diff --git a/go/tracing/main.go b/go/tracing/main.go index 6bb88933..41996aee 100644 --- a/go/tracing/main.go +++ b/go/tracing/main.go @@ -27,8 +27,18 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" ) +// logrusOtelErrorHandler routes OpenTelemetry SDK errors through logrus so they +// appear in our structured log pipeline (and therefore in Honeycomb) instead of +// being silently written to Go's default logger. +type logrusOtelErrorHandler struct{} + +func (logrusOtelErrorHandler) Handle(err error) { + log.WithError(err).Warn("OpenTelemetry SDK error") +} + const instrumentationName = "github.com/overmindtech/workspace" // the following vars will be set during the build using `ldflags`, eg: @@ -226,19 +236,36 @@ func InitTracerWithUpstreams(component, honeycombApiKey, sentryDSN string, opts return InitTracer(component, opts...) } +// batcherOpts are the shared BatchSpanProcessor options applied to every +// exporter. A large queue (8192, 4x the default 2048) reduces the chance of +// silent span drops during burst load. We intentionally avoid WithBlocking() +// because it causes test suites to hang when no collector is reachable (the +// common case in CI). The 60s export timeout aligns with the OTLP HTTP +// exporter's 1-minute retry budget. +var batcherOpts = []sdktrace.BatchSpanProcessorOption{ + sdktrace.WithMaxQueueSize(8192), + sdktrace.WithExportTimeout(60 * time.Second), +} + func InitTracer(component string, opts ...otlptracehttp.Option) error { - client := otlptracehttp.NewClient(opts...) - otlpExp, err := otlptrace.New(context.Background(), client) + otel.SetErrorHandler(logrusOtelErrorHandler{}) + + otlpExp, err := otlptrace.New(context.Background(), otlptracehttp.NewClient(opts...)) if err != nil { return fmt.Errorf("creating OTLP trace exporter: %w", err) } - // Create unified sampler for health checks and otelpgx spans + healthExp, err := otlptrace.New(context.Background(), otlptracehttp.NewClient(opts...)) + if err != nil { + return fmt.Errorf("creating health OTLP trace exporter: %w", err) + } + overmindSampler := NewOvermindSampler() + res := tracingResource(component) tracerOpts := []sdktrace.TracerProviderOption{ - sdktrace.WithBatcher(otlpExp), - sdktrace.WithResource(tracingResource(component)), + sdktrace.WithBatcher(otlpExp, batcherOpts...), + sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(overmindSampler)), } if viper.GetBool("stdout-trace-dump") { @@ -246,35 +273,63 @@ func InitTracer(component string, opts ...otlptracehttp.Option) error { if err != nil { return err } - tracerOpts = append(tracerOpts, sdktrace.WithBatcher(stdoutExp)) + tracerOpts = append(tracerOpts, sdktrace.WithBatcher(stdoutExp, batcherOpts...)) } tp = sdktrace.NewTracerProvider(tracerOpts...) - // Set the default tracer provider for all the libraries otel.SetTracerProvider(tp) - tracerOpts = append(tracerOpts, sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased((0.1))))) - healthTp = sdktrace.NewTracerProvider(tracerOpts...) + healthTp = sdktrace.NewTracerProvider( + sdktrace.WithBatcher(healthExp, batcherOpts...), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.1))), + ) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) return nil } func ShutdownTracer(ctx context.Context) { - // Flush buffered events before the program terminates. defer sentry.Flush(5 * time.Second) - // detach from the parent's cancellation, and ensure that we do not wait - // indefinitely on the trace provider shutdown - ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() + + var g errgroup.Group + + // Do not nil healthTp or tp here: concurrent callers (e.g. health check + // probes via HealthCheckTracerProvider) would panic on the nil guard. + // Calling Shutdown on an already-shutdown provider is a safe no-op. + if healthTp != nil { + localTp := healthTp + g.Go(func() error { + if err := localTp.ForceFlush(ctx); err != nil { + log.WithContext(ctx).WithError(err).Error("Error flushing health tracer provider") + } + if err := localTp.Shutdown(ctx); err != nil { + log.WithContext(ctx).WithError(err).Error("Error shutting down health tracer provider") + return err + } + return nil + }) + } + if tp != nil { - if err := tp.ForceFlush(ctx); err != nil { - log.WithContext(ctx).WithError(err).Error("Error flushing tracer provider") - } - if err := tp.Shutdown(ctx); err != nil { - log.WithContext(ctx).WithError(err).Error("Error shutting down tracer provider") - } + localTp := tp + g.Go(func() error { + if err := localTp.ForceFlush(ctx); err != nil { + log.WithContext(ctx).WithError(err).Error("Error flushing tracer provider") + } + if err := localTp.Shutdown(ctx); err != nil { + log.WithContext(ctx).WithError(err).Error("Error shutting down tracer provider") + return err + } + return nil + }) + } + + if err := g.Wait(); err != nil { + log.WithContext(ctx).WithError(err).Error("Error during tracer shutdown") } log.WithContext(ctx).Trace("tracing has shut down") } diff --git a/go/tracing/main_test.go b/go/tracing/main_test.go index 26bf36cf..3d277079 100644 --- a/go/tracing/main_test.go +++ b/go/tracing/main_test.go @@ -1,7 +1,17 @@ package tracing import ( + "bytes" + "context" + "fmt" + "os" "testing" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" ) func TestTracingResource(t *testing.T) { @@ -10,3 +20,91 @@ func TestTracingResource(t *testing.T) { t.Error("Could not initialize tracing resource. Check the log!") } } + +func TestShutdownBothProviders(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + + tp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp)) + healthTp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp)) + + if tp == nil || healthTp == nil { + t.Fatal("expected both tp and healthTp to be non-nil after init") + } + + ShutdownTracer(context.Background()) + + // After shutdown, calling Shutdown again on the providers should be a + // safe no-op (the SDK guards with stopOnce). We do NOT nil the package + // vars because concurrent callers (e.g. health probes) would panic. + if err := tp.Shutdown(context.Background()); err != nil { + t.Errorf("second tp.Shutdown should be a no-op, got: %v", err) + } + if err := healthTp.Shutdown(context.Background()); err != nil { + t.Errorf("second healthTp.Shutdown should be a no-op, got: %v", err) + } +} + +func TestShutdownIdempotent(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + + tp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp)) + healthTp = sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp)) + + ShutdownTracer(context.Background()) + ShutdownTracer(context.Background()) // must not panic +} + +func TestErrorHandlerRegistered(t *testing.T) { + otel.SetErrorHandler(logrusOtelErrorHandler{}) + + var buf bytes.Buffer + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + + otel.Handle(fmt.Errorf("test SDK error")) + + if !bytes.Contains(buf.Bytes(), []byte("OpenTelemetry SDK error")) { + t.Errorf("expected logrus to contain 'OpenTelemetry SDK error', got: %s", buf.String()) + } + if !bytes.Contains(buf.Bytes(), []byte("test SDK error")) { + t.Errorf("expected logrus to contain the original error, got: %s", buf.String()) + } +} + +func TestBatcherOptsQueueSize(t *testing.T) { + found := false + for _, opt := range batcherOpts { + // Apply each option to a zero-value struct and check the result. + var o sdktrace.BatchSpanProcessorOptions + opt(&o) + if o.MaxQueueSize == 8192 { + found = true + } + } + if !found { + t.Error("batcherOpts should set MaxQueueSize to 8192") + } +} + +func TestInitTracerSetsErrorHandler(t *testing.T) { + // Use a deliberately broken endpoint so the exporter creation succeeds + // but no actual spans are shipped. + err := InitTracer("test-component", + otlptracehttp.WithEndpoint("localhost:0"), + otlptracehttp.WithInsecure(), + ) + if err != nil { + t.Fatalf("InitTracer failed: %v", err) + } + t.Cleanup(func() { ShutdownTracer(context.Background()) }) + + var buf bytes.Buffer + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + + otel.Handle(fmt.Errorf("custom test error")) + + if !bytes.Contains(buf.Bytes(), []byte("OpenTelemetry SDK error")) { + t.Errorf("after InitTracer, OTel errors should be routed to logrus; got: %s", buf.String()) + } +} diff --git a/sources/.cursor/BUGBOT.md b/sources/.cursor/BUGBOT.md index 118360c6..a3f4363a 100644 --- a/sources/.cursor/BUGBOT.md +++ b/sources/.cursor/BUGBOT.md @@ -7,4 +7,21 @@ When reviewing newly created adapters, it is extremely important to ensure that There are also a couple of generic types that we should always create links for if the attributes are there. These are: * `ip`: Any attribute that would contain an IP address should create a LinkedItemQueries for an `ip` type. This should always use the scope of global, the method of GET and a query of the IP address itself -* `dns`: any attribute that contains a DNS name should create a LinkedItemQueries for a DNS type. The type should be `dns`, scope `global`, method SEARCH with the query being the DNS name itself \ No newline at end of file +* `dns`: any attribute that contains a DNS name should create a LinkedItemQueries for a DNS type. The type should be `dns`, scope `global`, method SEARCH with the query being the DNS name itself + +## IAMPermissions and PredefinedRole + +Every adapter must implement both `IAMPermissions()` and `PredefinedRole()`: + +* `IAMPermissions()` must return at least one permission string following the pattern `Microsoft.{Provider}/{resourcePath}/read`. The resource path must match the ARM resource type for the resource being adapted. For child resources, include the full path (e.g., `Microsoft.Batch/batchAccounts/applications/versions/read`, not just `Microsoft.Batch/batchAccounts/read`). The method should have a comment linking to the relevant Azure RBAC resource provider operations page. +* `PredefinedRole()` must return a non-empty string naming a valid Azure built-in role. If the service area has a specific reader role (e.g., `"Azure Batch Account Reader"` for Batch, `"Storage Blob Data Reader"` for Storage), use that. Otherwise, `"Reader"` is acceptable as the most restrictive general role. + +Flag any adapter missing either method, returning empty values, or using an incorrect resource provider path. + +## Azure ARM Get/List options + +For Azure adapters, only pass `*Options` fields (for example `$expand`) that the REST API for that resource and API version documents. Unsupported or mistyped query parameters can surface as `400 Bad Request` from malformed URLs. When in doubt, prefer `nil` options or cross-check the official REST reference for the operation. + +## PotentialLinks Completeness + +`PotentialLinks()` must include every resource type that appears in any `LinkedItemQuery` returned by the adapter's conversion function. If the adapter creates linked item queries for IP addresses, `PotentialLinks()` must include `stdlib.NetworkIP`. If it creates queries for DNS names, `PotentialLinks()` must include `stdlib.NetworkDNS`. Missing entries in `PotentialLinks()` break the Overmind dependency graph — linked items won't be discovered even though the queries exist in the adapter's output. diff --git a/sources/azure/clients/batch-application-package-client.go b/sources/azure/clients/batch-application-package-client.go new file mode 100644 index 00000000..39c4d350 --- /dev/null +++ b/sources/azure/clients/batch-application-package-client.go @@ -0,0 +1,35 @@ +package clients + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/batch/armbatch/v4" +) + +//go:generate mockgen -destination=../shared/mocks/mock_batch_application_package_client.go -package=mocks -source=batch-application-package-client.go + +// BatchApplicationPackagesPager is a type alias for the generic Pager interface with batch application package response type. +type BatchApplicationPackagesPager = Pager[armbatch.ApplicationPackageClientListResponse] + +// BatchApplicationPackagesClient is an interface for interacting with Azure Batch application packages. +type BatchApplicationPackagesClient interface { + Get(ctx context.Context, resourceGroupName string, accountName string, applicationName string, versionName string) (armbatch.ApplicationPackageClientGetResponse, error) + List(ctx context.Context, resourceGroupName string, accountName string, applicationName string) BatchApplicationPackagesPager +} + +type batchApplicationPackagesClient struct { + client *armbatch.ApplicationPackageClient +} + +func (c *batchApplicationPackagesClient) Get(ctx context.Context, resourceGroupName string, accountName string, applicationName string, versionName string) (armbatch.ApplicationPackageClientGetResponse, error) { + return c.client.Get(ctx, resourceGroupName, accountName, applicationName, versionName, nil) +} + +func (c *batchApplicationPackagesClient) List(ctx context.Context, resourceGroupName string, accountName string, applicationName string) BatchApplicationPackagesPager { + return c.client.NewListPager(resourceGroupName, accountName, applicationName, nil) +} + +// NewBatchApplicationPackagesClient creates a new BatchApplicationPackagesClient from the Azure SDK client. +func NewBatchApplicationPackagesClient(client *armbatch.ApplicationPackageClient) BatchApplicationPackagesClient { + return &batchApplicationPackagesClient{client: client} +} diff --git a/sources/azure/clients/dbforpostgresql-flexible-server-backup-client.go b/sources/azure/clients/dbforpostgresql-flexible-server-backup-client.go new file mode 100644 index 00000000..e6f21ee4 --- /dev/null +++ b/sources/azure/clients/dbforpostgresql-flexible-server-backup-client.go @@ -0,0 +1,32 @@ +package clients + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" +) + +//go:generate mockgen -destination=../shared/mocks/mock_dbforpostgresql_flexible_server_backup_client.go -package=mocks -source=dbforpostgresql-flexible-server-backup-client.go + +type DBforPostgreSQLFlexibleServerBackupPager = Pager[armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse] + +type DBforPostgreSQLFlexibleServerBackupClient interface { + ListByServer(ctx context.Context, resourceGroupName string, serverName string) DBforPostgreSQLFlexibleServerBackupPager + Get(ctx context.Context, resourceGroupName string, serverName string, backupName string) (armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse, error) +} + +type dbforPostgreSQLFlexibleServerBackupClient struct { + client *armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClient +} + +func (a *dbforPostgreSQLFlexibleServerBackupClient) ListByServer(ctx context.Context, resourceGroupName string, serverName string) DBforPostgreSQLFlexibleServerBackupPager { + return a.client.NewListByServerPager(resourceGroupName, serverName, nil) +} + +func (a *dbforPostgreSQLFlexibleServerBackupClient) Get(ctx context.Context, resourceGroupName string, serverName string, backupName string) (armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse, error) { + return a.client.Get(ctx, resourceGroupName, serverName, backupName, nil) +} + +func NewDBforPostgreSQLFlexibleServerBackupClient(client *armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClient) DBforPostgreSQLFlexibleServerBackupClient { + return &dbforPostgreSQLFlexibleServerBackupClient{client: client} +} diff --git a/sources/azure/clients/disk-accesses-client.go b/sources/azure/clients/disk-accesses-client.go index 5f7a1b20..f21b3892 100644 --- a/sources/azure/clients/disk-accesses-client.go +++ b/sources/azure/clients/disk-accesses-client.go @@ -33,4 +33,4 @@ func (a *diskAccessesClient) Get(ctx context.Context, resourceGroupName string, // NewDiskAccessesClient creates a new DiskAccessesClient from the Azure SDK client func NewDiskAccessesClient(client *armcompute.DiskAccessesClient) DiskAccessesClient { return &diskAccessesClient{client: client} -} \ No newline at end of file +} diff --git a/sources/azure/clients/flow-logs-client.go b/sources/azure/clients/flow-logs-client.go new file mode 100644 index 00000000..9c7414f4 --- /dev/null +++ b/sources/azure/clients/flow-logs-client.go @@ -0,0 +1,35 @@ +package clients + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" +) + +//go:generate mockgen -destination=../shared/mocks/mock_flow_logs_client.go -package=mocks -source=flow-logs-client.go + +// FlowLogsPager is a type alias for the generic Pager interface with flow logs list response type. +type FlowLogsPager = Pager[armnetwork.FlowLogsClientListResponse] + +// FlowLogsClient is an interface for interacting with Azure flow logs (child of network watcher). +type FlowLogsClient interface { + Get(ctx context.Context, resourceGroupName string, networkWatcherName string, flowLogName string, options *armnetwork.FlowLogsClientGetOptions) (armnetwork.FlowLogsClientGetResponse, error) + NewListPager(resourceGroupName string, networkWatcherName string, options *armnetwork.FlowLogsClientListOptions) FlowLogsPager +} + +type flowLogsClient struct { + client *armnetwork.FlowLogsClient +} + +func (a *flowLogsClient) Get(ctx context.Context, resourceGroupName string, networkWatcherName string, flowLogName string, options *armnetwork.FlowLogsClientGetOptions) (armnetwork.FlowLogsClientGetResponse, error) { + return a.client.Get(ctx, resourceGroupName, networkWatcherName, flowLogName, options) +} + +func (a *flowLogsClient) NewListPager(resourceGroupName string, networkWatcherName string, options *armnetwork.FlowLogsClientListOptions) FlowLogsPager { + return a.client.NewListPager(resourceGroupName, networkWatcherName, options) +} + +// NewFlowLogsClient creates a new FlowLogsClient from the Azure SDK client. +func NewFlowLogsClient(client *armnetwork.FlowLogsClient) FlowLogsClient { + return &flowLogsClient{client: client} +} diff --git a/sources/azure/clients/load-balancer-frontend-ip-configurations-client.go b/sources/azure/clients/load-balancer-frontend-ip-configurations-client.go new file mode 100644 index 00000000..ddae77f7 --- /dev/null +++ b/sources/azure/clients/load-balancer-frontend-ip-configurations-client.go @@ -0,0 +1,35 @@ +package clients + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" +) + +//go:generate mockgen -destination=../shared/mocks/mock_load_balancer_frontend_ip_configurations_client.go -package=mocks -source=load-balancer-frontend-ip-configurations-client.go + +// LoadBalancerFrontendIPConfigurationsPager is a type alias for the generic Pager interface. +type LoadBalancerFrontendIPConfigurationsPager = Pager[armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse] + +// LoadBalancerFrontendIPConfigurationsClient is an interface for interacting with Azure load balancer frontend IP configurations. +type LoadBalancerFrontendIPConfigurationsClient interface { + Get(ctx context.Context, resourceGroupName string, loadBalancerName string, frontendIPConfigurationName string) (armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse, error) + NewListPager(resourceGroupName string, loadBalancerName string) LoadBalancerFrontendIPConfigurationsPager +} + +type loadBalancerFrontendIPConfigurationsClient struct { + client *armnetwork.LoadBalancerFrontendIPConfigurationsClient +} + +func (a *loadBalancerFrontendIPConfigurationsClient) Get(ctx context.Context, resourceGroupName string, loadBalancerName string, frontendIPConfigurationName string) (armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse, error) { + return a.client.Get(ctx, resourceGroupName, loadBalancerName, frontendIPConfigurationName, nil) +} + +func (a *loadBalancerFrontendIPConfigurationsClient) NewListPager(resourceGroupName string, loadBalancerName string) LoadBalancerFrontendIPConfigurationsPager { + return a.client.NewListPager(resourceGroupName, loadBalancerName, nil) +} + +// NewLoadBalancerFrontendIPConfigurationsClient creates a new LoadBalancerFrontendIPConfigurationsClient from the Azure SDK client. +func NewLoadBalancerFrontendIPConfigurationsClient(client *armnetwork.LoadBalancerFrontendIPConfigurationsClient) LoadBalancerFrontendIPConfigurationsClient { + return &loadBalancerFrontendIPConfigurationsClient{client: client} +} diff --git a/sources/azure/integration-tests/README.md b/sources/azure/integration-tests/README.md index 02f47288..adb86869 100644 --- a/sources/azure/integration-tests/README.md +++ b/sources/azure/integration-tests/README.md @@ -63,6 +63,21 @@ The `Setup` and `Teardown` methods are idempotent, meaning they can be run multi We can easily run all `Setup` tests to create resources, then run all `Run` tests to execute the actual tests, and finally run all `Teardown` tests to clean up resources. +**Run after Setup:** `Run` subtests skip with a clear message when `Setup` did not complete successfully (for example Setup was skipped, failed, or you ran only `Run` without a prior successful Setup). That avoids noisy failures that are not adapter bugs. + +### Skips, quotas, and slow Azure operations + +Some tests intentionally call `t.Skip` for Azure conditions that are external to adapter correctness, for example: + +- Batch account quota exhaustion (`SubscriptionQuotaExceeded`) +- Transient VM/VMSS control-plane conflicts where create returns `409` but `Get` still cannot retrieve the resource +- **Gallery application version** (`compute-gallery-application-version_test.go`): requires env vars `AZURE_TEST_GALLERY_NAME`, `AZURE_TEST_GALLERY_APPLICATION_NAME`, and `AZURE_TEST_GALLERY_APPLICATION_VERSION` pointing at an existing gallery application version; if the version is missing (`404`), the test skips after preflight +- **Role assignments** (`authorization-role-assignment_test.go`): may wait for RBAC eventual consistency before asserting adapter behaviour + +This keeps integration runs stable without hiding adapter bugs. + +Also note that PostgreSQL Flexible Server creation and Key Vault purge/recreate can take many minutes. If a run times out, increase `go test -timeout` (for example `-timeout 30m`) before assuming the test is stuck. + From the `sources/azure/integration-tests` directory: For building up the infra for the Compute API resources. @@ -81,4 +96,47 @@ For tearing down the infra for the Compute API resources. ```bash go test ./integration-tests -run "TestCompute.*/Teardown" -v -``` \ No newline at end of file +``` + +## Running Integration Tests via Cloud Agents + +Cursor Cloud Agents can run Azure integration tests autonomously when configured with the correct credentials. + +### Prerequisites + +1. **1Password vault**: Azure credentials are stored in the "cursor" 1Password vault under the item "Azure Integration Tests" +2. **Cursor Cloud Agent secret**: Configure only `OP_SERVICE_ACCOUNT_TOKEN` in `https://cursor.com/dashboard/cloud-agents` +3. **Repo env files**: `op.azure-cloud-agent.secret` and `op.azure-cloud-agent.env` exist with required `op://...` references + +### How it works + +When a Cloud Agent picks up a Linear issue to create an Azure adapter: + +1. Cursor injects `OP_SERVICE_ACCOUNT_TOKEN` into the Cloud Agent environment +2. `inject-secrets` reads `op://...` references from env files using the 1Password SDK +3. `inject-secrets` writes resolved values to a local env file +4. The shell sources that file before test execution +5. The `DefaultAzureCredential` chain picks up `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, and `AZURE_TENANT_ID` from environment +6. Integration tests use `AZURE_SUBSCRIPTION_ID` and `RUN_AZURE_INTEGRATION_TESTS=true` + +To inject credentials manually (e.g. for debugging), run: + +```bash +go run build/inject-secrets/main.go \ + --no-ping \ + --secret-file .github/env/op.azure-cloud-agent.secret \ + --env-file .github/env/op.azure-cloud-agent.env \ + --output-file .env.azure-cloud-agent + +set -a +source .env.azure-cloud-agent +set +a +``` + +### Security + +- The service principal has **read-write access** scoped to the integration test subscription only +- Cloud Agent dashboard stores only the bootstrap token (`OP_SERVICE_ACCOUNT_TOKEN`) +- Azure credentials remain in 1Password and are resolved only at runtime via `inject-secrets` +- All test resources are created in the `overmind-integration-tests` resource group +- Teardown steps clean up created resources after each test run diff --git a/sources/azure/integration-tests/authorization-role-assignment_test.go b/sources/azure/integration-tests/authorization-role-assignment_test.go index 300d5588..6e3b5ea5 100644 --- a/sources/azure/integration-tests/authorization-role-assignment_test.go +++ b/sources/azure/integration-tests/authorization-role-assignment_test.go @@ -9,6 +9,7 @@ import ( "os/exec" "strings" "testing" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" @@ -26,6 +27,8 @@ import ( "github.com/overmindtech/cli/sources/shared" ) +var errRoleAssignmentConflictWithoutResource = errors.New("role assignment create returned conflict but role assignment could not be retrieved") + func TestAuthorizationRoleAssignmentIntegration(t *testing.T) { subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") if subscriptionID == "" { @@ -70,6 +73,7 @@ func TestAuthorizationRoleAssignmentIntegration(t *testing.T) { // Generate unique role assignment name (GUID) roleAssignmentName := uuid.New().String() azureScope := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s", subscriptionID, integrationTestResourceGroup) + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -83,13 +87,25 @@ func TestAuthorizationRoleAssignmentIntegration(t *testing.T) { // Create role assignment at resource group scope err = createRoleAssignment(ctx, roleAssignmentsClient, azureScope, roleAssignmentName, principalID, readerRoleDefinitionID) if err != nil { + if errors.Is(err, errRoleAssignmentConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure role-assignment control-plane conflict: %v", err) + } t.Fatalf("Failed to create role assignment: %v", err) } + err = waitForRoleAssignmentAvailable(ctx, roleAssignmentsClient, azureScope, roleAssignmentName) + if err != nil { + t.Fatalf("Failed waiting for role assignment to be available: %v", err) + } + setupCompleted = true log.Printf("Created role assignment %s at scope %s", roleAssignmentName, azureScope) }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetRoleAssignment", func(t *testing.T) { ctx := t.Context() @@ -387,8 +403,16 @@ func createRoleAssignment(ctx context.Context, client *armauthorization.RoleAssi var respErr *azcore.ResponseError if errors.As(err, &respErr) { if respErr.StatusCode == http.StatusConflict { - log.Printf("Role assignment %s already exists (conflict), skipping creation", roleAssignmentName) - return nil + existing, getErr := client.Get(ctx, scope, roleAssignmentName, nil) + if getErr == nil && existing.RoleAssignment.ID != nil && *existing.RoleAssignment.ID != "" { + log.Printf("Role assignment %s already exists (conflict), verified readable, skipping creation", roleAssignmentName) + return nil + } + var getRespErr *azcore.ResponseError + if errors.As(getErr, &getRespErr) && getRespErr.StatusCode == http.StatusNotFound { + return fmt.Errorf("%w: scope=%s roleAssignmentName=%s", errRoleAssignmentConflictWithoutResource, scope, roleAssignmentName) + } + return fmt.Errorf("role assignment conflict for %s and failed to verify existing role assignment: %w", roleAssignmentName, getErr) } if respErr.StatusCode == http.StatusForbidden { return fmt.Errorf("insufficient permissions to create role assignment: %w", err) @@ -421,3 +445,23 @@ func deleteRoleAssignment(ctx context.Context, client *armauthorization.RoleAssi log.Printf("Role assignment %s deleted successfully", roleAssignmentName) return nil } + +func waitForRoleAssignmentAvailable(ctx context.Context, client *armauthorization.RoleAssignmentsClient, scope, roleAssignmentName string) error { + maxAttempts := 20 + pollInterval := 5 * time.Second + + for attempt := 1; attempt <= maxAttempts; attempt++ { + _, err := client.Get(ctx, scope, roleAssignmentName, nil) + if err == nil { + return nil + } + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + time.Sleep(pollInterval) + continue + } + return fmt.Errorf("error checking role assignment availability: %w", err) + } + + return fmt.Errorf("timeout waiting for role assignment %s to be available", roleAssignmentName) +} diff --git a/sources/azure/integration-tests/batch-batch-accounts_test.go b/sources/azure/integration-tests/batch-batch-accounts_test.go index 4a2c02a2..421adde3 100644 --- a/sources/azure/integration-tests/batch-batch-accounts_test.go +++ b/sources/azure/integration-tests/batch-batch-accounts_test.go @@ -31,6 +31,8 @@ const ( integrationTestSANameForBatch = "ovm-integ-test-sa-batch" ) +var errBatchQuotaExceeded = errors.New("azure batch quota exceeded") + func TestBatchAccountIntegration(t *testing.T) { subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") if subscriptionID == "" { @@ -62,6 +64,7 @@ func TestBatchAccountIntegration(t *testing.T) { // Generate unique names (batch account names must be globally unique, 3-24 chars, lowercase alphanumeric) batchAccountName := generateBatchAccountName(integrationTestBatchAccountName) storageAccountName := generateStorageAccountName(integrationTestSANameForBatch) + setupCompleted := false var storageAccountID string @@ -96,6 +99,9 @@ func TestBatchAccountIntegration(t *testing.T) { // Create batch account err = createBatchAccount(ctx, batchClient, integrationTestResourceGroup, batchAccountName, integrationTestLocation, storageAccountID) if err != nil { + if errors.Is(err, errBatchQuotaExceeded) { + t.Skipf("Skipping Batch account integration test due to Azure subscription quota: %v", err) + } t.Fatalf("Failed to create batch account: %v", err) } @@ -104,9 +110,14 @@ func TestBatchAccountIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed waiting for batch account to be available: %v", err) } + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetBatchAccount", func(t *testing.T) { ctx := t.Context() @@ -334,15 +345,24 @@ func createBatchAccount(ctx context.Context, client *armbatch.AccountClient, res if err != nil { // Check if batch account already exists (conflict) var respErr *azcore.ResponseError - if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { - log.Printf("Batch account %s already exists (conflict), skipping creation", accountName) - return nil + if errors.As(err, &respErr) { + if respErr.StatusCode == http.StatusConflict { + log.Printf("Batch account %s already exists (conflict), skipping creation", accountName) + return nil + } + if respErr.ErrorCode == "SubscriptionQuotaExceeded" { + return fmt.Errorf("%w: %s", errBatchQuotaExceeded, respErr.Error()) + } } return fmt.Errorf("failed to begin creating batch account: %w", err) } resp, err := poller.PollUntilDone(ctx, nil) if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.ErrorCode == "SubscriptionQuotaExceeded" { + return fmt.Errorf("%w: %s", errBatchQuotaExceeded, respErr.Error()) + } return fmt.Errorf("failed to create batch account: %w", err) } diff --git a/sources/azure/integration-tests/batch-batch-application-package_test.go b/sources/azure/integration-tests/batch-batch-application-package_test.go new file mode 100644 index 00000000..d08b4424 --- /dev/null +++ b/sources/azure/integration-tests/batch-batch-application-package_test.go @@ -0,0 +1,416 @@ +package integrationtests + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/batch/armbatch/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3" + log "github.com/sirupsen/logrus" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +const ( + integrationTestBatchAppPkgAccountName = "ovm-integ-test-sa-pkg" + integrationTestBatchAppPkgBatchName = "ovm-integ-test-pkg" + integrationTestBatchAppName = "ovm-integ-test-app" + integrationTestBatchAppPkgVersion = "1.0" +) + +func TestBatchApplicationPackageIntegration(t *testing.T) { + subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + if subscriptionID == "" { + t.Skip("AZURE_SUBSCRIPTION_ID environment variable not set") + } + + cred, err := azureshared.NewAzureCredential(t.Context()) + if err != nil { + t.Fatalf("Failed to create Azure credential: %v", err) + } + + rgClient, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Resource Groups client: %v", err) + } + + saClient, err := armstorage.NewAccountsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Storage Accounts client: %v", err) + } + + batchAccountClient, err := armbatch.NewAccountClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Batch Account client: %v", err) + } + + batchAppClient, err := armbatch.NewApplicationClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Batch Application client: %v", err) + } + + batchAppPkgClient, err := armbatch.NewApplicationPackageClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Batch Application Package client: %v", err) + } + + storageAccountName := generateStorageAccountName(integrationTestBatchAppPkgAccountName) + batchAccountName := generateBatchAccountName(integrationTestBatchAppPkgBatchName) + setupCompleted := false + + t.Run("Setup", func(t *testing.T) { + ctx := t.Context() + + err := createResourceGroup(ctx, rgClient, integrationTestResourceGroup, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create resource group: %v", err) + } + + err = createStorageAccount(ctx, saClient, integrationTestResourceGroup, storageAccountName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create storage account: %v", err) + } + + err = waitForStorageAccountAvailable(ctx, saClient, integrationTestResourceGroup, storageAccountName) + if err != nil { + t.Fatalf("Failed waiting for storage account: %v", err) + } + + saResp, err := saClient.GetProperties(ctx, integrationTestResourceGroup, storageAccountName, nil) + if err != nil { + t.Fatalf("Failed to get storage account properties: %v", err) + } + storageAccountID := *saResp.ID + + err = createBatchAccount(ctx, batchAccountClient, integrationTestResourceGroup, batchAccountName, integrationTestLocation, storageAccountID) + if err != nil { + if errors.Is(err, errBatchQuotaExceeded) { + t.Skipf("Skipping Batch application package integration test due to Azure subscription quota: %v", err) + } + t.Fatalf("Failed to create batch account: %v", err) + } + + err = waitForBatchAccountAvailable(ctx, batchAccountClient, integrationTestResourceGroup, batchAccountName) + if err != nil { + t.Fatalf("Failed waiting for batch account: %v", err) + } + + err = createBatchApplication(ctx, batchAppClient, integrationTestResourceGroup, batchAccountName, integrationTestBatchAppName) + if err != nil { + t.Fatalf("Failed to create batch application: %v", err) + } + + err = createBatchApplicationPackage(ctx, batchAppPkgClient, integrationTestResourceGroup, batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + if err != nil { + t.Fatalf("Failed to create batch application package: %v", err) + } + setupCompleted = true + }) + + t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + + t.Run("GetApplicationPackage", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewBatchBatchApplicationPackage( + clients.NewBatchApplicationPackagesClient(batchAppPkgClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem == nil { + t.Fatalf("Expected sdpItem to be non-nil") + } + + expectedUnique := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + uniqueAttrKey := sdpItem.GetUniqueAttribute() + uniqueAttrValue, err := sdpItem.GetAttributes().Get(uniqueAttrKey) + if err != nil { + t.Fatalf("Failed to get unique attribute: %v", err) + } + + if uniqueAttrValue != expectedUnique { + t.Errorf("Expected unique attribute value %s, got %s", expectedUnique, uniqueAttrValue) + } + + log.Printf("Successfully retrieved application package %s", integrationTestBatchAppPkgVersion) + }) + + t.Run("SearchApplicationPackages", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewBatchBatchApplicationPackage( + clients.NewBatchApplicationPackagesClient(batchAppPkgClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + searchQuery := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName) + sdpItems, err := searchable.Search(ctx, scope, searchQuery, true) + if err != nil { + t.Fatalf("Failed to search application packages: %v", err) + } + + if len(sdpItems) < 1 { + t.Fatalf("Expected at least one application package, got %d", len(sdpItems)) + } + + expectedUnique := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + var found bool + for _, item := range sdpItems { + uniqueAttrKey := item.GetUniqueAttribute() + if v, getErr := item.GetAttributes().Get(uniqueAttrKey); getErr == nil && v == expectedUnique { + found = true + break + } + } + + if !found { + t.Fatalf("Expected to find application package %s in search results", integrationTestBatchAppPkgVersion) + } + + log.Printf("Found %d application packages in search results", len(sdpItems)) + }) + + t.Run("VerifyLinkedItems", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewBatchBatchApplicationPackage( + clients.NewBatchApplicationPackagesClient(batchAppPkgClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + linkedQueries := sdpItem.GetLinkedItemQueries() + if len(linkedQueries) == 0 { + t.Fatalf("Expected linked item queries, but got none") + } + + for _, liq := range linkedQueries { + q := liq.GetQuery() + if q.GetType() == "" { + t.Error("Linked item query has empty Type") + } + if q.GetQuery() == "" { + t.Error("Linked item query has empty Query") + } + if q.GetScope() == "" { + t.Error("Linked item query has empty Scope") + } + if q.GetMethod() != sdp.QueryMethod_GET && q.GetMethod() != sdp.QueryMethod_SEARCH { + t.Errorf("Linked item query has invalid Method: %s", q.GetMethod()) + } + } + + // Verify parent application link exists + var hasAppLink bool + for _, liq := range linkedQueries { + if liq.GetQuery().GetType() == azureshared.BatchBatchApplication.String() { + hasAppLink = true + break + } + } + if !hasAppLink { + t.Error("Expected linked query to parent BatchBatchApplication, but didn't find one") + } + + // Verify parent account link exists + var hasAccountLink bool + for _, liq := range linkedQueries { + if liq.GetQuery().GetType() == azureshared.BatchBatchAccount.String() { + hasAccountLink = true + break + } + } + if !hasAccountLink { + t.Error("Expected linked query to parent BatchBatchAccount, but didn't find one") + } + + log.Printf("Verified %d linked item queries", len(linkedQueries)) + }) + + t.Run("VerifyItemAttributes", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewBatchBatchApplicationPackage( + clients.NewBatchApplicationPackagesClient(batchAppPkgClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.BatchBatchApplicationPackage.String() { + t.Errorf("Expected type %s, got %s", azureshared.BatchBatchApplicationPackage.String(), sdpItem.GetType()) + } + + expectedScope := subscriptionID + "." + integrationTestResourceGroup + if sdpItem.GetScope() != expectedScope { + t.Errorf("Expected scope %s, got %s", expectedScope, sdpItem.GetScope()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Item validation failed: %v", err) + } + }) + }) + + t.Run("Teardown", func(t *testing.T) { + ctx := t.Context() + + err := deleteBatchApplicationPackage(ctx, batchAppPkgClient, integrationTestResourceGroup, batchAccountName, integrationTestBatchAppName, integrationTestBatchAppPkgVersion) + if err != nil { + t.Logf("Warning: failed to delete application package: %v", err) + } + + err = deleteBatchApplication(ctx, batchAppClient, integrationTestResourceGroup, batchAccountName, integrationTestBatchAppName) + if err != nil { + t.Logf("Warning: failed to delete batch application: %v", err) + } + + err = deleteBatchAccount(ctx, batchAccountClient, integrationTestResourceGroup, batchAccountName) + if err != nil { + t.Logf("Warning: failed to delete batch account: %v", err) + } + + err = deleteStorageAccount(ctx, saClient, integrationTestResourceGroup, storageAccountName) + if err != nil { + t.Logf("Warning: failed to delete storage account: %v", err) + } + }) +} + +func createBatchApplication(ctx context.Context, client *armbatch.ApplicationClient, resourceGroupName, accountName, applicationName string) error { + _, err := client.Get(ctx, resourceGroupName, accountName, applicationName, nil) + if err == nil { + log.Printf("Batch application %s already exists, skipping creation", applicationName) + return nil + } + + allowUpdates := true + _, err = client.Create(ctx, resourceGroupName, accountName, applicationName, armbatch.Application{ + Properties: &armbatch.ApplicationProperties{ + DisplayName: new("Integration Test Application"), + AllowUpdates: &allowUpdates, + }, + }, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("Batch application %s already exists (conflict), skipping creation", applicationName) + return nil + } + return fmt.Errorf("failed to create batch application: %w", err) + } + + log.Printf("Batch application %s created successfully", applicationName) + return nil +} + +func createBatchApplicationPackage(ctx context.Context, client *armbatch.ApplicationPackageClient, resourceGroupName, accountName, applicationName, versionName string) error { + _, err := client.Get(ctx, resourceGroupName, accountName, applicationName, versionName, nil) + if err == nil { + log.Printf("Batch application package %s already exists, skipping creation", versionName) + return nil + } + + _, err = client.Create(ctx, resourceGroupName, accountName, applicationName, versionName, armbatch.ApplicationPackage{}, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("Batch application package %s already exists (conflict), skipping creation", versionName) + return nil + } + return fmt.Errorf("failed to create batch application package: %w", err) + } + + log.Printf("Batch application package %s created successfully", versionName) + + // Wait briefly for the package to become available + maxAttempts := 10 + for attempt := 1; attempt <= maxAttempts; attempt++ { + _, getErr := client.Get(ctx, resourceGroupName, accountName, applicationName, versionName, nil) + if getErr == nil { + return nil + } + time.Sleep(2 * time.Second) + } + + return nil +} + +func deleteBatchApplicationPackage(ctx context.Context, client *armbatch.ApplicationPackageClient, resourceGroupName, accountName, applicationName, versionName string) error { + _, err := client.Delete(ctx, resourceGroupName, accountName, applicationName, versionName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Batch application package %s not found, skipping deletion", versionName) + return nil + } + return fmt.Errorf("failed to delete batch application package: %w", err) + } + + log.Printf("Batch application package %s deleted successfully", versionName) + return nil +} + +func deleteBatchApplication(ctx context.Context, client *armbatch.ApplicationClient, resourceGroupName, accountName, applicationName string) error { + _, err := client.Delete(ctx, resourceGroupName, accountName, applicationName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Batch application %s not found, skipping deletion", applicationName) + return nil + } + return fmt.Errorf("failed to delete batch application: %w", err) + } + + log.Printf("Batch application %s deleted successfully", applicationName) + return nil +} diff --git a/sources/azure/integration-tests/compute-availability-set_test.go b/sources/azure/integration-tests/compute-availability-set_test.go index 76f6f232..0b567e18 100644 --- a/sources/azure/integration-tests/compute-availability-set_test.go +++ b/sources/azure/integration-tests/compute-availability-set_test.go @@ -74,6 +74,7 @@ func TestComputeAvailabilitySetIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed to create Network Interfaces client: %v", err) } + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -128,6 +129,9 @@ func TestComputeAvailabilitySetIntegration(t *testing.T) { // Create virtual machine with availability set err = createVirtualMachineWithAvailabilitySet(ctx, vmClient, integrationTestResourceGroup, integrationTestVMForAVSetName, integrationTestLocation, *nicResp.ID, *avSetResp.ID) if err != nil { + if errors.Is(err, errVMConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure VM control-plane conflict: %v", err) + } t.Fatalf("Failed to create virtual machine: %v", err) } @@ -139,9 +143,14 @@ func TestComputeAvailabilitySetIntegration(t *testing.T) { // Wait a bit for the availability set to reflect the VM association time.Sleep(10 * time.Second) + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + // Check if availability set exists - if Setup failed, skip Run tests ctx := t.Context() _, err := avSetClient.Get(ctx, integrationTestResourceGroup, integrationTestAvailabilitySetName, nil) @@ -620,8 +629,20 @@ func createVirtualMachineWithAvailabilitySet(ctx context.Context, client *armcom // Check if VM already exists (conflict) var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { - log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) - return nil + existing, getErr := client.Get(ctx, resourceGroupName, vmName, nil) + if getErr == nil { + if existing.Properties != nil && existing.Properties.ProvisioningState != nil { + log.Printf("Virtual machine %s already exists (conflict) with state %s, skipping creation", vmName, *existing.Properties.ProvisioningState) + } else { + log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) + } + return nil + } + var getRespErr *azcore.ResponseError + if errors.As(getErr, &getRespErr) && getRespErr.StatusCode == http.StatusNotFound { + return fmt.Errorf("%w: vm=%s resourceGroup=%s", errVMConflictWithoutResource, vmName, resourceGroupName) + } + return fmt.Errorf("vm creation conflict for %s and failed to verify existing VM: %w", vmName, getErr) } return fmt.Errorf("failed to begin creating virtual machine: %w", err) } diff --git a/sources/azure/integration-tests/compute-disk-encryption-set_test.go b/sources/azure/integration-tests/compute-disk-encryption-set_test.go index e86b55a2..0d9200af 100644 --- a/sources/azure/integration-tests/compute-disk-encryption-set_test.go +++ b/sources/azure/integration-tests/compute-disk-encryption-set_test.go @@ -66,6 +66,7 @@ func TestComputeDiskEncryptionSetIntegration(t *testing.T) { var keyURL string var identityResourceID string var identityPrincipalID string + var setupCompleted bool t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -89,6 +90,14 @@ func TestComputeDiskEncryptionSetIntegration(t *testing.T) { if vault.ID == nil || *vault.ID == "" { t.Fatalf("Key Vault ID is nil/empty") } + if vault.Properties == nil || vault.Properties.EnablePurgeProtection == nil || !*vault.Properties.EnablePurgeProtection { + t.Skipf( + "Disk Encryption Set integration requires Key Vault purge protection enabled on %s; enable it once with: az keyvault update --name %s --resource-group %s --enable-purge-protection true", + integrationTestKeyVaultName, + integrationTestKeyVaultName, + integrationTestResourceGroup, + ) + } vaultID = *vault.ID // Ensure a user-assigned identity exists (shared with other tests) @@ -131,9 +140,15 @@ func TestComputeDiskEncryptionSetIntegration(t *testing.T) { if err := waitForDiskEncryptionSetAvailable(ctx, desClient, integrationTestResourceGroup, integrationTestDiskEncryptionSetName); err != nil { t.Fatalf("Failed waiting for Disk Encryption Set to be available: %v", err) } + + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetDiskEncryptionSet", func(t *testing.T) { ctx := t.Context() @@ -361,7 +376,7 @@ func createDiskEncryptionSet(ctx context.Context, client *armcompute.DiskEncrypt Identity: &armcompute.EncryptionSetIdentity{ Type: new(armcompute.DiskEncryptionSetIdentityTypeUserAssigned), UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ - userAssignedIdentityResourceID: &armcompute.UserAssignedIdentitiesValue{}, + userAssignedIdentityResourceID: {}, }, }, Properties: &armcompute.EncryptionSetProperties{ diff --git a/sources/azure/integration-tests/compute-gallery-application-version_test.go b/sources/azure/integration-tests/compute-gallery-application-version_test.go index 28959583..063cf3b4 100644 --- a/sources/azure/integration-tests/compute-gallery-application-version_test.go +++ b/sources/azure/integration-tests/compute-gallery-application-version_test.go @@ -1,10 +1,13 @@ package integrationtests import ( + "errors" "fmt" + "net/http" "os" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" log "github.com/sirupsen/logrus" @@ -78,6 +81,15 @@ func TestComputeGalleryApplicationVersionIntegration(t *testing.T) { t.Fatalf("Failed to create/verify resource group: %v", err) } + _, getErr := galleryApplicationVersionsClient.Get(ctx, resourceGroup, galleryName, applicationName, versionName, nil) + if getErr != nil { + var respErr *azcore.ResponseError + if errors.As(getErr, &respErr) && respErr.StatusCode == http.StatusNotFound { + t.Skipf("Skipping gallery application version integration test: resource %s/%s/%s not found in %s", galleryName, applicationName, versionName, resourceGroup) + } + t.Fatalf("Failed to verify gallery application version %s/%s/%s existence: %v", galleryName, applicationName, versionName, getErr) + } + t.Run("GetGalleryApplicationVersion", func(t *testing.T) { ctx := t.Context() diff --git a/sources/azure/integration-tests/compute-virtual-machine-extension_test.go b/sources/azure/integration-tests/compute-virtual-machine-extension_test.go index d2fdc25c..b58ef546 100644 --- a/sources/azure/integration-tests/compute-virtual-machine-extension_test.go +++ b/sources/azure/integration-tests/compute-virtual-machine-extension_test.go @@ -76,6 +76,7 @@ func TestComputeVirtualMachineExtensionIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed to create Network Interfaces client: %v", err) } + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -113,6 +114,9 @@ func TestComputeVirtualMachineExtensionIntegration(t *testing.T) { // Create virtual machine err = createVirtualMachineForExtension(ctx, vmClient, integrationTestResourceGroup, integrationTestExtensionVMName, integrationTestLocation, *nicResp.ID) if err != nil { + if errors.Is(err, errVMConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure VM control-plane conflict: %v", err) + } t.Fatalf("Failed to create virtual machine: %v", err) } @@ -133,9 +137,14 @@ func TestComputeVirtualMachineExtensionIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed waiting for extension to be available: %v", err) } + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetVirtualMachineExtension", func(t *testing.T) { ctx := t.Context() @@ -482,8 +491,20 @@ func createVirtualMachineForExtension(ctx context.Context, client *armcompute.Vi // Check if VM already exists (conflict) var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { - log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) - return nil + existing, getErr := client.Get(ctx, resourceGroupName, vmName, nil) + if getErr == nil { + if existing.Properties != nil && existing.Properties.ProvisioningState != nil { + log.Printf("Virtual machine %s already exists (conflict) with state %s, skipping creation", vmName, *existing.Properties.ProvisioningState) + } else { + log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) + } + return nil + } + var getRespErr *azcore.ResponseError + if errors.As(getErr, &getRespErr) && getRespErr.StatusCode == http.StatusNotFound { + return fmt.Errorf("%w: vm=%s resourceGroup=%s", errVMConflictWithoutResource, vmName, resourceGroupName) + } + return fmt.Errorf("vm creation conflict for %s and failed to verify existing VM: %w", vmName, getErr) } return fmt.Errorf("failed to begin creating virtual machine: %w", err) } @@ -511,20 +532,27 @@ func createVirtualMachineForExtension(ctx context.Context, client *armcompute.Vi func waitForVMAvailableForExtension(ctx context.Context, client *armcompute.VirtualMachinesClient, resourceGroupName, vmName string) error { maxAttempts := defaultMaxPollAttempts pollInterval := defaultPollInterval + maxNotFoundAttempts := 5 log.Printf("Waiting for VM %s to be available via API...", vmName) + notFoundCount := 0 for attempt := 1; attempt <= maxAttempts; attempt++ { resp, err := client.Get(ctx, resourceGroupName, vmName, nil) if err != nil { var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + notFoundCount++ + if notFoundCount >= maxNotFoundAttempts { + return fmt.Errorf("VM %s not found after %d attempts (possible stale conflict or failed creation)", vmName, notFoundCount) + } log.Printf("VM %s not yet available (attempt %d/%d), waiting %v...", vmName, attempt, maxAttempts, pollInterval) time.Sleep(pollInterval) continue } return fmt.Errorf("error checking VM availability: %w", err) } + notFoundCount = 0 // Check provisioning state if resp.Properties != nil && resp.Properties.ProvisioningState != nil { diff --git a/sources/azure/integration-tests/compute-virtual-machine-run-command_test.go b/sources/azure/integration-tests/compute-virtual-machine-run-command_test.go index 4cb7cd8f..9d3367d4 100644 --- a/sources/azure/integration-tests/compute-virtual-machine-run-command_test.go +++ b/sources/azure/integration-tests/compute-virtual-machine-run-command_test.go @@ -76,6 +76,7 @@ func TestComputeVirtualMachineRunCommandIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed to create Network Interfaces client: %v", err) } + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -113,6 +114,9 @@ func TestComputeVirtualMachineRunCommandIntegration(t *testing.T) { // Create virtual machine err = createVirtualMachineForRunCommand(ctx, vmClient, integrationTestResourceGroup, integrationTestRunCommandVMName, integrationTestLocation, *nicResp.ID) if err != nil { + if errors.Is(err, errVMConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure VM control-plane conflict: %v", err) + } t.Fatalf("Failed to create virtual machine: %v", err) } @@ -133,9 +137,14 @@ func TestComputeVirtualMachineRunCommandIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed waiting for run command to be available: %v", err) } + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetVirtualMachineRunCommand", func(t *testing.T) { ctx := t.Context() @@ -482,8 +491,20 @@ func createVirtualMachineForRunCommand(ctx context.Context, client *armcompute.V // Check if VM already exists (conflict) var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { - log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) - return nil + existing, getErr := client.Get(ctx, resourceGroupName, vmName, nil) + if getErr == nil { + if existing.Properties != nil && existing.Properties.ProvisioningState != nil { + log.Printf("Virtual machine %s already exists (conflict) with state %s, skipping creation", vmName, *existing.Properties.ProvisioningState) + } else { + log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) + } + return nil + } + var getRespErr *azcore.ResponseError + if errors.As(getErr, &getRespErr) && getRespErr.StatusCode == http.StatusNotFound { + return fmt.Errorf("%w: vm=%s resourceGroup=%s", errVMConflictWithoutResource, vmName, resourceGroupName) + } + return fmt.Errorf("vm creation conflict for %s and failed to verify existing VM: %w", vmName, getErr) } return fmt.Errorf("failed to begin creating virtual machine: %w", err) } @@ -511,20 +532,27 @@ func createVirtualMachineForRunCommand(ctx context.Context, client *armcompute.V func waitForVMAvailableForRunCommand(ctx context.Context, client *armcompute.VirtualMachinesClient, resourceGroupName, vmName string) error { maxAttempts := defaultMaxPollAttempts pollInterval := defaultPollInterval + maxNotFoundAttempts := 5 log.Printf("Waiting for VM %s to be available via API...", vmName) + notFoundCount := 0 for attempt := 1; attempt <= maxAttempts; attempt++ { resp, err := client.Get(ctx, resourceGroupName, vmName, nil) if err != nil { var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + notFoundCount++ + if notFoundCount >= maxNotFoundAttempts { + return fmt.Errorf("VM %s not found after %d attempts (possible stale conflict or failed creation)", vmName, notFoundCount) + } log.Printf("VM %s not yet available (attempt %d/%d), waiting %v...", vmName, attempt, maxAttempts, pollInterval) time.Sleep(pollInterval) continue } return fmt.Errorf("error checking VM availability: %w", err) } + notFoundCount = 0 // Check provisioning state if resp.Properties != nil && resp.Properties.ProvisioningState != nil { diff --git a/sources/azure/integration-tests/compute-virtual-machine-scale-set_test.go b/sources/azure/integration-tests/compute-virtual-machine-scale-set_test.go index 3a79bec0..186f57d2 100644 --- a/sources/azure/integration-tests/compute-virtual-machine-scale-set_test.go +++ b/sources/azure/integration-tests/compute-virtual-machine-scale-set_test.go @@ -31,6 +31,8 @@ const ( integrationTestVMSSSubnetName = "default" ) +var errVMSSConflictWithoutResource = errors.New("vmss conflict persisted without readable vmss resource") + func TestComputeVirtualMachineScaleSetIntegration(t *testing.T) { subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") if subscriptionID == "" { @@ -88,6 +90,9 @@ func TestComputeVirtualMachineScaleSetIntegration(t *testing.T) { // Create virtual machine scale set err = createVirtualMachineScaleSet(ctx, vmssClient, integrationTestResourceGroup, integrationTestVMSSName, integrationTestLocation, *subnetResp.ID) if err != nil { + if errors.Is(err, errVMSSConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure VMSS control-plane conflict: %v", err) + } t.Fatalf("Failed to create virtual machine scale set: %v", err) } @@ -551,7 +556,7 @@ func createVirtualMachineScaleSet(ctx context.Context, client *armcompute.Virtua // Still conflict - check if it exists now _, finalCheckErr := client.Get(ctx, resourceGroupName, vmssName, nil) if finalCheckErr != nil { - return fmt.Errorf("VMSS %s still returns conflict but doesn't exist after retry - may need manual cleanup", vmssName) + return fmt.Errorf("%w: vmss=%s resourceGroup=%s", errVMSSConflictWithoutResource, vmssName, resourceGroupName) } log.Printf("VMSS %s exists after retry conflict", vmssName) return nil diff --git a/sources/azure/integration-tests/compute-virtual-machine_test.go b/sources/azure/integration-tests/compute-virtual-machine_test.go index dfb76466..b4e47a77 100644 --- a/sources/azure/integration-tests/compute-virtual-machine_test.go +++ b/sources/azure/integration-tests/compute-virtual-machine_test.go @@ -34,6 +34,8 @@ const ( defaultPollInterval = 15 * time.Second ) +var errVMConflictWithoutResource = errors.New("vm create returned conflict but vm could not be retrieved") + func TestComputeVirtualMachineIntegration(t *testing.T) { subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") if subscriptionID == "" { @@ -71,6 +73,7 @@ func TestComputeVirtualMachineIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed to create Network Interfaces client: %v", err) } + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -108,6 +111,9 @@ func TestComputeVirtualMachineIntegration(t *testing.T) { // Create virtual machine err = createVirtualMachine(ctx, vmClient, integrationTestResourceGroup, integrationTestVMName, integrationTestLocation, *nicResp.ID) if err != nil { + if errors.Is(err, errVMConflictWithoutResource) { + t.Skipf("Skipping due to transient Azure VM control-plane conflict: %v", err) + } t.Fatalf("Failed to create virtual machine: %v", err) } @@ -116,9 +122,14 @@ func TestComputeVirtualMachineIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed waiting for VM to be available: %v", err) } + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetVirtualMachine", func(t *testing.T) { ctx := t.Context() @@ -451,8 +462,22 @@ func createVirtualMachine(ctx context.Context, client *armcompute.VirtualMachine // Check if VM already exists (conflict) var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { - log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) - return nil + // Azure can return conflict while the VM is in a stale/ghost state. + // Verify that the VM can actually be retrieved before treating this as success. + existing, getErr := client.Get(ctx, resourceGroupName, vmName, nil) + if getErr == nil { + if existing.Properties != nil && existing.Properties.ProvisioningState != nil { + log.Printf("Virtual machine %s already exists (conflict) with state %s, skipping creation", vmName, *existing.Properties.ProvisioningState) + } else { + log.Printf("Virtual machine %s already exists (conflict), skipping creation", vmName) + } + return nil + } + var getRespErr *azcore.ResponseError + if errors.As(getErr, &getRespErr) && getRespErr.StatusCode == http.StatusNotFound { + return fmt.Errorf("%w: vm=%s resourceGroup=%s", errVMConflictWithoutResource, vmName, resourceGroupName) + } + return fmt.Errorf("vm creation conflict for %s and failed to verify existing VM: %w", vmName, getErr) } return fmt.Errorf("failed to begin creating virtual machine: %w", err) } @@ -481,20 +506,27 @@ func createVirtualMachine(ctx context.Context, client *armcompute.VirtualMachine func waitForVMAvailable(ctx context.Context, client *armcompute.VirtualMachinesClient, resourceGroupName, vmName string) error { maxAttempts := defaultMaxPollAttempts pollInterval := defaultPollInterval + maxNotFoundAttempts := 5 log.Printf("Waiting for VM %s to be available via API...", vmName) + notFoundCount := 0 for attempt := 1; attempt <= maxAttempts; attempt++ { resp, err := client.Get(ctx, resourceGroupName, vmName, nil) if err != nil { var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + notFoundCount++ + if notFoundCount >= maxNotFoundAttempts { + return fmt.Errorf("VM %s not found after %d attempts (possible stale conflict or failed creation)", vmName, notFoundCount) + } log.Printf("VM %s not yet available (attempt %d/%d), waiting %v...", vmName, attempt, maxAttempts, pollInterval) time.Sleep(pollInterval) continue } return fmt.Errorf("error checking VM availability: %w", err) } + notFoundCount = 0 // Check provisioning state if resp.Properties != nil && resp.Properties.ProvisioningState != nil { @@ -598,7 +630,11 @@ func deleteVirtualNetwork(ctx context.Context, client *armnetwork.VirtualNetwork return fmt.Errorf("failed to begin deleting virtual network: %w", err) } - _, err = poller.PollUntilDone(ctx, nil) + // Bound teardown latency so one stuck ARM delete does not consume the full suite timeout. + deleteCtx, cancel := context.WithTimeout(ctx, 8*time.Minute) + defer cancel() + + _, err = poller.PollUntilDone(deleteCtx, nil) if err != nil { return fmt.Errorf("failed to delete virtual network: %w", err) } diff --git a/sources/azure/integration-tests/dbforpostgresql-database_test.go b/sources/azure/integration-tests/dbforpostgresql-database_test.go index 2b685d94..891773f7 100644 --- a/sources/azure/integration-tests/dbforpostgresql-database_test.go +++ b/sources/azure/integration-tests/dbforpostgresql-database_test.go @@ -307,7 +307,10 @@ func createPostgreSQLFlexibleServer(ctx context.Context, client *armpostgresqlfl // Create the PostgreSQL Flexible Server // Using Burstable tier for cost-effective testing - poller, err := client.BeginCreateOrUpdate(ctx, resourceGroupName, serverName, armpostgresqlflexibleservers.Server{ + opCtx, cancel := context.WithTimeout(ctx, 25*time.Minute) + defer cancel() + + poller, err := client.BeginCreateOrUpdate(opCtx, resourceGroupName, serverName, armpostgresqlflexibleservers.Server{ Location: new(location), Properties: &armpostgresqlflexibleservers.ServerProperties{ AdministratorLogin: new(adminLogin), @@ -337,7 +340,7 @@ func createPostgreSQLFlexibleServer(ctx context.Context, client *armpostgresqlfl return fmt.Errorf("failed to begin creating PostgreSQL Flexible Server: %w", err) } - resp, err := poller.PollUntilDone(ctx, nil) + resp, err := poller.PollUntilDone(opCtx, nil) if err != nil { return fmt.Errorf("failed to create PostgreSQL Flexible Server: %w", err) } @@ -353,8 +356,8 @@ func createPostgreSQLFlexibleServer(ctx context.Context, client *armpostgresqlfl // waitForPostgreSQLServerAvailable waits for a PostgreSQL Flexible Server to be fully available func waitForPostgreSQLServerAvailable(ctx context.Context, client *armpostgresqlflexibleservers.ServersClient, resourceGroupName, serverName string) error { - maxAttempts := 20 - pollInterval := 10 * time.Second + maxAttempts := 120 + pollInterval := 15 * time.Second log.Printf("Waiting for PostgreSQL Flexible Server %s to be available via API...", serverName) @@ -431,8 +434,8 @@ func createPostgreSQLDatabase(ctx context.Context, client *armpostgresqlflexible // waitForPostgreSQLDatabaseAvailable waits for a PostgreSQL Database to be fully available func waitForPostgreSQLDatabaseAvailable(ctx context.Context, client *armpostgresqlflexibleservers.DatabasesClient, resourceGroupName, serverName, databaseName string) error { - maxAttempts := 20 - pollInterval := 5 * time.Second + maxAttempts := 60 + pollInterval := 10 * time.Second log.Printf("Waiting for PostgreSQL database %s to be available via API...", databaseName) diff --git a/sources/azure/integration-tests/dbforpostgresql-flexible-server-backup_test.go b/sources/azure/integration-tests/dbforpostgresql-flexible-server-backup_test.go new file mode 100644 index 00000000..8bb94a29 --- /dev/null +++ b/sources/azure/integration-tests/dbforpostgresql-flexible-server-backup_test.go @@ -0,0 +1,406 @@ +package integrationtests + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" + log "github.com/sirupsen/logrus" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +const ( + integrationTestPGBackupServerName = "ovm-integ-test-pg-backup" + integrationTestPGBackupName = "ovm-integ-test-backup" +) + +func TestDBforPostgreSQLFlexibleServerBackupIntegration(t *testing.T) { + subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + if subscriptionID == "" { + t.Skip("AZURE_SUBSCRIPTION_ID environment variable not set") + } + + cred, err := azureshared.NewAzureCredential(t.Context()) + if err != nil { + t.Fatalf("Failed to create Azure credential: %v", err) + } + + postgreSQLServerClient, err := armpostgresqlflexibleservers.NewServersClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create PostgreSQL Flexible Servers client: %v", err) + } + + backupsClient, err := armpostgresqlflexibleservers.NewBackupsAutomaticAndOnDemandClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create PostgreSQL Backups client: %v", err) + } + + rgClient, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Resource Groups client: %v", err) + } + + pgServerName := generatePostgreSQLServerName(integrationTestPGBackupServerName) + + t.Run("Setup", func(t *testing.T) { + ctx := t.Context() + + err := createResourceGroup(ctx, rgClient, integrationTestResourceGroup, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create resource group: %v", err) + } + + err = createPostgreSQLFlexibleServerForBackup(ctx, postgreSQLServerClient, integrationTestResourceGroup, pgServerName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create PostgreSQL Flexible Server: %v", err) + } + + err = waitForPostgreSQLServerAvailable(ctx, postgreSQLServerClient, integrationTestResourceGroup, pgServerName) + if err != nil { + t.Fatalf("Failed waiting for PostgreSQL server to be available: %v", err) + } + + err = createOnDemandBackup(ctx, backupsClient, integrationTestResourceGroup, pgServerName, integrationTestPGBackupName) + if err != nil { + t.Fatalf("Failed to create on-demand backup: %v", err) + } + + err = waitForBackupAvailable(ctx, backupsClient, integrationTestResourceGroup, pgServerName, integrationTestPGBackupName) + if err != nil { + t.Fatalf("Failed waiting for backup to be available: %v", err) + } + }) + + t.Run("Run", func(t *testing.T) { + t.Run("GetPostgreSQLFlexibleServerBackup", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup( + clients.NewDBforPostgreSQLFlexibleServerBackupClient(backupsClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + query := shared.CompositeLookupKey(pgServerName, integrationTestPGBackupName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem == nil { + t.Fatalf("Expected sdpItem to be non-nil") + } + + if sdpItem.GetType() != azureshared.DBforPostgreSQLFlexibleServerBackup.String() { + t.Errorf("Expected type %s, got %s", azureshared.DBforPostgreSQLFlexibleServerBackup, sdpItem.GetType()) + } + + uniqueAttrKey := sdpItem.GetUniqueAttribute() + if uniqueAttrKey != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", uniqueAttrKey) + } + + uniqueAttrValue, err := sdpItem.GetAttributes().Get(uniqueAttrKey) + if err != nil { + t.Fatalf("Failed to get unique attribute: %v", err) + } + + expectedUniqueAttrValue := shared.CompositeLookupKey(pgServerName, integrationTestPGBackupName) + if uniqueAttrValue != expectedUniqueAttrValue { + t.Errorf("Expected unique attribute value %s, got %s", expectedUniqueAttrValue, uniqueAttrValue) + } + + if sdpItem.GetScope() != fmt.Sprintf("%s.%s", subscriptionID, integrationTestResourceGroup) { + t.Errorf("Expected scope %s.%s, got %s", subscriptionID, integrationTestResourceGroup, sdpItem.GetScope()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Item validation failed: %v", err) + } + + log.Printf("Successfully retrieved backup %s", integrationTestPGBackupName) + }) + + t.Run("SearchPostgreSQLFlexibleServerBackups", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup( + clients.NewDBforPostgreSQLFlexibleServerBackupClient(backupsClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, scope, pgServerName, true) + if err != nil { + t.Fatalf("Failed to search backups: %v", err) + } + + if len(sdpItems) < 1 { + t.Fatalf("Expected at least one backup, got %d", len(sdpItems)) + } + + var found bool + for _, item := range sdpItems { + uniqueAttrKey := item.GetUniqueAttribute() + if v, err := item.GetAttributes().Get(uniqueAttrKey); err == nil { + expectedValue := shared.CompositeLookupKey(pgServerName, integrationTestPGBackupName) + if v == expectedValue { + found = true + break + } + } + } + + if !found { + t.Fatalf("Expected to find backup %s in the search results", integrationTestPGBackupName) + } + + log.Printf("Found %d backups in search results", len(sdpItems)) + }) + + t.Run("VerifyLinkedItems", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup( + clients.NewDBforPostgreSQLFlexibleServerBackupClient(backupsClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + query := shared.CompositeLookupKey(pgServerName, integrationTestPGBackupName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + linkedQueries := sdpItem.GetLinkedItemQueries() + if len(linkedQueries) == 0 { + t.Fatalf("Expected linked item queries, but got none") + } + + var hasServerLink bool + for _, liq := range linkedQueries { + if liq.GetQuery().GetType() == azureshared.DBforPostgreSQLFlexibleServer.String() { + hasServerLink = true + if liq.GetQuery().GetQuery() != pgServerName { + t.Errorf("Expected linked query to server %s, got %s", pgServerName, liq.GetQuery().GetQuery()) + } + if liq.GetQuery().GetMethod() != sdp.QueryMethod_GET { + t.Errorf("Expected linked query method GET, got %s", liq.GetQuery().GetMethod()) + } + if liq.GetQuery().GetScope() != scope { + t.Errorf("Expected linked query scope %s, got %s", scope, liq.GetQuery().GetScope()) + } + break + } + } + + if !hasServerLink { + t.Error("Expected linked query to PostgreSQL Flexible Server, but didn't find one") + } + + log.Printf("Verified %d linked item queries for backup %s", len(linkedQueries), integrationTestPGBackupName) + }) + + t.Run("VerifyItemAttributes", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup( + clients.NewDBforPostgreSQLFlexibleServerBackupClient(backupsClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + query := shared.CompositeLookupKey(pgServerName, integrationTestPGBackupName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.DBforPostgreSQLFlexibleServerBackup.String() { + t.Errorf("Expected type %s, got %s", azureshared.DBforPostgreSQLFlexibleServerBackup, sdpItem.GetType()) + } + + expectedScope := fmt.Sprintf("%s.%s", subscriptionID, integrationTestResourceGroup) + if sdpItem.GetScope() != expectedScope { + t.Errorf("Expected scope %s, got %s", expectedScope, sdpItem.GetScope()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Item validation failed: %v", err) + } + }) + }) + + t.Run("Teardown", func(t *testing.T) { + ctx := t.Context() + + err := deleteOnDemandBackup(ctx, backupsClient, integrationTestResourceGroup, pgServerName, integrationTestPGBackupName) + if err != nil { + log.Printf("Warning: failed to delete backup (may have been auto-cleaned): %v", err) + } + + err = deletePostgreSQLFlexibleServer(ctx, postgreSQLServerClient, integrationTestResourceGroup, pgServerName) + if err != nil { + t.Fatalf("Failed to delete PostgreSQL Flexible Server: %v", err) + } + }) +} + +// createPostgreSQLFlexibleServerForBackup creates a GeneralPurpose-tier server +// because Azure does not allow on-demand backups on Burstable-tier servers. +func createPostgreSQLFlexibleServerForBackup(ctx context.Context, client *armpostgresqlflexibleservers.ServersClient, resourceGroupName, serverName, location string) error { + _, err := client.Get(ctx, resourceGroupName, serverName, nil) + if err == nil { + log.Printf("PostgreSQL Flexible Server %s already exists, skipping creation", serverName) + return nil + } + + adminLogin := os.Getenv("AZURE_POSTGRESQL_SERVER_ADMIN_LOGIN") + adminPassword := os.Getenv("AZURE_POSTGRESQL_SERVER_ADMIN_PASSWORD") + if adminLogin == "" || adminPassword == "" { + return fmt.Errorf("AZURE_POSTGRESQL_SERVER_ADMIN_LOGIN and AZURE_POSTGRESQL_SERVER_ADMIN_PASSWORD must be set") + } + + opCtx, cancel := context.WithTimeout(ctx, 25*time.Minute) + defer cancel() + + poller, err := client.BeginCreateOrUpdate(opCtx, resourceGroupName, serverName, armpostgresqlflexibleservers.Server{ + Location: new(location), + Properties: &armpostgresqlflexibleservers.ServerProperties{ + AdministratorLogin: new(adminLogin), + AdministratorLoginPassword: new(adminPassword), + Version: new(armpostgresqlflexibleservers.PostgresMajorVersion("14")), + Storage: &armpostgresqlflexibleservers.Storage{StorageSizeGB: new(int32(32))}, + Backup: &armpostgresqlflexibleservers.Backup{BackupRetentionDays: new(int32(7)), GeoRedundantBackup: new(armpostgresqlflexibleservers.GeographicallyRedundantBackupDisabled)}, + Network: &armpostgresqlflexibleservers.Network{PublicNetworkAccess: new(armpostgresqlflexibleservers.ServerPublicNetworkAccessStateEnabled)}, + }, + SKU: &armpostgresqlflexibleservers.SKU{ + Name: new("Standard_D2s_v3"), + Tier: new(armpostgresqlflexibleservers.SKUTierGeneralPurpose), + }, + Tags: map[string]*string{ + "purpose": new("overmind-integration-tests"), + "test": new("dbforpostgresql-backup"), + }, + }, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("PostgreSQL Flexible Server %s already exists, skipping creation", serverName) + return nil + } + return fmt.Errorf("failed to begin creating PostgreSQL Flexible Server: %w", err) + } + + _, err = poller.PollUntilDone(opCtx, nil) + if err != nil { + return fmt.Errorf("failed to create PostgreSQL Flexible Server: %w", err) + } + + log.Printf("PostgreSQL Flexible Server %s (GeneralPurpose) created successfully", serverName) + return nil +} + +func createOnDemandBackup(ctx context.Context, client *armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClient, resourceGroupName, serverName, backupName string) error { + _, err := client.Get(ctx, resourceGroupName, serverName, backupName, nil) + if err == nil { + log.Printf("Backup %s already exists, skipping creation", backupName) + return nil + } + + poller, err := client.BeginCreate(ctx, resourceGroupName, serverName, backupName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("Backup %s already exists (conflict), skipping", backupName) + return nil + } + return fmt.Errorf("failed to begin creating backup: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create backup: %w", err) + } + + log.Printf("Backup %s created successfully", backupName) + return nil +} + +func waitForBackupAvailable(ctx context.Context, client *armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClient, resourceGroupName, serverName, backupName string) error { + maxAttempts := 20 + pollInterval := 5 * time.Second + + for attempt := 1; attempt <= maxAttempts; attempt++ { + _, err := client.Get(ctx, resourceGroupName, serverName, backupName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Backup %s not yet available (attempt %d/%d), waiting...", backupName, attempt, maxAttempts) + time.Sleep(pollInterval) + continue + } + return fmt.Errorf("error checking backup availability: %w", err) + } + + log.Printf("Backup %s is available", backupName) + return nil + } + + return fmt.Errorf("timeout waiting for backup %s to be available", backupName) +} + +func deleteOnDemandBackup(ctx context.Context, client *armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClient, resourceGroupName, serverName, backupName string) error { + _, err := client.Get(ctx, resourceGroupName, serverName, backupName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Backup %s does not exist, skipping deletion", backupName) + return nil + } + return fmt.Errorf("error checking backup existence: %w", err) + } + + poller, err := client.BeginDelete(ctx, resourceGroupName, serverName, backupName, nil) + if err != nil { + return fmt.Errorf("failed to begin deleting backup: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to delete backup: %w", err) + } + + log.Printf("Backup %s deleted successfully", backupName) + return nil +} diff --git a/sources/azure/integration-tests/dbforpostgresql-flexible-server_test.go b/sources/azure/integration-tests/dbforpostgresql-flexible-server_test.go index d0a39a7f..9226436c 100644 --- a/sources/azure/integration-tests/dbforpostgresql-flexible-server_test.go +++ b/sources/azure/integration-tests/dbforpostgresql-flexible-server_test.go @@ -247,10 +247,10 @@ func TestDBforPostgreSQLFlexibleServerIntegration(t *testing.T) { } } - log.Printf("Verified %d linked item queries for PostgreSQL Flexible Server %s (hasSubnet: %v, hasVNet: %v, hasDNS: %v)", - len(linkedQueries), postgreSQLServerName, hasSubnetLink, hasVirtualNetworkLink, hasDNSLink) + log.Printf("Verified %d linked item queries for PostgreSQL Flexible Server %s (hasSubnet: %v, hasVNet: %v, hasDNS: %v)", + len(linkedQueries), postgreSQLServerName, hasSubnetLink, hasVirtualNetworkLink, hasDNSLink) + }) }) -}) t.Run("Teardown", func(t *testing.T) { ctx := t.Context() diff --git a/sources/azure/integration-tests/keyvault-secret_test.go b/sources/azure/integration-tests/keyvault-secret_test.go index bfca030b..7353fa83 100644 --- a/sources/azure/integration-tests/keyvault-secret_test.go +++ b/sources/azure/integration-tests/keyvault-secret_test.go @@ -7,7 +7,9 @@ import ( "net/http" "os" "os/exec" + "strings" "testing" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault/v2" @@ -58,6 +60,7 @@ func TestKeyVaultSecretIntegration(t *testing.T) { // Use the same Key Vault name as the vault integration test // Note: integrationTestKeyVaultName is defined in keyvault-vault_test.go vaultName := integrationTestKeyVaultName + setupCompleted := false t.Run("Setup", func(t *testing.T) { ctx := t.Context() @@ -108,9 +111,20 @@ func TestKeyVaultSecretIntegration(t *testing.T) { if err != nil { t.Fatalf("Failed to create Key Vault secret: %v", err) } + + // After create/recover, ARM control plane can lag briefly before GET is consistent. + err = waitForKeyVaultSecretAvailable(ctx, secretsClient, integrationTestResourceGroup, vaultName, integrationTestSecretName) + if err != nil { + t.Fatalf("Failed waiting for Key Vault secret to be available: %v", err) + } + setupCompleted = true }) t.Run("Run", func(t *testing.T) { + if !setupCompleted { + t.Skip("Skipping Run: Setup did not complete successfully") + } + t.Run("GetSecret", func(t *testing.T) { ctx := t.Context() @@ -320,6 +334,19 @@ func createKeyVaultSecret(ctx context.Context, vaultName, secretName string) err "--value", "test-secret-value") output, err := cmd.CombinedOutput() if err != nil { + if strings.Contains(string(output), "ObjectIsDeletedButRecoverable") { + log.Printf("Secret %s is deleted but recoverable, attempting recovery", secretName) + recoverCmd := exec.CommandContext(ctx, "az", "keyvault", "secret", "recover", + "--vault-name", vaultName, + "--name", secretName) + recoverOutput, recoverErr := recoverCmd.CombinedOutput() + if recoverErr != nil { + return fmt.Errorf("failed to recover deleted secret: %w, output: %s", recoverErr, string(recoverOutput)) + } + log.Printf("Secret %s recovered successfully", secretName) + return nil + } + // If the command failed, it might be because the secret already exists // Try to show it to confirm showCmd := exec.CommandContext(ctx, "az", "keyvault", "secret", "show", @@ -362,3 +389,25 @@ func deleteKeyVaultSecret(ctx context.Context, vaultName, secretName string) err log.Printf("Secret %s deleted successfully", secretName) return nil } + +func waitForKeyVaultSecretAvailable(ctx context.Context, client *armkeyvault.SecretsClient, resourceGroupName, vaultName, secretName string) error { + maxAttempts := 20 + pollInterval := 3 * time.Second + + for attempt := 1; attempt <= maxAttempts; attempt++ { + _, err := client.Get(ctx, resourceGroupName, vaultName, secretName, nil) + if err == nil { + return nil + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + time.Sleep(pollInterval) + continue + } + + return fmt.Errorf("error checking secret availability: %w", err) + } + + return fmt.Errorf("timeout waiting for secret %s in vault %s to be available", secretName, vaultName) +} diff --git a/sources/azure/integration-tests/keyvault-vault_test.go b/sources/azure/integration-tests/keyvault-vault_test.go index 614e9dfb..8ef54f3c 100644 --- a/sources/azure/integration-tests/keyvault-vault_test.go +++ b/sources/azure/integration-tests/keyvault-vault_test.go @@ -202,9 +202,10 @@ func createKeyVault(ctx context.Context, client *armkeyvault.VaultsClient, resou return fmt.Errorf("AZURE_TENANT_ID environment variable not set, required for Key Vault creation") } - // Create a context with timeout for the entire Key Vault creation operation + // Create a context with timeout for the entire Key Vault creation operation. + // Purging soft-deleted vaults can take several minutes in Azure. // Key Vault creation can hang if the Microsoft.KeyVault resource provider is not registered - createCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + createCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) defer cancel() // Create the Key Vault. diff --git a/sources/azure/integration-tests/network-application-gateway_test.go b/sources/azure/integration-tests/network-application-gateway_test.go index bbcbbefa..11fec167 100644 --- a/sources/azure/integration-tests/network-application-gateway_test.go +++ b/sources/azure/integration-tests/network-application-gateway_test.go @@ -133,6 +133,16 @@ func TestNetworkApplicationGatewayIntegration(t *testing.T) { }) t.Run("Run", func(t *testing.T) { + ctx := t.Context() + _, checkErr := agClient.Get(ctx, integrationTestResourceGroup, integrationTestAGName, nil) + if checkErr != nil { + var respErr *azcore.ResponseError + if errors.As(checkErr, &respErr) && respErr.StatusCode == http.StatusNotFound { + t.Skipf("Application Gateway %s does not exist (Setup may have been skipped). Skipping Run tests.", integrationTestAGName) + } + t.Fatalf("Failed preflight check for application gateway %s: %v", integrationTestAGName, checkErr) + } + t.Run("GetApplicationGateway", func(t *testing.T) { ctx := t.Context() diff --git a/sources/azure/integration-tests/network-dns-virtual-network-link_test.go b/sources/azure/integration-tests/network-dns-virtual-network-link_test.go index 03352387..95961c6f 100644 --- a/sources/azure/integration-tests/network-dns-virtual-network-link_test.go +++ b/sources/azure/integration-tests/network-dns-virtual-network-link_test.go @@ -258,6 +258,9 @@ func TestNetworkDNSVirtualNetworkLinkIntegration(t *testing.T) { t.Fatalf("Failed to delete virtual network link: %v", err) } + log.Printf("Waiting 30 seconds for VNet link deletion to propagate before deleting DNS zone...") + time.Sleep(30 * time.Second) + err = deletePrivateDNSZoneForLink(ctx, privateDNSZonesClient, integrationTestResourceGroup, integrationTestPrivateZoneName) if err != nil { t.Fatalf("Failed to delete private DNS zone: %v", err) diff --git a/sources/azure/integration-tests/network-flow-log_test.go b/sources/azure/integration-tests/network-flow-log_test.go new file mode 100644 index 00000000..1e2d7171 --- /dev/null +++ b/sources/azure/integration-tests/network-flow-log_test.go @@ -0,0 +1,478 @@ +package integrationtests + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "testing" + "time" + + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3" + log "github.com/sirupsen/logrus" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +const ( + integrationTestFlowLogName = "ovm-integ-test-flow-log" + integrationTestFlowLogNSGName = "ovm-integ-test-flow-log-nsg" + integrationTestFlowLogStorageName = "ovmintegflowlogstor" + integrationTestNetworkWatcherName = "NetworkWatcher_westus2" + integrationTestNetworkWatcherRG = "NetworkWatcherRG" +) + +func TestNetworkFlowLogIntegration(t *testing.T) { + subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + if subscriptionID == "" { + t.Skip("AZURE_SUBSCRIPTION_ID environment variable not set") + } + + cred, err := azureshared.NewAzureCredential(t.Context()) + if err != nil { + t.Fatalf("Failed to create Azure credential: %v", err) + } + + rgClient, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Resource Groups client: %v", err) + } + + nsgClient, err := armnetwork.NewSecurityGroupsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create NSG client: %v", err) + } + + storageClient, err := armstorage.NewAccountsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Storage Accounts client: %v", err) + } + + flowLogsSDKClient, err := armnetwork.NewFlowLogsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Flow Logs client: %v", err) + } + + t.Run("Setup", func(t *testing.T) { + ctx := t.Context() + + err := createResourceGroup(ctx, rgClient, integrationTestResourceGroup, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create resource group: %v", err) + } + + err = createResourceGroup(ctx, rgClient, integrationTestNetworkWatcherRG, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create NetworkWatcherRG: %v", err) + } + + err = createFlowLogNSG(ctx, nsgClient, integrationTestResourceGroup, integrationTestFlowLogNSGName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create NSG: %v", err) + } + + err = createFlowLogStorageAccount(ctx, storageClient, integrationTestResourceGroup, integrationTestFlowLogStorageName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create storage account: %v", err) + } + + err = waitForFlowLogStorageAccountAvailable(ctx, storageClient, integrationTestResourceGroup, integrationTestFlowLogStorageName) + if err != nil { + t.Fatalf("Failed waiting for storage account: %v", err) + } + + nsgID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s", + subscriptionID, integrationTestResourceGroup, integrationTestFlowLogNSGName) + storageID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", + subscriptionID, integrationTestResourceGroup, integrationTestFlowLogStorageName) + + err = createFlowLog(ctx, flowLogsSDKClient, integrationTestNetworkWatcherRG, integrationTestNetworkWatcherName, integrationTestFlowLogName, nsgID, storageID, integrationTestLocation) + if err != nil { + if strings.Contains(err.Error(), "NsgFlowLogCreationBlocked") { + t.Skipf("Skipping: Azure has retired new NSG flow log creation: %v", err) + } + t.Fatalf("Failed to create flow log: %v", err) + } + + err = waitForFlowLogAvailable(ctx, flowLogsSDKClient, integrationTestNetworkWatcherRG, integrationTestNetworkWatcherName, integrationTestFlowLogName) + if err != nil { + t.Fatalf("Failed waiting for flow log: %v", err) + } + }) + + t.Run("Run", func(t *testing.T) { + ctx := t.Context() + _, checkErr := flowLogsSDKClient.Get(ctx, integrationTestNetworkWatcherRG, integrationTestNetworkWatcherName, integrationTestFlowLogName, nil) + if checkErr != nil { + var respErr *azcore.ResponseError + if errors.As(checkErr, &respErr) && respErr.StatusCode == http.StatusNotFound { + t.Skipf("Flow log %s does not exist (Setup may have been skipped). Skipping Run tests.", integrationTestFlowLogName) + } + t.Fatalf("Failed preflight check for flow log %s: %v", integrationTestFlowLogName, checkErr) + } + + t.Run("GetFlowLog", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkFlowLog( + clients.NewFlowLogsClient(flowLogsSDKClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestNetworkWatcherRG)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(integrationTestNetworkWatcherName, integrationTestFlowLogName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem == nil { + t.Fatalf("Expected sdpItem to be non-nil") + } + + uniqueAttrKey := sdpItem.GetUniqueAttribute() + uniqueAttrValue, err := sdpItem.GetAttributes().Get(uniqueAttrKey) + if err != nil { + t.Fatalf("Failed to get unique attribute: %v", err) + } + + expectedUnique := shared.CompositeLookupKey(integrationTestNetworkWatcherName, integrationTestFlowLogName) + if uniqueAttrValue != expectedUnique { + t.Errorf("Expected unique attribute value %s, got %s", expectedUnique, uniqueAttrValue) + } + + log.Printf("Successfully retrieved flow log %s", integrationTestFlowLogName) + }) + + t.Run("SearchFlowLogs", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkFlowLog( + clients.NewFlowLogsClient(flowLogsSDKClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestNetworkWatcherRG)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, scope, integrationTestNetworkWatcherName, true) + if err != nil { + t.Fatalf("Failed to search flow logs: %v", err) + } + + if len(sdpItems) < 1 { + t.Fatalf("Expected at least one flow log, got %d", len(sdpItems)) + } + + var found bool + for _, item := range sdpItems { + uniqueAttrKey := item.GetUniqueAttribute() + if v, err := item.GetAttributes().Get(uniqueAttrKey); err == nil && v == shared.CompositeLookupKey(integrationTestNetworkWatcherName, integrationTestFlowLogName) { + found = true + break + } + } + + if !found { + t.Fatalf("Expected to find flow log %s in the search results", integrationTestFlowLogName) + } + + log.Printf("Found %d flow logs in search results", len(sdpItems)) + }) + + t.Run("VerifyLinkedItems", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkFlowLog( + clients.NewFlowLogsClient(flowLogsSDKClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestNetworkWatcherRG)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(integrationTestNetworkWatcherName, integrationTestFlowLogName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + linkedQueries := sdpItem.GetLinkedItemQueries() + if len(linkedQueries) == 0 { + t.Fatalf("Expected linked item queries, but got none") + } + + for _, liq := range linkedQueries { + q := liq.GetQuery() + if q.GetType() == "" { + t.Error("Linked item query has empty Type") + } + if q.GetQuery() == "" { + t.Errorf("Linked item query of type %s has empty Query", q.GetType()) + } + if q.GetScope() == "" { + t.Errorf("Linked item query of type %s has empty Scope", q.GetType()) + } + method := q.GetMethod() + if method != 1 && method != 2 { // GET=1, SEARCH=2 + t.Errorf("Linked item query of type %s has unexpected Method %d", q.GetType(), method) + } + } + + log.Printf("Verified %d linked item queries for flow log %s", len(linkedQueries), integrationTestFlowLogName) + }) + + t.Run("VerifyItemAttributes", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkFlowLog( + clients.NewFlowLogsClient(flowLogsSDKClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestNetworkWatcherRG)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(integrationTestNetworkWatcherName, integrationTestFlowLogName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.NetworkFlowLog.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkFlowLog.String(), sdpItem.GetType()) + } + + expectedScope := subscriptionID + "." + integrationTestNetworkWatcherRG + if sdpItem.GetScope() != expectedScope { + t.Errorf("Expected scope %s, got %s", expectedScope, sdpItem.GetScope()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + if err := sdpItem.Validate(); err != nil { + t.Errorf("Expected item to validate, got: %v", err) + } + }) + }) + + t.Run("Teardown", func(t *testing.T) { + ctx := t.Context() + + err := deleteFlowLog(ctx, flowLogsSDKClient, integrationTestNetworkWatcherRG, integrationTestNetworkWatcherName, integrationTestFlowLogName) + if err != nil { + t.Fatalf("Failed to delete flow log: %v", err) + } + + err = deleteFlowLogStorageAccount(ctx, storageClient, integrationTestResourceGroup, integrationTestFlowLogStorageName) + if err != nil { + t.Fatalf("Failed to delete storage account: %v", err) + } + + err = deleteFlowLogNSG(ctx, nsgClient, integrationTestResourceGroup, integrationTestFlowLogNSGName) + if err != nil { + t.Fatalf("Failed to delete NSG: %v", err) + } + }) +} + +func createFlowLogNSG(ctx context.Context, client *armnetwork.SecurityGroupsClient, rg, name, location string) error { + _, err := client.Get(ctx, rg, name, nil) + if err == nil { + log.Printf("NSG %s already exists, skipping creation", name) + return nil + } + + poller, err := client.BeginCreateOrUpdate(ctx, rg, name, armnetwork.SecurityGroup{ + Location: &location, + }, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("NSG %s already exists (conflict), skipping", name) + return nil + } + return fmt.Errorf("failed to create NSG: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create NSG: %w", err) + } + log.Printf("NSG %s created successfully", name) + return nil +} + +func createFlowLogStorageAccount(ctx context.Context, client *armstorage.AccountsClient, rg, name, location string) error { + _, err := client.GetProperties(ctx, rg, name, nil) + if err == nil { + log.Printf("Storage account %s already exists, skipping creation", name) + return nil + } + + poller, err := client.BeginCreate(ctx, rg, name, armstorage.AccountCreateParameters{ + Location: &location, + Kind: new(armstorage.KindStorageV2), + SKU: &armstorage.SKU{ + Name: new(armstorage.SKUNameStandardLRS), + }, + }, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("Storage account %s already exists (conflict), skipping", name) + return nil + } + return fmt.Errorf("failed to create storage account: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create storage account: %w", err) + } + log.Printf("Storage account %s created successfully", name) + return nil +} + +func waitForFlowLogStorageAccountAvailable(ctx context.Context, client *armstorage.AccountsClient, rg, name string) error { + maxAttempts := 20 + pollInterval := 5 * time.Second + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err := client.GetProperties(ctx, rg, name, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + time.Sleep(pollInterval) + continue + } + return fmt.Errorf("error checking storage account: %w", err) + } + if resp.Properties != nil && resp.Properties.ProvisioningState != nil && *resp.Properties.ProvisioningState == armstorage.ProvisioningStateSucceeded { + return nil + } + time.Sleep(pollInterval) + } + return fmt.Errorf("timeout waiting for storage account %s", name) +} + +func createFlowLog(ctx context.Context, client *armnetwork.FlowLogsClient, rg, networkWatcherName, flowLogName, nsgID, storageID, location string) error { + _, err := client.Get(ctx, rg, networkWatcherName, flowLogName, nil) + if err == nil { + log.Printf("Flow log %s already exists, skipping creation", flowLogName) + return nil + } + + enabled := true + poller, err := client.BeginCreateOrUpdate(ctx, rg, networkWatcherName, flowLogName, armnetwork.FlowLog{ + Location: &location, + Properties: &armnetwork.FlowLogPropertiesFormat{ + TargetResourceID: &nsgID, + StorageID: &storageID, + Enabled: &enabled, + RetentionPolicy: &armnetwork.RetentionPolicyParameters{ + Enabled: &enabled, + Days: new(int32(7)), + }, + }, + }, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusConflict { + log.Printf("Flow log %s already exists (conflict), skipping", flowLogName) + return nil + } + return fmt.Errorf("failed to create flow log: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create flow log: %w", err) + } + log.Printf("Flow log %s created successfully", flowLogName) + return nil +} + +func waitForFlowLogAvailable(ctx context.Context, client *armnetwork.FlowLogsClient, rg, networkWatcherName, flowLogName string) error { + maxAttempts := 20 + pollInterval := 5 * time.Second + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err := client.Get(ctx, rg, networkWatcherName, flowLogName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + time.Sleep(pollInterval) + continue + } + return fmt.Errorf("error checking flow log: %w", err) + } + if resp.Properties != nil && resp.Properties.ProvisioningState != nil && string(*resp.Properties.ProvisioningState) == "Succeeded" { + return nil + } + time.Sleep(pollInterval) + } + return fmt.Errorf("timeout waiting for flow log %s", flowLogName) +} + +func deleteFlowLog(ctx context.Context, client *armnetwork.FlowLogsClient, rg, networkWatcherName, flowLogName string) error { + poller, err := client.BeginDelete(ctx, rg, networkWatcherName, flowLogName, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Flow log %s not found, skipping deletion", flowLogName) + return nil + } + return fmt.Errorf("failed to delete flow log: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to delete flow log: %w", err) + } + log.Printf("Flow log %s deleted successfully", flowLogName) + return nil +} + +func deleteFlowLogStorageAccount(ctx context.Context, client *armstorage.AccountsClient, rg, name string) error { + _, err := client.Delete(ctx, rg, name, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("Storage account %s not found, skipping deletion", name) + return nil + } + return fmt.Errorf("failed to delete storage account: %w", err) + } + log.Printf("Storage account %s deleted successfully", name) + return nil +} + +func deleteFlowLogNSG(ctx context.Context, client *armnetwork.SecurityGroupsClient, rg, name string) error { + poller, err := client.BeginDelete(ctx, rg, name, nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { + log.Printf("NSG %s not found, skipping deletion", name) + return nil + } + return fmt.Errorf("failed to delete NSG: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to delete NSG: %w", err) + } + log.Printf("NSG %s deleted successfully", name) + return nil +} diff --git a/sources/azure/integration-tests/network-load-balancer-frontend-ip-configuration_test.go b/sources/azure/integration-tests/network-load-balancer-frontend-ip-configuration_test.go new file mode 100644 index 00000000..bd94b89c --- /dev/null +++ b/sources/azure/integration-tests/network-load-balancer-frontend-ip-configuration_test.go @@ -0,0 +1,322 @@ +package integrationtests + +import ( + "fmt" + "os" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" + log "github.com/sirupsen/logrus" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +const ( + integrationTestFrontendIPLBName = "ovm-integ-test-lb-fip" + integrationTestFrontendIPPublicIPName = "ovm-integ-test-pip-fip" + integrationTestFrontendIPConfigName = "frontend-ip-config" + integrationTestFrontendIPVNetName = "ovm-integ-test-vnet-fip" + integrationTestFrontendIPSubnetName = "default" + integrationTestFrontendIPInternalLBName = "ovm-integ-test-lb-fip-internal" + integrationTestFrontendIPInternalName = "frontend-ip-internal" +) + +func TestNetworkLoadBalancerFrontendIPConfigurationIntegration(t *testing.T) { + subscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + if subscriptionID == "" { + t.Skip("AZURE_SUBSCRIPTION_ID environment variable not set") + } + + cred, err := azureshared.NewAzureCredential(t.Context()) + if err != nil { + t.Fatalf("Failed to create Azure credential: %v", err) + } + + rgClient, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Resource Groups client: %v", err) + } + + publicIPClient, err := armnetwork.NewPublicIPAddressesClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Public IP Addresses client: %v", err) + } + + lbClient, err := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Load Balancers client: %v", err) + } + + frontendIPClient, err := armnetwork.NewLoadBalancerFrontendIPConfigurationsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Frontend IP Configurations client: %v", err) + } + + vnetClient, err := armnetwork.NewVirtualNetworksClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Virtual Networks client: %v", err) + } + + subnetClient, err := armnetwork.NewSubnetsClient(subscriptionID, cred, nil) + if err != nil { + t.Fatalf("Failed to create Subnets client: %v", err) + } + + t.Run("Setup", func(t *testing.T) { + ctx := t.Context() + + err := createResourceGroup(ctx, rgClient, integrationTestResourceGroup, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create resource group: %v", err) + } + + // Create public IP for public LB + err = createPublicIPForLB(ctx, publicIPClient, integrationTestResourceGroup, integrationTestFrontendIPPublicIPName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create public IP address: %v", err) + } + + publicIPResp, err := publicIPClient.Get(ctx, integrationTestResourceGroup, integrationTestFrontendIPPublicIPName, nil) + if err != nil { + t.Fatalf("Failed to get public IP address: %v", err) + } + + // Create public LB with frontend IP config + err = createPublicLoadBalancer(ctx, lbClient, integrationTestResourceGroup, integrationTestFrontendIPLBName, integrationTestLocation, *publicIPResp.ID) + if err != nil { + t.Fatalf("Failed to create public load balancer: %v", err) + } + + // Create VNet + subnet for internal LB + err = createVirtualNetworkForLB(ctx, vnetClient, integrationTestResourceGroup, integrationTestFrontendIPVNetName, integrationTestLocation) + if err != nil { + t.Fatalf("Failed to create virtual network: %v", err) + } + + subnetResp, err := subnetClient.Get(ctx, integrationTestResourceGroup, integrationTestFrontendIPVNetName, integrationTestFrontendIPSubnetName, nil) + if err != nil { + t.Fatalf("Failed to get subnet: %v", err) + } + + // Create internal LB + err = createInternalLoadBalancer(ctx, lbClient, integrationTestResourceGroup, integrationTestFrontendIPInternalLBName, integrationTestLocation, *subnetResp.ID) + if err != nil { + t.Fatalf("Failed to create internal load balancer: %v", err) + } + + log.Printf("Setup completed for frontend IP configuration integration tests") + }) + + t.Run("Run", func(t *testing.T) { + t.Run("GetFrontendIPConfiguration", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration( + clients.NewLoadBalancerFrontendIPConfigurationsClient(frontendIPClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + // The public LB has a frontend IP config named "frontend-ip-config-public" + query := shared.CompositeLookupKey(integrationTestFrontendIPLBName, "frontend-ip-config-public") + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.NetworkLoadBalancerFrontendIPConfiguration.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkLoadBalancerFrontendIPConfiguration, sdpItem.GetType()) + } + + expectedUniqueValue := shared.CompositeLookupKey(integrationTestFrontendIPLBName, "frontend-ip-config-public") + if sdpItem.UniqueAttributeValue() != expectedUniqueValue { + t.Errorf("Expected unique value %s, got %s", expectedUniqueValue, sdpItem.UniqueAttributeValue()) + } + + log.Printf("Successfully retrieved frontend IP configuration for LB %s", integrationTestFrontendIPLBName) + }) + + t.Run("SearchFrontendIPConfigurations", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration( + clients.NewLoadBalancerFrontendIPConfigurationsClient(frontendIPClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, scope, integrationTestFrontendIPLBName, true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) < 1 { + t.Fatalf("Expected at least 1 frontend IP configuration, got: %d", len(sdpItems)) + } + + for _, item := range sdpItems { + if item.GetType() != azureshared.NetworkLoadBalancerFrontendIPConfiguration.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkLoadBalancerFrontendIPConfiguration, item.GetType()) + } + } + + log.Printf("Successfully searched %d frontend IP configurations for LB %s", len(sdpItems), integrationTestFrontendIPLBName) + }) + + t.Run("VerifyLinkedItems", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration( + clients.NewLoadBalancerFrontendIPConfigurationsClient(frontendIPClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + // Verify public frontend IP config links + t.Run("PublicFrontendIP", func(t *testing.T) { + query := shared.CompositeLookupKey(integrationTestFrontendIPLBName, "frontend-ip-config-public") + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + linkedQueries := sdpItem.GetLinkedItemQueries() + for _, liq := range linkedQueries { + q := liq.GetQuery() + if q.GetType() == "" { + t.Error("Linked item query has empty Type") + } + if q.GetQuery() == "" { + t.Errorf("Linked item query of type %s has empty Query", q.GetType()) + } + if q.GetScope() == "" { + t.Errorf("Linked item query of type %s has empty Scope", q.GetType()) + } + } + + expectedTypes := map[string]bool{ + azureshared.NetworkLoadBalancer.String(): false, + azureshared.NetworkPublicIPAddress.String(): false, + } + + for _, liq := range linkedQueries { + if _, exists := expectedTypes[liq.GetQuery().GetType()]; exists { + expectedTypes[liq.GetQuery().GetType()] = true + } + } + + for linkedType, found := range expectedTypes { + if !found { + t.Errorf("Expected linked query to %s, but didn't find one", linkedType) + } + } + }) + + // Verify internal frontend IP config links + t.Run("InternalFrontendIP", func(t *testing.T) { + query := shared.CompositeLookupKey(integrationTestFrontendIPInternalLBName, "frontend-ip-config-internal") + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + linkedQueries := sdpItem.GetLinkedItemQueries() + + expectedTypes := map[string]bool{ + azureshared.NetworkLoadBalancer.String(): false, + azureshared.NetworkSubnet.String(): false, + "ip": false, + } + + for _, liq := range linkedQueries { + if _, exists := expectedTypes[liq.GetQuery().GetType()]; exists { + expectedTypes[liq.GetQuery().GetType()] = true + } + } + + for linkedType, found := range expectedTypes { + if !found { + t.Errorf("Expected linked query to %s, but didn't find one", linkedType) + } + } + }) + }) + + t.Run("VerifyItemAttributes", func(t *testing.T) { + ctx := t.Context() + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration( + clients.NewLoadBalancerFrontendIPConfigurationsClient(frontendIPClient), + []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, integrationTestResourceGroup)}, + ) + scope := wrapper.Scopes()[0] + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(integrationTestFrontendIPLBName, "frontend-ip-config-public") + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.NetworkLoadBalancerFrontendIPConfiguration.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkLoadBalancerFrontendIPConfiguration, sdpItem.GetType()) + } + + expectedScope := fmt.Sprintf("%s.%s", subscriptionID, integrationTestResourceGroup) + if sdpItem.GetScope() != expectedScope { + t.Errorf("Expected scope %s, got %s", expectedScope, sdpItem.GetScope()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + if err := sdpItem.Validate(); err != nil { + t.Errorf("Expected no validation error, got: %v", err) + } + }) + }) + + t.Run("Teardown", func(t *testing.T) { + ctx := t.Context() + + // Delete public LB + err := deleteLoadBalancer(ctx, lbClient, integrationTestResourceGroup, integrationTestFrontendIPLBName) + if err != nil { + t.Fatalf("Failed to delete public load balancer: %v", err) + } + + // Delete internal LB + err = deleteLoadBalancer(ctx, lbClient, integrationTestResourceGroup, integrationTestFrontendIPInternalLBName) + if err != nil { + t.Fatalf("Failed to delete internal load balancer: %v", err) + } + + // Delete public IP + err = deletePublicIPForLB(ctx, publicIPClient, integrationTestResourceGroup, integrationTestFrontendIPPublicIPName) + if err != nil { + t.Fatalf("Failed to delete public IP address: %v", err) + } + + // Delete VNet + err = deleteVirtualNetworkForLB(ctx, vnetClient, integrationTestResourceGroup, integrationTestFrontendIPVNetName) + if err != nil { + t.Fatalf("Failed to delete virtual network: %v", err) + } + }) +} diff --git a/sources/azure/integration-tests/sql-server_test.go b/sources/azure/integration-tests/sql-server_test.go index f670a2f1..fadff130 100644 --- a/sources/azure/integration-tests/sql-server_test.go +++ b/sources/azure/integration-tests/sql-server_test.go @@ -249,9 +249,9 @@ func TestSQLServerIntegration(t *testing.T) { } } - log.Printf("Verified %d linked item queries for SQL server %s", len(linkedQueries), sqlServerName) + log.Printf("Verified %d linked item queries for SQL server %s", len(linkedQueries), sqlServerName) + }) }) -}) t.Run("Teardown", func(t *testing.T) { ctx := t.Context() diff --git a/sources/azure/integration-tests/storage-blob-container_test.go b/sources/azure/integration-tests/storage-blob-container_test.go index ffda244e..7c742a86 100644 --- a/sources/azure/integration-tests/storage-blob-container_test.go +++ b/sources/azure/integration-tests/storage-blob-container_test.go @@ -120,8 +120,9 @@ func TestStorageBlobContainerIntegration(t *testing.T) { t.Fatalf("Failed to get unique attribute: %v", err) } - if uniqueAttrValue != integrationTestContainerName { - t.Fatalf("Expected unique attribute value to be %s, got %s", integrationTestContainerName, uniqueAttrValue) + expectedUniqueValue := shared.CompositeLookupKey(storageAccountName, integrationTestContainerName) + if uniqueAttrValue != expectedUniqueValue { + t.Fatalf("Expected unique attribute value to be %s, got %s", expectedUniqueValue, uniqueAttrValue) } log.Printf("Successfully retrieved blob container %s", integrationTestContainerName) @@ -155,10 +156,11 @@ func TestStorageBlobContainerIntegration(t *testing.T) { t.Fatalf("Expected at least one blob container, got %d", len(sdpItems)) } + expectedUniqueValue := shared.CompositeLookupKey(storageAccountName, integrationTestContainerName) var found bool for _, item := range sdpItems { uniqueAttrKey := item.GetUniqueAttribute() - if v, err := item.GetAttributes().Get(uniqueAttrKey); err == nil && v == integrationTestContainerName { + if v, err := item.GetAttributes().Get(uniqueAttrKey); err == nil && v == expectedUniqueValue { found = true break } diff --git a/sources/azure/integration-tests/storage-queues_test.go b/sources/azure/integration-tests/storage-queues_test.go index 24dc4d14..0cae249f 100644 --- a/sources/azure/integration-tests/storage-queues_test.go +++ b/sources/azure/integration-tests/storage-queues_test.go @@ -199,8 +199,8 @@ func TestStorageQueuesIntegration(t *testing.T) { } // Verify unique attribute - if sdpItem.GetUniqueAttribute() != "id" { - t.Errorf("Expected unique attribute 'id', got %s", sdpItem.GetUniqueAttribute()) + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) } // Verify item validation diff --git a/sources/azure/integration-tests/storage-table_test.go b/sources/azure/integration-tests/storage-table_test.go index 2a0bae7e..716cdade 100644 --- a/sources/azure/integration-tests/storage-table_test.go +++ b/sources/azure/integration-tests/storage-table_test.go @@ -23,7 +23,7 @@ import ( ) const ( - integrationTestTableName = "ovm-integ-test-table" + integrationTestTableName = "ovmintegtesttable" ) func TestStorageTableIntegration(t *testing.T) { @@ -199,8 +199,8 @@ func TestStorageTableIntegration(t *testing.T) { } // Verify unique attribute - if sdpItem.GetUniqueAttribute() != "id" { - t.Errorf("Expected unique attribute 'id', got %s", sdpItem.GetUniqueAttribute()) + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) } // Verify item validation diff --git a/sources/azure/manual/adapters.go b/sources/azure/manual/adapters.go index 0367a752..77b70da6 100644 --- a/sources/azure/manual/adapters.go +++ b/sources/azure/manual/adapters.go @@ -10,12 +10,12 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmos/armcosmos/v3" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/elasticsan/armelasticsan" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault/v2" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/elasticsan/armelasticsan" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql/v2" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3" @@ -183,6 +183,11 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred return nil, fmt.Errorf("failed to create load balancers client: %w", err) } + loadBalancerFrontendIPConfigurationsClient, err := armnetwork.NewLoadBalancerFrontendIPConfigurationsClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create load balancer frontend IP configurations client: %w", err) + } + privateEndpointsClient, err := armnetwork.NewPrivateEndpointsClient(subscriptionID, cred, nil) if err != nil { return nil, fmt.Errorf("failed to create private endpoints client: %w", err) @@ -203,6 +208,11 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred return nil, fmt.Errorf("failed to create batch pool client: %w", err) } + batchApplicationPackageClient, err := armbatch.NewApplicationPackageClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create batch application package client: %w", err) + } + virtualMachineScaleSetsClient, err := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil) if err != nil { return nil, fmt.Errorf("failed to create virtual machine scale sets client: %w", err) @@ -262,6 +272,11 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred return nil, fmt.Errorf("failed to create nat gateways client: %w", err) } + flowLogsClient, err := armnetwork.NewFlowLogsClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create flow logs client: %w", err) + } + managedHSMsClient, err := armkeyvault.NewManagedHsmsClient(subscriptionID, cred, nil) if err != nil { return nil, fmt.Errorf("failed to create managed hsms client: %w", err) @@ -312,6 +327,11 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred return nil, fmt.Errorf("failed to create postgresql flexible server private endpoint connections client: %w", err) } + postgresqlBackupsClient, err := armpostgresqlflexibleservers.NewBackupsAutomaticAndOnDemandClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create postgresql flexible server backups client: %w", err) + } + secretsClient, err := armkeyvault.NewSecretsClient(subscriptionID, cred, nil) if err != nil { return nil, fmt.Errorf("failed to create secrets client: %w", err) @@ -553,6 +573,10 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred clients.NewLoadBalancersClient(loadBalancersClient), resourceGroupScopes, ), cache), + sources.WrapperToAdapter(NewNetworkLoadBalancerFrontendIPConfiguration( + clients.NewLoadBalancerFrontendIPConfigurationsClient(loadBalancerFrontendIPConfigurationsClient), + resourceGroupScopes, + ), cache), sources.WrapperToAdapter(NewNetworkPrivateEndpoint( clients.NewPrivateEndpointsClient(privateEndpointsClient), resourceGroupScopes, @@ -585,6 +609,10 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred clients.NewBatchPoolsClient(batchPoolClient), resourceGroupScopes, ), cache), + sources.WrapperToAdapter(NewBatchBatchApplicationPackage( + clients.NewBatchApplicationPackagesClient(batchApplicationPackageClient), + resourceGroupScopes, + ), cache), sources.WrapperToAdapter(NewComputeVirtualMachineScaleSet( clients.NewVirtualMachineScaleSetsClient(virtualMachineScaleSetsClient), resourceGroupScopes, @@ -633,6 +661,10 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred clients.NewNatGatewaysClient(natGatewaysClient), resourceGroupScopes, ), cache), + sources.WrapperToAdapter(NewNetworkFlowLog( + clients.NewFlowLogsClient(flowLogsClient), + resourceGroupScopes, + ), cache), sources.WrapperToAdapter(NewSqlServer( clients.NewSqlServersClient(sqlServersClient), resourceGroupScopes, @@ -649,6 +681,10 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred clients.NewDBforPostgreSQLFlexibleServerPrivateEndpointConnectionsClient(postgresqlPrivateEndpointConnectionsClient), resourceGroupScopes, ), cache), + sources.WrapperToAdapter(NewDBforPostgreSQLFlexibleServerBackup( + clients.NewDBforPostgreSQLFlexibleServerBackupClient(postgresqlBackupsClient), + resourceGroupScopes, + ), cache), sources.WrapperToAdapter(NewKeyVaultSecret( clients.NewSecretsClient(secretsClient), resourceGroupScopes, @@ -790,6 +826,7 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred sources.WrapperToAdapter(NewNetworkPublicIPPrefix(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkDdosProtectionPlan(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkLoadBalancer(nil, placeholderResourceGroupScopes), noOpCache), + sources.WrapperToAdapter(NewNetworkLoadBalancerFrontendIPConfiguration(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkZone(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkPrivateDNSZone(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkDNSRecordSet(nil, placeholderResourceGroupScopes), noOpCache), @@ -797,6 +834,7 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred sources.WrapperToAdapter(NewBatchAccount(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewBatchBatchApplication(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewBatchBatchPool(nil, placeholderResourceGroupScopes), noOpCache), + sources.WrapperToAdapter(NewBatchBatchApplicationPackage(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewComputeVirtualMachineScaleSet(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewComputeAvailabilitySet(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewComputeDisk(nil, placeholderResourceGroupScopes), noOpCache), @@ -808,10 +846,12 @@ func Adapters(ctx context.Context, subscriptionID string, regions []string, cred sources.WrapperToAdapter(NewNetworkApplicationGateway(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkVirtualNetworkGateway(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewNetworkNatGateway(nil, placeholderResourceGroupScopes), noOpCache), + sources.WrapperToAdapter(NewNetworkFlowLog(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewSqlServer(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewDBforPostgreSQLFlexibleServer(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewDBforPostgreSQLFlexibleServerFirewallRule(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewDBforPostgreSQLFlexibleServerPrivateEndpointConnection(nil, placeholderResourceGroupScopes), noOpCache), + sources.WrapperToAdapter(NewDBforPostgreSQLFlexibleServerBackup(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewKeyVaultSecret(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewKeyVaultKey(nil, placeholderResourceGroupScopes), noOpCache), sources.WrapperToAdapter(NewManagedIdentityUserAssignedIdentity(nil, placeholderResourceGroupScopes), noOpCache), diff --git a/sources/azure/manual/authorization-role-assignment.go b/sources/azure/manual/authorization-role-assignment.go index 48d0f07a..ab96e11c 100644 --- a/sources/azure/manual/authorization-role-assignment.go +++ b/sources/azure/manual/authorization-role-assignment.go @@ -6,14 +6,15 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" + "github.com/overmindtech/cli/go/discovery" "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/sources" "github.com/overmindtech/cli/sources/azure/clients" azureshared "github.com/overmindtech/cli/sources/azure/shared" "github.com/overmindtech/cli/sources/shared" - "github.com/overmindtech/cli/go/sdpcache" - "github.com/overmindtech/cli/go/discovery" ) + var AuthorizationRoleAssignmentLookupByName = shared.NewItemTypeLookup("name", azureshared.AuthorizationRoleAssignment) type authorizationRoleAssignmentWrapper struct { @@ -86,6 +87,7 @@ func (a authorizationRoleAssignmentWrapper) ListStream(ctx context.Context, stre } } } + func (a authorizationRoleAssignmentWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { if scope == "" { return nil, azureshared.QueryError(errors.New("scope cannot be empty"), scope, a.Type()) diff --git a/sources/azure/manual/authorization-role-assignment_test.go b/sources/azure/manual/authorization-role-assignment_test.go index 7a7fec01..141ded9f 100644 --- a/sources/azure/manual/authorization-role-assignment_test.go +++ b/sources/azure/manual/authorization-role-assignment_test.go @@ -73,7 +73,8 @@ func TestAuthorizationRoleAssignment(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "b24988ac-6180-42a0-ab88-20f7382dd24c", ExpectedScope: subscriptionID, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -503,7 +504,8 @@ func TestAuthorizationRoleAssignment(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-identity", ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/batch-batch-accounts.go b/sources/azure/manual/batch-batch-accounts.go index b0e2383c..d751f400 100644 --- a/sources/azure/manual/batch-batch-accounts.go +++ b/sources/azure/manual/batch-batch-accounts.go @@ -91,6 +91,7 @@ func (b batchAccountWrapper) ListStream(ctx context.Context, stream discovery.Qu } } } + func (b batchAccountWrapper) azureBatchAccountToSDPItem(account *armbatch.Account, scope string) (*sdp.Item, *sdp.QueryError) { if account.Name == nil { return nil, azureshared.QueryError(errors.New("name is nil"), scope, b.Type()) diff --git a/sources/azure/manual/batch-batch-accounts_test.go b/sources/azure/manual/batch-batch-accounts_test.go index 0d37f1a2..8d95f22c 100644 --- a/sources/azure/manual/batch-batch-accounts_test.go +++ b/sources/azure/manual/batch-batch-accounts_test.go @@ -146,7 +146,8 @@ func TestBatchAccount(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: accountName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/batch-batch-application-package.go b/sources/azure/manual/batch-batch-application-package.go new file mode 100644 index 00000000..27fd0ced --- /dev/null +++ b/sources/azure/manual/batch-batch-application-package.go @@ -0,0 +1,245 @@ +package manual + +import ( + "context" + "errors" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/batch/armbatch/v4" + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" + "github.com/overmindtech/cli/sources/stdlib" +) + +var BatchBatchApplicationPackageLookupByName = shared.NewItemTypeLookup("name", azureshared.BatchBatchApplicationPackage) + +type batchBatchApplicationPackageWrapper struct { + client clients.BatchApplicationPackagesClient + *azureshared.MultiResourceGroupBase +} + +// NewBatchBatchApplicationPackage returns a SearchableWrapper for Azure Batch application packages +// (child of Batch application, grandchild of Batch account). +func NewBatchBatchApplicationPackage(client clients.BatchApplicationPackagesClient, resourceGroupScopes []azureshared.ResourceGroupScope) sources.SearchableWrapper { + return &batchBatchApplicationPackageWrapper{ + client: client, + MultiResourceGroupBase: azureshared.NewMultiResourceGroupBase( + resourceGroupScopes, + sdp.AdapterCategory_ADAPTER_CATEGORY_COMPUTE_APPLICATION, + azureshared.BatchBatchApplicationPackage, + ), + } +} + +func (c batchBatchApplicationPackageWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 3 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Get requires 3 query parts: accountName, applicationName, and versionName", + Scope: scope, + ItemType: c.Type(), + } + } + accountName := queryParts[0] + applicationName := queryParts[1] + versionName := queryParts[2] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + resp, err := c.client.Get(ctx, rgScope.ResourceGroup, accountName, applicationName, versionName) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + return c.azureApplicationPackageToSDPItem(&resp.ApplicationPackage, accountName, applicationName, versionName, scope) +} + +func (c batchBatchApplicationPackageWrapper) GetLookups() sources.ItemTypeLookups { + return sources.ItemTypeLookups{ + BatchAccountLookupByName, + BatchBatchApplicationLookupByName, + BatchBatchApplicationPackageLookupByName, + } +} + +func (c batchBatchApplicationPackageWrapper) Search(ctx context.Context, scope string, queryParts ...string) ([]*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 2 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Search requires 2 query parts: accountName and applicationName", + Scope: scope, + ItemType: c.Type(), + } + } + accountName := queryParts[0] + applicationName := queryParts[1] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + pager := c.client.List(ctx, rgScope.ResourceGroup, accountName, applicationName) + + var items []*sdp.Item + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + for _, pkg := range page.Value { + if pkg == nil || pkg.Name == nil { + continue + } + item, sdpErr := c.azureApplicationPackageToSDPItem(pkg, accountName, applicationName, *pkg.Name, scope) + if sdpErr != nil { + return nil, sdpErr + } + items = append(items, item) + } + } + + return items, nil +} + +func (c batchBatchApplicationPackageWrapper) SearchStream(ctx context.Context, stream discovery.QueryResultStream, cache sdpcache.Cache, cacheKey sdpcache.CacheKey, scope string, queryParts ...string) { + if len(queryParts) < 2 { + stream.SendError(azureshared.QueryError(errors.New("Search requires 2 query parts: accountName and applicationName"), scope, c.Type())) + return + } + accountName := queryParts[0] + applicationName := queryParts[1] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + pager := c.client.List(ctx, rgScope.ResourceGroup, accountName, applicationName) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + for _, pkg := range page.Value { + if pkg == nil || pkg.Name == nil { + continue + } + item, sdpErr := c.azureApplicationPackageToSDPItem(pkg, accountName, applicationName, *pkg.Name, scope) + if sdpErr != nil { + stream.SendError(sdpErr) + continue + } + cache.StoreItem(ctx, item, shared.DefaultCacheDuration, cacheKey) + stream.SendItem(item) + } + } +} + +func (c batchBatchApplicationPackageWrapper) SearchLookups() []sources.ItemTypeLookups { + return []sources.ItemTypeLookups{ + { + BatchAccountLookupByName, + BatchBatchApplicationLookupByName, + }, + } +} + +func (c batchBatchApplicationPackageWrapper) azureApplicationPackageToSDPItem(pkg *armbatch.ApplicationPackage, accountName, applicationName, versionName, scope string) (*sdp.Item, *sdp.QueryError) { + if pkg.Name == nil { + return nil, azureshared.QueryError(errors.New("application package name is nil"), scope, c.Type()) + } + attributes, err := shared.ToAttributesWithExclude(pkg, "tags") + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + if err := attributes.Set("uniqueAttr", shared.CompositeLookupKey(accountName, applicationName, versionName)); err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + sdpItem := &sdp.Item{ + Type: azureshared.BatchBatchApplicationPackage.String(), + UniqueAttribute: "uniqueAttr", + Attributes: attributes, + Scope: scope, + Tags: azureshared.ConvertAzureTags(pkg.Tags), + LinkedItemQueries: []*sdp.LinkedItemQuery{}, + } + + // Health status from package state + if pkg.Properties != nil && pkg.Properties.State != nil { + switch *pkg.Properties.State { + case armbatch.PackageStateActive: + sdpItem.Health = sdp.Health_HEALTH_OK.Enum() + case armbatch.PackageStatePending: + sdpItem.Health = sdp.Health_HEALTH_PENDING.Enum() + default: + sdpItem.Health = sdp.Health_HEALTH_UNKNOWN.Enum() + } + } + + // Link to parent Batch Application + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.BatchBatchApplication.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(accountName, applicationName), + Scope: scope, + }, + }) + + // Link to parent Batch Account + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.BatchBatchAccount.String(), + Method: sdp.QueryMethod_GET, + Query: accountName, + Scope: scope, + }, + }) + + // Link to StorageURL DNS name (Azure Storage blob endpoint hosting the package) + if pkg.Properties != nil && pkg.Properties.StorageURL != nil && *pkg.Properties.StorageURL != "" { + u, parseErr := url.Parse(*pkg.Properties.StorageURL) + if parseErr == nil && u.Hostname() != "" { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: stdlib.NetworkDNS.String(), + Method: sdp.QueryMethod_SEARCH, + Query: u.Hostname(), + Scope: "global", + }, + }) + } + } + + return sdpItem, nil +} + +func (c batchBatchApplicationPackageWrapper) PotentialLinks() map[shared.ItemType]bool { + return map[shared.ItemType]bool{ + azureshared.BatchBatchApplication: true, + azureshared.BatchBatchAccount: true, + stdlib.NetworkDNS: true, + } +} + +// ref: https://learn.microsoft.com/en-us/azure/role-based-access-control/resource-provider-operations#microsoftbatch +func (c batchBatchApplicationPackageWrapper) IAMPermissions() []string { + return []string{ + "Microsoft.Batch/batchAccounts/applications/versions/read", + } +} + +func (c batchBatchApplicationPackageWrapper) PredefinedRole() string { + return "Azure Batch Account Reader" +} diff --git a/sources/azure/manual/batch-batch-application-package_test.go b/sources/azure/manual/batch-batch-application-package_test.go new file mode 100644 index 00000000..3fe15b71 --- /dev/null +++ b/sources/azure/manual/batch-batch-application-package_test.go @@ -0,0 +1,394 @@ +package manual_test + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/batch/armbatch/v4" + "go.uber.org/mock/gomock" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/azure/shared/mocks" + "github.com/overmindtech/cli/sources/shared" + "github.com/overmindtech/cli/sources/stdlib" +) + +type mockBatchApplicationPackagesPager struct { + pages []armbatch.ApplicationPackageClientListResponse + index int +} + +func (m *mockBatchApplicationPackagesPager) More() bool { + return m.index < len(m.pages) +} + +func (m *mockBatchApplicationPackagesPager) NextPage(ctx context.Context) (armbatch.ApplicationPackageClientListResponse, error) { + if m.index >= len(m.pages) { + return armbatch.ApplicationPackageClientListResponse{}, errors.New("no more pages") + } + page := m.pages[m.index] + m.index++ + return page, nil +} + +type errorBatchApplicationPackagesPager struct{} + +func (e *errorBatchApplicationPackagesPager) More() bool { + return true +} + +func (e *errorBatchApplicationPackagesPager) NextPage(ctx context.Context) (armbatch.ApplicationPackageClientListResponse, error) { + return armbatch.ApplicationPackageClientListResponse{}, errors.New("pager error") +} + +type testBatchApplicationPackagesClient struct { + *mocks.MockBatchApplicationPackagesClient + pager clients.BatchApplicationPackagesPager +} + +func (t *testBatchApplicationPackagesClient) List(ctx context.Context, resourceGroupName, accountName, applicationName string) clients.BatchApplicationPackagesPager { + if t.pager != nil { + return t.pager + } + return t.MockBatchApplicationPackagesClient.List(ctx, resourceGroupName, accountName, applicationName) +} + +func createAzureBatchApplicationPackage(versionName string) *armbatch.ApplicationPackage { + state := armbatch.PackageStateActive + return &armbatch.ApplicationPackage{ + ID: new("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Batch/batchAccounts/acc/applications/app/versions/" + versionName), + Name: new(versionName), + Type: new("Microsoft.Batch/batchAccounts/applications/versions"), + Properties: &armbatch.ApplicationPackageProperties{ + State: &state, + Format: new("zip"), + StorageURL: new("https://teststorage.blob.core.windows.net/packages/" + versionName + ".zip"), + }, + Tags: map[string]*string{"env": new("test")}, + } +} + +func TestBatchBatchApplicationPackage(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subscriptionID := "test-subscription" + resourceGroup := "test-rg" + scope := subscriptionID + "." + resourceGroup + accountName := "test-batch-account" + applicationName := "test-app" + versionName := "1.0" + + t.Run("Get", func(t *testing.T) { + pkg := createAzureBatchApplicationPackage(versionName) + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, accountName, applicationName, versionName).Return( + armbatch.ApplicationPackageClientGetResponse{ + ApplicationPackage: *pkg, + }, nil) + + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(accountName, applicationName, versionName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.BatchBatchApplicationPackage.String() { + t.Errorf("Expected type %s, got %s", azureshared.BatchBatchApplicationPackage.String(), sdpItem.GetType()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + expectedUnique := shared.CompositeLookupKey(accountName, applicationName, versionName) + if sdpItem.UniqueAttributeValue() != expectedUnique { + t.Errorf("Expected unique attribute value %s, got %s", expectedUnique, sdpItem.UniqueAttributeValue()) + } + + if sdpItem.GetScope() != scope { + t.Errorf("Expected scope %s, got %s", scope, sdpItem.GetScope()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Expected valid item, got: %v", err) + } + + if sdpItem.GetHealth() != sdp.Health_HEALTH_OK { + t.Errorf("Expected health OK for active package, got %s", sdpItem.GetHealth()) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + {ExpectedType: azureshared.BatchBatchApplication.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: shared.CompositeLookupKey(accountName, applicationName), ExpectedScope: scope}, + {ExpectedType: azureshared.BatchBatchAccount.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: accountName, ExpectedScope: scope}, + {ExpectedType: stdlib.NetworkDNS.String(), ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "teststorage.blob.core.windows.net", ExpectedScope: "global"}, + } + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("Get_InvalidQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + // Only 2 parts instead of 3 + query := shared.CompositeLookupKey(accountName, applicationName) + _, qErr := adapter.Get(ctx, scope, query, true) + if qErr == nil { + t.Error("Expected error when Get with insufficient query parts, but got nil") + } + }) + + t.Run("Get_ClientError", func(t *testing.T) { + expectedErr := errors.New("application package not found") + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, accountName, applicationName, "nonexistent").Return( + armbatch.ApplicationPackageClientGetResponse{}, expectedErr) + + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(accountName, applicationName, "nonexistent") + _, qErr := adapter.Get(ctx, scope, query, true) + if qErr == nil { + t.Error("Expected error when client returns error, but got nil") + } + }) + + t.Run("Search", func(t *testing.T) { + pkg1 := createAzureBatchApplicationPackage("1.0") + pkg2 := createAzureBatchApplicationPackage("2.0") + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + pages := []armbatch.ApplicationPackageClientListResponse{ + { + ListApplicationPackagesResult: armbatch.ListApplicationPackagesResult{ + Value: []*armbatch.ApplicationPackage{pkg1, pkg2}, + }, + }, + } + mockPager := &mockBatchApplicationPackagesPager{pages: pages} + testClient := &testBatchApplicationPackagesClient{ + MockBatchApplicationPackagesClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewBatchBatchApplicationPackage(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, scope, shared.CompositeLookupKey(accountName, applicationName), true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) != 2 { + t.Fatalf("Expected 2 items, got: %d", len(sdpItems)) + } + + for _, item := range sdpItems { + if err := item.Validate(); err != nil { + t.Errorf("Expected valid item, got: %v", err) + } + } + }) + + t.Run("SearchStream", func(t *testing.T) { + pkg1 := createAzureBatchApplicationPackage("1.0") + pkg2 := createAzureBatchApplicationPackage("2.0") + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + pages := []armbatch.ApplicationPackageClientListResponse{ + { + ListApplicationPackagesResult: armbatch.ListApplicationPackagesResult{ + Value: []*armbatch.ApplicationPackage{pkg1, pkg2}, + }, + }, + } + mockPager := &mockBatchApplicationPackagesPager{pages: pages} + testClient := &testBatchApplicationPackagesClient{ + MockBatchApplicationPackagesClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewBatchBatchApplicationPackage(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchStreamable, ok := adapter.(discovery.SearchStreamableAdapter) + if !ok { + t.Fatalf("Adapter does not support SearchStream operation") + } + + var items []*sdp.Item + var errs []error + stream := discovery.NewQueryResultStream( + func(item *sdp.Item) { items = append(items, item) }, + func(err error) { errs = append(errs, err) }, + ) + + searchStreamable.SearchStream(ctx, scope, shared.CompositeLookupKey(accountName, applicationName), true, stream) + + if len(errs) != 0 { + t.Fatalf("Expected no errors, got: %v", errs) + } + + if len(items) != 2 { + t.Fatalf("Expected 2 items, got: %d", len(items)) + } + }) + + t.Run("Search_InvalidQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + + // No query parts + _, qErr := wrapper.Search(ctx, scope) + if qErr == nil { + t.Error("Expected error when Search with no query parts, but got nil") + } + }) + + t.Run("Search_PagerError", func(t *testing.T) { + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + errorPager := &errorBatchApplicationPackagesPager{} + testClient := &testBatchApplicationPackagesClient{ + MockBatchApplicationPackagesClient: mockClient, + pager: errorPager, + } + + wrapper := manual.NewBatchBatchApplicationPackage(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + _, qErr := wrapper.Search(ctx, scope, accountName, applicationName) + if qErr == nil { + t.Error("Expected error when pager returns error, but got nil") + } + }) + + t.Run("Search_NilNameSkipped", func(t *testing.T) { + validPkg := createAzureBatchApplicationPackage("1.0") + nilNamePkg := &armbatch.ApplicationPackage{ + Name: nil, + } + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + pages := []armbatch.ApplicationPackageClientListResponse{ + { + ListApplicationPackagesResult: armbatch.ListApplicationPackagesResult{ + Value: []*armbatch.ApplicationPackage{nilNamePkg, validPkg}, + }, + }, + } + mockPager := &mockBatchApplicationPackagesPager{pages: pages} + testClient := &testBatchApplicationPackagesClient{ + MockBatchApplicationPackagesClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewBatchBatchApplicationPackage(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + items, qErr := wrapper.Search(ctx, scope, accountName, applicationName) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if len(items) != 1 { + t.Fatalf("Expected 1 item (nil-name skipped), got: %d", len(items)) + } + }) + + t.Run("PotentialLinks", func(t *testing.T) { + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + + links := wrapper.PotentialLinks() + if !links[azureshared.BatchBatchApplication] { + t.Error("PotentialLinks() should include BatchBatchApplication") + } + if !links[azureshared.BatchBatchAccount] { + t.Error("PotentialLinks() should include BatchBatchAccount") + } + if !links[stdlib.NetworkDNS] { + t.Error("PotentialLinks() should include stdlib.NetworkDNS") + } + }) + + t.Run("HealthPending", func(t *testing.T) { + pkg := createAzureBatchApplicationPackage(versionName) + state := armbatch.PackageStatePending + pkg.Properties.State = &state + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, accountName, applicationName, versionName).Return( + armbatch.ApplicationPackageClientGetResponse{ + ApplicationPackage: *pkg, + }, nil) + + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(accountName, applicationName, versionName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetHealth() != sdp.Health_HEALTH_PENDING { + t.Errorf("Expected health PENDING for pending package, got %s", sdpItem.GetHealth()) + } + }) + + t.Run("GetWithoutStorageURL", func(t *testing.T) { + pkg := createAzureBatchApplicationPackage(versionName) + pkg.Properties.StorageURL = nil + + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, accountName, applicationName, versionName).Return( + armbatch.ApplicationPackageClientGetResponse{ + ApplicationPackage: *pkg, + }, nil) + + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(accountName, applicationName, versionName) + sdpItem, qErr := adapter.Get(ctx, scope, query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + // Should have 2 linked queries (application + account) but no DNS link + linkedQueries := sdpItem.GetLinkedItemQueries() + for _, liq := range linkedQueries { + if liq.GetQuery().GetType() == stdlib.NetworkDNS.String() { + t.Error("Expected no DNS linked query when StorageURL is nil") + } + } + }) + + t.Run("ImplementsSearchableAdapter", func(t *testing.T) { + mockClient := mocks.NewMockBatchApplicationPackagesClient(ctrl) + wrapper := manual.NewBatchBatchApplicationPackage(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + _, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Error("Adapter should implement SearchableAdapter interface") + } + }) +} diff --git a/sources/azure/manual/batch-batch-application.go b/sources/azure/manual/batch-batch-application.go index a551e962..c4cf1fdf 100644 --- a/sources/azure/manual/batch-batch-application.go +++ b/sources/azure/manual/batch-batch-application.go @@ -205,8 +205,8 @@ func (b batchBatchApplicationWrapper) azureApplicationToSDPItem(app *armbatch.Ap func (b batchBatchApplicationWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.BatchBatchAccount: true, - azureshared.BatchBatchApplicationPackage: true, + azureshared.BatchBatchAccount: true, + azureshared.BatchBatchApplicationPackage: true, } } diff --git a/sources/azure/manual/batch-batch-application_test.go b/sources/azure/manual/batch-batch-application_test.go index 32f38ee2..8a0ce298 100644 --- a/sources/azure/manual/batch-batch-application_test.go +++ b/sources/azure/manual/batch-batch-application_test.go @@ -69,8 +69,8 @@ func createAzureBatchApplication(name string) *armbatch.Application { Name: new(name), Type: new("Microsoft.Batch/batchAccounts/applications"), Properties: &armbatch.ApplicationProperties{ - DisplayName: new("Test application " + name), - AllowUpdates: &allowUpdates, + DisplayName: new("Test application " + name), + AllowUpdates: &allowUpdates, }, Tags: map[string]*string{"env": new("test")}, } diff --git a/sources/azure/manual/batch-batch-pool_test.go b/sources/azure/manual/batch-batch-pool_test.go index 93648924..9a9a5592 100644 --- a/sources/azure/manual/batch-batch-pool_test.go +++ b/sources/azure/manual/batch-batch-pool_test.go @@ -215,7 +215,7 @@ func TestBatchBatchPool(t *testing.T) { errorPager := &errorBatchPoolsPager{} testClient := &testBatchPoolsClient{ MockBatchPoolsClient: mockClient, - pager: errorPager, + pager: errorPager, } wrapper := manual.NewBatchBatchPool(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) diff --git a/sources/azure/manual/compute-availability-set_test.go b/sources/azure/manual/compute-availability-set_test.go index dfcae477..bdafd8c8 100644 --- a/sources/azure/manual/compute-availability-set_test.go +++ b/sources/azure/manual/compute-availability-set_test.go @@ -82,7 +82,8 @@ func TestComputeAvailabilitySet(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-vm-2", ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/compute-capacity-reservation-group.go b/sources/azure/manual/compute-capacity-reservation-group.go index 25caaa53..b2fb69e6 100644 --- a/sources/azure/manual/compute-capacity-reservation-group.go +++ b/sources/azure/manual/compute-capacity-reservation-group.go @@ -34,10 +34,7 @@ func NewComputeCapacityReservationGroup(client clients.CapacityReservationGroups } func capacityReservationGroupGetOptions() *armcompute.CapacityReservationGroupsClientGetOptions { - expand := armcompute.CapacityReservationGroupInstanceViewTypes(armcompute.ExpandTypesForGetCapacityReservationGroupsVirtualMachinesRef) - return &armcompute.CapacityReservationGroupsClientGetOptions{ - Expand: &expand, - } + return nil } func capacityReservationGroupListOptions() *armcompute.CapacityReservationGroupsClientListByResourceGroupOptions { @@ -195,6 +192,7 @@ func (c *computeCapacityReservationGroupWrapper) azureCapacityReservationGroupTo return sdpItem, nil } + func (c *computeCapacityReservationGroupWrapper) GetLookups() sources.ItemTypeLookups { return sources.ItemTypeLookups{ ComputeCapacityReservationGroupLookupByName, diff --git a/sources/azure/manual/compute-capacity-reservation-group_test.go b/sources/azure/manual/compute-capacity-reservation-group_test.go index 6dfb123f..e916d166 100644 --- a/sources/azure/manual/compute-capacity-reservation-group_test.go +++ b/sources/azure/manual/compute-capacity-reservation-group_test.go @@ -315,10 +315,7 @@ func TestComputeCapacityReservationGroup(t *testing.T) { } func capacityReservationGroupGetOptions() *armcompute.CapacityReservationGroupsClientGetOptions { - expand := armcompute.CapacityReservationGroupInstanceViewTypes(armcompute.ExpandTypesForGetCapacityReservationGroupsVirtualMachinesRef) - return &armcompute.CapacityReservationGroupsClientGetOptions{ - Expand: &expand, - } + return nil } func capacityReservationGroupListOptions() *armcompute.CapacityReservationGroupsClientListByResourceGroupOptions { diff --git a/sources/azure/manual/compute-capacity-reservation.go b/sources/azure/manual/compute-capacity-reservation.go index 697853bb..77f927a1 100644 --- a/sources/azure/manual/compute-capacity-reservation.go +++ b/sources/azure/manual/compute-capacity-reservation.go @@ -262,7 +262,7 @@ func (c *computeCapacityReservationWrapper) SearchLookups() []sources.ItemTypeLo func (c *computeCapacityReservationWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ azureshared.ComputeCapacityReservationGroup: true, - azureshared.ComputeVirtualMachine: true, + azureshared.ComputeVirtualMachine: true, } } diff --git a/sources/azure/manual/compute-capacity-reservation_test.go b/sources/azure/manual/compute-capacity-reservation_test.go index 515e7285..cdb46be1 100644 --- a/sources/azure/manual/compute-capacity-reservation_test.go +++ b/sources/azure/manual/compute-capacity-reservation_test.go @@ -53,8 +53,8 @@ func createAzureCapacityReservationWithVMs(reservationName, groupName, subscript Capacity: new(int64(1)), }, Properties: &armcompute.CapacityReservationProperties{ - ProvisioningState: new("Succeeded"), - VirtualMachinesAssociated: vms, + ProvisioningState: new("Succeeded"), + VirtualMachinesAssociated: vms, }, } } @@ -248,7 +248,7 @@ func TestComputeCapacityReservation(t *testing.T) { } testClient := &testCapacityReservationsClient{ MockCapacityReservationsClient: mockClient, - pager: pager, + pager: pager, } wrapper := NewComputeCapacityReservation(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) @@ -300,7 +300,7 @@ func TestComputeCapacityReservation(t *testing.T) { errorPager := &errorCapacityReservationsPager{} testClient := &testCapacityReservationsClient{ MockCapacityReservationsClient: mockClient, - pager: errorPager, + pager: errorPager, } wrapper := NewComputeCapacityReservation(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) @@ -324,7 +324,7 @@ func TestComputeCapacityReservation(t *testing.T) { links := wrapper.PotentialLinks() expected := map[shared.ItemType]bool{ azureshared.ComputeCapacityReservationGroup: true, - azureshared.ComputeVirtualMachine: true, + azureshared.ComputeVirtualMachine: true, } for itemType, want := range expected { if got := links[itemType]; got != want { diff --git a/sources/azure/manual/compute-dedicated-host-group_test.go b/sources/azure/manual/compute-dedicated-host-group_test.go index 605de8e2..7660cfd7 100644 --- a/sources/azure/manual/compute-dedicated-host-group_test.go +++ b/sources/azure/manual/compute-dedicated-host-group_test.go @@ -99,7 +99,8 @@ func TestComputeDedicatedHostGroup(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: shared.CompositeLookupKey(hostGroupName, "host-2"), ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) }) diff --git a/sources/azure/manual/compute-dedicated-host_test.go b/sources/azure/manual/compute-dedicated-host_test.go index b3f1204b..3c5cab4f 100644 --- a/sources/azure/manual/compute-dedicated-host_test.go +++ b/sources/azure/manual/compute-dedicated-host_test.go @@ -248,7 +248,7 @@ func TestComputeDedicatedHost(t *testing.T) { } testClient := &testDedicatedHostsClient{ MockDedicatedHostsClient: mockClient, - pager: pager, + pager: pager, } wrapper := NewComputeDedicatedHost(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) diff --git a/sources/azure/manual/compute-disk-access-private-endpoint-connection.go b/sources/azure/manual/compute-disk-access-private-endpoint-connection.go index 429364d9..ddba2d82 100644 --- a/sources/azure/manual/compute-disk-access-private-endpoint-connection.go +++ b/sources/azure/manual/compute-disk-access-private-endpoint-connection.go @@ -153,7 +153,7 @@ func (s computeDiskAccessPrivateEndpointConnectionWrapper) SearchLookups() []sou func (s computeDiskAccessPrivateEndpointConnectionWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.ComputeDiskAccess: true, + azureshared.ComputeDiskAccess: true, azureshared.NetworkPrivateEndpoint: true, } } diff --git a/sources/azure/manual/compute-disk-access.go b/sources/azure/manual/compute-disk-access.go index 15b821cd..2ba39749 100644 --- a/sources/azure/manual/compute-disk-access.go +++ b/sources/azure/manual/compute-disk-access.go @@ -105,6 +105,7 @@ func (c *computeDiskAccessWrapper) ListStream(ctx context.Context, stream discov } } } + func (c *computeDiskAccessWrapper) azureDiskAccessToSDPItem(diskAccess *armcompute.DiskAccess, scope string) (*sdp.Item, *sdp.QueryError) { if diskAccess.Name == nil { return nil, azureshared.QueryError(errors.New("name is nil"), scope, c.Type()) diff --git a/sources/azure/manual/compute-disk-access_test.go b/sources/azure/manual/compute-disk-access_test.go index 128f0c90..5066e54f 100644 --- a/sources/azure/manual/compute-disk-access_test.go +++ b/sources/azure/manual/compute-disk-access_test.go @@ -71,7 +71,8 @@ func TestComputeDiskAccess(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: diskAccessName, ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) }) @@ -113,7 +114,8 @@ func TestComputeDiskAccess(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-private-endpoint-other-rg", ExpectedScope: subscriptionID + ".other-rg", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) }) diff --git a/sources/azure/manual/compute-disk_test.go b/sources/azure/manual/compute-disk_test.go index f503c364..36ff1e5a 100644 --- a/sources/azure/manual/compute-disk_test.go +++ b/sources/azure/manual/compute-disk_test.go @@ -197,7 +197,8 @@ func TestComputeDisk(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "test-keyvault-2.vault.azure.net", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/compute-gallery-application-version.go b/sources/azure/manual/compute-gallery-application-version.go index dc863f58..6848c841 100644 --- a/sources/azure/manual/compute-gallery-application-version.go +++ b/sources/azure/manual/compute-gallery-application-version.go @@ -16,9 +16,7 @@ import ( "github.com/overmindtech/cli/sources/stdlib" ) -var ( - ComputeGalleryApplicationVersionLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryApplicationVersion) -) +var ComputeGalleryApplicationVersionLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryApplicationVersion) type computeGalleryApplicationVersionWrapper struct { client clients.GalleryApplicationVersionsClient @@ -370,7 +368,7 @@ func (c computeGalleryApplicationVersionWrapper) TerraformMappings() []*sdp.Terr return []*sdp.TerraformMapping{ { TerraformMethod: sdp.QueryMethod_SEARCH, - //example id: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group1/providers/Microsoft.Compute/galleries/gallery1/applications/galleryApplication1/versions/galleryApplicationVersion1 + // example id: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group1/providers/Microsoft.Compute/galleries/gallery1/applications/galleryApplication1/versions/galleryApplicationVersion1 TerraformQueryMap: "azurerm_gallery_application_version.id", }, } diff --git a/sources/azure/manual/compute-gallery-application.go b/sources/azure/manual/compute-gallery-application.go index 5fa2e23d..e75f5dae 100644 --- a/sources/azure/manual/compute-gallery-application.go +++ b/sources/azure/manual/compute-gallery-application.go @@ -15,9 +15,7 @@ import ( "github.com/overmindtech/cli/sources/stdlib" ) -var ( - ComputeGalleryApplicationLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryApplication) -) +var ComputeGalleryApplicationLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryApplication) type computeGalleryApplicationWrapper struct { client clients.GalleryApplicationsClient diff --git a/sources/azure/manual/compute-gallery-image.go b/sources/azure/manual/compute-gallery-image.go index 2da9df24..880e17bd 100644 --- a/sources/azure/manual/compute-gallery-image.go +++ b/sources/azure/manual/compute-gallery-image.go @@ -6,8 +6,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/overmindtech/cli/go/discovery" - "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/sources" "github.com/overmindtech/cli/sources/azure/clients" azureshared "github.com/overmindtech/cli/sources/azure/shared" @@ -15,9 +15,7 @@ import ( "github.com/overmindtech/cli/sources/stdlib" ) -var ( - ComputeGalleryImageLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryImage) -) +var ComputeGalleryImageLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeGalleryImage) type computeGalleryImageWrapper struct { client clients.GalleryImagesClient diff --git a/sources/azure/manual/compute-image_test.go b/sources/azure/manual/compute-image_test.go index 475248a6..74fff56c 100644 --- a/sources/azure/manual/compute-image_test.go +++ b/sources/azure/manual/compute-image_test.go @@ -162,7 +162,8 @@ func TestComputeImage(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-source-vm", ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -623,8 +624,7 @@ func (m *mockImagesPager) NextPage(ctx context.Context) (armcompute.ImagesClient } // errorImagesPager is a mock pager that always returns an error -type errorImagesPager struct { -} +type errorImagesPager struct{} func newErrorImagesPager(ctrl *gomock.Controller) clients.ImagesPager { return &errorImagesPager{} diff --git a/sources/azure/manual/compute-proximity-placement-group.go b/sources/azure/manual/compute-proximity-placement-group.go index c6684149..df67ec14 100644 --- a/sources/azure/manual/compute-proximity-placement-group.go +++ b/sources/azure/manual/compute-proximity-placement-group.go @@ -5,14 +5,15 @@ import ( "errors" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/overmindtech/cli/go/discovery" "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/sources" "github.com/overmindtech/cli/sources/azure/clients" azureshared "github.com/overmindtech/cli/sources/azure/shared" "github.com/overmindtech/cli/sources/shared" - "github.com/overmindtech/cli/go/sdpcache" - "github.com/overmindtech/cli/go/discovery" ) + var ComputeProximityPlacementGroupLookupByName = shared.NewItemTypeLookup("name", azureshared.ComputeProximityPlacementGroup) type computeProximityPlacementGroupWrapper struct { diff --git a/sources/azure/manual/compute-proximity-placement-group_test.go b/sources/azure/manual/compute-proximity-placement-group_test.go index 464a8d82..cd7b89ff 100644 --- a/sources/azure/manual/compute-proximity-placement-group_test.go +++ b/sources/azure/manual/compute-proximity-placement-group_test.go @@ -79,7 +79,8 @@ func TestComputeProximityPlacementGroup(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-vmss", ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -251,7 +252,6 @@ func TestComputeProximityPlacementGroup(t *testing.T) { t.Error("Expected error when getting proximity placement group with empty name, but got nil") } }) - } func createAzureProximityPlacementGroup(ppgName, subscriptionID, resourceGroup string) *armcompute.ProximityPlacementGroup { diff --git a/sources/azure/manual/compute-snapshot_test.go b/sources/azure/manual/compute-snapshot_test.go index 3208b2f1..642a0cb6 100644 --- a/sources/azure/manual/compute-snapshot_test.go +++ b/sources/azure/manual/compute-snapshot_test.go @@ -74,19 +74,22 @@ func TestComputeSnapshot(t *testing.T) { ExpectedType: azureshared.ComputeDiskAccess.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-disk-access", - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, { // Properties.Encryption.DiskEncryptionSetID ExpectedType: azureshared.ComputeDiskEncryptionSet.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-des", - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, { // Properties.CreationData.SourceResourceID (disk) ExpectedType: azureshared.ComputeDisk.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "source-disk", - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, } shared.RunStaticTests(t, adapter, sdpItem, queryTests) @@ -118,7 +121,8 @@ func TestComputeSnapshot(t *testing.T) { ExpectedType: azureshared.ComputeSnapshot.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "source-snapshot", - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, } shared.RunStaticTests(t, adapter, sdpItem, queryTests) @@ -150,25 +154,29 @@ func TestComputeSnapshot(t *testing.T) { ExpectedType: azureshared.StorageAccount.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "teststorageaccount", - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, { // Properties.CreationData.SourceURI → Blob Container ExpectedType: azureshared.StorageBlobContainer.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: shared.CompositeLookupKey("teststorageaccount", "vhds"), - ExpectedScope: subscriptionID + "." + resourceGroup}, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, { // Properties.CreationData.SourceURI → HTTP ExpectedType: stdlib.NetworkHTTP.String(), ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "https://teststorageaccount.blob.core.windows.net/vhds/my-disk.vhd", - ExpectedScope: "global"}, + ExpectedScope: "global", + }, { // Properties.CreationData.SourceURI → DNS ExpectedType: stdlib.NetworkDNS.String(), ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "teststorageaccount.blob.core.windows.net", - ExpectedScope: "global"}, + ExpectedScope: "global", + }, } shared.RunStaticTests(t, adapter, sdpItem, queryTests) @@ -200,13 +208,15 @@ func TestComputeSnapshot(t *testing.T) { ExpectedType: stdlib.NetworkHTTP.String(), ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "https://10.0.0.1/vhds/my-disk.vhd", - ExpectedScope: "global"}, + ExpectedScope: "global", + }, { // Properties.CreationData.SourceURI → IP (host is IP address) ExpectedType: stdlib.NetworkIP.String(), ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "10.0.0.1", - ExpectedScope: "global"}, + ExpectedScope: "global", + }, } shared.RunStaticTests(t, adapter, sdpItem, queryTests) diff --git a/sources/azure/manual/compute-virtual-machine-extension_test.go b/sources/azure/manual/compute-virtual-machine-extension_test.go index 8cbff84e..6c43ffec 100644 --- a/sources/azure/manual/compute-virtual-machine-extension_test.go +++ b/sources/azure/manual/compute-virtual-machine-extension_test.go @@ -69,7 +69,8 @@ func TestComputeVirtualMachineExtension(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: vmName, ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/compute-virtual-machine-run-command_test.go b/sources/azure/manual/compute-virtual-machine-run-command_test.go index 33b68cee..cd66fcf2 100644 --- a/sources/azure/manual/compute-virtual-machine-run-command_test.go +++ b/sources/azure/manual/compute-virtual-machine-run-command_test.go @@ -149,7 +149,8 @@ func TestComputeVirtualMachineRunCommand(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: vmName, ExpectedScope: scope, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/compute-virtual-machine-scale-set_test.go b/sources/azure/manual/compute-virtual-machine-scale-set_test.go index 22487b1d..af69ea63 100644 --- a/sources/azure/manual/compute-virtual-machine-scale-set_test.go +++ b/sources/azure/manual/compute-virtual-machine-scale-set_test.go @@ -206,7 +206,8 @@ func TestComputeVirtualMachineScaleSet(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-keyvault-ext", ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/compute-virtual-machine_test.go b/sources/azure/manual/compute-virtual-machine_test.go index f565919d..53bac3a7 100644 --- a/sources/azure/manual/compute-virtual-machine_test.go +++ b/sources/azure/manual/compute-virtual-machine_test.go @@ -103,7 +103,8 @@ func TestComputeVirtualMachine(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: vmName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/dbforpostgresql-database_test.go b/sources/azure/manual/dbforpostgresql-database_test.go index eba76125..fdc59aa8 100644 --- a/sources/azure/manual/dbforpostgresql-database_test.go +++ b/sources/azure/manual/dbforpostgresql-database_test.go @@ -119,7 +119,8 @@ func TestDBforPostgreSQLDatabase(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: serverName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/dbforpostgresql-flexible-server-backup.go b/sources/azure/manual/dbforpostgresql-flexible-server-backup.go new file mode 100644 index 00000000..67cfd2c8 --- /dev/null +++ b/sources/azure/manual/dbforpostgresql-flexible-server-backup.go @@ -0,0 +1,230 @@ +package manual + +import ( + "context" + "errors" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +var DBforPostgreSQLFlexibleServerBackupLookupByName = shared.NewItemTypeLookup("name", azureshared.DBforPostgreSQLFlexibleServerBackup) + +type dbforPostgreSQLFlexibleServerBackupWrapper struct { + client clients.DBforPostgreSQLFlexibleServerBackupClient + + *azureshared.MultiResourceGroupBase +} + +func NewDBforPostgreSQLFlexibleServerBackup(client clients.DBforPostgreSQLFlexibleServerBackupClient, resourceGroupScopes []azureshared.ResourceGroupScope) sources.SearchableWrapper { + return &dbforPostgreSQLFlexibleServerBackupWrapper{ + client: client, + MultiResourceGroupBase: azureshared.NewMultiResourceGroupBase( + resourceGroupScopes, + sdp.AdapterCategory_ADAPTER_CATEGORY_DATABASE, + azureshared.DBforPostgreSQLFlexibleServerBackup, + ), + } +} + +// ref: https://learn.microsoft.com/en-us/rest/api/postgresql/backups-automatic-and-on-demand/get?view=rest-postgresql-2025-08-01 +func (s dbforPostgreSQLFlexibleServerBackupWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 2 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Get requires 2 query parts: serverName and backupName", + Scope: scope, + ItemType: s.Type(), + } + } + serverName := queryParts[0] + backupName := queryParts[1] + if serverName == "" { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "serverName cannot be empty", + Scope: scope, + ItemType: s.Type(), + } + } + if backupName == "" { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "backupName cannot be empty", + Scope: scope, + ItemType: s.Type(), + } + } + + rgScope, err := s.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + resp, err := s.client.Get(ctx, rgScope.ResourceGroup, serverName, backupName) + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + + return s.azureBackupToSDPItem(&resp.BackupAutomaticAndOnDemand, serverName, backupName, scope) +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) azureBackupToSDPItem(backup *armpostgresqlflexibleservers.BackupAutomaticAndOnDemand, serverName, backupName, scope string) (*sdp.Item, *sdp.QueryError) { + if backup.Name == nil { + return nil, azureshared.QueryError(errors.New("backup name is nil"), scope, s.Type()) + } + + attributes, err := shared.ToAttributesWithExclude(backup, "tags") + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + + err = attributes.Set("uniqueAttr", shared.CompositeLookupKey(serverName, backupName)) + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + + sdpItem := &sdp.Item{ + Type: azureshared.DBforPostgreSQLFlexibleServerBackup.String(), + UniqueAttribute: "uniqueAttr", + Attributes: attributes, + Scope: scope, + Tags: nil, + } + + // Link to parent PostgreSQL Flexible Server + if backup.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*backup.ID, []string{"flexibleServers"}) + if len(params) > 0 { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.DBforPostgreSQLFlexibleServer.String(), + Method: sdp.QueryMethod_GET, + Query: params[0], + Scope: scope, + }, + }) + } + } else { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.DBforPostgreSQLFlexibleServer.String(), + Method: sdp.QueryMethod_GET, + Query: serverName, + Scope: scope, + }, + }) + } + + return sdpItem, nil +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) GetLookups() sources.ItemTypeLookups { + return sources.ItemTypeLookups{ + DBforPostgreSQLFlexibleServerLookupByName, + DBforPostgreSQLFlexibleServerBackupLookupByName, + } +} + +// ref: https://learn.microsoft.com/en-us/rest/api/postgresql/backups-automatic-and-on-demand/list-by-server?view=rest-postgresql-2025-08-01 +func (s dbforPostgreSQLFlexibleServerBackupWrapper) Search(ctx context.Context, scope string, queryParts ...string) ([]*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 1 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Search requires 1 query part: serverName", + Scope: scope, + ItemType: s.Type(), + } + } + serverName := queryParts[0] + + rgScope, err := s.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + pager := s.client.ListByServer(ctx, rgScope.ResourceGroup, serverName) + + var items []*sdp.Item + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, azureshared.QueryError(err, scope, s.Type()) + } + for _, backup := range page.Value { + if backup.Name == nil { + continue + } + item, sdpErr := s.azureBackupToSDPItem(backup, serverName, *backup.Name, scope) + if sdpErr != nil { + return nil, sdpErr + } + items = append(items, item) + } + } + + return items, nil +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) SearchStream(ctx context.Context, stream discovery.QueryResultStream, cache sdpcache.Cache, cacheKey sdpcache.CacheKey, scope string, queryParts ...string) { + if len(queryParts) < 1 { + stream.SendError(azureshared.QueryError(errors.New("Search requires 1 query part: serverName"), scope, s.Type())) + return + } + serverName := queryParts[0] + + rgScope, err := s.ResourceGroupScopeFromScope(scope) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, s.Type())) + return + } + pager := s.client.ListByServer(ctx, rgScope.ResourceGroup, serverName) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, s.Type())) + return + } + for _, backup := range page.Value { + if backup.Name == nil { + continue + } + item, sdpErr := s.azureBackupToSDPItem(backup, serverName, *backup.Name, scope) + if sdpErr != nil { + stream.SendError(sdpErr) + continue + } + cache.StoreItem(ctx, item, shared.DefaultCacheDuration, cacheKey) + stream.SendItem(item) + } + } +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) SearchLookups() []sources.ItemTypeLookups { + return []sources.ItemTypeLookups{ + { + DBforPostgreSQLFlexibleServerLookupByName, + }, + } +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) PotentialLinks() map[shared.ItemType]bool { + return map[shared.ItemType]bool{ + azureshared.DBforPostgreSQLFlexibleServer: true, + } +} + +// ref: https://learn.microsoft.com/en-us/azure/role-based-access-control/resource-provider-operations#microsoftdbforpostgresql +func (s dbforPostgreSQLFlexibleServerBackupWrapper) IAMPermissions() []string { + return []string{ + "Microsoft.DBforPostgreSQL/flexibleServers/backups/read", + } +} + +func (s dbforPostgreSQLFlexibleServerBackupWrapper) PredefinedRole() string { + return "Reader" +} diff --git a/sources/azure/manual/dbforpostgresql-flexible-server-backup_test.go b/sources/azure/manual/dbforpostgresql-flexible-server-backup_test.go new file mode 100644 index 00000000..a32ffae1 --- /dev/null +++ b/sources/azure/manual/dbforpostgresql-flexible-server-backup_test.go @@ -0,0 +1,295 @@ +package manual_test + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" + "go.uber.org/mock/gomock" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/azure/shared/mocks" + "github.com/overmindtech/cli/sources/shared" +) + +type mockDBforPostgreSQLFlexibleServerBackupPager struct { + pages []armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse + index int +} + +func (m *mockDBforPostgreSQLFlexibleServerBackupPager) More() bool { + return m.index < len(m.pages) +} + +func (m *mockDBforPostgreSQLFlexibleServerBackupPager) NextPage(ctx context.Context) (armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse, error) { + if m.index >= len(m.pages) { + return armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse{}, errors.New("no more pages") + } + page := m.pages[m.index] + m.index++ + return page, nil +} + +type errorDBforPostgreSQLFlexibleServerBackupPager struct{} + +func (e *errorDBforPostgreSQLFlexibleServerBackupPager) More() bool { + return true +} + +func (e *errorDBforPostgreSQLFlexibleServerBackupPager) NextPage(ctx context.Context) (armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse, error) { + return armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse{}, errors.New("pager error") +} + +type testDBforPostgreSQLFlexibleServerBackupClient struct { + *mocks.MockDBforPostgreSQLFlexibleServerBackupClient + pager clients.DBforPostgreSQLFlexibleServerBackupPager +} + +func (t *testDBforPostgreSQLFlexibleServerBackupClient) ListByServer(ctx context.Context, resourceGroupName, serverName string) clients.DBforPostgreSQLFlexibleServerBackupPager { + return t.pager +} + +func TestDBforPostgreSQLFlexibleServerBackup(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subscriptionID := "test-subscription" + resourceGroup := "test-rg" + serverName := "test-server" + backupName := "test-backup" + + t.Run("Get", func(t *testing.T) { + backup := createAzurePostgreSQLFlexibleServerBackup(serverName, backupName) + + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, serverName, backupName).Return( + armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse{ + BackupAutomaticAndOnDemand: *backup, + }, nil) + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(serverName, backupName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.DBforPostgreSQLFlexibleServerBackup.String() { + t.Errorf("Expected type %s, got %s", azureshared.DBforPostgreSQLFlexibleServerBackup, sdpItem.GetType()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + expectedUniqueAttrValue := shared.CompositeLookupKey(serverName, backupName) + if sdpItem.UniqueAttributeValue() != expectedUniqueAttrValue { + t.Errorf("Expected unique attribute value %s, got %s", expectedUniqueAttrValue, sdpItem.UniqueAttributeValue()) + } + + if sdpItem.GetScope() != subscriptionID+"."+resourceGroup { + t.Errorf("Expected scope %s, got %s", subscriptionID+"."+resourceGroup, sdpItem.GetScope()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Expected no validation error, got: %v", err) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + { + ExpectedType: azureshared.DBforPostgreSQLFlexibleServer.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: serverName, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, + } + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("GetWithInsufficientQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], serverName, true) + if qErr == nil { + t.Error("Expected error when providing only serverName (1 query part), but got nil") + } + }) + + t.Run("GetWithEmptyServerName", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey("", backupName) + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when serverName is empty, but got nil") + } + }) + + t.Run("GetWithEmptyBackupName", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(serverName, "") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when backupName is empty, but got nil") + } + }) + + t.Run("Search", func(t *testing.T) { + backup1 := createAzurePostgreSQLFlexibleServerBackup(serverName, "backup1") + backup2 := createAzurePostgreSQLFlexibleServerBackup(serverName, "backup2") + + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + pager := &mockDBforPostgreSQLFlexibleServerBackupPager{ + pages: []armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse{ + { + BackupAutomaticAndOnDemandList: armpostgresqlflexibleservers.BackupAutomaticAndOnDemandList{ + Value: []*armpostgresqlflexibleservers.BackupAutomaticAndOnDemand{backup1, backup2}, + }, + }, + }, + } + + testClient := &testDBforPostgreSQLFlexibleServerBackupClient{ + MockDBforPostgreSQLFlexibleServerBackupClient: mockClient, + pager: pager, + } + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + items, qErr := searchable.Search(ctx, wrapper.Scopes()[0], serverName, true) + if qErr != nil { + t.Fatalf("Expected no error from Search, got: %v", qErr) + } + if len(items) != 2 { + t.Errorf("Expected 2 items from Search, got %d", len(items)) + } + }) + + t.Run("SearchStream", func(t *testing.T) { + backup1 := createAzurePostgreSQLFlexibleServerBackup(serverName, "backup1") + + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + pager := &mockDBforPostgreSQLFlexibleServerBackupPager{ + pages: []armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientListByServerResponse{ + { + BackupAutomaticAndOnDemandList: armpostgresqlflexibleservers.BackupAutomaticAndOnDemandList{ + Value: []*armpostgresqlflexibleservers.BackupAutomaticAndOnDemand{backup1}, + }, + }, + }, + } + + testClient := &testDBforPostgreSQLFlexibleServerBackupClient{ + MockDBforPostgreSQLFlexibleServerBackupClient: mockClient, + pager: pager, + } + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchStreamable, ok := adapter.(discovery.SearchStreamableAdapter) + if !ok { + t.Fatalf("Adapter does not support SearchStream operation") + } + + stream := discovery.NewRecordingQueryResultStream() + searchStreamable.SearchStream(ctx, wrapper.Scopes()[0], serverName, true, stream) + items := stream.GetItems() + errs := stream.GetErrors() + if len(errs) > 0 { + t.Fatalf("Expected no errors from SearchStream, got: %v", errs) + } + if len(items) != 1 { + t.Errorf("Expected 1 item from SearchStream, got %d", len(items)) + } + }) + + t.Run("SearchWithInsufficientQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + + _, qErr := wrapper.Search(ctx, wrapper.Scopes()[0]) + if qErr == nil { + t.Error("Expected error when providing no query parts, but got nil") + } + }) + + t.Run("ErrorHandling_Get", func(t *testing.T) { + expectedErr := errors.New("backup not found") + + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, serverName, "nonexistent-backup").Return( + armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse{}, expectedErr) + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(serverName, "nonexistent-backup") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when getting non-existent backup, but got nil") + } + }) + + t.Run("ErrorHandling_Search", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + errorPager := &errorDBforPostgreSQLFlexibleServerBackupPager{} + testClient := &testDBforPostgreSQLFlexibleServerBackupClient{ + MockDBforPostgreSQLFlexibleServerBackupClient: mockClient, + pager: errorPager, + } + + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + _, qErr := wrapper.Search(ctx, wrapper.Scopes()[0], serverName) + if qErr == nil { + t.Error("Expected error from Search when pager returns error, but got nil") + } + }) + + t.Run("PotentialLinks", func(t *testing.T) { + mockClient := mocks.NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl) + wrapper := manual.NewDBforPostgreSQLFlexibleServerBackup(&testDBforPostgreSQLFlexibleServerBackupClient{MockDBforPostgreSQLFlexibleServerBackupClient: mockClient}, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + potentialLinks := wrapper.PotentialLinks() + + if !potentialLinks[azureshared.DBforPostgreSQLFlexibleServer] { + t.Error("Expected PotentialLinks to include DBforPostgreSQLFlexibleServer") + } + }) +} + +func createAzurePostgreSQLFlexibleServerBackup(serverName, backupName string) *armpostgresqlflexibleservers.BackupAutomaticAndOnDemand { + backupID := "/subscriptions/test-subscription/resourceGroups/test-rg/providers/Microsoft.DBforPostgreSQL/flexibleServers/" + serverName + "/backups/" + backupName + backupType := armpostgresqlflexibleservers.BackupTypeFull + return &armpostgresqlflexibleservers.BackupAutomaticAndOnDemand{ + Name: new(backupName), + ID: new(backupID), + Type: new("Microsoft.DBforPostgreSQL/flexibleServers/backups"), + Properties: &armpostgresqlflexibleservers.BackupAutomaticAndOnDemandProperties{ + BackupType: &backupType, + Source: new("Automatic"), + }, + } +} diff --git a/sources/azure/manual/dbforpostgresql-flexible-server-firewall-rule.go b/sources/azure/manual/dbforpostgresql-flexible-server-firewall-rule.go index eb583710..3b5549bd 100644 --- a/sources/azure/manual/dbforpostgresql-flexible-server-firewall-rule.go +++ b/sources/azure/manual/dbforpostgresql-flexible-server-firewall-rule.go @@ -226,7 +226,7 @@ func (s dbforPostgreSQLFlexibleServerFirewallRuleWrapper) SearchLookups() []sour func (s dbforPostgreSQLFlexibleServerFirewallRuleWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ azureshared.DBforPostgreSQLFlexibleServer: true, - stdlib.NetworkIP: true, + stdlib.NetworkIP: true, } } diff --git a/sources/azure/manual/dbforpostgresql-flexible-server-private-endpoint-connection.go b/sources/azure/manual/dbforpostgresql-flexible-server-private-endpoint-connection.go index 00bdb154..27f35516 100644 --- a/sources/azure/manual/dbforpostgresql-flexible-server-private-endpoint-connection.go +++ b/sources/azure/manual/dbforpostgresql-flexible-server-private-endpoint-connection.go @@ -155,7 +155,7 @@ func (s dbforpostgresqlFlexibleServerPrivateEndpointConnectionWrapper) SearchLoo func (s dbforpostgresqlFlexibleServerPrivateEndpointConnectionWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ azureshared.DBforPostgreSQLFlexibleServer: true, - azureshared.NetworkPrivateEndpoint: true, + azureshared.NetworkPrivateEndpoint: true, } } diff --git a/sources/azure/manual/dbforpostgresql-flexible-server_test.go b/sources/azure/manual/dbforpostgresql-flexible-server_test.go index beb4448c..3819152e 100644 --- a/sources/azure/manual/dbforpostgresql-flexible-server_test.go +++ b/sources/azure/manual/dbforpostgresql-flexible-server_test.go @@ -161,7 +161,8 @@ func TestDBforPostgreSQLFlexibleServer(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: serverName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/documentdb-database-accounts_test.go b/sources/azure/manual/documentdb-database-accounts_test.go index 2c6a3e51..4cb492a6 100644 --- a/sources/azure/manual/documentdb-database-accounts_test.go +++ b/sources/azure/manual/documentdb-database-accounts_test.go @@ -153,7 +153,8 @@ func TestDocumentDBDatabaseAccounts(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "identity-rg", ExpectedScope: subscriptionID + ".identity-rg", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/elastic-san-volume-group.go b/sources/azure/manual/elastic-san-volume-group.go index 16dde194..e2dc33d6 100644 --- a/sources/azure/manual/elastic-san-volume-group.go +++ b/sources/azure/manual/elastic-san-volume-group.go @@ -334,14 +334,14 @@ func (e elasticSanVolumeGroupWrapper) azureVolumeGroupToSDPItem(vg *armelasticsa func (e elasticSanVolumeGroupWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.ElasticSan: true, - azureshared.ElasticSanVolume: true, - azureshared.ElasticSanVolumeSnapshot: true, - azureshared.NetworkPrivateEndpoint: true, - azureshared.NetworkSubnet: true, - azureshared.KeyVaultVault: true, + azureshared.ElasticSan: true, + azureshared.ElasticSanVolume: true, + azureshared.ElasticSanVolumeSnapshot: true, + azureshared.NetworkPrivateEndpoint: true, + azureshared.NetworkSubnet: true, + azureshared.KeyVaultVault: true, azureshared.ManagedIdentityUserAssignedIdentity: true, - stdlib.NetworkDNS: true, + stdlib.NetworkDNS: true, } } diff --git a/sources/azure/manual/elastic-san-volume-snapshot.go b/sources/azure/manual/elastic-san-volume-snapshot.go index 0c0bdbad..6f50db57 100644 --- a/sources/azure/manual/elastic-san-volume-snapshot.go +++ b/sources/azure/manual/elastic-san-volume-snapshot.go @@ -245,9 +245,9 @@ func (s elasticSanVolumeSnapshotWrapper) azureSnapshotToSDPItem(snapshot *armela func (s elasticSanVolumeSnapshotWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.ElasticSan: true, - azureshared.ElasticSanVolumeGroup: true, - azureshared.ElasticSanVolume: true, + azureshared.ElasticSan: true, + azureshared.ElasticSanVolumeGroup: true, + azureshared.ElasticSanVolume: true, } } diff --git a/sources/azure/manual/elastic-san_test.go b/sources/azure/manual/elastic-san_test.go index 670214ec..d21dade0 100644 --- a/sources/azure/manual/elastic-san_test.go +++ b/sources/azure/manual/elastic-san_test.go @@ -31,9 +31,9 @@ func createAzureElasticSan(name string) *armelasticsan.ElasticSan { Tags: map[string]*string{"env": new("test")}, Properties: &armelasticsan.Properties{ BaseSizeTiB: &baseSize, - ExtendedCapacitySizeTiB: &extendedSize, - ProvisioningState: &provisioningState, - VolumeGroupCount: new(int64(0)), + ExtendedCapacitySizeTiB: &extendedSize, + ProvisioningState: &provisioningState, + VolumeGroupCount: new(int64(0)), }, } } diff --git a/sources/azure/manual/keyvault-key_test.go b/sources/azure/manual/keyvault-key_test.go index 7c5462e2..1fc3e1f0 100644 --- a/sources/azure/manual/keyvault-key_test.go +++ b/sources/azure/manual/keyvault-key_test.go @@ -22,7 +22,6 @@ import ( "github.com/overmindtech/cli/sources/stdlib" ) - type mockKeysPager struct { pages []armkeyvault.KeysClientListResponse index int @@ -125,7 +124,8 @@ func TestKeyVaultKey(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: fmt.Sprintf("https://%s.vault.azure.net/keys/%s", vaultName, keyName), ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/keyvault-managed-hsm-private-endpoint-connection.go b/sources/azure/manual/keyvault-managed-hsm-private-endpoint-connection.go index 70d80c38..8cdbafc5 100644 --- a/sources/azure/manual/keyvault-managed-hsm-private-endpoint-connection.go +++ b/sources/azure/manual/keyvault-managed-hsm-private-endpoint-connection.go @@ -154,8 +154,8 @@ func (s keyvaultManagedHSMPrivateEndpointConnectionWrapper) SearchLookups() []so func (s keyvaultManagedHSMPrivateEndpointConnectionWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.KeyVaultManagedHSM: true, - azureshared.NetworkPrivateEndpoint: true, + azureshared.KeyVaultManagedHSM: true, + azureshared.NetworkPrivateEndpoint: true, azureshared.ManagedIdentityUserAssignedIdentity: true, } } diff --git a/sources/azure/manual/keyvault-managed-hsm_test.go b/sources/azure/manual/keyvault-managed-hsm_test.go index 8a7fb39a..49d81ab9 100644 --- a/sources/azure/manual/keyvault-managed-hsm_test.go +++ b/sources/azure/manual/keyvault-managed-hsm_test.go @@ -179,7 +179,8 @@ func TestKeyVaultManagedHSM(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "10.0.0.0/24", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/keyvault-secret_test.go b/sources/azure/manual/keyvault-secret_test.go index c8814996..4d2252e6 100644 --- a/sources/azure/manual/keyvault-secret_test.go +++ b/sources/azure/manual/keyvault-secret_test.go @@ -135,7 +135,8 @@ func TestKeyVaultSecret(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: fmt.Sprintf("https://%s.vault.azure.net/secrets/%s", vaultName, secretName), ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/keyvault-vault_test.go b/sources/azure/manual/keyvault-vault_test.go index bb8ac8d0..f134231a 100644 --- a/sources/azure/manual/keyvault-vault_test.go +++ b/sources/azure/manual/keyvault-vault_test.go @@ -166,7 +166,8 @@ func TestKeyVaultVault(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "https://test-keyvault.vault.azure.net/", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/managedidentity-user-assigned-identity_test.go b/sources/azure/manual/managedidentity-user-assigned-identity_test.go index c4e6ff92..c952d3bb 100644 --- a/sources/azure/manual/managedidentity-user-assigned-identity_test.go +++ b/sources/azure/manual/managedidentity-user-assigned-identity_test.go @@ -70,7 +70,8 @@ func TestManagedIdentityUserAssignedIdentity(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: identityName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/network-application-gateway_test.go b/sources/azure/manual/network-application-gateway_test.go index 9bb62f88..fd910bed 100644 --- a/sources/azure/manual/network-application-gateway_test.go +++ b/sources/azure/manual/network-application-gateway_test.go @@ -209,7 +209,8 @@ func TestNetworkApplicationGateway(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-identity", ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/network-application-security-group.go b/sources/azure/manual/network-application-security-group.go index 6aec9794..186b2e41 100644 --- a/sources/azure/manual/network-application-security-group.go +++ b/sources/azure/manual/network-application-security-group.go @@ -106,7 +106,7 @@ func (n networkApplicationSecurityGroupWrapper) azureApplicationSecurityGroupToS LinkedItemQueries: []*sdp.LinkedItemQuery{}, } - //no links - https://learn.microsoft.com/en-us/rest/api/virtualnetwork/application-security-groups/get?view=rest-virtualnetwork-2025-05-01&tabs=HTTP + // no links - https://learn.microsoft.com/en-us/rest/api/virtualnetwork/application-security-groups/get?view=rest-virtualnetwork-2025-05-01&tabs=HTTP // Health from provisioning state if asg.Properties != nil && asg.Properties.ProvisioningState != nil { diff --git a/sources/azure/manual/network-ddos-protection-plan.go b/sources/azure/manual/network-ddos-protection-plan.go index 0e65fd1a..0089e967 100644 --- a/sources/azure/manual/network-ddos-protection-plan.go +++ b/sources/azure/manual/network-ddos-protection-plan.go @@ -121,11 +121,11 @@ func (n networkDdosProtectionPlanWrapper) azureDdosProtectionPlanToSDPItem(plan } sdpItem := &sdp.Item{ - Type: azureshared.NetworkDdosProtectionPlan.String(), - UniqueAttribute: "name", - Attributes: attributes, - Scope: scope, - Tags: azureshared.ConvertAzureTags(plan.Tags), + Type: azureshared.NetworkDdosProtectionPlan.String(), + UniqueAttribute: "name", + Attributes: attributes, + Scope: scope, + Tags: azureshared.ConvertAzureTags(plan.Tags), LinkedItemQueries: []*sdp.LinkedItemQuery{}, } diff --git a/sources/azure/manual/network-dns-record-set.go b/sources/azure/manual/network-dns-record-set.go index 00869b7e..867c0fc2 100644 --- a/sources/azure/manual/network-dns-record-set.go +++ b/sources/azure/manual/network-dns-record-set.go @@ -16,8 +16,10 @@ import ( "github.com/overmindtech/cli/sources/stdlib" ) -var NetworkDNSRecordSetLookupByRecordType = shared.NewItemTypeLookup("recordType", azureshared.NetworkDNSRecordSet) -var NetworkDNSRecordSetLookupByName = shared.NewItemTypeLookup("name", azureshared.NetworkDNSRecordSet) +var ( + NetworkDNSRecordSetLookupByRecordType = shared.NewItemTypeLookup("recordType", azureshared.NetworkDNSRecordSet) + NetworkDNSRecordSetLookupByName = shared.NewItemTypeLookup("name", azureshared.NetworkDNSRecordSet) +) type networkDNSRecordSetWrapper struct { client clients.RecordSetsClient @@ -48,7 +50,7 @@ func recordTypeFromResourceType(resourceType string) string { return "" } -//ref: https://learn.microsoft.com/en-us/rest/api/dns/record-sets/get?view=rest-dns-2018-05-01&tabs=HTTP +// ref: https://learn.microsoft.com/en-us/rest/api/dns/record-sets/get?view=rest-dns-2018-05-01&tabs=HTTP func (n networkDNSRecordSetWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { if len(queryParts) < 3 { return nil, azureshared.QueryError(errors.New("Get requires 3 query parts: zoneName, recordType, and relativeRecordSetName"), scope, n.Type()) diff --git a/sources/azure/manual/network-dns-virtual-network-link_test.go b/sources/azure/manual/network-dns-virtual-network-link_test.go index b2e25f2b..408fed6e 100644 --- a/sources/azure/manual/network-dns-virtual-network-link_test.go +++ b/sources/azure/manual/network-dns-virtual-network-link_test.go @@ -30,9 +30,9 @@ func createAzureVirtualNetworkLink(name, privateZoneName, subscriptionID, resour Location: new("global"), Tags: map[string]*string{"env": new("test")}, Properties: &armprivatedns.VirtualNetworkLinkProperties{ - ProvisioningState: &provisioningState, + ProvisioningState: &provisioningState, VirtualNetworkLinkState: &linkState, - RegistrationEnabled: ®istrationEnabled, + RegistrationEnabled: ®istrationEnabled, VirtualNetwork: &armprivatedns.SubResource{ ID: new("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/virtualNetworks/test-vnet"), }, diff --git a/sources/azure/manual/network-flow-log.go b/sources/azure/manual/network-flow-log.go new file mode 100644 index 00000000..36f8085d --- /dev/null +++ b/sources/azure/manual/network-flow-log.go @@ -0,0 +1,339 @@ +package manual + +import ( + "context" + "errors" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" +) + +var ( + NetworkWatcherLookupByName = shared.NewItemTypeLookup("name", azureshared.NetworkNetworkWatcher) + NetworkFlowLogLookupByUniqueAttr = shared.NewItemTypeLookup("uniqueAttr", azureshared.NetworkFlowLog) +) + +type networkFlowLogWrapper struct { + client clients.FlowLogsClient + *azureshared.MultiResourceGroupBase +} + +// NewNetworkFlowLog creates a new networkFlowLogWrapper instance (SearchableWrapper: child of network watcher). +func NewNetworkFlowLog(client clients.FlowLogsClient, resourceGroupScopes []azureshared.ResourceGroupScope) sources.SearchableWrapper { + return &networkFlowLogWrapper{ + client: client, + MultiResourceGroupBase: azureshared.NewMultiResourceGroupBase( + resourceGroupScopes, + sdp.AdapterCategory_ADAPTER_CATEGORY_NETWORK, + azureshared.NetworkFlowLog, + ), + } +} + +// ref: https://learn.microsoft.com/en-us/rest/api/network-watcher/flow-logs/get +func (c networkFlowLogWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 2 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Get requires 2 query parts: networkWatcherName and flowLogName", + Scope: scope, + ItemType: c.Type(), + } + } + networkWatcherName := queryParts[0] + flowLogName := queryParts[1] + if networkWatcherName == "" { + return nil, azureshared.QueryError(errors.New("networkWatcherName cannot be empty"), scope, c.Type()) + } + if flowLogName == "" { + return nil, azureshared.QueryError(errors.New("flowLogName cannot be empty"), scope, c.Type()) + } + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + resp, err := c.client.Get(ctx, rgScope.ResourceGroup, networkWatcherName, flowLogName, nil) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + return c.azureFlowLogToSDPItem(&resp.FlowLog, networkWatcherName, flowLogName, scope) +} + +func (c networkFlowLogWrapper) GetLookups() sources.ItemTypeLookups { + return sources.ItemTypeLookups{ + NetworkWatcherLookupByName, + NetworkFlowLogLookupByUniqueAttr, + } +} + +// ref: https://learn.microsoft.com/en-us/rest/api/network-watcher/flow-logs/list +func (c networkFlowLogWrapper) Search(ctx context.Context, scope string, queryParts ...string) ([]*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 1 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Search requires 1 query part: networkWatcherName", + Scope: scope, + ItemType: c.Type(), + } + } + networkWatcherName := queryParts[0] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + pager := c.client.NewListPager(rgScope.ResourceGroup, networkWatcherName, nil) + + var items []*sdp.Item + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + for _, flowLog := range page.Value { + if flowLog == nil || flowLog.Name == nil { + continue + } + item, sdpErr := c.azureFlowLogToSDPItem(flowLog, networkWatcherName, *flowLog.Name, scope) + if sdpErr != nil { + return nil, sdpErr + } + items = append(items, item) + } + } + return items, nil +} + +func (c networkFlowLogWrapper) SearchStream(ctx context.Context, stream discovery.QueryResultStream, cache sdpcache.Cache, cacheKey sdpcache.CacheKey, scope string, queryParts ...string) { + if len(queryParts) < 1 { + stream.SendError(azureshared.QueryError(errors.New("Search requires 1 query part: networkWatcherName"), scope, c.Type())) + return + } + networkWatcherName := queryParts[0] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + pager := c.client.NewListPager(rgScope.ResourceGroup, networkWatcherName, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + for _, flowLog := range page.Value { + if flowLog == nil || flowLog.Name == nil { + continue + } + item, sdpErr := c.azureFlowLogToSDPItem(flowLog, networkWatcherName, *flowLog.Name, scope) + if sdpErr != nil { + stream.SendError(sdpErr) + continue + } + cache.StoreItem(ctx, item, shared.DefaultCacheDuration, cacheKey) + stream.SendItem(item) + } + } +} + +func (c networkFlowLogWrapper) SearchLookups() []sources.ItemTypeLookups { + return []sources.ItemTypeLookups{ + {NetworkWatcherLookupByName}, + } +} + +func (c networkFlowLogWrapper) azureFlowLogToSDPItem(flowLog *armnetwork.FlowLog, networkWatcherName, flowLogName, scope string) (*sdp.Item, *sdp.QueryError) { + if flowLog.Name == nil { + return nil, azureshared.QueryError(errors.New("resource name is nil"), scope, c.Type()) + } + + attributes, err := shared.ToAttributesWithExclude(flowLog, "tags") + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + err = attributes.Set("uniqueAttr", shared.CompositeLookupKey(networkWatcherName, flowLogName)) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + sdpItem := &sdp.Item{ + Type: azureshared.NetworkFlowLog.String(), + UniqueAttribute: "uniqueAttr", + Attributes: attributes, + Scope: scope, + Tags: azureshared.ConvertAzureTags(flowLog.Tags), + } + + if flowLog.Properties != nil { + // Health mapping from ProvisioningState + if flowLog.Properties.ProvisioningState != nil { + switch *flowLog.Properties.ProvisioningState { + case armnetwork.ProvisioningStateSucceeded: + sdpItem.Health = sdp.Health_HEALTH_OK.Enum() + case armnetwork.ProvisioningStateCreating, armnetwork.ProvisioningStateUpdating, armnetwork.ProvisioningStateDeleting: + sdpItem.Health = sdp.Health_HEALTH_PENDING.Enum() + case armnetwork.ProvisioningStateFailed, armnetwork.ProvisioningStateCanceled: + sdpItem.Health = sdp.Health_HEALTH_ERROR.Enum() + default: + sdpItem.Health = sdp.Health_HEALTH_UNKNOWN.Enum() + } + } + + // Link to TargetResourceID (polymorphic: NSG, VNet, or Subnet) + if flowLog.Properties.TargetResourceID != nil && *flowLog.Properties.TargetResourceID != "" { + targetID := *flowLog.Properties.TargetResourceID + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(targetID); extractedScope != "" { + linkedScope = extractedScope + } + + switch { + case strings.Contains(targetID, "/networkSecurityGroups/"): + nsgName := azureshared.ExtractResourceName(targetID) + if nsgName != "" { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkNetworkSecurityGroup.String(), + Method: sdp.QueryMethod_GET, + Query: nsgName, + Scope: linkedScope, + }, + }) + } + case strings.Contains(targetID, "/subnets/"): + params := azureshared.ExtractPathParamsFromResourceID(targetID, []string{"virtualNetworks", "subnets"}) + if len(params) >= 2 { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkSubnet.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + case strings.Contains(targetID, "/virtualNetworks/"): + vnetName := azureshared.ExtractResourceName(targetID) + if vnetName != "" { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkVirtualNetwork.String(), + Method: sdp.QueryMethod_GET, + Query: vnetName, + Scope: linkedScope, + }, + }) + } + } + } + + // Link to StorageID (storage account) + if flowLog.Properties.StorageID != nil && *flowLog.Properties.StorageID != "" { + storageAccountName := azureshared.ExtractResourceName(*flowLog.Properties.StorageID) + if storageAccountName != "" { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*flowLog.Properties.StorageID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.StorageAccount.String(), + Method: sdp.QueryMethod_GET, + Query: storageAccountName, + Scope: linkedScope, + }, + }) + } + } + + // Link to Traffic Analytics workspace + if flowLog.Properties.FlowAnalyticsConfiguration != nil && + flowLog.Properties.FlowAnalyticsConfiguration.NetworkWatcherFlowAnalyticsConfiguration != nil && + flowLog.Properties.FlowAnalyticsConfiguration.NetworkWatcherFlowAnalyticsConfiguration.WorkspaceResourceID != nil && + *flowLog.Properties.FlowAnalyticsConfiguration.NetworkWatcherFlowAnalyticsConfiguration.WorkspaceResourceID != "" { + workspaceName := azureshared.ExtractResourceName(*flowLog.Properties.FlowAnalyticsConfiguration.NetworkWatcherFlowAnalyticsConfiguration.WorkspaceResourceID) + if workspaceName != "" { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*flowLog.Properties.FlowAnalyticsConfiguration.NetworkWatcherFlowAnalyticsConfiguration.WorkspaceResourceID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.OperationalInsightsWorkspace.String(), + Method: sdp.QueryMethod_GET, + Query: workspaceName, + Scope: linkedScope, + }, + }) + } + } + } + + // Link to parent NetworkWatcher + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkNetworkWatcher.String(), + Method: sdp.QueryMethod_GET, + Query: networkWatcherName, + Scope: scope, + }, + }) + + // Link to user-assigned managed identities + if flowLog.Identity != nil && flowLog.Identity.UserAssignedIdentities != nil { + for identityID := range flowLog.Identity.UserAssignedIdentities { + identityName := azureshared.ExtractResourceName(identityID) + if identityName != "" { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(identityID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.ManagedIdentityUserAssignedIdentity.String(), + Method: sdp.QueryMethod_GET, + Query: identityName, + Scope: linkedScope, + }, + }) + } + } + } + + return sdpItem, nil +} + +func (c networkFlowLogWrapper) PotentialLinks() map[shared.ItemType]bool { + return shared.NewItemTypesSet( + azureshared.NetworkNetworkWatcher, + azureshared.NetworkNetworkSecurityGroup, + azureshared.NetworkVirtualNetwork, + azureshared.NetworkSubnet, + azureshared.StorageAccount, + azureshared.OperationalInsightsWorkspace, + azureshared.ManagedIdentityUserAssignedIdentity, + ) +} + +// ref: https://learn.microsoft.com/en-us/azure/role-based-access-control/resource-provider-operations#microsoftnetwork +func (c networkFlowLogWrapper) IAMPermissions() []string { + return []string{ + "Microsoft.Network/networkWatchers/flowLogs/read", + } +} + +func (c networkFlowLogWrapper) PredefinedRole() string { + return "Reader" +} diff --git a/sources/azure/manual/network-flow-log_test.go b/sources/azure/manual/network-flow-log_test.go new file mode 100644 index 00000000..c6134621 --- /dev/null +++ b/sources/azure/manual/network-flow-log_test.go @@ -0,0 +1,542 @@ +package manual_test + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "go.uber.org/mock/gomock" + + "github.com/overmindtech/cli/go/discovery" + sdp "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/azure/shared/mocks" + "github.com/overmindtech/cli/sources/shared" +) + +type mockFlowLogsPager struct { + pages []armnetwork.FlowLogsClientListResponse + index int +} + +func (m *mockFlowLogsPager) More() bool { + return m.index < len(m.pages) +} + +func (m *mockFlowLogsPager) NextPage(ctx context.Context) (armnetwork.FlowLogsClientListResponse, error) { + if m.index >= len(m.pages) { + return armnetwork.FlowLogsClientListResponse{}, errors.New("no more pages") + } + page := m.pages[m.index] + m.index++ + return page, nil +} + +type errorFlowLogsPager struct{} + +func (e *errorFlowLogsPager) More() bool { + return true +} + +func (e *errorFlowLogsPager) NextPage(ctx context.Context) (armnetwork.FlowLogsClientListResponse, error) { + return armnetwork.FlowLogsClientListResponse{}, errors.New("pager error") +} + +type testFlowLogsClient struct { + *mocks.MockFlowLogsClient + pager clients.FlowLogsPager +} + +func (t *testFlowLogsClient) NewListPager(resourceGroupName, networkWatcherName string, options *armnetwork.FlowLogsClientListOptions) clients.FlowLogsPager { + return t.pager +} + +func TestNetworkFlowLog(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subscriptionID := "test-subscription" + resourceGroup := "test-rg" + networkWatcherName := "test-watcher" + flowLogName := "test-flow-log" + + t.Run("Get", func(t *testing.T) { + flowLog := createAzureFlowLog(flowLogName, networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, flowLogName, nil).Return( + armnetwork.FlowLogsClientGetResponse{ + FlowLog: *flowLog, + }, nil) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, flowLogName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.NetworkFlowLog.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkFlowLog, sdpItem.GetType()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + if sdpItem.UniqueAttributeValue() != shared.CompositeLookupKey(networkWatcherName, flowLogName) { + t.Errorf("Expected unique attribute value %s, got %s", shared.CompositeLookupKey(networkWatcherName, flowLogName), sdpItem.UniqueAttributeValue()) + } + + if sdpItem.GetScope() != subscriptionID+"."+resourceGroup { + t.Errorf("Expected scope %s, got %s", subscriptionID+"."+resourceGroup, sdpItem.GetScope()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Expected no validation error, got: %v", err) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + { + ExpectedType: azureshared.NetworkNetworkSecurityGroup.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "test-nsg", + ExpectedScope: subscriptionID + "." + resourceGroup, + }, + { + ExpectedType: azureshared.StorageAccount.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "teststorageaccount", + ExpectedScope: subscriptionID + "." + resourceGroup, + }, + { + ExpectedType: azureshared.OperationalInsightsWorkspace.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "test-workspace", + ExpectedScope: subscriptionID + ".test-workspace-rg", + }, + { + ExpectedType: azureshared.NetworkNetworkWatcher.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: networkWatcherName, + ExpectedScope: subscriptionID + "." + resourceGroup, + }, + { + ExpectedType: azureshared.ManagedIdentityUserAssignedIdentity.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "test-identity", + ExpectedScope: subscriptionID + "." + resourceGroup, + }, + } + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("Get_VNetTarget", func(t *testing.T) { + flowLog := createAzureFlowLogWithVNetTarget(flowLogName, networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, flowLogName, nil).Return( + armnetwork.FlowLogsClientGetResponse{ + FlowLog: *flowLog, + }, nil) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, flowLogName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + found := false + for _, link := range sdpItem.GetLinkedItemQueries() { + if link.GetQuery().GetType() == azureshared.NetworkVirtualNetwork.String() { + found = true + if link.GetQuery().GetQuery() != "test-vnet" { + t.Errorf("Expected VNet query 'test-vnet', got %s", link.GetQuery().GetQuery()) + } + } + } + if !found { + t.Error("Expected a linked item query for VirtualNetwork, but none found") + } + }) + + t.Run("Get_SubnetTarget", func(t *testing.T) { + flowLog := createAzureFlowLogWithSubnetTarget(flowLogName, networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, flowLogName, nil).Return( + armnetwork.FlowLogsClientGetResponse{ + FlowLog: *flowLog, + }, nil) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, flowLogName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + found := false + for _, link := range sdpItem.GetLinkedItemQueries() { + if link.GetQuery().GetType() == azureshared.NetworkSubnet.String() { + found = true + expectedQuery := shared.CompositeLookupKey("test-vnet", "test-subnet") + if link.GetQuery().GetQuery() != expectedQuery { + t.Errorf("Expected Subnet query %s, got %s", expectedQuery, link.GetQuery().GetQuery()) + } + } + } + if !found { + t.Error("Expected a linked item query for Subnet, but none found") + } + }) + + t.Run("Get_EmptyFlowLogName", func(t *testing.T) { + mockClient := mocks.NewMockFlowLogsClient(ctrl) + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, "") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when flow log name is empty, but got nil") + } + }) + + t.Run("Get_EmptyNetworkWatcherName", func(t *testing.T) { + mockClient := mocks.NewMockFlowLogsClient(ctrl) + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey("", flowLogName) + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when network watcher name is empty, but got nil") + } + }) + + t.Run("Get_InsufficientQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockFlowLogsClient(ctrl) + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], networkWatcherName, true) + if qErr == nil { + t.Error("Expected error when providing insufficient query parts, but got nil") + } + }) + + t.Run("Search", func(t *testing.T) { + flowLog1 := createAzureFlowLog("flow-log-1", networkWatcherName, subscriptionID, resourceGroup) + flowLog2 := createAzureFlowLog("flow-log-2", networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockPager := &mockFlowLogsPager{ + pages: []armnetwork.FlowLogsClientListResponse{ + { + FlowLogListResult: armnetwork.FlowLogListResult{ + Value: []*armnetwork.FlowLog{flowLog1, flowLog2}, + }, + }, + }, + } + + testClient := &testFlowLogsClient{ + MockFlowLogsClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, wrapper.Scopes()[0], networkWatcherName, true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) != 2 { + t.Fatalf("Expected 2 items, got: %d", len(sdpItems)) + } + + for _, item := range sdpItems { + if err := item.Validate(); err != nil { + t.Fatalf("Expected no validation error, got: %v", err) + } + if item.GetType() != azureshared.NetworkFlowLog.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkFlowLog, item.GetType()) + } + } + }) + + t.Run("Search_InvalidQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockFlowLogsClient(ctrl) + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + + _, qErr := wrapper.Search(ctx, wrapper.Scopes()[0]) + if qErr == nil { + t.Error("Expected error when providing no query parts, but got nil") + } + }) + + t.Run("Search_FlowLogWithNilName", func(t *testing.T) { + validFlowLog := createAzureFlowLog("valid-flow-log", networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockPager := &mockFlowLogsPager{ + pages: []armnetwork.FlowLogsClientListResponse{ + { + FlowLogListResult: armnetwork.FlowLogListResult{ + Value: []*armnetwork.FlowLog{ + {Name: nil, ID: new("/some/id")}, + validFlowLog, + }, + }, + }, + }, + } + + testClient := &testFlowLogsClient{ + MockFlowLogsClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable := adapter.(discovery.SearchableAdapter) + sdpItems, err := searchable.Search(ctx, wrapper.Scopes()[0], networkWatcherName, true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) != 1 { + t.Fatalf("Expected 1 item (nil name skipped), got: %d", len(sdpItems)) + } + if sdpItems[0].UniqueAttributeValue() != shared.CompositeLookupKey(networkWatcherName, "valid-flow-log") { + t.Errorf("Expected unique value %s, got %s", shared.CompositeLookupKey(networkWatcherName, "valid-flow-log"), sdpItems[0].UniqueAttributeValue()) + } + }) + + t.Run("ErrorHandling_Get", func(t *testing.T) { + expectedErr := errors.New("flow log not found") + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, "nonexistent", nil).Return( + armnetwork.FlowLogsClientGetResponse{}, expectedErr) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, "nonexistent") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when getting non-existent flow log, but got nil") + } + }) + + t.Run("ErrorHandling_Search", func(t *testing.T) { + mockClient := mocks.NewMockFlowLogsClient(ctrl) + testClient := &testFlowLogsClient{ + MockFlowLogsClient: mockClient, + pager: &errorFlowLogsPager{}, + } + + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable := adapter.(discovery.SearchableAdapter) + _, err := searchable.Search(ctx, wrapper.Scopes()[0], networkWatcherName, true) + if err == nil { + t.Error("Expected error from pager when NextPage returns an error, but got nil") + } + }) + + t.Run("HealthMapping", func(t *testing.T) { + tests := []struct { + name string + state armnetwork.ProvisioningState + expectedHealth sdp.Health + }{ + {"Succeeded", armnetwork.ProvisioningStateSucceeded, sdp.Health_HEALTH_OK}, + {"Updating", armnetwork.ProvisioningStateUpdating, sdp.Health_HEALTH_PENDING}, + {"Failed", armnetwork.ProvisioningStateFailed, sdp.Health_HEALTH_ERROR}, + {"Unknown", armnetwork.ProvisioningState("SomeOtherState"), sdp.Health_HEALTH_UNKNOWN}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + flowLog := createAzureFlowLog(flowLogName, networkWatcherName, subscriptionID, resourceGroup) + flowLog.Properties.ProvisioningState = &tc.state + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, flowLogName, nil).Return( + armnetwork.FlowLogsClientGetResponse{ + FlowLog: *flowLog, + }, nil) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, flowLogName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetHealth() != tc.expectedHealth { + t.Errorf("Expected health %s, got %s", tc.expectedHealth, sdpItem.GetHealth()) + } + }) + } + }) + + t.Run("Get_NoLinks", func(t *testing.T) { + flowLog := createAzureFlowLogWithoutLinks(flowLogName, networkWatcherName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockFlowLogsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, networkWatcherName, flowLogName, nil).Return( + armnetwork.FlowLogsClientGetResponse{ + FlowLog: *flowLog, + }, nil) + + testClient := &testFlowLogsClient{MockFlowLogsClient: mockClient} + wrapper := manual.NewNetworkFlowLog(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(networkWatcherName, flowLogName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + // Should only have the parent NetworkWatcher link + if len(sdpItem.GetLinkedItemQueries()) != 1 { + t.Errorf("Expected 1 linked query (parent only), got %d", len(sdpItem.GetLinkedItemQueries())) + } + if sdpItem.GetLinkedItemQueries()[0].GetQuery().GetType() != azureshared.NetworkNetworkWatcher.String() { + t.Errorf("Expected parent link to NetworkWatcher, got %s", sdpItem.GetLinkedItemQueries()[0].GetQuery().GetType()) + } + }) +} + +func createAzureFlowLog(name, networkWatcherName, subscriptionID, resourceGroup string) *armnetwork.FlowLog { + provisioningState := armnetwork.ProvisioningStateSucceeded + enabled := true + nsgID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + storageID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Storage/storageAccounts/teststorageaccount" + workspaceResourceID := "/subscriptions/" + subscriptionID + "/resourceGroups/test-workspace-rg/providers/Microsoft.OperationalInsights/workspaces/test-workspace" + identityID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity" + + return &armnetwork.FlowLog{ + ID: new("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/networkWatchers/" + networkWatcherName + "/flowLogs/" + name), + Name: &name, + Type: new("Microsoft.Network/networkWatchers/flowLogs"), + Location: new("eastus"), + Tags: map[string]*string{ + "env": new("test"), + }, + Identity: &armnetwork.ManagedServiceIdentity{ + UserAssignedIdentities: map[string]*armnetwork.Components1Jq1T4ISchemasManagedserviceidentityPropertiesUserassignedidentitiesAdditionalproperties{ + identityID: {}, + }, + }, + Properties: &armnetwork.FlowLogPropertiesFormat{ + TargetResourceID: &nsgID, + StorageID: &storageID, + Enabled: &enabled, + ProvisioningState: &provisioningState, + FlowAnalyticsConfiguration: &armnetwork.TrafficAnalyticsProperties{ + NetworkWatcherFlowAnalyticsConfiguration: &armnetwork.TrafficAnalyticsConfigurationProperties{ + Enabled: &enabled, + WorkspaceResourceID: &workspaceResourceID, + }, + }, + RetentionPolicy: &armnetwork.RetentionPolicyParameters{ + Enabled: &enabled, + Days: new(int32(90)), + }, + }, + } +} + +func createAzureFlowLogWithVNetTarget(name, networkWatcherName, subscriptionID, resourceGroup string) *armnetwork.FlowLog { + provisioningState := armnetwork.ProvisioningStateSucceeded + enabled := true + vnetID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/virtualNetworks/test-vnet" + storageID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Storage/storageAccounts/teststorageaccount" + + return &armnetwork.FlowLog{ + ID: new("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/networkWatchers/" + networkWatcherName + "/flowLogs/" + name), + Name: &name, + Type: new("Microsoft.Network/networkWatchers/flowLogs"), + Location: new("eastus"), + Properties: &armnetwork.FlowLogPropertiesFormat{ + TargetResourceID: &vnetID, + StorageID: &storageID, + Enabled: &enabled, + ProvisioningState: &provisioningState, + }, + } +} + +func createAzureFlowLogWithSubnetTarget(name, networkWatcherName, subscriptionID, resourceGroup string) *armnetwork.FlowLog { + provisioningState := armnetwork.ProvisioningStateSucceeded + enabled := true + subnetID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + storageID := "/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Storage/storageAccounts/teststorageaccount" + + return &armnetwork.FlowLog{ + ID: new("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/networkWatchers/" + networkWatcherName + "/flowLogs/" + name), + Name: &name, + Type: new("Microsoft.Network/networkWatchers/flowLogs"), + Location: new("eastus"), + Properties: &armnetwork.FlowLogPropertiesFormat{ + TargetResourceID: &subnetID, + StorageID: &storageID, + Enabled: &enabled, + ProvisioningState: &provisioningState, + }, + } +} + +func createAzureFlowLogWithoutLinks(name, networkWatcherName, subscriptionID, resourceGroup string) *armnetwork.FlowLog { + return &armnetwork.FlowLog{ + ID: new("/subscriptions/" + subscriptionID + "/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/networkWatchers/" + networkWatcherName + "/flowLogs/" + name), + Name: &name, + Type: new("Microsoft.Network/networkWatchers/flowLogs"), + Location: new("eastus"), + } +} diff --git a/sources/azure/manual/network-load-balancer-frontend-ip-configuration.go b/sources/azure/manual/network-load-balancer-frontend-ip-configuration.go new file mode 100644 index 00000000..c5a02aaf --- /dev/null +++ b/sources/azure/manual/network-load-balancer-frontend-ip-configuration.go @@ -0,0 +1,416 @@ +package manual + +import ( + "context" + "errors" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/shared" + "github.com/overmindtech/cli/sources/stdlib" +) + +var NetworkLoadBalancerFrontendIPConfigurationLookupByUniqueAttr = shared.NewItemTypeLookup("uniqueAttr", azureshared.NetworkLoadBalancerFrontendIPConfiguration) + +type networkLoadBalancerFrontendIPConfigurationWrapper struct { + client clients.LoadBalancerFrontendIPConfigurationsClient + + *azureshared.MultiResourceGroupBase +} + +func NewNetworkLoadBalancerFrontendIPConfiguration(client clients.LoadBalancerFrontendIPConfigurationsClient, resourceGroupScopes []azureshared.ResourceGroupScope) sources.SearchableWrapper { + return &networkLoadBalancerFrontendIPConfigurationWrapper{ + client: client, + MultiResourceGroupBase: azureshared.NewMultiResourceGroupBase( + resourceGroupScopes, + sdp.AdapterCategory_ADAPTER_CATEGORY_NETWORK, + azureshared.NetworkLoadBalancerFrontendIPConfiguration, + ), + } +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) Get(ctx context.Context, scope string, queryParts ...string) (*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 2 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Get requires 2 query parts: loadBalancerName and frontendIPConfigurationName", + Scope: scope, + ItemType: c.Type(), + } + } + loadBalancerName := queryParts[0] + frontendIPConfigName := queryParts[1] + + if loadBalancerName == "" { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "loadBalancerName cannot be empty", + Scope: scope, + ItemType: c.Type(), + } + } + if frontendIPConfigName == "" { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "frontendIPConfigurationName cannot be empty", + Scope: scope, + ItemType: c.Type(), + } + } + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + resp, err := c.client.Get(ctx, rgScope.ResourceGroup, loadBalancerName, frontendIPConfigName) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + return c.azureFrontendIPConfigToSDPItem(&resp.FrontendIPConfiguration, loadBalancerName, scope) +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) Search(ctx context.Context, scope string, queryParts ...string) ([]*sdp.Item, *sdp.QueryError) { + if len(queryParts) < 1 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_OTHER, + ErrorString: "Search requires 1 query part: loadBalancerName", + Scope: scope, + ItemType: c.Type(), + } + } + loadBalancerName := queryParts[0] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + pager := c.client.NewListPager(rgScope.ResourceGroup, loadBalancerName) + + var items []*sdp.Item + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + for _, frontendIPConfig := range page.Value { + if frontendIPConfig == nil || frontendIPConfig.Name == nil { + continue + } + item, sdpErr := c.azureFrontendIPConfigToSDPItem(frontendIPConfig, loadBalancerName, scope) + if sdpErr != nil { + return nil, sdpErr + } + items = append(items, item) + } + } + + return items, nil +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) SearchStream(ctx context.Context, stream discovery.QueryResultStream, cache sdpcache.Cache, cacheKey sdpcache.CacheKey, scope string, queryParts ...string) { + if len(queryParts) < 1 { + stream.SendError(azureshared.QueryError(errors.New("Search requires 1 query part: loadBalancerName"), scope, c.Type())) + return + } + loadBalancerName := queryParts[0] + + rgScope, err := c.ResourceGroupScopeFromScope(scope) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + pager := c.client.NewListPager(rgScope.ResourceGroup, loadBalancerName) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + stream.SendError(azureshared.QueryError(err, scope, c.Type())) + return + } + for _, frontendIPConfig := range page.Value { + if frontendIPConfig == nil || frontendIPConfig.Name == nil { + continue + } + item, sdpErr := c.azureFrontendIPConfigToSDPItem(frontendIPConfig, loadBalancerName, scope) + if sdpErr != nil { + stream.SendError(sdpErr) + continue + } + cache.StoreItem(ctx, item, shared.DefaultCacheDuration, cacheKey) + stream.SendItem(item) + } + } +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) GetLookups() sources.ItemTypeLookups { + return sources.ItemTypeLookups{ + NetworkLoadBalancerLookupByName, + NetworkLoadBalancerFrontendIPConfigurationLookupByUniqueAttr, + } +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) SearchLookups() []sources.ItemTypeLookups { + return []sources.ItemTypeLookups{ + { + NetworkLoadBalancerLookupByName, + }, + } +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) azureFrontendIPConfigToSDPItem(frontendIPConfig *armnetwork.FrontendIPConfiguration, loadBalancerName string, scope string) (*sdp.Item, *sdp.QueryError) { + if frontendIPConfig.Name == nil { + return nil, azureshared.QueryError(errors.New("frontend IP configuration name is nil"), scope, c.Type()) + } + + frontendIPConfigName := *frontendIPConfig.Name + + attributes, err := shared.ToAttributesWithExclude(frontendIPConfig, "tags") + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + err = attributes.Set("uniqueAttr", shared.CompositeLookupKey(loadBalancerName, frontendIPConfigName)) + if err != nil { + return nil, azureshared.QueryError(err, scope, c.Type()) + } + + sdpItem := &sdp.Item{ + Type: azureshared.NetworkLoadBalancerFrontendIPConfiguration.String(), + UniqueAttribute: "uniqueAttr", + Attributes: attributes, + Scope: scope, + } + + // Health status from provisioning state + if frontendIPConfig.Properties != nil && frontendIPConfig.Properties.ProvisioningState != nil { + switch *frontendIPConfig.Properties.ProvisioningState { + case armnetwork.ProvisioningStateSucceeded: + sdpItem.Health = sdp.Health_HEALTH_OK.Enum() + case armnetwork.ProvisioningStateCreating, armnetwork.ProvisioningStateUpdating, armnetwork.ProvisioningStateDeleting: + sdpItem.Health = sdp.Health_HEALTH_PENDING.Enum() + case armnetwork.ProvisioningStateFailed, armnetwork.ProvisioningStateCanceled: + sdpItem.Health = sdp.Health_HEALTH_ERROR.Enum() + default: + sdpItem.Health = sdp.Health_HEALTH_UNKNOWN.Enum() + } + } + + // Link to parent Load Balancer + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancer.String(), + Method: sdp.QueryMethod_GET, + Query: loadBalancerName, + Scope: scope, + }, + }) + + if frontendIPConfig.Properties != nil { + // Link to Public IP Address + if frontendIPConfig.Properties.PublicIPAddress != nil && frontendIPConfig.Properties.PublicIPAddress.ID != nil { + publicIPName := azureshared.ExtractResourceName(*frontendIPConfig.Properties.PublicIPAddress.ID) + if publicIPName != "" { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*frontendIPConfig.Properties.PublicIPAddress.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkPublicIPAddress.String(), + Method: sdp.QueryMethod_GET, + Query: publicIPName, + Scope: linkedScope, + }, + }) + } + } + + // Link to Subnet + if frontendIPConfig.Properties.Subnet != nil && frontendIPConfig.Properties.Subnet.ID != nil { + subnetID := *frontendIPConfig.Properties.Subnet.ID + params := azureshared.ExtractPathParamsFromResourceID(subnetID, []string{"virtualNetworks", "subnets"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(subnetID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkSubnet.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + + // Link to Public IP Prefix + if frontendIPConfig.Properties.PublicIPPrefix != nil && frontendIPConfig.Properties.PublicIPPrefix.ID != nil { + publicIPPrefixName := azureshared.ExtractResourceName(*frontendIPConfig.Properties.PublicIPPrefix.ID) + if publicIPPrefixName != "" { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*frontendIPConfig.Properties.PublicIPPrefix.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkPublicIPPrefix.String(), + Method: sdp.QueryMethod_GET, + Query: publicIPPrefixName, + Scope: linkedScope, + }, + }) + } + } + + // Link to Gateway Load Balancer Frontend IP Configuration + if frontendIPConfig.Properties.GatewayLoadBalancer != nil && frontendIPConfig.Properties.GatewayLoadBalancer.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*frontendIPConfig.Properties.GatewayLoadBalancer.ID, []string{"loadBalancers", "frontendIPConfigurations"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*frontendIPConfig.Properties.GatewayLoadBalancer.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancerFrontendIPConfiguration.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + + // Link to Inbound NAT Rules (read-only references) + for _, natRule := range frontendIPConfig.Properties.InboundNatRules { + if natRule != nil && natRule.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*natRule.ID, []string{"loadBalancers", "inboundNatRules"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*natRule.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancerInboundNatRule.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + } + + // Link to Inbound NAT Pools (read-only references) + for _, natPool := range frontendIPConfig.Properties.InboundNatPools { + if natPool != nil && natPool.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*natPool.ID, []string{"loadBalancers", "inboundNatPools"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*natPool.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancerInboundNatPool.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + } + + // Link to Outbound Rules (read-only references) + for _, outboundRule := range frontendIPConfig.Properties.OutboundRules { + if outboundRule != nil && outboundRule.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*outboundRule.ID, []string{"loadBalancers", "outboundRules"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*outboundRule.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancerOutboundRule.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + } + + // Link to Load Balancing Rules (read-only references) + for _, lbRule := range frontendIPConfig.Properties.LoadBalancingRules { + if lbRule != nil && lbRule.ID != nil { + params := azureshared.ExtractPathParamsFromResourceID(*lbRule.ID, []string{"loadBalancers", "loadBalancingRules"}) + if len(params) >= 2 { + linkedScope := scope + if extractedScope := azureshared.ExtractScopeFromResourceID(*lbRule.ID); extractedScope != "" { + linkedScope = extractedScope + } + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: azureshared.NetworkLoadBalancerLoadBalancingRule.String(), + Method: sdp.QueryMethod_GET, + Query: shared.CompositeLookupKey(params[0], params[1]), + Scope: linkedScope, + }, + }) + } + } + } + + // Link to Private IP Address (stdlib) + if frontendIPConfig.Properties.PrivateIPAddress != nil && *frontendIPConfig.Properties.PrivateIPAddress != "" { + sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: stdlib.NetworkIP.String(), + Method: sdp.QueryMethod_GET, + Query: *frontendIPConfig.Properties.PrivateIPAddress, + Scope: "global", + }, + }) + } + } + + return sdpItem, nil +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) PotentialLinks() map[shared.ItemType]bool { + return map[shared.ItemType]bool{ + azureshared.NetworkLoadBalancer: true, + azureshared.NetworkPublicIPAddress: true, + azureshared.NetworkSubnet: true, + azureshared.NetworkPublicIPPrefix: true, + azureshared.NetworkLoadBalancerFrontendIPConfiguration: true, + azureshared.NetworkLoadBalancerInboundNatRule: true, + azureshared.NetworkLoadBalancerInboundNatPool: true, + azureshared.NetworkLoadBalancerOutboundRule: true, + azureshared.NetworkLoadBalancerLoadBalancingRule: true, + stdlib.NetworkIP: true, + } +} + +// ref: https://learn.microsoft.com/en-us/azure/role-based-access-control/permissions/networking +func (c networkLoadBalancerFrontendIPConfigurationWrapper) IAMPermissions() []string { + return []string{ + "Microsoft.Network/loadBalancers/frontendIPConfigurations/read", + } +} + +func (c networkLoadBalancerFrontendIPConfigurationWrapper) PredefinedRole() string { + return "Reader" +} diff --git a/sources/azure/manual/network-load-balancer-frontend-ip-configuration_test.go b/sources/azure/manual/network-load-balancer-frontend-ip-configuration_test.go new file mode 100644 index 00000000..1ccea60a --- /dev/null +++ b/sources/azure/manual/network-load-balancer-frontend-ip-configuration_test.go @@ -0,0 +1,498 @@ +package manual_test + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "go.uber.org/mock/gomock" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources" + "github.com/overmindtech/cli/sources/azure/clients" + "github.com/overmindtech/cli/sources/azure/manual" + azureshared "github.com/overmindtech/cli/sources/azure/shared" + "github.com/overmindtech/cli/sources/azure/shared/mocks" + "github.com/overmindtech/cli/sources/shared" +) + +type mockFrontendIPConfigPager struct { + pages []armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse + index int +} + +func (m *mockFrontendIPConfigPager) More() bool { + return m.index < len(m.pages) +} + +func (m *mockFrontendIPConfigPager) NextPage(ctx context.Context) (armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse, error) { + if m.index >= len(m.pages) { + return armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse{}, errors.New("no more pages") + } + page := m.pages[m.index] + m.index++ + return page, nil +} + +type errorFrontendIPConfigPager struct{} + +func (e *errorFrontendIPConfigPager) More() bool { + return true +} + +func (e *errorFrontendIPConfigPager) NextPage(ctx context.Context) (armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse, error) { + return armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse{}, errors.New("pager error") +} + +type testFrontendIPConfigClient struct { + *mocks.MockLoadBalancerFrontendIPConfigurationsClient + pager clients.LoadBalancerFrontendIPConfigurationsPager +} + +func (t *testFrontendIPConfigClient) NewListPager(resourceGroupName, loadBalancerName string) clients.LoadBalancerFrontendIPConfigurationsPager { + return t.pager +} + +func TestNetworkLoadBalancerFrontendIPConfiguration(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subscriptionID := "test-subscription" + resourceGroup := "test-rg" + loadBalancerName := "test-lb" + frontendIPConfigName := "test-frontend-ip" + + t.Run("Get", func(t *testing.T) { + frontendIPConfig := createAzureFrontendIPConfiguration(frontendIPConfigName, loadBalancerName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, loadBalancerName, frontendIPConfigName).Return( + armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse{ + FrontendIPConfiguration: *frontendIPConfig, + }, nil) + + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(loadBalancerName, frontendIPConfigName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + if sdpItem.GetType() != azureshared.NetworkLoadBalancerFrontendIPConfiguration.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkLoadBalancerFrontendIPConfiguration, sdpItem.GetType()) + } + + if sdpItem.GetUniqueAttribute() != "uniqueAttr" { + t.Errorf("Expected unique attribute 'uniqueAttr', got %s", sdpItem.GetUniqueAttribute()) + } + + expectedUniqueValue := shared.CompositeLookupKey(loadBalancerName, frontendIPConfigName) + if sdpItem.UniqueAttributeValue() != expectedUniqueValue { + t.Errorf("Expected unique attribute value %s, got %s", expectedUniqueValue, sdpItem.UniqueAttributeValue()) + } + + if sdpItem.GetScope() != subscriptionID+"."+resourceGroup { + t.Errorf("Expected scope %s, got %s", subscriptionID+"."+resourceGroup, sdpItem.GetScope()) + } + + if err := sdpItem.Validate(); err != nil { + t.Fatalf("Expected no validation error, got: %v", err) + } + + if sdpItem.GetHealth() != sdp.Health_HEALTH_OK { + t.Errorf("Expected health OK, got %s", sdpItem.GetHealth()) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + { + ExpectedType: azureshared.NetworkLoadBalancer.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: loadBalancerName, + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkPublicIPAddress.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "test-public-ip", + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkSubnet.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey("test-vnet", "test-subnet"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkPublicIPPrefix.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "test-ip-prefix", + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkLoadBalancerFrontendIPConfiguration.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey("gateway-lb", "gateway-frontend"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkLoadBalancerInboundNatRule.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey(loadBalancerName, "nat-rule-1"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkLoadBalancerInboundNatPool.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey(loadBalancerName, "nat-pool-1"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkLoadBalancerOutboundRule.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey(loadBalancerName, "outbound-rule-1"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: azureshared.NetworkLoadBalancerLoadBalancingRule.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey(loadBalancerName, "lb-rule-1"), + ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), + }, + { + ExpectedType: "ip", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "10.0.0.5", + ExpectedScope: "global", + }, + } + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("Get_WithInsufficientQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], loadBalancerName, true) + if qErr == nil { + t.Error("Expected error when providing insufficient query parts, but got nil") + } + }) + + t.Run("Get_WithEmptyLoadBalancerName", func(t *testing.T) { + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey("", frontendIPConfigName) + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when loadBalancerName is empty, but got nil") + } + }) + + t.Run("Get_WithEmptyFrontendIPConfigName", func(t *testing.T) { + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(loadBalancerName, "") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when frontendIPConfigurationName is empty, but got nil") + } + }) + + t.Run("Search", func(t *testing.T) { + frontendIP1 := createAzureFrontendIPConfigurationMinimal("frontend-1", loadBalancerName, subscriptionID, resourceGroup) + frontendIP2 := createAzureFrontendIPConfigurationMinimal("frontend-2", loadBalancerName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockPager := &mockFrontendIPConfigPager{ + pages: []armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse{ + { + LoadBalancerFrontendIPConfigurationListResult: armnetwork.LoadBalancerFrontendIPConfigurationListResult{ + Value: []*armnetwork.FrontendIPConfiguration{frontendIP1, frontendIP2}, + }, + }, + }, + } + + testClient := &testFrontendIPConfigClient{ + MockLoadBalancerFrontendIPConfigurationsClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter does not support Search operation") + } + + sdpItems, err := searchable.Search(ctx, wrapper.Scopes()[0], loadBalancerName, true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) != 2 { + t.Fatalf("Expected 2 items, got: %d", len(sdpItems)) + } + + for _, item := range sdpItems { + if err := item.Validate(); err != nil { + t.Fatalf("Expected no validation error, got: %v", err) + } + if item.GetType() != azureshared.NetworkLoadBalancerFrontendIPConfiguration.String() { + t.Errorf("Expected type %s, got %s", azureshared.NetworkLoadBalancerFrontendIPConfiguration, item.GetType()) + } + } + }) + + t.Run("Search_WithNilName", func(t *testing.T) { + validFrontendIP := createAzureFrontendIPConfigurationMinimal("valid-frontend", loadBalancerName, subscriptionID, resourceGroup) + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockPager := &mockFrontendIPConfigPager{ + pages: []armnetwork.LoadBalancerFrontendIPConfigurationsClientListResponse{ + { + LoadBalancerFrontendIPConfigurationListResult: armnetwork.LoadBalancerFrontendIPConfigurationListResult{ + Value: []*armnetwork.FrontendIPConfiguration{ + {Name: nil, ID: new("/some/id")}, + validFrontendIP, + }, + }, + }, + }, + } + + testClient := &testFrontendIPConfigClient{ + MockLoadBalancerFrontendIPConfigurationsClient: mockClient, + pager: mockPager, + } + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable := adapter.(discovery.SearchableAdapter) + sdpItems, err := searchable.Search(ctx, wrapper.Scopes()[0], loadBalancerName, true) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(sdpItems) != 1 { + t.Fatalf("Expected 1 item (nil name skipped), got: %d", len(sdpItems)) + } + expectedValue := shared.CompositeLookupKey(loadBalancerName, "valid-frontend") + if sdpItems[0].UniqueAttributeValue() != expectedValue { + t.Errorf("Expected unique value %s, got %s", expectedValue, sdpItems[0].UniqueAttributeValue()) + } + }) + + t.Run("Search_InvalidQueryParts", func(t *testing.T) { + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + + _, qErr := wrapper.Search(ctx, wrapper.Scopes()[0]) + if qErr == nil { + t.Error("Expected error when providing no query parts, but got nil") + } + }) + + t.Run("ErrorHandling_Get", func(t *testing.T) { + expectedErr := errors.New("frontend IP config not found") + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, loadBalancerName, "nonexistent-frontend").Return( + armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse{}, expectedErr) + + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(loadBalancerName, "nonexistent-frontend") + _, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr == nil { + t.Error("Expected error when getting non-existent frontend IP config, but got nil") + } + }) + + t.Run("ErrorHandling_Search", func(t *testing.T) { + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + testClient := &testFrontendIPConfigClient{ + MockLoadBalancerFrontendIPConfigurationsClient: mockClient, + pager: &errorFrontendIPConfigPager{}, + } + + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + searchable := adapter.(discovery.SearchableAdapter) + _, err := searchable.Search(ctx, wrapper.Scopes()[0], loadBalancerName, true) + if err == nil { + t.Error("Expected error from pager when NextPage returns an error, but got nil") + } + }) + + t.Run("Get_CrossResourceGroupLinks", func(t *testing.T) { + frontendIPConfig := createAzureFrontendIPConfigCrossRG(frontendIPConfigName, loadBalancerName, "other-sub", "other-rg") + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, loadBalancerName, frontendIPConfigName).Return( + armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse{ + FrontendIPConfiguration: *frontendIPConfig, + }, nil) + + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(loadBalancerName, frontendIPConfigName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + found := false + for _, linkedQuery := range sdpItem.GetLinkedItemQueries() { + if linkedQuery.GetQuery().GetType() == azureshared.NetworkPublicIPAddress.String() { + found = true + expectedScope := "other-sub.other-rg" + if linkedQuery.GetQuery().GetScope() != expectedScope { + t.Errorf("Expected PublicIPAddress scope to be %s, got: %s", expectedScope, linkedQuery.GetQuery().GetScope()) + } + break + } + } + if !found { + t.Error("Expected to find PublicIPAddress linked query") + } + }) + + t.Run("Get_NoProperties", func(t *testing.T) { + frontendIPConfig := &armnetwork.FrontendIPConfiguration{ + ID: new(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", subscriptionID, resourceGroup, loadBalancerName, frontendIPConfigName)), + Name: new(frontendIPConfigName), + Type: new("Microsoft.Network/loadBalancers/frontendIPConfigurations"), + } + + mockClient := mocks.NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl) + mockClient.EXPECT().Get(ctx, resourceGroup, loadBalancerName, frontendIPConfigName).Return( + armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse{ + FrontendIPConfiguration: *frontendIPConfig, + }, nil) + + testClient := &testFrontendIPConfigClient{MockLoadBalancerFrontendIPConfigurationsClient: mockClient} + wrapper := manual.NewNetworkLoadBalancerFrontendIPConfiguration(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) + adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) + + query := shared.CompositeLookupKey(loadBalancerName, frontendIPConfigName) + sdpItem, qErr := adapter.Get(ctx, wrapper.Scopes()[0], query, true) + if qErr != nil { + t.Fatalf("Expected no error, got: %v", qErr) + } + + // Should only have the parent load balancer link + linkedQueries := sdpItem.GetLinkedItemQueries() + if len(linkedQueries) != 1 { + t.Errorf("Expected 1 linked query (parent LB only), got %d", len(linkedQueries)) + } + if linkedQueries[0].GetQuery().GetType() != azureshared.NetworkLoadBalancer.String() { + t.Errorf("Expected parent LB link, got type %s", linkedQueries[0].GetQuery().GetType()) + } + }) +} + +func createAzureFrontendIPConfiguration(name, lbName, subscriptionID, resourceGroup string) *armnetwork.FrontendIPConfiguration { + provisioningState := armnetwork.ProvisioningStateSucceeded + publicIPID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicIPAddresses/test-public-ip", subscriptionID, resourceGroup) + subnetID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet", subscriptionID, resourceGroup) + publicIPPrefixID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicIPPrefixes/test-ip-prefix", subscriptionID, resourceGroup) + gatewayLBFrontendID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/gateway-lb/frontendIPConfigurations/gateway-frontend", subscriptionID, resourceGroup) + natRuleID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/inboundNatRules/nat-rule-1", subscriptionID, resourceGroup, lbName) + natPoolID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/inboundNatPools/nat-pool-1", subscriptionID, resourceGroup, lbName) + outboundRuleID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/outboundRules/outbound-rule-1", subscriptionID, resourceGroup, lbName) + lbRuleID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/loadBalancingRules/lb-rule-1", subscriptionID, resourceGroup, lbName) + + return &armnetwork.FrontendIPConfiguration{ + ID: new(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", subscriptionID, resourceGroup, lbName, name)), + Name: new(name), + Type: new("Microsoft.Network/loadBalancers/frontendIPConfigurations"), + Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ + ProvisioningState: &provisioningState, + PublicIPAddress: &armnetwork.PublicIPAddress{ + ID: new(publicIPID), + }, + Subnet: &armnetwork.Subnet{ + ID: new(subnetID), + }, + PublicIPPrefix: &armnetwork.SubResource{ + ID: new(publicIPPrefixID), + }, + GatewayLoadBalancer: &armnetwork.SubResource{ + ID: new(gatewayLBFrontendID), + }, + PrivateIPAddress: new("10.0.0.5"), + InboundNatRules: []*armnetwork.SubResource{ + {ID: new(natRuleID)}, + }, + InboundNatPools: []*armnetwork.SubResource{ + {ID: new(natPoolID)}, + }, + OutboundRules: []*armnetwork.SubResource{ + {ID: new(outboundRuleID)}, + }, + LoadBalancingRules: []*armnetwork.SubResource{ + {ID: new(lbRuleID)}, + }, + }, + } +} + +func createAzureFrontendIPConfigurationMinimal(name, lbName, subscriptionID, resourceGroup string) *armnetwork.FrontendIPConfiguration { + provisioningState := armnetwork.ProvisioningStateSucceeded + return &armnetwork.FrontendIPConfiguration{ + ID: new(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", subscriptionID, resourceGroup, lbName, name)), + Name: new(name), + Type: new("Microsoft.Network/loadBalancers/frontendIPConfigurations"), + Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ + ProvisioningState: &provisioningState, + }, + } +} + +func createAzureFrontendIPConfigCrossRG(name, lbName, otherSub, otherRG string) *armnetwork.FrontendIPConfiguration { + provisioningState := armnetwork.ProvisioningStateSucceeded + publicIPID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicIPAddresses/cross-rg-ip", otherSub, otherRG) + + return &armnetwork.FrontendIPConfiguration{ + ID: new(fmt.Sprintf("/subscriptions/test-subscription/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", lbName, name)), + Name: new(name), + Type: new("Microsoft.Network/loadBalancers/frontendIPConfigurations"), + Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ + ProvisioningState: &provisioningState, + PublicIPAddress: &armnetwork.PublicIPAddress{ + ID: new(publicIPID), + }, + }, + } +} diff --git a/sources/azure/manual/network-load-balancer_test.go b/sources/azure/manual/network-load-balancer_test.go index 6f028cf6..f61ae1f4 100644 --- a/sources/azure/manual/network-load-balancer_test.go +++ b/sources/azure/manual/network-load-balancer_test.go @@ -131,7 +131,8 @@ func TestNetworkLoadBalancer(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: shared.CompositeLookupKey(lbName, "nat-pool"), ExpectedScope: fmt.Sprintf("%s.%s", subscriptionID, resourceGroup), - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -306,7 +307,7 @@ func TestNetworkLoadBalancer(t *testing.T) { wrapper := manual.NewNetworkLoadBalancer(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) diff --git a/sources/azure/manual/network-nat-gateway.go b/sources/azure/manual/network-nat-gateway.go index e6f6f98f..db10b89e 100644 --- a/sources/azure/manual/network-nat-gateway.go +++ b/sources/azure/manual/network-nat-gateway.go @@ -128,8 +128,8 @@ func (n networkNatGatewayWrapper) azureNatGatewayToSDPItem(ng *armnetwork.NatGat sdpItem := &sdp.Item{ Type: azureshared.NetworkNatGateway.String(), - UniqueAttribute: "name", - Attributes: attributes, + UniqueAttribute: "name", + Attributes: attributes, Scope: scope, Tags: azureshared.ConvertAzureTags(ng.Tags), LinkedItemQueries: []*sdp.LinkedItemQuery{}, @@ -256,10 +256,10 @@ func (n networkNatGatewayWrapper) GetLookups() sources.ItemTypeLookups { func (n networkNatGatewayWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.NetworkPublicIPAddress: true, - azureshared.NetworkPublicIPPrefix: true, - azureshared.NetworkSubnet: true, - azureshared.NetworkVirtualNetwork: true, + azureshared.NetworkPublicIPAddress: true, + azureshared.NetworkPublicIPPrefix: true, + azureshared.NetworkSubnet: true, + azureshared.NetworkVirtualNetwork: true, } } diff --git a/sources/azure/manual/network-network-interface_test.go b/sources/azure/manual/network-network-interface_test.go index 413aa50b..206bc780 100644 --- a/sources/azure/manual/network-network-interface_test.go +++ b/sources/azure/manual/network-network-interface_test.go @@ -89,7 +89,8 @@ func TestNetworkNetworkInterface(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: nicName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -142,7 +143,8 @@ func TestNetworkNetworkInterface(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "dns.internal", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) }) @@ -321,7 +323,7 @@ func TestNetworkNetworkInterface(t *testing.T) { wrapper := manual.NewNetworkNetworkInterface(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) @@ -361,7 +363,6 @@ func TestNetworkNetworkInterface(t *testing.T) { if !foundMapping { t.Error("Expected TerraformMappings to include 'azurerm_network_interface.name' mapping") } - }) } diff --git a/sources/azure/manual/network-network-security-group_test.go b/sources/azure/manual/network-network-security-group_test.go index 4080c289..27e656e7 100644 --- a/sources/azure/manual/network-network-security-group_test.go +++ b/sources/azure/manual/network-network-security-group_test.go @@ -107,7 +107,8 @@ func TestNetworkNetworkSecurityGroup(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-asg-default-source", ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -375,7 +376,7 @@ func TestNetworkNetworkSecurityGroup(t *testing.T) { wrapper := manual.NewNetworkNetworkSecurityGroup(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) diff --git a/sources/azure/manual/network-private-dns-zone_test.go b/sources/azure/manual/network-private-dns-zone_test.go index 4901d7a3..c5444d30 100644 --- a/sources/azure/manual/network-private-dns-zone_test.go +++ b/sources/azure/manual/network-private-dns-zone_test.go @@ -365,7 +365,7 @@ func createAzurePrivateZone(zoneName string) *armprivatedns.PrivateZone { "project": new("testing"), }, Properties: &armprivatedns.PrivateZoneProperties{ - ProvisioningState: &state, + ProvisioningState: &state, MaxNumberOfRecordSets: new(int64(5000)), NumberOfRecordSets: new(int64(0)), }, diff --git a/sources/azure/manual/network-public-ip-address_test.go b/sources/azure/manual/network-public-ip-address_test.go index fb7b0559..a69c1a61 100644 --- a/sources/azure/manual/network-public-ip-address_test.go +++ b/sources/azure/manual/network-public-ip-address_test.go @@ -95,7 +95,8 @@ func TestNetworkPublicIPAddress(t *testing.T) { ExpectedMethod: sdp.QueryMethod_GET, ExpectedQuery: "test-ddos-plan", ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -373,7 +374,7 @@ func TestNetworkPublicIPAddress(t *testing.T) { wrapper := manual.NewNetworkPublicIPAddress(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) diff --git a/sources/azure/manual/network-public-ip-prefix.go b/sources/azure/manual/network-public-ip-prefix.go index 050ea99c..1a80b528 100644 --- a/sources/azure/manual/network-public-ip-prefix.go +++ b/sources/azure/manual/network-public-ip-prefix.go @@ -128,7 +128,7 @@ func (n networkPublicIPPrefixWrapper) azurePublicIPPrefixToSDPItem(prefix *armne Attributes: attributes, Scope: scope, Tags: azureshared.ConvertAzureTags(prefix.Tags), - LinkedItemQueries: []*sdp.LinkedItemQuery{}, + LinkedItemQueries: []*sdp.LinkedItemQuery{}, } // Link to Custom Location when ExtendedLocation.Name is a custom location resource ID (Microsoft.ExtendedLocation/customLocations) diff --git a/sources/azure/manual/network-route-table.go b/sources/azure/manual/network-route-table.go index 5c6cedc5..f1b50706 100644 --- a/sources/azure/manual/network-route-table.go +++ b/sources/azure/manual/network-route-table.go @@ -5,15 +5,16 @@ import ( "errors" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + "github.com/overmindtech/cli/go/discovery" "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/sources" "github.com/overmindtech/cli/sources/azure/clients" azureshared "github.com/overmindtech/cli/sources/azure/shared" "github.com/overmindtech/cli/sources/shared" "github.com/overmindtech/cli/sources/stdlib" - "github.com/overmindtech/cli/go/sdpcache" - "github.com/overmindtech/cli/go/discovery" ) + var NetworkRouteTableLookupByName = shared.NewItemTypeLookup("name", azureshared.NetworkRouteTable) type networkRouteTableWrapper struct { diff --git a/sources/azure/manual/network-subnet.go b/sources/azure/manual/network-subnet.go index 031a9134..cd05b31c 100644 --- a/sources/azure/manual/network-subnet.go +++ b/sources/azure/manual/network-subnet.go @@ -149,15 +149,15 @@ func (n networkSubnetWrapper) SearchLookups() []sources.ItemTypeLookups { func (n networkSubnetWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.NetworkVirtualNetwork: true, - azureshared.NetworkNetworkSecurityGroup: true, - azureshared.NetworkRouteTable: true, - azureshared.NetworkNatGateway: true, - azureshared.NetworkPrivateEndpoint: true, - azureshared.NetworkServiceEndpointPolicy: true, - azureshared.NetworkIpAllocation: true, - azureshared.NetworkNetworkInterface: true, - azureshared.NetworkApplicationGateway: true, + azureshared.NetworkVirtualNetwork: true, + azureshared.NetworkNetworkSecurityGroup: true, + azureshared.NetworkRouteTable: true, + azureshared.NetworkNatGateway: true, + azureshared.NetworkPrivateEndpoint: true, + azureshared.NetworkServiceEndpointPolicy: true, + azureshared.NetworkIpAllocation: true, + azureshared.NetworkNetworkInterface: true, + azureshared.NetworkApplicationGateway: true, } } diff --git a/sources/azure/manual/network-virtual-network-gateway.go b/sources/azure/manual/network-virtual-network-gateway.go index a08c3785..3ecf208e 100644 --- a/sources/azure/manual/network-virtual-network-gateway.go +++ b/sources/azure/manual/network-virtual-network-gateway.go @@ -131,12 +131,12 @@ func (n networkVirtualNetworkGatewayWrapper) azureVirtualNetworkGatewayToSDPItem } sdpItem := &sdp.Item{ - Type: azureshared.NetworkVirtualNetworkGateway.String(), - UniqueAttribute: "name", - Attributes: attributes, - Scope: scope, - Tags: azureshared.ConvertAzureTags(gw.Tags), - LinkedItemQueries: []*sdp.LinkedItemQuery{}, + Type: azureshared.NetworkVirtualNetworkGateway.String(), + UniqueAttribute: "name", + Attributes: attributes, + Scope: scope, + Tags: azureshared.ConvertAzureTags(gw.Tags), + LinkedItemQueries: []*sdp.LinkedItemQuery{}, } // Health from provisioning state @@ -441,15 +441,15 @@ func (n networkVirtualNetworkGatewayWrapper) GetLookups() sources.ItemTypeLookup func (n networkVirtualNetworkGatewayWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.NetworkSubnet: true, - azureshared.NetworkPublicIPAddress: true, - azureshared.NetworkLocalNetworkGateway: true, - azureshared.NetworkVirtualNetworkGatewayConnection: true, - azureshared.ExtendedLocationCustomLocation: true, - azureshared.ManagedIdentityUserAssignedIdentity: true, - azureshared.NetworkVirtualNetwork: true, - stdlib.NetworkIP: true, - stdlib.NetworkDNS: true, + azureshared.NetworkSubnet: true, + azureshared.NetworkPublicIPAddress: true, + azureshared.NetworkLocalNetworkGateway: true, + azureshared.NetworkVirtualNetworkGatewayConnection: true, + azureshared.ExtendedLocationCustomLocation: true, + azureshared.ManagedIdentityUserAssignedIdentity: true, + azureshared.NetworkVirtualNetwork: true, + stdlib.NetworkIP: true, + stdlib.NetworkDNS: true, } } diff --git a/sources/azure/manual/network-virtual-network_test.go b/sources/azure/manual/network-virtual-network_test.go index c25b2bc7..8eda47f9 100644 --- a/sources/azure/manual/network-virtual-network_test.go +++ b/sources/azure/manual/network-virtual-network_test.go @@ -77,7 +77,8 @@ func TestNetworkVirtualNetwork(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: vnetName, ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -128,7 +129,8 @@ func TestNetworkVirtualNetwork(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "dns.internal", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) }) @@ -302,7 +304,7 @@ func TestNetworkVirtualNetwork(t *testing.T) { wrapper := manual.NewNetworkVirtualNetwork(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) diff --git a/sources/azure/manual/network-zone_test.go b/sources/azure/manual/network-zone_test.go index 9c3a7a41..e9493008 100644 --- a/sources/azure/manual/network-zone_test.go +++ b/sources/azure/manual/network-zone_test.go @@ -102,7 +102,8 @@ func TestNetworkZone(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: "ns2.example.com", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -328,7 +329,7 @@ func TestNetworkZone(t *testing.T) { wrapper := manual.NewNetworkZone(mockClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) // Verify wrapper implements ListableWrapper interface - var _ = wrapper + _ = wrapper // Cast to sources.Wrapper to access interface methods w := wrapper.(sources.Wrapper) diff --git a/sources/azure/manual/sql-database_test.go b/sources/azure/manual/sql-database_test.go index 5e976be7..4edf1864 100644 --- a/sources/azure/manual/sql-database_test.go +++ b/sources/azure/manual/sql-database_test.go @@ -126,7 +126,8 @@ func TestSqlDatabase(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: shared.CompositeLookupKey(serverName, databaseName), ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) @@ -172,7 +173,8 @@ func TestSqlDatabase(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: shared.CompositeLookupKey(serverName, databaseName), ExpectedScope: subscriptionID + "." + resourceGroup, - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/sql-elastic-pool.go b/sources/azure/manual/sql-elastic-pool.go index 85423f20..893b3921 100644 --- a/sources/azure/manual/sql-elastic-pool.go +++ b/sources/azure/manual/sql-elastic-pool.go @@ -235,7 +235,7 @@ func (s sqlElasticPoolWrapper) SearchLookups() []sources.ItemTypeLookups { func (s sqlElasticPoolWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ azureshared.SQLServer: true, - azureshared.SQLDatabase: true, + azureshared.SQLDatabase: true, azureshared.MaintenanceMaintenanceConfiguration: true, } } diff --git a/sources/azure/manual/sql-server-private-endpoint-connection.go b/sources/azure/manual/sql-server-private-endpoint-connection.go index 9665cb3c..d4639ad8 100644 --- a/sources/azure/manual/sql-server-private-endpoint-connection.go +++ b/sources/azure/manual/sql-server-private-endpoint-connection.go @@ -154,7 +154,7 @@ func (s sqlServerPrivateEndpointConnectionWrapper) SearchLookups() []sources.Ite func (s sqlServerPrivateEndpointConnectionWrapper) PotentialLinks() map[shared.ItemType]bool { return map[shared.ItemType]bool{ - azureshared.SQLServer: true, + azureshared.SQLServer: true, azureshared.NetworkPrivateEndpoint: true, } } diff --git a/sources/azure/manual/sql-server-virtual-network-rule_test.go b/sources/azure/manual/sql-server-virtual-network-rule_test.go index 9182f287..5c65953b 100644 --- a/sources/azure/manual/sql-server-virtual-network-rule_test.go +++ b/sources/azure/manual/sql-server-virtual-network-rule_test.go @@ -203,7 +203,7 @@ func TestSqlServerVirtualNetworkRule(t *testing.T) { testClient := &testSqlServerVirtualNetworkRuleClient{ MockSqlServerVirtualNetworkRuleClient: mockClient, - pager: pager, + pager: pager, } wrapper := manual.NewSqlServerVirtualNetworkRule(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) @@ -238,7 +238,7 @@ func TestSqlServerVirtualNetworkRule(t *testing.T) { testClient := &testSqlServerVirtualNetworkRuleClient{ MockSqlServerVirtualNetworkRuleClient: mockClient, - pager: pager, + pager: pager, } wrapper := manual.NewSqlServerVirtualNetworkRule(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) adapter := sources.WrapperToAdapter(wrapper, sdpcache.NewNoOpCache()) @@ -292,7 +292,7 @@ func TestSqlServerVirtualNetworkRule(t *testing.T) { errorPager := &errorSqlServerVirtualNetworkRulePager{} testClient := &testSqlServerVirtualNetworkRuleClient{ MockSqlServerVirtualNetworkRuleClient: mockClient, - pager: errorPager, + pager: errorPager, } wrapper := manual.NewSqlServerVirtualNetworkRule(testClient, []azureshared.ResourceGroupScope{azureshared.NewResourceGroupScope(subscriptionID, resourceGroup)}) @@ -348,8 +348,8 @@ func TestSqlServerVirtualNetworkRule(t *testing.T) { func createAzureSqlServerVirtualNetworkRule(serverName, ruleName, subnetID string) *armsql.VirtualNetworkRule { ruleID := "/subscriptions/test-subscription/resourceGroups/test-rg/providers/Microsoft.Sql/servers/" + serverName + "/virtualNetworkRules/" + ruleName rule := &armsql.VirtualNetworkRule{ - Name: &ruleName, - ID: &ruleID, + Name: &ruleName, + ID: &ruleID, Properties: &armsql.VirtualNetworkRuleProperties{}, } if subnetID != "" { diff --git a/sources/azure/manual/sql-server_test.go b/sources/azure/manual/sql-server_test.go index 9e1ad621..dff19ec1 100644 --- a/sources/azure/manual/sql-server_test.go +++ b/sources/azure/manual/sql-server_test.go @@ -254,7 +254,8 @@ func TestSqlServer(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: serverName + ".database.windows.net", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/storage-account.go b/sources/azure/manual/storage-account.go index b5e89808..fb8dcec8 100644 --- a/sources/azure/manual/storage-account.go +++ b/sources/azure/manual/storage-account.go @@ -5,15 +5,16 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3" + "github.com/overmindtech/cli/go/discovery" "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" "github.com/overmindtech/cli/sources" "github.com/overmindtech/cli/sources/azure/clients" azureshared "github.com/overmindtech/cli/sources/azure/shared" "github.com/overmindtech/cli/sources/shared" "github.com/overmindtech/cli/sources/stdlib" - "github.com/overmindtech/cli/go/discovery" - "github.com/overmindtech/cli/go/sdpcache" ) + var StorageAccountLookupByName = shared.NewItemTypeLookup("name", azureshared.StorageAccount) type storageAccountWrapper struct { diff --git a/sources/azure/manual/storage-account_test.go b/sources/azure/manual/storage-account_test.go index 860b0b75..ac950039 100644 --- a/sources/azure/manual/storage-account_test.go +++ b/sources/azure/manual/storage-account_test.go @@ -122,7 +122,8 @@ func TestStorageAccount(t *testing.T) { ExpectedMethod: sdp.QueryMethod_SEARCH, ExpectedQuery: accountName + ".file.core.windows.net", ExpectedScope: "global", - }} + }, + } shared.RunStaticTests(t, adapter, sdpItem, queryTests) }) diff --git a/sources/azure/manual/storage-encryption-scope.go b/sources/azure/manual/storage-encryption-scope.go index ea8dcb0e..d670713b 100644 --- a/sources/azure/manual/storage-encryption-scope.go +++ b/sources/azure/manual/storage-encryption-scope.go @@ -156,8 +156,8 @@ func (s storageEncryptionScopeWrapper) PotentialLinks() map[shared.ItemType]bool return map[shared.ItemType]bool{ azureshared.StorageAccount: true, azureshared.KeyVaultVault: true, - azureshared.KeyVaultKey: true, - stdlib.NetworkDNS: true, + azureshared.KeyVaultKey: true, + stdlib.NetworkDNS: true, } } diff --git a/sources/azure/shared/item-types.go b/sources/azure/shared/item-types.go index 6f05bbc0..1b471196 100644 --- a/sources/azure/shared/item-types.go +++ b/sources/azure/shared/item-types.go @@ -87,11 +87,12 @@ var ( NetworkNetworkInterfaceTapConfiguration = shared.NewItemType(Azure, Network, NetworkInterfaceTapConfiguration) NetworkServiceEndpointPolicy = shared.NewItemType(Azure, Network, ServiceEndpointPolicy) NetworkIpAllocation = shared.NewItemType(Azure, Network, IpAllocation) + NetworkNetworkWatcher = shared.NewItemType(Azure, Network, NetworkWatcher) // ExtendedLocation item types ExtendedLocationCustomLocation = shared.NewItemType(Azure, ExtendedLocation, CustomLocation) - //Storage item types + // Storage item types StorageAccount = shared.NewItemType(Azure, Storage, Account) StorageBlobContainer = shared.NewItemType(Azure, Storage, BlobContainer) StorageEncryptionScope = shared.NewItemType(Azure, Storage, EncryptionScope) @@ -176,11 +177,14 @@ var ( BatchBatchDetector = shared.NewItemType(Azure, Batch, BatchDetector) // ElasticSAN item types - ElasticSan = shared.NewItemType(Azure, ElasticSAN, ElasticSanResource) - ElasticSanVolumeGroup = shared.NewItemType(Azure, ElasticSAN, VolumeGroup) - ElasticSanVolume = shared.NewItemType(Azure, ElasticSAN, Volume) + ElasticSan = shared.NewItemType(Azure, ElasticSAN, ElasticSanResource) + ElasticSanVolumeGroup = shared.NewItemType(Azure, ElasticSAN, VolumeGroup) + ElasticSanVolume = shared.NewItemType(Azure, ElasticSAN, Volume) ElasticSanVolumeSnapshot = shared.NewItemType(Azure, ElasticSAN, VolumeSnapshot) + // OperationalInsights item types + OperationalInsightsWorkspace = shared.NewItemType(Azure, OperationalInsights, Workspace) + // Authorization item types AuthorizationRoleAssignment = shared.NewItemType(Azure, Authorization, RoleAssignment) AuthorizationRoleDefinition = shared.NewItemType(Azure, Authorization, RoleDefinition) diff --git a/sources/azure/shared/mocks/mock_batch_application_package_client.go b/sources/azure/shared/mocks/mock_batch_application_package_client.go new file mode 100644 index 00000000..1b088aea --- /dev/null +++ b/sources/azure/shared/mocks/mock_batch_application_package_client.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: batch-application-package-client.go +// +// Generated by this command: +// +// mockgen -destination=../shared/mocks/mock_batch_application_package_client.go -package=mocks -source=batch-application-package-client.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + armbatch "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/batch/armbatch/v4" + clients "github.com/overmindtech/cli/sources/azure/clients" + gomock "go.uber.org/mock/gomock" +) + +// MockBatchApplicationPackagesClient is a mock of BatchApplicationPackagesClient interface. +type MockBatchApplicationPackagesClient struct { + ctrl *gomock.Controller + recorder *MockBatchApplicationPackagesClientMockRecorder + isgomock struct{} +} + +// MockBatchApplicationPackagesClientMockRecorder is the mock recorder for MockBatchApplicationPackagesClient. +type MockBatchApplicationPackagesClientMockRecorder struct { + mock *MockBatchApplicationPackagesClient +} + +// NewMockBatchApplicationPackagesClient creates a new mock instance. +func NewMockBatchApplicationPackagesClient(ctrl *gomock.Controller) *MockBatchApplicationPackagesClient { + mock := &MockBatchApplicationPackagesClient{ctrl: ctrl} + mock.recorder = &MockBatchApplicationPackagesClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBatchApplicationPackagesClient) EXPECT() *MockBatchApplicationPackagesClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockBatchApplicationPackagesClient) Get(ctx context.Context, resourceGroupName, accountName, applicationName, versionName string) (armbatch.ApplicationPackageClientGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, accountName, applicationName, versionName) + ret0, _ := ret[0].(armbatch.ApplicationPackageClientGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockBatchApplicationPackagesClientMockRecorder) Get(ctx, resourceGroupName, accountName, applicationName, versionName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockBatchApplicationPackagesClient)(nil).Get), ctx, resourceGroupName, accountName, applicationName, versionName) +} + +// List mocks base method. +func (m *MockBatchApplicationPackagesClient) List(ctx context.Context, resourceGroupName, accountName, applicationName string) clients.BatchApplicationPackagesPager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, resourceGroupName, accountName, applicationName) + ret0, _ := ret[0].(clients.BatchApplicationPackagesPager) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockBatchApplicationPackagesClientMockRecorder) List(ctx, resourceGroupName, accountName, applicationName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockBatchApplicationPackagesClient)(nil).List), ctx, resourceGroupName, accountName, applicationName) +} diff --git a/sources/azure/shared/mocks/mock_dbforpostgresql_flexible_server_backup_client.go b/sources/azure/shared/mocks/mock_dbforpostgresql_flexible_server_backup_client.go new file mode 100644 index 00000000..6552c1b5 --- /dev/null +++ b/sources/azure/shared/mocks/mock_dbforpostgresql_flexible_server_backup_client.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: dbforpostgresql-flexible-server-backup-client.go +// +// Generated by this command: +// +// mockgen -destination=../shared/mocks/mock_dbforpostgresql_flexible_server_backup_client.go -package=mocks -source=dbforpostgresql-flexible-server-backup-client.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + armpostgresqlflexibleservers "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v5" + clients "github.com/overmindtech/cli/sources/azure/clients" + gomock "go.uber.org/mock/gomock" +) + +// MockDBforPostgreSQLFlexibleServerBackupClient is a mock of DBforPostgreSQLFlexibleServerBackupClient interface. +type MockDBforPostgreSQLFlexibleServerBackupClient struct { + ctrl *gomock.Controller + recorder *MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder + isgomock struct{} +} + +// MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder is the mock recorder for MockDBforPostgreSQLFlexibleServerBackupClient. +type MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder struct { + mock *MockDBforPostgreSQLFlexibleServerBackupClient +} + +// NewMockDBforPostgreSQLFlexibleServerBackupClient creates a new mock instance. +func NewMockDBforPostgreSQLFlexibleServerBackupClient(ctrl *gomock.Controller) *MockDBforPostgreSQLFlexibleServerBackupClient { + mock := &MockDBforPostgreSQLFlexibleServerBackupClient{ctrl: ctrl} + mock.recorder = &MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDBforPostgreSQLFlexibleServerBackupClient) EXPECT() *MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDBforPostgreSQLFlexibleServerBackupClient) Get(ctx context.Context, resourceGroupName, serverName, backupName string) (armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, serverName, backupName) + ret0, _ := ret[0].(armpostgresqlflexibleservers.BackupsAutomaticAndOnDemandClientGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder) Get(ctx, resourceGroupName, serverName, backupName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDBforPostgreSQLFlexibleServerBackupClient)(nil).Get), ctx, resourceGroupName, serverName, backupName) +} + +// ListByServer mocks base method. +func (m *MockDBforPostgreSQLFlexibleServerBackupClient) ListByServer(ctx context.Context, resourceGroupName, serverName string) clients.DBforPostgreSQLFlexibleServerBackupPager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListByServer", ctx, resourceGroupName, serverName) + ret0, _ := ret[0].(clients.DBforPostgreSQLFlexibleServerBackupPager) + return ret0 +} + +// ListByServer indicates an expected call of ListByServer. +func (mr *MockDBforPostgreSQLFlexibleServerBackupClientMockRecorder) ListByServer(ctx, resourceGroupName, serverName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByServer", reflect.TypeOf((*MockDBforPostgreSQLFlexibleServerBackupClient)(nil).ListByServer), ctx, resourceGroupName, serverName) +} diff --git a/sources/azure/shared/mocks/mock_flow_logs_client.go b/sources/azure/shared/mocks/mock_flow_logs_client.go new file mode 100644 index 00000000..2e0d0066 --- /dev/null +++ b/sources/azure/shared/mocks/mock_flow_logs_client.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: flow-logs-client.go +// +// Generated by this command: +// +// mockgen -destination=../shared/mocks/mock_flow_logs_client.go -package=mocks -source=flow-logs-client.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + armnetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + clients "github.com/overmindtech/cli/sources/azure/clients" + gomock "go.uber.org/mock/gomock" +) + +// MockFlowLogsClient is a mock of FlowLogsClient interface. +type MockFlowLogsClient struct { + ctrl *gomock.Controller + recorder *MockFlowLogsClientMockRecorder + isgomock struct{} +} + +// MockFlowLogsClientMockRecorder is the mock recorder for MockFlowLogsClient. +type MockFlowLogsClientMockRecorder struct { + mock *MockFlowLogsClient +} + +// NewMockFlowLogsClient creates a new mock instance. +func NewMockFlowLogsClient(ctrl *gomock.Controller) *MockFlowLogsClient { + mock := &MockFlowLogsClient{ctrl: ctrl} + mock.recorder = &MockFlowLogsClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFlowLogsClient) EXPECT() *MockFlowLogsClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockFlowLogsClient) Get(ctx context.Context, resourceGroupName, networkWatcherName, flowLogName string, options *armnetwork.FlowLogsClientGetOptions) (armnetwork.FlowLogsClientGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, networkWatcherName, flowLogName, options) + ret0, _ := ret[0].(armnetwork.FlowLogsClientGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockFlowLogsClientMockRecorder) Get(ctx, resourceGroupName, networkWatcherName, flowLogName, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockFlowLogsClient)(nil).Get), ctx, resourceGroupName, networkWatcherName, flowLogName, options) +} + +// NewListPager mocks base method. +func (m *MockFlowLogsClient) NewListPager(resourceGroupName, networkWatcherName string, options *armnetwork.FlowLogsClientListOptions) clients.FlowLogsPager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewListPager", resourceGroupName, networkWatcherName, options) + ret0, _ := ret[0].(clients.FlowLogsPager) + return ret0 +} + +// NewListPager indicates an expected call of NewListPager. +func (mr *MockFlowLogsClientMockRecorder) NewListPager(resourceGroupName, networkWatcherName, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewListPager", reflect.TypeOf((*MockFlowLogsClient)(nil).NewListPager), resourceGroupName, networkWatcherName, options) +} diff --git a/sources/azure/shared/mocks/mock_load_balancer_frontend_ip_configurations_client.go b/sources/azure/shared/mocks/mock_load_balancer_frontend_ip_configurations_client.go new file mode 100644 index 00000000..c1c1c6ac --- /dev/null +++ b/sources/azure/shared/mocks/mock_load_balancer_frontend_ip_configurations_client.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: load-balancer-frontend-ip-configurations-client.go +// +// Generated by this command: +// +// mockgen -destination=../shared/mocks/mock_load_balancer_frontend_ip_configurations_client.go -package=mocks -source=load-balancer-frontend-ip-configurations-client.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + armnetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" + clients "github.com/overmindtech/cli/sources/azure/clients" + gomock "go.uber.org/mock/gomock" +) + +// MockLoadBalancerFrontendIPConfigurationsClient is a mock of LoadBalancerFrontendIPConfigurationsClient interface. +type MockLoadBalancerFrontendIPConfigurationsClient struct { + ctrl *gomock.Controller + recorder *MockLoadBalancerFrontendIPConfigurationsClientMockRecorder + isgomock struct{} +} + +// MockLoadBalancerFrontendIPConfigurationsClientMockRecorder is the mock recorder for MockLoadBalancerFrontendIPConfigurationsClient. +type MockLoadBalancerFrontendIPConfigurationsClientMockRecorder struct { + mock *MockLoadBalancerFrontendIPConfigurationsClient +} + +// NewMockLoadBalancerFrontendIPConfigurationsClient creates a new mock instance. +func NewMockLoadBalancerFrontendIPConfigurationsClient(ctrl *gomock.Controller) *MockLoadBalancerFrontendIPConfigurationsClient { + mock := &MockLoadBalancerFrontendIPConfigurationsClient{ctrl: ctrl} + mock.recorder = &MockLoadBalancerFrontendIPConfigurationsClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLoadBalancerFrontendIPConfigurationsClient) EXPECT() *MockLoadBalancerFrontendIPConfigurationsClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockLoadBalancerFrontendIPConfigurationsClient) Get(ctx context.Context, resourceGroupName, loadBalancerName, frontendIPConfigurationName string) (armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, loadBalancerName, frontendIPConfigurationName) + ret0, _ := ret[0].(armnetwork.LoadBalancerFrontendIPConfigurationsClientGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockLoadBalancerFrontendIPConfigurationsClientMockRecorder) Get(ctx, resourceGroupName, loadBalancerName, frontendIPConfigurationName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockLoadBalancerFrontendIPConfigurationsClient)(nil).Get), ctx, resourceGroupName, loadBalancerName, frontendIPConfigurationName) +} + +// NewListPager mocks base method. +func (m *MockLoadBalancerFrontendIPConfigurationsClient) NewListPager(resourceGroupName, loadBalancerName string) clients.LoadBalancerFrontendIPConfigurationsPager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewListPager", resourceGroupName, loadBalancerName) + ret0, _ := ret[0].(clients.LoadBalancerFrontendIPConfigurationsPager) + return ret0 +} + +// NewListPager indicates an expected call of NewListPager. +func (mr *MockLoadBalancerFrontendIPConfigurationsClientMockRecorder) NewListPager(resourceGroupName, loadBalancerName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewListPager", reflect.TypeOf((*MockLoadBalancerFrontendIPConfigurationsClient)(nil).NewListPager), resourceGroupName, loadBalancerName) +} diff --git a/sources/azure/shared/models.go b/sources/azure/shared/models.go index 2dab0456..2ed1b1ef 100644 --- a/sources/azure/shared/models.go +++ b/sources/azure/shared/models.go @@ -50,6 +50,9 @@ const ( // Resources (subscriptions, resource groups) Resources shared.API = "resources" // Microsoft.Resources + // OperationalInsights + OperationalInsights shared.API = "operationalinsights" // Microsoft.OperationalInsights + // ExtendedLocation (custom locations, edge zones) ExtendedLocation shared.API = "extendedlocation" // Microsoft.ExtendedLocation ) @@ -138,6 +141,7 @@ const ( NetworkInterfaceTapConfiguration shared.Resource = "network-interface-tap-configuration" ServiceEndpointPolicy shared.Resource = "service-endpoint-policy" IpAllocation shared.Resource = "ip-allocation" + NetworkWatcher shared.Resource = "network-watcher" // Storage resources Account shared.Resource = "account" @@ -223,15 +227,18 @@ const ( BatchDetector shared.Resource = "batch-detector" // ElasticSAN resources - ElasticSanResource shared.Resource = "elastic-san" - VolumeGroup shared.Resource = "volume-group" - Volume shared.Resource = "volume" - VolumeSnapshot shared.Resource = "elastic-san-volume-snapshot" + ElasticSanResource shared.Resource = "elastic-san" + VolumeGroup shared.Resource = "volume-group" + Volume shared.Resource = "volume" + VolumeSnapshot shared.Resource = "elastic-san-volume-snapshot" // Authorization resources RoleAssignment shared.Resource = "role-assignment" RoleDefinition shared.Resource = "role-definition" + // OperationalInsights resources + Workspace shared.Resource = "workspace" + // ExtendedLocation resources CustomLocation shared.Resource = "custom-location" ) diff --git a/sources/azure/shared/scope.go b/sources/azure/shared/scope.go index bed1d144..5956a541 100644 --- a/sources/azure/shared/scope.go +++ b/sources/azure/shared/scope.go @@ -11,14 +11,14 @@ import ( // It is used by multi-scope adapters to handle multiple resource groups. type ResourceGroupScope struct { SubscriptionID string - ResourceGroup string + ResourceGroup string } // NewResourceGroupScope creates a ResourceGroupScope for the given subscription and resource group. func NewResourceGroupScope(subscriptionID, resourceGroup string) ResourceGroupScope { return ResourceGroupScope{ SubscriptionID: subscriptionID, - ResourceGroup: resourceGroup, + ResourceGroup: resourceGroup, } } diff --git a/sources/azure/shared/utils.go b/sources/azure/shared/utils.go index d1b52df7..09346736 100644 --- a/sources/azure/shared/utils.go +++ b/sources/azure/shared/utils.go @@ -18,44 +18,48 @@ import ( func GetResourceIDPathKeys(resourceType string) []string { // Map of resource types to their path keys in the order they appear in GetLookups() pathKeysMap := map[string][]string{ - "azure-storage-queue": {"storageAccounts", "queues"}, - "azure-storage-blob-container": {"storageAccounts", "containers"}, - "azure-storage-encryption-scope": {"storageAccounts", "encryptionScopes"}, - "azure-storage-file-share": {"storageAccounts", "shares"}, - "azure-storage-storage-account-private-endpoint-connection": {"storageAccounts", "privateEndpointConnections"}, - "azure-documentdb-private-endpoint-connection": {"databaseAccounts", "privateEndpointConnections"}, - "azure-storage-table": {"storageAccounts", "tables"}, - "azure-sql-database": {"servers", "databases"}, // "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/Default-SQL-SouthEastAsia/providers/Microsoft.Sql/servers/testsvr/databases/testdb", - "azure-sql-elastic-pool": {"servers", "elasticPools"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/elasticPools/{elasticPoolName}", - "azure-sql-server-firewall-rule": {"servers", "firewallRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/firewallRules/{ruleName}", - "azure-sql-server-virtual-network-rule": {"servers", "virtualNetworkRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/virtualNetworkRules/{ruleName}", - "azure-sql-server-private-endpoint-connection": {"servers", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/privateEndpointConnections/{connectionName}", - "azure-dbforpostgresql-database": {"flexibleServers", "databases"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/databases/{db}", - "azure-dbforpostgresql-flexible-server-firewall-rule": {"flexibleServers", "firewallRules"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/firewallRules/{rule}", - "azure-dbforpostgresql-flexible-server-private-endpoint-connection": {"flexibleServers", "privateEndpointConnections"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/privateEndpointConnections/{connectionName}", - "azure-keyvault-secret": {"vaults", "secrets"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/vaults/{vaultName}/secrets/{secretName}", - "azure-keyvault-key": {"vaults", "keys"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/vaults/{vaultName}/keys/{keyName}", - "azure-keyvault-managed-hsm-private-endpoint-connection": {"managedHSMs", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/managedHSMs/{name}/privateEndpointConnections/{connectionName}", - "azure-authorization-role-assignment": {"roleAssignments"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/roleAssignments/{roleAssignmentName}", - "azure-compute-virtual-machine-run-command": {"virtualMachines", "runCommands"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines/{virtualMachineName}/runCommands/{runCommandName}", - "azure-compute-virtual-machine-extension": {"virtualMachines", "extensions"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines/{virtualMachineName}/extensions/{extensionName}", - "azure-compute-gallery-application-version": {"galleries", "applications", "versions"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/applications/{applicationName}/versions/{versionName}", - "azure-compute-gallery-application": {"galleries", "applications"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/applications/{applicationName}", - "azure-compute-gallery-image": {"galleries", "images"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/images/{imageName}", - "azure-compute-dedicated-host": {"hostGroups", "hosts"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/hostGroups/{hostGroupName}/hosts/{hostName}", - "azure-compute-capacity-reservation": {"capacityReservationGroups", "capacityReservations"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/capacityReservationGroups/{groupName}/capacityReservations/{reservationName}", - "azure-network-subnet": {"virtualNetworks", "subnets"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}", - "azure-network-virtual-network-peering": {"virtualNetworks", "virtualNetworkPeerings"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnetName}/virtualNetworkPeerings/{peeringName}", - "azure-network-route": {"routeTables", "routes"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/routeTables/{routeTableName}/routes/{routeName}", - "azure-network-security-rule": {"networkSecurityGroups", "securityRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/networkSecurityGroups/{nsgName}/securityRules/{ruleName}", - "azure-network-default-security-rule": {"networkSecurityGroups", "defaultSecurityRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/networkSecurityGroups/{nsgName}/defaultSecurityRules/{ruleName}", - "azure-batch-batch-application": {"batchAccounts", "applications"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Batch/batchAccounts/{accountName}/applications/{applicationName}", - "azure-batch-batch-pool": {"batchAccounts", "pools"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Batch/batchAccounts/{accountName}/pools/{poolName}", - "azure-network-dns-record-set": {"dnszones"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/dnszones/{zoneName}/{recordType}/{relativeRecordSetName}" - "azure-elasticsan-elastic-san-volume-group": {"elasticSans", "volumegroups"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ElasticSan/elasticSans/{elasticSanName}/volumegroups/{volumeGroupName}" - "azure-elasticsan-elastic-san-volume-snapshot": {"elasticSans", "volumegroups", "snapshots"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ElasticSan/elasticSans/{elasticSanName}/volumegroups/{volumeGroupName}/snapshots/{snapshotName}" - "azure-compute-disk-access-private-endpoint-connection": {"diskAccesses", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/diskAccesses/{diskAccessName}/privateEndpointConnections/{connectionName}" - "azure-network-dns-virtual-network-link": {"privateDnsZones", "virtualNetworkLinks"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/privateDnsZones/{zoneName}/virtualNetworkLinks/{linkName}" + "azure-storage-queue": {"storageAccounts", "queues"}, + "azure-storage-blob-container": {"storageAccounts", "containers"}, + "azure-storage-encryption-scope": {"storageAccounts", "encryptionScopes"}, + "azure-storage-file-share": {"storageAccounts", "shares"}, + "azure-storage-storage-account-private-endpoint-connection": {"storageAccounts", "privateEndpointConnections"}, + "azure-documentdb-private-endpoint-connection": {"databaseAccounts", "privateEndpointConnections"}, + "azure-storage-table": {"storageAccounts", "tables"}, + "azure-sql-database": {"servers", "databases"}, // "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/Default-SQL-SouthEastAsia/providers/Microsoft.Sql/servers/testsvr/databases/testdb", + "azure-sql-elastic-pool": {"servers", "elasticPools"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/elasticPools/{elasticPoolName}", + "azure-sql-server-firewall-rule": {"servers", "firewallRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/firewallRules/{ruleName}", + "azure-sql-server-virtual-network-rule": {"servers", "virtualNetworkRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/virtualNetworkRules/{ruleName}", + "azure-sql-server-private-endpoint-connection": {"servers", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Sql/servers/{serverName}/privateEndpointConnections/{connectionName}", + "azure-dbforpostgresql-database": {"flexibleServers", "databases"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/databases/{db}", + "azure-dbforpostgresql-flexible-server-firewall-rule": {"flexibleServers", "firewallRules"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/firewallRules/{rule}", + "azure-dbforpostgresql-flexible-server-private-endpoint-connection": {"flexibleServers", "privateEndpointConnections"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/privateEndpointConnections/{connectionName}", + "azure-dbforpostgresql-flexible-server-backup": {"flexibleServers", "backups"}, // "/subscriptions/.../Microsoft.DBforPostgreSQL/flexibleServers/{server}/backups/{backupName}", + "azure-keyvault-secret": {"vaults", "secrets"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/vaults/{vaultName}/secrets/{secretName}", + "azure-keyvault-key": {"vaults", "keys"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/vaults/{vaultName}/keys/{keyName}", + "azure-keyvault-managed-hsm-private-endpoint-connection": {"managedHSMs", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.KeyVault/managedHSMs/{name}/privateEndpointConnections/{connectionName}", + "azure-authorization-role-assignment": {"roleAssignments"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/roleAssignments/{roleAssignmentName}", + "azure-compute-virtual-machine-run-command": {"virtualMachines", "runCommands"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines/{virtualMachineName}/runCommands/{runCommandName}", + "azure-compute-virtual-machine-extension": {"virtualMachines", "extensions"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines/{virtualMachineName}/extensions/{extensionName}", + "azure-compute-gallery-application-version": {"galleries", "applications", "versions"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/applications/{applicationName}/versions/{versionName}", + "azure-compute-gallery-application": {"galleries", "applications"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/applications/{applicationName}", + "azure-compute-gallery-image": {"galleries", "images"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/galleries/{galleryName}/images/{imageName}", + "azure-compute-dedicated-host": {"hostGroups", "hosts"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/hostGroups/{hostGroupName}/hosts/{hostName}", + "azure-compute-capacity-reservation": {"capacityReservationGroups", "capacityReservations"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/capacityReservationGroups/{groupName}/capacityReservations/{reservationName}", + "azure-network-subnet": {"virtualNetworks", "subnets"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}", + "azure-network-virtual-network-peering": {"virtualNetworks", "virtualNetworkPeerings"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnetName}/virtualNetworkPeerings/{peeringName}", + "azure-network-route": {"routeTables", "routes"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/routeTables/{routeTableName}/routes/{routeName}", + "azure-network-security-rule": {"networkSecurityGroups", "securityRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/networkSecurityGroups/{nsgName}/securityRules/{ruleName}", + "azure-network-default-security-rule": {"networkSecurityGroups", "defaultSecurityRules"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/networkSecurityGroups/{nsgName}/defaultSecurityRules/{ruleName}", + "azure-batch-batch-application": {"batchAccounts", "applications"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Batch/batchAccounts/{accountName}/applications/{applicationName}", + "azure-batch-batch-application-package": {"batchAccounts", "applications", "versions"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Batch/batchAccounts/{accountName}/applications/{applicationName}/versions/{versionName}", + "azure-batch-batch-pool": {"batchAccounts", "pools"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Batch/batchAccounts/{accountName}/pools/{poolName}", + "azure-network-dns-record-set": {"dnszones"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/dnszones/{zoneName}/{recordType}/{relativeRecordSetName}" + "azure-elasticsan-elastic-san-volume-group": {"elasticSans", "volumegroups"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ElasticSan/elasticSans/{elasticSanName}/volumegroups/{volumeGroupName}" + "azure-elasticsan-elastic-san-volume-snapshot": {"elasticSans", "volumegroups", "snapshots"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ElasticSan/elasticSans/{elasticSanName}/volumegroups/{volumeGroupName}/snapshots/{snapshotName}" + "azure-compute-disk-access-private-endpoint-connection": {"diskAccesses", "privateEndpointConnections"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/diskAccesses/{diskAccessName}/privateEndpointConnections/{connectionName}" + "azure-network-dns-virtual-network-link": {"privateDnsZones", "virtualNetworkLinks"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/privateDnsZones/{zoneName}/virtualNetworkLinks/{linkName}" + "azure-network-load-balancer-frontend-ip-configuration": {"loadBalancers", "frontendIPConfigurations"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/loadBalancers/{lbName}/frontendIPConfigurations/{frontendIPConfigName}" + "azure-network-flow-log": {"networkWatchers", "flowLogs"}, // "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/networkWatchers/{networkWatcherName}/flowLogs/{flowLogName}" } if keys, ok := pathKeysMap[resourceType]; ok { diff --git a/sources/gcp/dynamic/adapter-listable.go b/sources/gcp/dynamic/adapter-listable.go index a2b1cfe6..7ad9582a 100644 --- a/sources/gcp/dynamic/adapter-listable.go +++ b/sources/gcp/dynamic/adapter-listable.go @@ -17,6 +17,7 @@ import ( // ListableAdapter implements discovery.ListableAdapter for GCP dynamic adapters. type ListableAdapter struct { listEndpointFunc gcpshared.ListEndpointFunc + listFilterFunc gcpshared.ListFilterFunc Adapter } @@ -24,6 +25,7 @@ type ListableAdapter struct { func NewListableAdapter(listEndpointFunc gcpshared.ListEndpointFunc, config *AdapterConfig, cache sdpcache.Cache) discovery.ListableAdapter { return ListableAdapter{ listEndpointFunc: listEndpointFunc, + listFilterFunc: config.ListFilterFunc, Adapter: Adapter{ locations: config.Locations, httpCli: config.HTTPClient, @@ -111,6 +113,16 @@ func (g ListableAdapter) List(ctx context.Context, scope string, ignoreCache boo return nil, err } + if g.listFilterFunc != nil { + filtered := make([]*sdp.Item, 0, len(items)) + for _, item := range items { + if g.listFilterFunc(item) { + filtered = append(filtered, item) + } + } + items = filtered + } + if len(items) == 0 { // Cache not-found when no items were found notFoundErr := &sdp.QueryError{ @@ -133,6 +145,20 @@ func (g ListableAdapter) List(ctx context.Context, scope string, ignoreCache boo } func (g ListableAdapter) ListStream(ctx context.Context, scope string, ignoreCache bool, stream discovery.QueryResultStream) { + // When a post-filter is configured, fall back to the non-streaming List + // so we can filter before sending items to the stream. + if g.listFilterFunc != nil { + items, err := g.List(ctx, scope, ignoreCache) + if err != nil { + stream.SendError(err) + return + } + for _, item := range items { + stream.SendItem(item) + } + return + } + location, err := g.validateScope(scope) if err != nil { stream.SendError(err) diff --git a/sources/gcp/dynamic/adapter-searchable-listable.go b/sources/gcp/dynamic/adapter-searchable-listable.go index 98ce9ba9..7ddd4d93 100644 --- a/sources/gcp/dynamic/adapter-searchable-listable.go +++ b/sources/gcp/dynamic/adapter-searchable-listable.go @@ -36,6 +36,7 @@ func NewSearchableListableAdapter(searchURLFunc gcpshared.EndpointFunc, listEndp searchFilterFunc: config.SearchFilterFunc, ListableAdapter: ListableAdapter{ listEndpointFunc: listEndpointFunc, + listFilterFunc: config.ListFilterFunc, Adapter: Adapter{ locations: config.Locations, httpCli: config.HTTPClient, diff --git a/sources/gcp/dynamic/adapter.go b/sources/gcp/dynamic/adapter.go index 106b3764..0b7a5a76 100644 --- a/sources/gcp/dynamic/adapter.go +++ b/sources/gcp/dynamic/adapter.go @@ -31,6 +31,7 @@ type AdapterConfig struct { NameSelector string // By default, it is `name`, but can be overridden for outlier cases ListResponseSelector string SearchFilterFunc gcpshared.SearchFilterFunc + ListFilterFunc gcpshared.ListFilterFunc } // Adapter implements discovery.ListableAdapter for GCP dynamic adapters. diff --git a/sources/gcp/dynamic/adapters.go b/sources/gcp/dynamic/adapters.go index 03074ac9..9318921e 100644 --- a/sources/gcp/dynamic/adapters.go +++ b/sources/gcp/dynamic/adapters.go @@ -136,6 +136,7 @@ func MakeAdapter(sdpItemType shared.ItemType, linker *gcpshared.Linker, httpCli NameSelector: meta.NameSelector, ListResponseSelector: meta.ListResponseSelector, SearchFilterFunc: meta.SearchFilterFunc, + ListFilterFunc: meta.ListFilterFunc, } switch adapterType(meta) { diff --git a/sources/gcp/dynamic/adapters/cloudfunctions-function.go b/sources/gcp/dynamic/adapters/cloudfunctions-function.go index a0d4dfc9..f16770c2 100644 --- a/sources/gcp/dynamic/adapters/cloudfunctions-function.go +++ b/sources/gcp/dynamic/adapters/cloudfunctions-function.go @@ -18,6 +18,10 @@ var cloudFunctionAdapter = registerableAdapter{ //nolint:unused GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( "https://cloudfunctions.googleapis.com/v2/projects/%s/locations/%s/functions/%s", ), + // LIST all functions across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://cloudfunctions.googleapis.com/v2/projects/%s/locations/-/functions", + ), // Use SearchEndpointFunc since caller supplies a location to enumerate functions SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( "https://cloudfunctions.googleapis.com/v2/projects/%s/locations/%s/functions", diff --git a/sources/gcp/dynamic/adapters/cloudfunctions-function_test.go b/sources/gcp/dynamic/adapters/cloudfunctions-function_test.go index 7f05bcf1..aee2326c 100644 --- a/sources/gcp/dynamic/adapters/cloudfunctions-function_test.go +++ b/sources/gcp/dynamic/adapters/cloudfunctions-function_test.go @@ -120,6 +120,10 @@ func TestCloudFunctionsFunction(t *testing.T) { StatusCode: http.StatusOK, Body: cloudFunctionsList, }, + fmt.Sprintf("https://cloudfunctions.googleapis.com/v2/projects/%s/locations/-/functions", projectID): { + StatusCode: http.StatusOK, + Body: cloudFunctionsList, + }, } t.Run("Get", func(t *testing.T) { @@ -311,6 +315,39 @@ func TestCloudFunctionsFunction(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Cloud Functions: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Cloud Functions, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + expectedUniqueAttr := shared.CompositeLookupKey(location, functionName) + if item.UniqueAttributeValue() != expectedUniqueAttr { + t.Errorf("Expected unique attribute value '%s', got %s", expectedUniqueAttr, item.UniqueAttributeValue()) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { // Test with error responses to simulate API errors errorResponses := map[string]shared.MockResponse{ diff --git a/sources/gcp/dynamic/adapters/container-cluster.go b/sources/gcp/dynamic/adapters/container-cluster.go index c637be9a..16dbe18d 100644 --- a/sources/gcp/dynamic/adapters/container-cluster.go +++ b/sources/gcp/dynamic/adapters/container-cluster.go @@ -20,6 +20,10 @@ var _ = registerableAdapter{ GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( "https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s", ), + // LIST all clusters across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://container.googleapis.com/v1/projects/%s/locations/-/clusters", + ), // LIST https://container.googleapis.com/v1/projects/{project}/locations/{location}/clusters SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( "https://container.googleapis.com/v1/projects/%s/locations/%s/clusters", diff --git a/sources/gcp/dynamic/adapters/container-cluster_test.go b/sources/gcp/dynamic/adapters/container-cluster_test.go index 5c4f967b..724282f0 100644 --- a/sources/gcp/dynamic/adapters/container-cluster_test.go +++ b/sources/gcp/dynamic/adapters/container-cluster_test.go @@ -99,6 +99,10 @@ func TestContainerCluster(t *testing.T) { StatusCode: http.StatusOK, Body: clusterList, }, + fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/-/clusters", projectID): { + StatusCode: http.StatusOK, + Body: clusterList, + }, } t.Run("Get", func(t *testing.T) { @@ -310,6 +314,38 @@ func TestContainerCluster(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list clusters: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 clusters, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { // Test with error responses to simulate API errors errorResponses := map[string]shared.MockResponse{ diff --git a/sources/gcp/dynamic/adapters/dataflow-job.go b/sources/gcp/dynamic/adapters/dataflow-job.go new file mode 100644 index 00000000..5b8a5e3f --- /dev/null +++ b/sources/gcp/dynamic/adapters/dataflow-job.go @@ -0,0 +1,84 @@ +package adapters + +import ( + "github.com/overmindtech/cli/go/sdp-go" + gcpshared "github.com/overmindtech/cli/sources/gcp/shared" +) + +// Dataflow Job adapter for Google Cloud Dataflow jobs. +// Reference: https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.jobs#Job +// GET: https://dataflow.googleapis.com/v1b3/projects/{project}/locations/{location}/jobs/{jobId} +// LIST: https://dataflow.googleapis.com/v1b3/projects/{project}/jobs:aggregated +// SEARCH: https://dataflow.googleapis.com/v1b3/projects/{project}/locations/{location}/jobs +var _ = registerableAdapter{ + sdpType: gcpshared.DataflowJob, + meta: gcpshared.AdapterMeta{ + SDPAdapterCategory: sdp.AdapterCategory_ADAPTER_CATEGORY_COMPUTE_APPLICATION, + LocationLevel: gcpshared.ProjectLevel, + GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( + "https://dataflow.googleapis.com/v1b3/projects/%s/locations/%s/jobs/%s", + ), + SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( + "https://dataflow.googleapis.com/v1b3/projects/%s/locations/%s/jobs", + ), + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://dataflow.googleapis.com/v1b3/projects/%s/jobs:aggregated", + ), + UniqueAttributeKeys: []string{"locations", "jobs"}, + IAMPermissions: []string{"dataflow.jobs.get", "dataflow.jobs.list"}, + PredefinedRole: "roles/dataflow.viewer", + }, + linkRules: map[string]*gcpshared.Impact{ + // Pub/Sub links (critical for ENG-3217 outage detection) + "jobMetadata.pubsubDetails.topic": { + ToSDPItemType: gcpshared.PubSubTopic, + Description: "If the Pub/Sub Topic is deleted or misconfigured: The Dataflow job may fail to read/write messages. If the Dataflow job changes: The topic remains unaffected.", + }, + "jobMetadata.pubsubDetails.subscription": { + ToSDPItemType: gcpshared.PubSubSubscription, + Description: "If the Pub/Sub Subscription is deleted or misconfigured: The Dataflow job may fail to consume messages. If the Dataflow job changes: The subscription remains unaffected.", + }, + + // BigQuery links + "jobMetadata.bigqueryDetails.table": { + ToSDPItemType: gcpshared.BigQueryTable, + Description: "If the BigQuery Table is deleted or misconfigured: The Dataflow job may fail to read/write data. If the Dataflow job changes: The table remains unaffected.", + }, + "jobMetadata.bigqueryDetails.dataset": { + ToSDPItemType: gcpshared.BigQueryDataset, + Description: "If the BigQuery Dataset is deleted or misconfigured: The Dataflow job may fail to access tables. If the Dataflow job changes: The dataset remains unaffected.", + }, + + // Spanner links + "jobMetadata.spannerDetails.instanceId": { + ToSDPItemType: gcpshared.SpannerInstance, + Description: "If the Spanner Instance is deleted or misconfigured: The Dataflow job may fail to read/write data. If the Dataflow job changes: The instance remains unaffected.", + }, + // Bigtable links + "jobMetadata.bigTableDetails.instanceId": { + ToSDPItemType: gcpshared.BigTableAdminInstance, + Description: "If the Bigtable Instance is deleted or misconfigured: The Dataflow job may fail to read/write data. If the Dataflow job changes: The instance remains unaffected.", + }, + // Environment/infra links + "environment.serviceAccountEmail": gcpshared.IAMServiceAccountImpactInOnly, + "environment.serviceKmsKeyName": gcpshared.CryptoKeyImpactInOnly, + "environment.workerPools.network": gcpshared.ComputeNetworkImpactInOnly, + "environment.workerPools.subnetwork": { + ToSDPItemType: gcpshared.ComputeSubnetwork, + Description: "If the Compute Subnetwork is deleted or misconfigured: Dataflow workers may lose connectivity or fail to start. If the Dataflow job changes: The subnetwork remains unaffected.", + }, + }, + terraformMapping: gcpshared.TerraformMapping{ + Reference: "https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dataflow_job", + Mappings: []*sdp.TerraformMapping{ + { + TerraformMethod: sdp.QueryMethod_GET, + TerraformQueryMap: "google_dataflow_job.job_id", + }, + { + TerraformMethod: sdp.QueryMethod_GET, + TerraformQueryMap: "google_dataflow_flex_template_job.job_id", + }, + }, + }, +}.Register() diff --git a/sources/gcp/dynamic/adapters/dataflow-job_test.go b/sources/gcp/dynamic/adapters/dataflow-job_test.go new file mode 100644 index 00000000..cccee9c3 --- /dev/null +++ b/sources/gcp/dynamic/adapters/dataflow-job_test.go @@ -0,0 +1,333 @@ +package adapters_test + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources/gcp/dynamic" + gcpshared "github.com/overmindtech/cli/sources/gcp/shared" + "github.com/overmindtech/cli/sources/shared" +) + +func TestDataflowJob(t *testing.T) { + ctx := context.Background() + projectID := "test-project" + linker := gcpshared.NewLinker() + location := "us-central1" + jobID := "2024-01-15_test-job-id-123" + + dataflowJob := map[string]any{ + "id": jobID, + "name": fmt.Sprintf("projects/%s/locations/%s/jobs/%s", projectID, location, jobID), + "type": "JOB_TYPE_STREAMING", + "currentState": "JOB_STATE_RUNNING", + "currentStateTime": "2024-01-15T10:30:00Z", + "environment": map[string]any{ + "serviceAccountEmail": fmt.Sprintf("dataflow-sa@%s.iam.gserviceaccount.com", projectID), + "serviceKmsKeyName": fmt.Sprintf("projects/%s/locations/%s/keyRings/dataflow-ring/cryptoKeys/dataflow-key", projectID, location), + "workerPools": []any{ + map[string]any{ + "network": fmt.Sprintf("projects/%s/global/networks/dataflow-network", projectID), + "subnetwork": fmt.Sprintf("projects/%s/regions/%s/subnetworks/dataflow-subnet", projectID, location), + "machineType": "n1-standard-4", + "numWorkers": float64(3), + }, + }, + }, + "jobMetadata": map[string]any{ + "pubsubDetails": []any{ + map[string]any{ + "topic": fmt.Sprintf("projects/%s/topics/input-topic", projectID), + "subscription": fmt.Sprintf("projects/%s/subscriptions/input-subscription", projectID), + }, + map[string]any{ + "topic": fmt.Sprintf("projects/%s/topics/output-topic", projectID), + "subscription": fmt.Sprintf("projects/%s/subscriptions/output-subscription", projectID), + }, + }, + "bigqueryDetails": []any{ + map[string]any{ + "table": fmt.Sprintf("projects/%s/datasets/analytics/tables/events", projectID), + "dataset": fmt.Sprintf("projects/%s/datasets/analytics", projectID), + }, + }, + "spannerDetails": []any{ + map[string]any{ + "instanceId": "spanner-instance-1", + }, + }, + "bigTableDetails": []any{ + map[string]any{ + "instanceId": "bigtable-instance-1", + }, + }, + }, + } + + jobID2 := "2024-01-15_test-job-id-456" + dataflowJob2 := map[string]any{ + "id": jobID2, + "name": fmt.Sprintf("projects/%s/locations/%s/jobs/%s", projectID, location, jobID2), + "type": "JOB_TYPE_BATCH", + "currentState": "JOB_STATE_DONE", + } + + dataflowJobsList := map[string]any{ + "jobs": []any{dataflowJob, dataflowJob2}, + } + + sdpItemType := gcpshared.DataflowJob + + expectedCallAndResponses := map[string]shared.MockResponse{ + fmt.Sprintf("https://dataflow.googleapis.com/v1b3/projects/%s/locations/%s/jobs/%s", projectID, location, jobID): { + StatusCode: http.StatusOK, + Body: dataflowJob, + }, + fmt.Sprintf("https://dataflow.googleapis.com/v1b3/projects/%s/locations/%s/jobs", projectID, location): { + StatusCode: http.StatusOK, + Body: dataflowJobsList, + }, + fmt.Sprintf("https://dataflow.googleapis.com/v1b3/projects/%s/jobs:aggregated", projectID): { + StatusCode: http.StatusOK, + Body: dataflowJobsList, + }, + } + + t.Run("Get", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + getQuery := shared.CompositeLookupKey(location, jobID) + sdpItem, err := adapter.Get(ctx, projectID, getQuery, true) + if err != nil { + t.Fatalf("Failed to get Dataflow Job: %v", err) + } + + if sdpItem.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), sdpItem.GetType()) + } + if sdpItem.UniqueAttributeValue() != getQuery { + t.Errorf("Expected unique attribute value '%s', got %s", getQuery, sdpItem.UniqueAttributeValue()) + } + if sdpItem.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, sdpItem.GetScope()) + } + + val, err := sdpItem.GetAttributes().Get("name") + if err != nil { + t.Fatalf("Failed to get 'name' attribute: %v", err) + } + expectedName := fmt.Sprintf("projects/%s/locations/%s/jobs/%s", projectID, location, jobID) + if val != expectedName { + t.Errorf("Expected name '%s', got %s", expectedName, val) + } + + val, err = sdpItem.GetAttributes().Get("currentState") + if err != nil { + t.Fatalf("Failed to get 'currentState' attribute: %v", err) + } + if val != "JOB_STATE_RUNNING" { + t.Errorf("Expected currentState 'JOB_STATE_RUNNING', got %s", val) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + // Pub/Sub topic links (from pubsubDetails array) + { + ExpectedType: gcpshared.PubSubTopic.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "input-topic", + ExpectedScope: projectID, + }, + { + ExpectedType: gcpshared.PubSubTopic.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "output-topic", + ExpectedScope: projectID, + }, + // Pub/Sub subscription links (from pubsubDetails array) + { + ExpectedType: gcpshared.PubSubSubscription.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "input-subscription", + ExpectedScope: projectID, + }, + { + ExpectedType: gcpshared.PubSubSubscription.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "output-subscription", + ExpectedScope: projectID, + }, + // BigQuery links + { + ExpectedType: gcpshared.BigQueryTable.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey("analytics", "events"), + ExpectedScope: projectID, + }, + { + ExpectedType: gcpshared.BigQueryDataset.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "analytics", + ExpectedScope: projectID, + }, + // Spanner instance link (plain name resolves for single-key types) + { + ExpectedType: gcpshared.SpannerInstance.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "spanner-instance-1", + ExpectedScope: projectID, + }, + // Bigtable instance link (plain name resolves for single-key types) + { + ExpectedType: gcpshared.BigTableAdminInstance.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "bigtable-instance-1", + ExpectedScope: projectID, + }, + // IAM service account link + { + ExpectedType: gcpshared.IAMServiceAccount.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: fmt.Sprintf("dataflow-sa@%s.iam.gserviceaccount.com", projectID), + ExpectedScope: projectID, + }, + // KMS crypto key link + { + ExpectedType: gcpshared.CloudKMSCryptoKey.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: shared.CompositeLookupKey(location, "dataflow-ring", "dataflow-key"), + ExpectedScope: projectID, + }, + // Compute network link + { + ExpectedType: gcpshared.ComputeNetwork.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "dataflow-network", + ExpectedScope: projectID, + }, + // Compute subnetwork link (regional — scope includes region) + { + ExpectedType: gcpshared.ComputeSubnetwork.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "dataflow-subnet", + ExpectedScope: fmt.Sprintf("%s.%s", projectID, location), + }, + } + + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("Search", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter is not a SearchableAdapter") + } + + searchQuery := location + sdpItems, err := searchable.Search(ctx, projectID, searchQuery, true) + if err != nil { + t.Fatalf("Failed to search Dataflow Jobs: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Dataflow Jobs, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + expectedUniqueAttr := shared.CompositeLookupKey(location, jobID) + if item.UniqueAttributeValue() != expectedUniqueAttr { + t.Errorf("Expected unique attribute value '%s', got %s", expectedUniqueAttr, item.UniqueAttributeValue()) + } + } + + if len(sdpItems) >= 2 { + item := sdpItems[1] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + expectedUniqueAttr2 := shared.CompositeLookupKey(location, jobID2) + if item.UniqueAttributeValue() != expectedUniqueAttr2 { + t.Errorf("Expected unique attribute value '%s', got %s", expectedUniqueAttr2, item.UniqueAttributeValue()) + } + } + }) + + t.Run("ErrorHandling", func(t *testing.T) { + errorResponses := map[string]shared.MockResponse{ + fmt.Sprintf("https://dataflow.googleapis.com/v1b3/projects/%s/locations/%s/jobs/%s", projectID, location, jobID): { + StatusCode: http.StatusNotFound, + Body: map[string]any{"error": "Job not found"}, + }, + } + + httpCli := shared.NewMockHTTPClientProvider(errorResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + getQuery := shared.CompositeLookupKey(location, jobID) + _, err = adapter.Get(ctx, projectID, getQuery, true) + if err == nil { + t.Error("Expected error when getting non-existent Dataflow Job, but got nil") + } + }) + + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter is not a ListableAdapter") + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Dataflow Jobs: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Dataflow Jobs, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + expectedUniqueAttr := shared.CompositeLookupKey(location, jobID) + if item.UniqueAttributeValue() != expectedUniqueAttr { + t.Errorf("Expected unique attribute value '%s', got %s", expectedUniqueAttr, item.UniqueAttributeValue()) + } + } + + if len(sdpItems) >= 2 { + item := sdpItems[1] + expectedUniqueAttr2 := shared.CompositeLookupKey(location, jobID2) + if item.UniqueAttributeValue() != expectedUniqueAttr2 { + t.Errorf("Expected unique attribute value '%s', got %s", expectedUniqueAttr2, item.UniqueAttributeValue()) + } + } + }) +} diff --git a/sources/gcp/dynamic/adapters/eventarc-trigger.go b/sources/gcp/dynamic/adapters/eventarc-trigger.go index fe6ba4c5..80c0b819 100644 --- a/sources/gcp/dynamic/adapters/eventarc-trigger.go +++ b/sources/gcp/dynamic/adapters/eventarc-trigger.go @@ -19,6 +19,10 @@ var eventarcTriggerAdapter = registerableAdapter{ //nolint:unused GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( "https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers/%s", ), + // LIST all triggers across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://eventarc.googleapis.com/v1/projects/%s/locations/-/triggers", + ), // List requires only the location (region or global) besides project. SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( "https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers", diff --git a/sources/gcp/dynamic/adapters/eventarc-trigger_test.go b/sources/gcp/dynamic/adapters/eventarc-trigger_test.go new file mode 100644 index 00000000..416d1daf --- /dev/null +++ b/sources/gcp/dynamic/adapters/eventarc-trigger_test.go @@ -0,0 +1,191 @@ +package adapters_test + +import ( + "context" + "fmt" + "net/http" + "testing" + + "cloud.google.com/go/eventarc/apiv1/eventarcpb" + + "github.com/overmindtech/cli/go/discovery" + "github.com/overmindtech/cli/go/sdp-go" + "github.com/overmindtech/cli/go/sdpcache" + "github.com/overmindtech/cli/sources/gcp/dynamic" + gcpshared "github.com/overmindtech/cli/sources/gcp/shared" + "github.com/overmindtech/cli/sources/shared" +) + +func TestEventarcTrigger(t *testing.T) { + ctx := context.Background() + projectID := "test-project" + location := "us-central1" + linker := gcpshared.NewLinker() + triggerName := "test-trigger" + + trigger := &eventarcpb.Trigger{ + Name: fmt.Sprintf("projects/%s/locations/%s/triggers/%s", projectID, location, triggerName), + ServiceAccount: fmt.Sprintf("test-sa@%s.iam.gserviceaccount.com", projectID), + } + + triggerName2 := "test-trigger-2" + trigger2 := &eventarcpb.Trigger{ + Name: fmt.Sprintf("projects/%s/locations/%s/triggers/%s", projectID, location, triggerName2), + } + + triggerList := &eventarcpb.ListTriggersResponse{ + Triggers: []*eventarcpb.Trigger{trigger, trigger2}, + } + + sdpItemType := gcpshared.EventarcTrigger + + expectedCallAndResponses := map[string]shared.MockResponse{ + fmt.Sprintf("https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers/%s", projectID, location, triggerName): { + StatusCode: http.StatusOK, + Body: trigger, + }, + fmt.Sprintf("https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers/%s", projectID, location, triggerName2): { + StatusCode: http.StatusOK, + Body: trigger2, + }, + fmt.Sprintf("https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers", projectID, location): { + StatusCode: http.StatusOK, + Body: triggerList, + }, + fmt.Sprintf("https://eventarc.googleapis.com/v1/projects/%s/locations/-/triggers", projectID): { + StatusCode: http.StatusOK, + Body: triggerList, + }, + } + + t.Run("Get", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + combinedQuery := shared.CompositeLookupKey(location, triggerName) + sdpItem, err := adapter.Get(ctx, projectID, combinedQuery, true) + if err != nil { + t.Fatalf("Failed to get Eventarc trigger: %v", err) + } + + if sdpItem.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), sdpItem.GetType()) + } + if sdpItem.UniqueAttributeValue() != combinedQuery { + t.Errorf("Expected unique attribute value '%s', got %s", combinedQuery, sdpItem.UniqueAttributeValue()) + } + if sdpItem.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, sdpItem.GetScope()) + } + + val, err := sdpItem.GetAttributes().Get("name") + if err != nil { + t.Fatalf("Failed to get 'name' attribute: %v", err) + } + expectedName := fmt.Sprintf("projects/%s/locations/%s/triggers/%s", projectID, location, triggerName) + if val != expectedName { + t.Errorf("Expected name field to be '%s', got %s", expectedName, val) + } + + t.Run("StaticTests", func(t *testing.T) { + queryTests := shared.QueryTests{ + { + ExpectedType: gcpshared.IAMServiceAccount.String(), + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: fmt.Sprintf("test-sa@%s.iam.gserviceaccount.com", projectID), + ExpectedScope: projectID, + }, + } + + shared.RunStaticTests(t, adapter, sdpItem, queryTests) + }) + }) + + t.Run("Search", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + searchable, ok := adapter.(discovery.SearchableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement SearchableAdapter", sdpItemType) + } + + sdpItems, err := searchable.Search(ctx, projectID, location, true) + if err != nil { + t.Fatalf("Failed to search Eventarc triggers: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Eventarc triggers, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Eventarc triggers: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Eventarc triggers, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + + t.Run("ErrorHandling", func(t *testing.T) { + errorResponses := map[string]shared.MockResponse{ + fmt.Sprintf("https://eventarc.googleapis.com/v1/projects/%s/locations/%s/triggers/%s", projectID, location, triggerName): { + StatusCode: http.StatusNotFound, + Body: map[string]any{"error": "Trigger not found"}, + }, + } + + httpCli := shared.NewMockHTTPClientProvider(errorResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + combinedQuery := shared.CompositeLookupKey(location, triggerName) + _, err = adapter.Get(ctx, projectID, combinedQuery, true) + if err == nil { + t.Error("Expected error when getting non-existent Eventarc trigger, but got nil") + } + }) +} diff --git a/sources/gcp/dynamic/adapters/file-instance.go b/sources/gcp/dynamic/adapters/file-instance.go index d4768491..a9ad5b5d 100644 --- a/sources/gcp/dynamic/adapters/file-instance.go +++ b/sources/gcp/dynamic/adapters/file-instance.go @@ -21,6 +21,10 @@ var _ = registerableAdapter{ GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( "https://file.googleapis.com/v1/projects/%s/locations/%s/instances/%s", ), + // LIST all instances across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://file.googleapis.com/v1/projects/%s/locations/-/instances", + ), // Search (per-location) https://file.googleapis.com/v1/projects/{project}/locations/{location}/instances SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( "https://file.googleapis.com/v1/projects/%s/locations/%s/instances", diff --git a/sources/gcp/dynamic/adapters/file-instance_test.go b/sources/gcp/dynamic/adapters/file-instance_test.go index 190c03a2..bc06ebbd 100644 --- a/sources/gcp/dynamic/adapters/file-instance_test.go +++ b/sources/gcp/dynamic/adapters/file-instance_test.go @@ -67,6 +67,10 @@ func TestFileInstance(t *testing.T) { StatusCode: http.StatusOK, Body: instanceList, }, + fmt.Sprintf("https://file.googleapis.com/v1/projects/%s/locations/-/instances", projectID): { + StatusCode: http.StatusOK, + Body: instanceList, + }, } t.Run("Get", func(t *testing.T) { @@ -201,6 +205,38 @@ func TestFileInstance(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Filestore instances: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Filestore instances, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { // Test with error responses to simulate API errors errorResponses := map[string]shared.MockResponse{ diff --git a/sources/gcp/dynamic/adapters/logging-bucket.go b/sources/gcp/dynamic/adapters/logging-bucket.go index b3bb259c..26d4c4b1 100644 --- a/sources/gcp/dynamic/adapters/logging-bucket.go +++ b/sources/gcp/dynamic/adapters/logging-bucket.go @@ -17,6 +17,8 @@ var _ = registerableAdapter{ // GET https://logging.googleapis.com/v2/projects/*/locations/*/buckets/* // IAM permissions: logging.buckets.get GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries("https://logging.googleapis.com/v2/projects/%s/locations/%s/buckets/%s"), + // LIST all buckets across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc("https://logging.googleapis.com/v2/projects/%s/locations/-/buckets"), // Reference: https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.locations.buckets/list // GET https://logging.googleapis.com/v2/projects/*/locations/*/buckets // IAM permissions: logging.buckets.list diff --git a/sources/gcp/dynamic/adapters/logging-bucket_test.go b/sources/gcp/dynamic/adapters/logging-bucket_test.go index 8621cf59..707a737f 100644 --- a/sources/gcp/dynamic/adapters/logging-bucket_test.go +++ b/sources/gcp/dynamic/adapters/logging-bucket_test.go @@ -47,6 +47,10 @@ func TestLoggingBucket(t *testing.T) { StatusCode: http.StatusOK, Body: bucketList, }, + fmt.Sprintf("https://logging.googleapis.com/v2/projects/%s/locations/-/buckets", projectID): { + StatusCode: http.StatusOK, + Body: bucketList, + }, } t.Run("Get", func(t *testing.T) { @@ -117,6 +121,38 @@ func TestLoggingBucket(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list logging buckets: %v", err) + } + + if len(sdpItems) != 1 { + t.Errorf("Expected 1 logging bucket, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { errorResponses := map[string]shared.MockResponse{ fmt.Sprintf("https://logging.googleapis.com/v2/projects/%s/locations/%s/buckets/%s", projectID, location, bucketName): { diff --git a/sources/gcp/dynamic/adapters/logging-saved-query.go b/sources/gcp/dynamic/adapters/logging-saved-query.go index 0e8567bb..a9066568 100644 --- a/sources/gcp/dynamic/adapters/logging-saved-query.go +++ b/sources/gcp/dynamic/adapters/logging-saved-query.go @@ -15,6 +15,8 @@ var _ = registerableAdapter{ // GET https://logging.googleapis.com/v2/projects/*/locations/*/savedQueries/* // IAM permissions: logging.savedQueries.get GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries("https://logging.googleapis.com/v2/projects/%s/locations/%s/savedQueries/%s"), + // LIST all saved queries across all locations using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc("https://logging.googleapis.com/v2/projects/%s/locations/-/savedQueries"), // Reference: https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.locations.savedQueries/list // GET https://logging.googleapis.com/v2/projects/*/locations/*/savedQueries // IAM permissions: logging.savedQueries.list diff --git a/sources/gcp/dynamic/adapters/logging-saved-query_test.go b/sources/gcp/dynamic/adapters/logging-saved-query_test.go index 22c873e9..fa609e18 100644 --- a/sources/gcp/dynamic/adapters/logging-saved-query_test.go +++ b/sources/gcp/dynamic/adapters/logging-saved-query_test.go @@ -42,6 +42,10 @@ func TestLoggingSavedQuery(t *testing.T) { StatusCode: http.StatusOK, Body: queryList, }, + fmt.Sprintf("https://logging.googleapis.com/v2/projects/%s/locations/-/savedQueries", projectID): { + StatusCode: http.StatusOK, + Body: queryList, + }, } t.Run("Get", func(t *testing.T) { @@ -84,6 +88,38 @@ func TestLoggingSavedQuery(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list saved queries: %v", err) + } + + if len(sdpItems) != 1 { + t.Errorf("Expected 1 saved query, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { errorResponses := map[string]shared.MockResponse{ fmt.Sprintf("https://logging.googleapis.com/v2/projects/%s/locations/%s/savedQueries/%s", projectID, location, queryName): { diff --git a/sources/gcp/dynamic/adapters/redis-instance.go b/sources/gcp/dynamic/adapters/redis-instance.go index de20dac9..1b2296e5 100644 --- a/sources/gcp/dynamic/adapters/redis-instance.go +++ b/sources/gcp/dynamic/adapters/redis-instance.go @@ -1,10 +1,27 @@ package adapters import ( + "strings" + "github.com/overmindtech/cli/go/sdp-go" gcpshared "github.com/overmindtech/cli/sources/gcp/shared" ) +// redisInstanceListFilter filters out placeholder entries that GCP returns +// for unavailable locations when using wildcard location queries. +// Placeholder entries have names ending in "/instances/-" with error status. +func redisInstanceListFilter(item *sdp.Item) bool { + name, err := item.GetAttributes().Get("name") + if err != nil { + return true + } + nameStr, ok := name.(string) + if !ok { + return true + } + return !strings.HasSuffix(nameStr, "/instances/-") +} + // GCP Cloud Memorystore Redis Instance adapter. // Cloud Memorystore for Redis provides a fully managed Redis service that is highly available and scalable. // GCP Ref: @@ -24,6 +41,14 @@ var _ = registerableAdapter{ GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries( "https://redis.googleapis.com/v1/projects/%s/locations/%s/instances/%s", ), + // LIST all instances across all locations using wildcard + // Note: wildcard list may include placeholder entries for unavailable locations + // (entries with name ending in "/instances/-" and error status) + ListEndpointFunc: gcpshared.ProjectLevelListFunc( + "https://redis.googleapis.com/v1/projects/%s/locations/-/instances", + ), + // Filter out placeholder entries from LIST results + ListFilterFunc: redisInstanceListFilter, // Reference: https://cloud.google.com/memorystore/docs/redis/reference/rest/v1/projects.locations.instances/list // GET https://redis.googleapis.com/v1/projects/{project}/locations/{location}/instances SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery( diff --git a/sources/gcp/dynamic/adapters/redis-instance_test.go b/sources/gcp/dynamic/adapters/redis-instance_test.go index dc2a9309..6673e6db 100644 --- a/sources/gcp/dynamic/adapters/redis-instance_test.go +++ b/sources/gcp/dynamic/adapters/redis-instance_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "testing" "cloud.google.com/go/redis/apiv1/redispb" @@ -69,6 +70,10 @@ func TestRedisInstance(t *testing.T) { StatusCode: http.StatusOK, Body: instanceList, }, + fmt.Sprintf("https://redis.googleapis.com/v1/projects/%s/locations/-/instances", projectID): { + StatusCode: http.StatusOK, + Body: instanceList, + }, } t.Run("Get", func(t *testing.T) { @@ -218,6 +223,92 @@ func TestRedisInstance(t *testing.T) { } }) + t.Run("List", func(t *testing.T) { + httpCli := shared.NewMockHTTPClientProvider(expectedCallAndResponses) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Redis instances: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Redis instances, got %d", len(sdpItems)) + } + + if len(sdpItems) >= 1 { + item := sdpItems[0] + if item.GetType() != sdpItemType.String() { + t.Errorf("Expected type %s, got %s", sdpItemType.String(), item.GetType()) + } + if item.GetScope() != projectID { + t.Errorf("Expected scope '%s', got %s", projectID, item.GetScope()) + } + } + }) + + t.Run("List filters out placeholder entries", func(t *testing.T) { + placeholder := &redispb.Instance{ + Name: fmt.Sprintf("projects/%s/locations/us-west1/instances/-", projectID), + LocationId: "us-west1", + } + + instanceListWithPlaceholder := &redispb.ListInstancesResponse{ + Instances: []*redispb.Instance{instance, placeholder, instance2}, + } + + responsesWithPlaceholder := map[string]shared.MockResponse{ + fmt.Sprintf("https://redis.googleapis.com/v1/projects/%s/locations/-/instances", projectID): { + StatusCode: http.StatusOK, + Body: instanceListWithPlaceholder, + }, + } + + httpCli := shared.NewMockHTTPClientProvider(responsesWithPlaceholder) + adapter, err := dynamic.MakeAdapter(sdpItemType, linker, httpCli, sdpcache.NewNoOpCache(), []gcpshared.LocationInfo{gcpshared.NewProjectLocation(projectID)}) + if err != nil { + t.Fatalf("Failed to create adapter for %s: %v", sdpItemType, err) + } + + listable, ok := adapter.(discovery.ListableAdapter) + if !ok { + t.Fatalf("Adapter for %s does not implement ListableAdapter", sdpItemType) + } + + sdpItems, err := listable.List(ctx, projectID, true) + if err != nil { + t.Fatalf("Failed to list Redis instances: %v", err) + } + + if len(sdpItems) != 2 { + t.Errorf("Expected 2 Redis instances (placeholder filtered out), got %d", len(sdpItems)) + } + + for _, item := range sdpItems { + name, err := item.GetAttributes().Get("name") + if err != nil { + t.Errorf("Failed to get name attribute: %v", err) + continue + } + nameStr, ok := name.(string) + if !ok { + t.Errorf("Name is not a string: %T", name) + continue + } + if strings.HasSuffix(nameStr, "/instances/-") { + t.Errorf("Placeholder entry was not filtered out: %s", nameStr) + } + } + }) + t.Run("ErrorHandling", func(t *testing.T) { // Test with error responses to simulate API errors errorResponses := map[string]shared.MockResponse{ diff --git a/sources/gcp/dynamic/adapters/sql-admin-backup-run.go b/sources/gcp/dynamic/adapters/sql-admin-backup-run.go index 16d5aa06..7c3efe42 100644 --- a/sources/gcp/dynamic/adapters/sql-admin-backup-run.go +++ b/sources/gcp/dynamic/adapters/sql-admin-backup-run.go @@ -14,6 +14,8 @@ var _ = registerableAdapter{ // Reference: https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/backupRuns/get // GET https://sqladmin.googleapis.com/v1/projects/{project}/instances/{instance}/backupRuns/{id} GetEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithTwoQueries("https://sqladmin.googleapis.com/v1/projects/%s/instances/%s/backupRuns/%s"), + // LIST all backup runs across all instances using wildcard + ListEndpointFunc: gcpshared.ProjectLevelListFunc("https://sqladmin.googleapis.com/v1/projects/%s/instances/-/backupRuns"), // Reference: https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/backupRuns/list // GET https://sqladmin.googleapis.com/v1/projects/{project}/instances/{instance}/backupRuns SearchEndpointFunc: gcpshared.ProjectLevelEndpointFuncWithSingleQuery("https://sqladmin.googleapis.com/v1/projects/%s/instances/%s/backupRuns"), diff --git a/sources/gcp/dynamic/adapters_test.go b/sources/gcp/dynamic/adapters_test.go index cc56d16f..52e88860 100644 --- a/sources/gcp/dynamic/adapters_test.go +++ b/sources/gcp/dynamic/adapters_test.go @@ -83,10 +83,10 @@ func Test_addAdapter(t *testing.T) { searchableListable: true, }, { - name: "Searchable adapter", - sdpType: gcpshared.SQLAdminBackupRun, - locations: projectLocation, - searchable: true, + name: "Searchable adapter", + sdpType: gcpshared.SQLAdminBackupRun, + locations: projectLocation, + searchableListable: true, }, { name: "SearchableListable adapter", diff --git a/sources/gcp/setup/scripts/overmind-gcp-roles.sh b/sources/gcp/setup/scripts/overmind-gcp-roles.sh index aedeee82..b49121c2 100644 --- a/sources/gcp/setup/scripts/overmind-gcp-roles.sh +++ b/sources/gcp/setup/scripts/overmind-gcp-roles.sh @@ -15,6 +15,7 @@ ROLES=( "roles/dataform.viewer" "roles/dataplex.catalogViewer" "roles/dataplex.viewer" + "roles/dataflow.viewer" "roles/dataproc.viewer" "roles/dns.reader" "roles/essentialcontacts.viewer" diff --git a/sources/gcp/shared/adapter-meta.go b/sources/gcp/shared/adapter-meta.go index b2c24b37..55d4cd40 100644 --- a/sources/gcp/shared/adapter-meta.go +++ b/sources/gcp/shared/adapter-meta.go @@ -13,6 +13,11 @@ import ( // where the GCP API does not support server-side filtering. type SearchFilterFunc func(query string, item *sdp.Item) bool +// ListFilterFunc filters items returned by LIST. Takes an SDP item and returns +// true to keep the item. Used to filter out placeholder/phantom entries that +// some GCP APIs return when using wildcard location queries. +type ListFilterFunc func(item *sdp.Item) bool + // LocationLevel defines at which level of the GCP hierarchy a resource is located. type LocationLevel string @@ -57,6 +62,10 @@ type AdapterMeta struct { // to keep only items matching the query. Used for tag-based SEARCH where // the API has no server-side filter. SearchFilterFunc SearchFilterFunc + // ListFilterFunc, if set, is applied after fetching items during LIST + // to filter out unwanted entries. Used to exclude placeholder/phantom + // entries that some GCP APIs return with wildcard location queries. + ListFilterFunc ListFilterFunc } // ============================================= diff --git a/sources/gcp/shared/item-types.go b/sources/gcp/shared/item-types.go index 9b3e6ccb..a9ae74fa 100644 --- a/sources/gcp/shared/item-types.go +++ b/sources/gcp/shared/item-types.go @@ -201,4 +201,5 @@ var ( ComputeBgpRoute = shared.NewItemType(GCP, Compute, BgpRoute) // Router BGP Route child resource NetworkServicesMesh = shared.NewItemType(GCP, NetworkServices, Mesh) // https://cloud.google.com/service-mesh/docs/reference/network-services/rest/v1/projects.locations.meshes/get BinaryAuthorizationPlatformPolicy = shared.NewItemType(GCP, BinaryAuthorization, BinaryAuthorizationPolicy) // https://cloud.google.com/binary-authorization/docs/reference/rest/v1/projects.platforms.policies/get + DataflowJob = shared.NewItemType(GCP, Dataflow, Job) // https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.jobs/get ) diff --git a/sources/gcp/shared/models.go b/sources/gcp/shared/models.go index 337480bd..4c72181d 100644 --- a/sources/gcp/shared/models.go +++ b/sources/gcp/shared/models.go @@ -50,6 +50,7 @@ const ( File shared.API = "file" // Added for File (file.googleapis.com) CertificateManager shared.API = "certificate-manager" // Added for Certificate Manager (certificatemanager.googleapis.com) BinaryAuthorization shared.API = "binary-authorization" // Added for Binary Authorization (binaryauthorization.googleapis.com) + Dataflow shared.API = "dataflow" // Added for Dataflow (dataflow.googleapis.com) ) @@ -231,4 +232,5 @@ const ( WorkflowInvocation shared.Resource = "workflow-invocation" // Dataform Workflow Invocation child resource Mesh shared.Resource = "mesh" // Network Services API Mesh BinaryAuthorizationPolicy shared.Resource = "binary-authorization-policy" // Binary Authorization API Platform Policy + Job shared.Resource = "job" // Dataflow Job ) diff --git a/sources/gcp/shared/predefined-roles.go b/sources/gcp/shared/predefined-roles.go index f5966f79..d8399fb7 100644 --- a/sources/gcp/shared/predefined-roles.go +++ b/sources/gcp/shared/predefined-roles.go @@ -220,6 +220,14 @@ var PredefinedRoles = map[string]role{ "container.clusters.list", }, }, + "roles/dataflow.viewer": { + Role: "roles/dataflow.viewer", + Link: "https://cloud.google.com/iam/docs/roles-permissions/dataflow#dataflow.viewer", + IAMPermissions: []string{ + "dataflow.jobs.get", + "dataflow.jobs.list", + }, + }, "roles/dataproc.viewer": { Role: "roles/dataproc.viewer", // Provides read-only access to Dataproc resources. diff --git a/sources/snapshot/README.md b/sources/snapshot/README.md index cf949ef7..ff58c688 100644 --- a/sources/snapshot/README.md +++ b/sources/snapshot/README.md @@ -37,6 +37,91 @@ The snapshot source requires a snapshot file or URL to be specified: --health-check-port # Health check port (default: 8089) ``` +### Running with Docker + +#### Build the Docker image + +Build the snapshot source Docker image: + +```bash +docker buildx bake snapshot +``` + +Or build directly with docker build: + +```bash +docker build -f sources/snapshot/build/package/Dockerfile \ + --build-arg BUILD_VERSION=dev \ + --build-arg BUILD_COMMIT=$(git rev-parse HEAD) \ + -t snapshot-source:local . +``` + +#### Run the Docker container + +Run the container with a mounted snapshot file: + +**Local/dev environment (unauthenticated):** + +```bash +docker run --rm \ + -v /path/to/snapshot.json:/data/snapshot.json:ro \ + -e SNAPSHOT_SOURCE=/data/snapshot.json \ + -e OVERMIND_MANAGED_SOURCE=true \ + -e NATS_SERVICE_HOST=nats \ + -e NATS_SERVICE_PORT=4222 \ + -e ALLOW_UNAUTHENTICATED=true \ + --network=host \ + ghcr.io/overmindtech/workspace/snapshot-source:dev +``` + +> ⚠️ **WARNING**: `ALLOW_UNAUTHENTICATED=true` is for local/dev testing only. Do not use in production. + +**Production environment (authenticated):** + +```bash +docker run --rm \ + -v /path/to/snapshot.json:/data/snapshot.json:ro \ + -e SNAPSHOT_SOURCE=/data/snapshot.json \ + -e OVERMIND_MANAGED_SOURCE=true \ + -e API_KEY=your-api-key \ + -e NATS_SERVICE_HOST=nats \ + -e NATS_SERVICE_PORT=4222 \ + --network=host \ + ghcr.io/overmindtech/workspace/snapshot-source:dev +``` + +Or use with docker-compose (local/dev): + +```yaml +services: + snapshot-source: + image: ghcr.io/overmindtech/workspace/snapshot-source:dev + volumes: + - ./snapshot.json:/data/snapshot.json:ro + environment: + SNAPSHOT_SOURCE: /data/snapshot.json + OVERMIND_MANAGED_SOURCE: "true" + NATS_SERVICE_HOST: nats + NATS_SERVICE_PORT: 4222 + ALLOW_UNAUTHENTICATED: "true" # WARNING: local/dev only + depends_on: + - nats +``` + +For production, replace `ALLOW_UNAUTHENTICATED: "true"` with `API_KEY: ${API_KEY}` and set the API key via environment variable or secrets management. + +#### Health check + +The container exposes health check endpoints on port 8089: + +```bash +# Liveness probe - checks NATS connection +curl http://localhost:8089/healthz/alive + +# Readiness probe - checks adapter initialization +curl http://localhost:8089/healthz/ready +``` + ### Running Locally #### Option 1: With backend services (recommended) diff --git a/sources/snapshot/build/package/Dockerfile b/sources/snapshot/build/package/Dockerfile new file mode 100644 index 00000000..8780ed41 --- /dev/null +++ b/sources/snapshot/build/package/Dockerfile @@ -0,0 +1,26 @@ +# Build the source binary +FROM golang:1.26-alpine AS builder +ARG TARGETOS +ARG TARGETARCH +ARG BUILD_VERSION +ARG BUILD_COMMIT + +# required for generating the version descriptor +RUN apk upgrade --no-cache && apk add --no-cache git + +WORKDIR /workspace + +# Copy the go source +COPY . . + +# Build +RUN --mount=type=cache,target=/go/pkg \ + --mount=type=cache,target=/root/.cache/go-build \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w -X github.com/overmindtech/cli/go/tracing.version=${BUILD_VERSION} -X github.com/overmindtech/cli/go/tracing.commit=${BUILD_COMMIT}" -o source sources/snapshot/main.go + +FROM alpine:3.23.3 +WORKDIR / +COPY --from=builder /workspace/source . +USER 65534:65534 + +ENTRYPOINT ["/source"]