diff --git a/.cursor/rules/discovery-indexers.mdc b/.cursor/rules/discovery-indexers.mdc new file mode 100644 index 000000000..dec611957 --- /dev/null +++ b/.cursor/rules/discovery-indexers.mdc @@ -0,0 +1,71 @@ +--- +description: Contract for RunWhen Local discovery indexers (native SDK + CloudQuery). Enforces registry parity, collector registration, normalizer shape, and docs/test/generation-rule definition of done. +globs: src/indexers/**,src/enrichers/azure.py,src/enrichers/gcp.py,src/enrichers/aws.py,scripts/azure/**,scripts/gcp/** +alwaysApply: false +--- + +# Discovery indexer contract + +RunWhen Local discovers resources via native SDK indexers (`azureapi`, +`gcpapi`, future `awsapi`) plus the legacy `cloudquery` indexer. The native +indexers exist to remove the CloudQuery dependency entirely. When editing any +indexer, enforce every rule below. + +For full workflows use the `extend-discovery-type` and `add-discovery-type` +skills. Worked examples: `docs/architecture/{azure,gcp}-indexer-internals.md`. + +## CloudQuery parity is the public contract + +- The **CloudQuery table name** (e.g. `gcp_compute_instances`) is the + `resource_type` used in generation rules. Native indexers MUST normalize + payloads into the same CloudQuery-shaped dict so rules don't change when the + backend flips. +- Every CloudQuery table for a platform MUST resolve via the registry by its + canonical name or a registered alias. Don't drop parity to simplify. + +## Generated registry — never hand-edit + +`src/indexers/*_resource_type_registry.yaml` is GENERATED. To change a mapping: + +1. Edit the overrides YAML (`scripts//_resource_type_overrides.yaml`). +2. Re-run `scripts//sync__resource_type_registry.py`. +3. Commit both the overrides and the regenerated YAML; the sync `--dry-run` + must report 0 drift. + +Use `null` for the native type of tables with no API equivalent so the generic +pass skips them. + +## Typed vs generic collectors + +- The **generic collector** (Azure `resources.list()`, GCP Cloud Asset + Inventory, AWS Cloud Control) provides broad parity. +- **Typed collectors** are enrichment for high-value types. Register them in + `_TYPED_COLLECTORS` keyed by the canonical CloudQuery table name. A typed type + is automatically excluded from the generic pass — write each resource once. +- Lazy-import SDK clients inside the collector so importing the module never + hard-requires the SDK. + +## Discovery is selective + anchored + +- Collect only types referenced by loaded generation rules, plus the mandatory + anchor (GCP `gcp_projects`, Azure `azure_resources_resource_groups`). +- Emit the anchor first so child resources link to it at parse time. +- Respect Level-of-Detail: scopes resolving to `none` are skipped entirely. + +## Normalizer shape + +Every resource flows: `normalize -> tag filter -> handler.parse_resource_data +-> writer.add_resource`. Normalizers MUST map cloud labels/tags into the +canonical `tags` field and preserve the qualifier fields the tag/hierarchy +templates expect. Persist only via the `ResourceWriter`, never directly. + +## Definition of done (every indexer change) + +A change is incomplete until ALL of these are updated together: + +- [ ] Code + registry (regenerated, not hand-edited) +- [ ] Unit tests: registry loader, normalizer+handler round-trip, selective dispatch +- [ ] Docs: `docs/authoring/indexed-resources/.md` and the indexer-internals doc +- [ ] Generation-rule contract verified: the type resolves by CloudQuery table + name and the normalized dict carries the fields the templates need +- [ ] `cd src && python -m unittest discover -s indexers -p 'test_*.py'` passes diff --git a/.cursor/rules/resource-type-registry.mdc b/.cursor/rules/resource-type-registry.mdc new file mode 100644 index 000000000..6ef56a327 --- /dev/null +++ b/.cursor/rules/resource-type-registry.mdc @@ -0,0 +1,38 @@ +--- +description: Guardrail for generated resource-type registry YAMLs and their sync scripts. Edit overrides + re-run sync; never hand-edit the generated file. +globs: src/indexers/*_resource_type_registry.yaml,scripts/azure/*_overrides.yaml,scripts/gcp/*_overrides.yaml,scripts/azure/sync_*.py,scripts/gcp/sync_*.py,scripts/azure/*_cloudquery_tables.txt,scripts/gcp/*_cloudquery_tables.txt +alwaysApply: false +--- + +# Resource-type registry is generated + +`src/indexers/*_resource_type_registry.yaml` is produced by +`scripts//sync__resource_type_registry.py` from three +inputs. **Do not hand-edit the generated YAML.** + +## To change a mapping + +1. Edit the right input: + - **Native-type wrong / missing** -> `*_overrides.yaml` + (`arm_type_overrides` for Azure, `cai_type_overrides` for GCP; use `null` + when there is no native API equivalent). + - **Alternate name** -> `aliases` in the overrides. + - **Rich payload wanted** -> add the table to `typed_collectors`. + - **Service -> API host remap** -> `service_api_hosts` (GCP). + - **New parity tables** -> refresh the `*_cloudquery_tables.txt` snapshot + from the CloudQuery hub (record source + version in its header). +2. Regenerate: + +```bash +python scripts/gcp/sync_gcp_resource_type_registry.py # or azure/... +``` + +3. Review the `added / removed / changed` summary and the YAML diff; commit the + overrides AND the regenerated YAML together. + +## Parity invariant + +Every CloudQuery table for the platform must resolve via the registry by +canonical name or alias. The sync `--dry-run` must report 0 drift after your +change. Removing a table from parity requires an explicit reason, not +convenience. diff --git a/.cursor/skills/add-discovery-type/SKILL.md b/.cursor/skills/add-discovery-type/SKILL.md new file mode 100644 index 000000000..6a876c910 --- /dev/null +++ b/.cursor/skills/add-discovery-type/SKILL.md @@ -0,0 +1,158 @@ +--- +name: add-discovery-type +description: >- + Stand up a brand-new RunWhen Local discovery platform / indexer from scratch + (a new cloud or resource source) following the native SDK pattern used by + azureapi and gcpapi. Use when asked to add a new platform, new cloud provider, + or new top-level discovery source. Covers the parity registry, indexer + modules, platform handler, tag/hierarchy templates, pipeline wiring, tests, + and docs. +--- + +# Add a new discovery type + +Use this to add a **new platform** (a new cloud provider or discovery source). +To add a single resource type to an existing indexer, use the +`extend-discovery-type` skill instead. + +`gcpapi` is the most recent end-to-end example — read +`docs/architecture/gcp-indexer-internals.md` first; it documents every piece you +are about to replicate. `azureapi` is the second reference. The mandate is to +build native SDK indexers so CloudQuery can eventually be removed entirely. + +## Architecture (what you are building) + +``` +scripts// +├── _cloudquery_tables.txt # parity source (CloudQuery hub table list) +├── _resource_type_overrides.yaml # hand-curated overrides +└── sync__resource_type_registry.py # registry generator + +src/indexers/ +├── _resource_type_registry.yaml # GENERATED catalog (never hand-edit) +├── _resource_type_registry.py # read-only loader +├── _common.py # credentials + scope resolution + tag filters +├── api_normalizers.py # raw SDK/API -> CloudQuery-shaped dict +├── api_resource_types.py # generic collector + typed collectors + specs +├── api.py # orchestration loop +└── test_*.py # unit tests + +src/enrichers/.py # PlatformHandler.parse_resource_data +src/templates/-tags.yaml # tag template +src/templates/-hierarchy.yaml # hierarchy template +docs/architecture/-indexer-internals.md +docs/authoring/indexed-resources/.md +``` + +## Core principles + +1. **Parity first.** Seed the registry from the upstream CloudQuery plugin's + table list (use the CloudQuery hub, not the old in-container plugin version). + Every CloudQuery table must resolve via the registry by canonical name/alias. +2. **CloudQuery table name is the public contract.** Generation rules reference + it as `resource_type`. The native indexer normalizes payloads into the same + shape so rules don't change when the backend flips. +3. **Generic collector is the workhorse**, typed collectors are enrichment. Pick + the broadest first-party API for the generic pass (Azure + `resources.list()`, GCP Cloud Asset Inventory, AWS Cloud Control API). +4. **Selective, generation-rule-driven discovery.** Only collect types + referenced by loaded generation rules (plus the mandatory anchor), and + respect Level-of-Detail scoping. +5. **Never hand-edit the generated registry YAML.** + +## Workflow + +``` +- [ ] 1. Obtain upstream CloudQuery table list -> scripts/

/

_cloudquery_tables.txt +- [ ] 2. Author the overrides YAML (type remaps, aliases, typed flags, anchor, nulls) +- [ ] 3. Write the sync script (heuristic table-name -> native-type mapping) +- [ ] 4. Generate the registry YAML + write the read-only loader +- [ ] 5. Add SDK deps to pyproject + regenerate lock in python:3.14-slim +- [ ] 6. Write

_common.py (credentials, scope/LOD, tag filters) +- [ ] 7. Write

api_normalizers.py (raw -> CloudQuery-shaped dict) +- [ ] 8. Write

api_resource_types.py (generic collector + typed + specs) +- [ ] 9. Write

api.py orchestrator (phases: bootstrap -> anchors -> typed -> generic) +- [ ] 10. Add/confirm the enricher PlatformHandler.parse_resource_data +- [ ] 11. Add src/templates/

-tags.yaml +

-hierarchy.yaml +- [ ] 12. Wire pipeline: component.py, run.sh, run.py setting, cloudquery skip +- [ ] 13. Tests: registry + normalizers + selective +- [ ] 14. Docs: indexer-internals + indexed-resources + READMEs +- [ ] 15. Run the full indexer test suite +``` + +### 1–4. Registry pipeline + +Mirror `scripts/gcp/`: + +- **Table list**: scrape the CloudQuery hub plugin table reference into a + `.txt` with a header noting source + version. This is the parity source. +- **Overrides**: a YAML with native-type remaps, `aliases`, `typed_collectors`, + the **mandatory anchor** type (e.g. GCP `gcp_projects`, Azure + `azure_resources_resource_groups`), and `null` native types for tables with + no API equivalent. +- **Sync script**: heuristic `__` -> + `/`, plus override application. Copy + `sync_gcp_resource_type_registry.py` and adapt the heuristic. +- **Loader** (`_resource_type_registry.py`): copy the GCP loader; + provide `load_registry`, `find`, and a `find_by__type` lookup. + +### 6–9. Indexer modules + +- **`_common.py`**: resolve credentials (K8s secret, inline key, ADC / + env) and the account/project/subscription scope; provide `has_included_tags` / + `has_excluded_tags` label filters. +- **`api_normalizers.py`**: convert raw SDK/API objects to the + CloudQuery-shaped dict the handler expects; hoist the full payload, map cloud + labels/tags to `tags`, normalize names/IDs/regions. Include a + `make__resource_data` helper for the synthesized anchor. +- **`api_resource_types.py`**: a `*ResourceTypeSpec` dataclass, the + generic collector over the broad API, typed collectors (lazy-import SDK + clients), `_TYPED_COLLECTORS` keyed by canonical table name, and + `find_spec` / `find_spec_by__type`. +- **`api.py`** `index(ctx)` phases: + 1. Bootstrap: check `IndexerBackend`, resolve creds + scope, mirror + into the enricher, resolve LOD (skip scopes with LOD `none`). + 2. Phase 0: emit the synthesized **anchor** resource first so children link to + it at parse time. + 3. Phase 1: typed collectors for accessed types that have one. + 4. Phase 2: one generic pass per scope, filtered to accessed native types not + already covered by a typed collector. + Every resource flows: normalize -> tag filter -> `handler.parse_resource_data` + -> `writer.add_resource`. + +### 10–11. Handler + templates + +- Confirm/extend `src/enrichers/.py` `parse_resource_data` consumes + the normalized dict and links children to the anchor. +- Add `src/templates/-tags.yaml` and `-hierarchy.yaml` following the + **`template-tags-hierarchy` rule** exactly (hierarchy ends with + `resource_name`; emit `platform`, `resource_type`, `resource_name`, + `child_resource`; specificity-ordered `resource_name`; dedup loop). + +### 12. Pipeline wiring + +- `src/component.py`: add `"api"` to the `INDEXER` stage. +- `src/run.sh`: include `api` in `COMPONENTS`. +- `src/run.py`: coalesce `IndexerBackend` from `workspaceInfo` or + `WB__INDEXER_BACKEND` into request data. +- `src/indexers/cloudquery.py`: skip this platform when the native backend is + selected (mutual exclusion). + +### 13–14. Tests + docs (required) + +- Tests mirroring `test_gcp*.py`: registry loader contract, normalizer + + handler round-trip, selective discovery + generic-pass filter dispatch. +- `docs/architecture/-indexer-internals.md` (link it from + `docs/architecture/README.md`). +- `docs/authoring/indexed-resources/.md` documenting both backends, + the matchable types, and the stable generation-rule contract. +- Update top-level `README.md` / docs indexes if the platform is newly listed. + +### 15. Verify + +```bash +cd src && python -m unittest discover -s indexers -p 'test_*.py' +python scripts//sync__resource_type_registry.py --dry-run +``` + +The sync dry-run must report 0 drift (registry matches overrides + table list). diff --git a/.cursor/skills/extend-discovery-type/SKILL.md b/.cursor/skills/extend-discovery-type/SKILL.md new file mode 100644 index 000000000..04f8c5e27 --- /dev/null +++ b/.cursor/skills/extend-discovery-type/SKILL.md @@ -0,0 +1,175 @@ +--- +name: extend-discovery-type +description: >- + Add or enrich a resource type in an existing RunWhen Local discovery indexer + (Azure azureapi, GCP gcpapi, AWS, or Kubernetes). Use when asked to support a + new cloud resource type, add a typed SDK collector, fix a resource type + mapping, or extend what an indexer discovers. Enforces the registry sync, + collector registration, normalizer, tests, docs, and generation-rule contract. +--- + +# Extend a discovery type + +Use this when an **existing** indexer should discover a new resource type or +return richer data for one it already covers. To stand up a brand-new platform, +use the `add-discovery-type` skill instead. + +The native SDK indexers (`azureapi`, `gcpapi`, future `awsapi`) all share the +same shape: a **generated registry** maps CloudQuery table names to native +cloud API types, a **generic collector** provides broad parity, and a small set +of **typed collectors** provide rich payloads for high-value types. Read the +canonical worked examples before editing: + +- Azure: `docs/architecture/azure-indexer-internals.md` +- GCP: `docs/architecture/gcp-indexer-internals.md` + +## Golden rules + +1. **Never hand-edit a generated registry YAML** (`src/indexers/*_resource_type_registry.yaml`). + Edit the overrides file and re-run the sync script. +2. **Maintain CloudQuery parity.** The CloudQuery table name is the public + `resource_type` contract used in generation rules. New types must resolve by + their canonical CloudQuery table name (or a registered alias). +3. **Done means done:** code + registry + tests + docs + generation-rule + verification. A change that doesn't update docs is not complete. + +## Workflow + +Copy this checklist and track it: + +``` +- [ ] 1. Identify the target indexer + canonical CloudQuery table name +- [ ] 2. Edit the overrides YAML (type mapping / alias / typed_collector flag) +- [ ] 3. Regenerate the registry via the sync script (never hand-edit the YAML) +- [ ] 4. (typed only) Implement collector + register in _TYPED_COLLECTORS +- [ ] 5. (if needed) Extend the normalizer for the new payload shape +- [ ] 6. Add/extend the SDK dependency (pyproject + locked) if required +- [ ] 7. Update tests (registry + normalizer + selective) +- [ ] 8. Update docs (indexed-resources + indexer internals) +- [ ] 9. Verify the generation-rule contract still holds +- [ ] 10. Run the unit tests +``` + +### 1. Identify target + table name + +Determine which backend (`azureapi` / `gcpapi` / `aws` / Kubernetes) owns the +type and the **canonical CloudQuery table name** (e.g. `gcp_compute_instances`, +`azure_compute_disks`). This name is the registry key and the generation-rule +`resource_type`. + +### 2. Edit the overrides YAML + +Per platform: + +| Platform | Overrides file | +|----------|----------------| +| Azure | `scripts/azure/azure_resource_type_overrides.yaml` | +| GCP | `scripts/gcp/gcp_resource_type_overrides.yaml` | + +Edit only what the heuristic gets wrong: + +- **Wrong native type**: add an entry under `arm_type_overrides` (Azure) or + `cai_type_overrides` (GCP). Set it to `null` if the table has no native API + equivalent (so the generic pass skips it). +- **Alias**: add under `aliases` so older/alternate names still resolve. +- **Typed collector**: append the table name to `typed_collectors`. +- **Service host remap** (GCP): add under `service_api_hosts`. + +### 3. Regenerate the registry + +```bash +# GCP (round-trips existing table list, picks up override changes): +python scripts/gcp/sync_gcp_resource_type_registry.py + +# Azure: +python scripts/azure/sync_azure_resource_type_registry.py +``` + +The script reports `added / removed / changed`. Inspect the diff in the +generated YAML; it should reflect only your intended change. + +### 4. Implement a typed collector (only for rich-payload types) + +Skip this for types that are fine coming from the generic pass (Azure +`resources.list()` envelope, GCP Cloud Asset Inventory full payload). + +**GCP** (`src/indexers/gcpapi_resource_types.py`): + +```python +def _collect_(credentials, project_id): + from google.cloud import # lazy import + client = (credentials=credentials) + return list(client.list_(project=project_id)) +``` + +Register it keyed by canonical table name: + +```python +_TYPED_COLLECTORS: dict[str, GcpCollector] = { + "gcp_compute_instances": _collect_compute_instances, + "gcp__": _collect_, +} +``` + +**Azure** (`src/indexers/azureapi_resource_types.py`): implement both +`_collect__all(credential, subscription_id)` and +`_collect__in_rg(credential, subscription_id, rg_name)`, then register in +`_TYPED_COLLECTORS` keyed by canonical table name (pass `None` for the second +slot only if no per-RG SDK call exists). + +A typed type is automatically excluded from the generic pass, so the resource is +written exactly once. + +### 5. Extend the normalizer if the shape is new + +Normalizers (`*_normalizers.py`) convert raw SDK/API objects into the +CloudQuery-shaped dict the platform handler expects. Reuse the existing +`normalize_*` helpers when possible. Only add code for genuinely new fields, and +always map cloud labels/tags into the canonical `tags` field. + +### 6. Dependencies + +If the typed collector needs an SDK that isn't already a dependency, add it to +`src/pyproject.toml` and regenerate the lock inside the pinned Python container: + +```bash +docker run --rm -v "$PWD/src:/app" -w /app python:3.14-slim \ + bash -lc "pip install poetry && poetry lock" +``` + +Lazy-import SDK clients inside the collector so importing the indexer module +never hard-requires the SDK. + +### 7. Tests (required) + +- `test__resource_type_registry.py`: assert the new type resolves and, + if typed, appears in the typed-collector set. +- `test_api_normalizers.py`: a round-trip through the platform handler + for the new payload shape. +- `test_api_selective.py`: if dispatch/selection logic changed. + +### 8. Docs (required) + +- `docs/authoring/indexed-resources/.md`: list the new matchable type + and note whether it's typed (rich) or generic. +- The indexer internals doc if collector counts or behavior changed. + +### 9. Generation-rule contract (required) + +Generation rules reference the type by its CloudQuery table name as +`resource_type`. Confirm: + +- The new type resolves via the registry's `find`/`find_spec` by that exact + table name (and any aliases you added). +- The normalized dict carries the qualifier fields the tag/hierarchy templates + expect (`resource_name`, scope qualifiers, `tags`). See the + `template-tags-hierarchy` rule for the tag/hierarchy contract. + +If a contrib generation rule should start matching the new type, update or note +it; do not silently change matching behavior for existing rules. + +### 10. Run tests + +```bash +cd src && python -m unittest discover -s indexers -p 'test_*.py' +``` diff --git a/.github/workflows/ado-ci-test.yml b/.github/workflows/ado-ci-test.yml index 8166ebf38..61e309321 100644 --- a/.github/workflows/ado-ci-test.yml +++ b/.github/workflows/ado-ci-test.yml @@ -7,6 +7,11 @@ on: - "src/enrichers/azure_devops.py" - "src/templates/azure-devops-auth.yaml" - "src/indexers/cloudquery.py" + - "src/indexers/azureapi.py" + - "src/indexers/azureapi_resource_types.py" + - "src/indexers/azureapi_normalizers.py" + - "src/indexers/azure_common.py" + - "src/indexers/resource_writer.py" - ".test/azure/ado/**" - ".github/workflows/ado-ci-test.yml" workflow_dispatch: diff --git a/.github/workflows/merge_to_main.yaml b/.github/workflows/merge_to_main.yaml index 0b5a4abe4..5b6e0ef27 100644 --- a/.github/workflows/merge_to_main.yaml +++ b/.github/workflows/merge_to_main.yaml @@ -22,7 +22,6 @@ env: DEFAULT_BRANCH: "origin/${{ github.event.repository.default_branch }}" CONTAINER_NAME: "runwhen-local" SHARED_ARTIFACT_REPOSITORY_PATH: "us-docker.pkg.dev/runwhen-nonprod-shared/public-images" - RW_LOCAL_MKDOCS_CONFIG: "src/cheat-sheet-docs/mkdocs.yml" SANDBOX_DEPLOYMENT_NAME: "runwhen-local" SANDBOX_DEPLOYMENT_NAMESPACE: "runwhen-local" APP_LABEL: "app=runwhen-local" @@ -60,7 +59,6 @@ jobs: env: IMAGE: runwhen-local SHARED_ARTIFACT_REPOSITORY_PATH: us-docker.pkg.dev/runwhen-nonprod-shared/public-images - RW_LOCAL_MKDOCS_CONFIG: src/cheat-sheet-docs/mkdocs.yml GHCR_ORG: "runwhen-contrib" outputs: version: ${{ steps.version.outputs.version }} @@ -70,9 +68,6 @@ jobs: run: | echo "VERSION=$(cat src/VERSION | jq -r .version)" | tee -a $GITHUB_ENV echo "version=$VERSION" >> $GITHUB_OUTPUT - - run: | - sed -i "s/date: today/date: $(date +'%Y-%m-%d %H:%M')/g" ${RW_LOCAL_MKDOCS_CONFIG} - sed -i "s/version: 0\.1/version: ${VERSION}/g" ${RW_LOCAL_MKDOCS_CONFIG} - uses: google-github-actions/auth@v2 with: workload_identity_provider: ${{ secrets.RUNWHEN_NONPROD_SHARED_WI_PROVIDER }} @@ -112,14 +107,10 @@ jobs: env: IMAGE: runwhen-local SHARED_ARTIFACT_REPOSITORY_PATH: us-docker.pkg.dev/runwhen-nonprod-shared/public-images - RW_LOCAL_MKDOCS_CONFIG: src/cheat-sheet-docs/mkdocs.yml GHCR_ORG: "runwhen-contrib" steps: - uses: actions/checkout@v4 - run: echo "VERSION=$(cat src/VERSION | jq -r .version)" | tee -a $GITHUB_ENV - - run: | - sed -i "s/date: today/date: $(date +'%Y-%m-%d %H:%M')/g" ${RW_LOCAL_MKDOCS_CONFIG} - sed -i "s/version: 0\.1/version: ${VERSION}/g" ${RW_LOCAL_MKDOCS_CONFIG} - uses: google-github-actions/auth@v2 with: workload_identity_provider: ${{ secrets.RUNWHEN_NONPROD_SHARED_WI_PROVIDER }} diff --git a/.github/workflows/pr_open.yaml b/.github/workflows/pr_open.yaml index b62d4963d..58d718376 100644 --- a/.github/workflows/pr_open.yaml +++ b/.github/workflows/pr_open.yaml @@ -19,7 +19,6 @@ env: DEFAULT_BRANCH: "origin/${{ github.event.repository.default_branch }}" CONTAINER_NAME: "runwhen-local" SHARED_ARTIFACT_REPOSITORY_PATH: "us-docker.pkg.dev/runwhen-nonprod-shared/public-images" - RW_LOCAL_MKDOCS_CONFIG: "src/cheat-sheet-docs/mkdocs.yml" SANDBOX_DEPLOYMENT_NAME: "runwhen-local" SANDBOX_DEPLOYMENT_NAMESPACE: "runwhen-local" APP_LABEL: "app=runwhen-local" @@ -52,7 +51,6 @@ jobs: env: IMAGE: runwhen-local SHARED_ARTIFACT_REPOSITORY_PATH: us-docker.pkg.dev/runwhen-nonprod-shared/public-images - RW_LOCAL_MKDOCS_CONFIG: src/cheat-sheet-docs/mkdocs.yml outputs: tag: ${{ steps.tag.outputs.tag }} steps: @@ -61,9 +59,6 @@ jobs: run: | echo "TAG=$(echo $GITHUB_REF_NAME | sed 's/[^a-zA-Z0-9]/-/g')-${GITHUB_SHA::8}" | tee -a $GITHUB_ENV echo "tag=$TAG" >> $GITHUB_OUTPUT - - run: | - sed -i "s/date: today/date: $(date +'%Y-%m-%d %H:%M')/g" ${RW_LOCAL_MKDOCS_CONFIG} - sed -i "s/version: 0\.1/version: ${TAG}/g" ${RW_LOCAL_MKDOCS_CONFIG} - uses: google-github-actions/auth@v2 with: workload_identity_provider: ${{ secrets.RUNWHEN_NONPROD_SHARED_WI_PROVIDER }} @@ -90,13 +85,9 @@ jobs: env: IMAGE: runwhen-local SHARED_ARTIFACT_REPOSITORY_PATH: us-docker.pkg.dev/runwhen-nonprod-shared/public-images - RW_LOCAL_MKDOCS_CONFIG: src/cheat-sheet-docs/mkdocs.yml steps: - uses: actions/checkout@v4 - run: echo "TAG=$(echo $GITHUB_REF_NAME | sed 's/[^a-zA-Z0-9]/-/g')-${GITHUB_SHA::8}" | tee -a $GITHUB_ENV - - run: | - sed -i "s/date: today/date: $(date +'%Y-%m-%d %H:%M')/g" ${RW_LOCAL_MKDOCS_CONFIG} - sed -i "s/version: 0\.1/version: ${TAG}/g" ${RW_LOCAL_MKDOCS_CONFIG} - uses: google-github-actions/auth@v2 with: workload_identity_provider: ${{ secrets.RUNWHEN_NONPROD_SHARED_WI_PROVIDER }} diff --git a/.github/workflows/test-aws-indexer.yaml b/.github/workflows/test-aws-indexer.yaml new file mode 100644 index 000000000..2a037462e --- /dev/null +++ b/.github/workflows/test-aws-indexer.yaml @@ -0,0 +1,107 @@ +# Smoke-test the native awsapi indexer end-to-end against a static AWS sandbox +# account. Like the GCP fixture there is no Terraform: the sandbox account +# already exists, so the job just writes the credentials, builds the RWL test +# image, runs discovery with awsIndexerBackend=awsapi + the SQLite resource +# store, and asserts the persisted resources directly. Designed to be cheap so +# we can run it on every PR that touches src/** or the fixture. +# +# Required repo/org secrets: +# AWS_ACCESS_KEY_ID - the sandbox IAM user's access key id. +# AWS_SECRET_ACCESS_KEY - the sandbox IAM user's secret access key. +# (Same secrets already used by .github/workflows/test-aws-auth.yaml.) +name: Discovery AWS Indexer Tests + +on: + workflow_dispatch: + pull_request: + paths: + - "src/**" + - ".test/aws/awsapi-baseline/**" + - ".github/workflows/test-aws-indexer.yaml" + - "!src/VERSION" + +permissions: + contents: "read" + id-token: "write" + actions: "read" + +jobs: + awsapi-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: .test/aws/awsapi-baseline + steps: + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Create buildx builder + run: | + docker buildx create --use --name=mybuilder + docker buildx inspect --bootstrap + + - name: Write aws.secret from sandbox credentials + run: | + set -euo pipefail + # Credentials come from repo secrets; nothing is committed or echoed. + cat > aws.secret <> $GITHUB_ENV + else + echo "status=✅ *Success*" >> $GITHUB_ENV + fi + + - name: Send Slack Notification + uses: slackapi/slack-github-action@v1 + with: + channel-id: "#runwhen-local" + slack-message: "${{ env.status }} - Workflow *${{ github.workflow }}* in repo *${{ github.repository }}* on branch *${{ github.ref_name }}*.\nSee the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/test-azure-indexer.yaml b/.github/workflows/test-azure-indexer.yaml new file mode 100644 index 000000000..4eeec9c6d --- /dev/null +++ b/.github/workflows/test-azure-indexer.yaml @@ -0,0 +1,135 @@ +# Smoke-test the native azureapi indexer end-to-end against a small, +# AKS-free Azure fixture. Provisions a couple of resource groups + storage +# accounts + a key vault via Terraform (local state, single job), runs the +# baseline / selective / tag-filter discovery tasks, and tears the infra +# back down. Designed to be cheap so we can run it on every PR that +# touches src/** or the fixture. +name: Discovery Azure Indexer Tests + +on: + workflow_dispatch: + pull_request: + paths: + - "src/**" + - ".test/azure/no-aks-resources/**" + - ".github/workflows/test-azure-indexer.yaml" + - "!src/VERSION" + +permissions: + contents: "read" + id-token: "write" + actions: "read" + +env: + TF_IN_AUTOMATION: "true" + +jobs: + azureapi-no-aks-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: .test/azure/no-aks-resources + steps: + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Create buildx builder + run: | + docker buildx create --use --name=mybuilder + docker buildx inspect --bootstrap + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZ_SANDBOX_CREDS }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Write terraform/tf.secret from sandbox credentials + run: | + set -euo pipefail + echo '${{ secrets.AZ_SANDBOX_CREDS }}' > /tmp/azure_creds.json + AZ_CLIENT_ID=$(jq -r '.clientId' /tmp/azure_creds.json) + AZ_CLIENT_SECRET=$(jq -r '.clientSecret' /tmp/azure_creds.json) + AZ_SUB_ID=$(jq -r '.subscriptionId' /tmp/azure_creds.json) + AZ_TENANT_ID=$(jq -r '.tenantId' /tmp/azure_creds.json) + SP_PRINCIPAL_ID=$(az ad sp list \ + --filter "appId eq '$AZ_CLIENT_ID'" \ + --query '[0].id' -o tsv) + + cat > terraform/tf.secret <> $GITHUB_ENV + else + echo "status=✅ *Success*" >> $GITHUB_ENV + fi + + - name: Send Slack Notification + uses: slackapi/slack-github-action@v1 + with: + channel-id: "#runwhen-local" + slack-message: "${{ env.status }} - Workflow *${{ github.workflow }}* in repo *${{ github.repository }}* on branch *${{ github.ref_name }}*.\nSee the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/test-gcp-indexer.yaml b/.github/workflows/test-gcp-indexer.yaml new file mode 100644 index 000000000..d1ac3edc8 --- /dev/null +++ b/.github/workflows/test-gcp-indexer.yaml @@ -0,0 +1,119 @@ +# Smoke-test the native gcpapi indexer end-to-end against a static GCP +# sandbox project. Unlike the Azure fixture there is no Terraform: the sandbox +# project (a GKE cluster + a handful of storage buckets) already exists, so the +# job just writes the service-account credentials, builds the RWL test image, +# runs discovery with gcpIndexerBackend=gcpapi + the SQLite resource store, and +# asserts the persisted resources directly. Designed to be cheap so we can run +# it on every PR that touches src/** or the fixture. +# +# Required repo/org secret: +# GCP_SANDBOX_SA_JSON - the raw, single-line service-account key JSON for the +# sandbox SA (project_id is read back out of it by the +# Taskfile). +# +# Required cloud-side setup on the sandbox service account (per-service viewer +# roles — this is the supported functional baseline for native GCP discovery): +# * roles/compute.viewer, roles/storage.objectViewer, roles/container.viewer, +# roles/pubsub.viewer, roles/iam.serviceAccountViewer (or a convenience +# superset like roles/viewer). These gate the typed SDK collectors that the +# baseline asserts (project anchor, storage buckets, GKE clusters, and the +# always-present default compute network / firewall rules). +# +# OPTIONAL accelerator (NOT required; CI passes without it): +# * Enabling the Cloud Asset Inventory API (cloudasset.googleapis.com) with a +# CAI viewer role broadens coverage to resource types that have no typed +# collector. Its absence is normal: the indexer logs an informational +# GCP_CAI_PERMISSION_DENIED note and discovery proceeds on the typed +# collectors. The baseline assert does NOT fail when CAI is unavailable. +name: Discovery GCP Indexer Tests + +on: + workflow_dispatch: + pull_request: + paths: + - "src/**" + - ".test/gcp/gcp-and-k8s/**" + - ".github/workflows/test-gcp-indexer.yaml" + - "!src/VERSION" + +permissions: + contents: "read" + id-token: "write" + actions: "read" + +jobs: + gcpapi-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: .test/gcp/gcp-and-k8s + steps: + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Create buildx builder + run: | + docker buildx create --use --name=mybuilder + docker buildx inspect --bootstrap + + - name: Write gcp.secret from sandbox service-account JSON + run: | + set -euo pipefail + # The SA JSON is committed nowhere; it comes from the repo secret. + # Single-quote wrapping is safe because service-account JSON never + # contains single quotes. + echo '${{ secrets.GCP_SANDBOX_SA_JSON }}' > gcp.secret + # Sanity-check it parses and never echo the key material. + python3 -c "import json; d=json.load(open('gcp.secret')); assert d.get('project_id'); print('gcp.secret OK for project', d['project_id'])" + + - name: Run baseline discovery (gcpIndexerBackend=gcpapi, SQLite store assertions) + run: task ci-test-gcpapi-baseline + + - name: Upload discovery artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gcpapi-logs-${{ github.run_id }} + path: | + .test/gcp/gcp-and-k8s/output/ + .test/gcp/gcp-and-k8s/run_sh_output.log + .test/gcp/gcp-and-k8s/container_logs.log + retention-days: 7 + + - name: Remove gcp.secret + if: always() + run: rm -f gcp.secret + + notify: + runs-on: ubuntu-latest + needs: [gcpapi-test] + if: always() + steps: + - name: Set Workflow Status + id: status + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "status=❌ *Failure*" >> $GITHUB_ENV + else + echo "status=✅ *Success*" >> $GITHUB_ENV + fi + + - name: Send Slack Notification + uses: slackapi/slack-github-action@v1 + with: + channel-id: "#runwhen-local" + slack-message: "${{ env.status }} - Workflow *${{ github.workflow }}* in repo *${{ github.repository }}* on branch *${{ github.ref_name }}*.\nSee the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/test-upload.yaml b/.github/workflows/test-upload.yaml index 235737cd3..7fc5e507b 100644 --- a/.github/workflows/test-upload.yaml +++ b/.github/workflows/test-upload.yaml @@ -1,118 +1,118 @@ -# Dedicated Upload Test Workflow - Runs on Cron Schedule -name: Upload Test +# # Dedicated Upload Test Workflow - Runs on Cron Schedule +# name: Upload Test -on: - workflow_dispatch: - pull_request: - paths: - - "src/**" - - ".github/workflows/test-upload.yaml" - - "!src/VERSION" - schedule: - # Run every day at 2 AM UTC (adjust as needed) - - cron: '0 2 * * *' +# on: +# workflow_dispatch: +# pull_request: +# paths: +# - "src/**" +# - ".github/workflows/test-upload.yaml" +# - "!src/VERSION" +# schedule: +# # Run every day at 2 AM UTC (adjust as needed) +# - cron: '0 2 * * *' -permissions: - contents: "read" - id-token: "write" - security-events: "write" - actions: "read" +# permissions: +# contents: "read" +# id-token: "write" +# security-events: "write" +# actions: "read" -env: - PROJECT_ID: runwhen-nonprod-shared - DEFAULT_BRANCH: "origin/${{ github.event.repository.default_branch }}" +# env: +# PROJECT_ID: runwhen-nonprod-shared +# DEFAULT_BRANCH: "origin/${{ github.event.repository.default_branch }}" -jobs: - build-image: - runs-on: ubuntu-latest - outputs: - image-tag: ${{ github.sha }} - steps: - - name: Install Task - uses: arduino/setup-task@v2 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v2 - name: Checkout +# jobs: +# build-image: +# runs-on: ubuntu-latest +# outputs: +# image-tag: ${{ github.sha }} +# steps: +# - name: Install Task +# uses: arduino/setup-task@v2 +# with: +# version: 3.x +# repo-token: ${{ secrets.GITHUB_TOKEN }} +# - uses: actions/checkout@v2 +# name: Checkout - - name: Build Image - run: |- - cd .test/k8s/upload - docker buildx create --use --name=mybuilder - docker buildx inspect --bootstrap - task build-rwl - docker save -o image.tar runwhen-local:test || (echo "Image not found!" && exit 1) +# - name: Build Image +# run: |- +# cd .test/k8s/upload +# docker buildx create --use --name=mybuilder +# docker buildx inspect --bootstrap +# task build-rwl +# docker save -o image.tar runwhen-local:test || (echo "Image not found!" && exit 1) - - name: Upload Image as Artifact - uses: actions/upload-artifact@v4 - with: - name: container-image - path: .test/k8s/upload/image.tar - retention-days: 1 +# - name: Upload Image as Artifact +# uses: actions/upload-artifact@v4 +# with: +# name: container-image +# path: .test/k8s/upload/image.tar +# retention-days: 1 - k8s-upload-test: - needs: build-image - runs-on: ubuntu-latest - steps: - - name: Install Task - uses: arduino/setup-task@v2 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} +# k8s-upload-test: +# needs: build-image +# runs-on: ubuntu-latest +# steps: +# - name: Install Task +# uses: arduino/setup-task@v2 +# with: +# version: 3.x +# repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v2 - name: Checkout +# - uses: actions/checkout@v2 +# name: Checkout - - name: Download Image Artifact - uses: actions/download-artifact@v4 - with: - name: container-image - path: .test/k8s/upload +# - name: Download Image Artifact +# uses: actions/download-artifact@v4 +# with: +# name: container-image +# path: .test/k8s/upload - - name: Load Docker Image - run: | - docker load -i .test/k8s/upload/image.tar +# - name: Load Docker Image +# run: | +# docker load -i .test/k8s/upload/image.tar - - name: Setup Upload Configuration - env: - CI_VERIFY_UPLOADINFO: ${{ secrets.CI_VERIFY_UPLOADINFO }} - KUBECONFIG_SECRET: ${{ secrets.SANDBOX_KUBECONFIG }} - run: |- - cd .test/k8s/upload - echo "$KUBECONFIG_SECRET" > kubeconfig.secret - echo "$CI_VERIFY_UPLOADINFO" > uploadInfo.yaml +# - name: Setup Upload Configuration +# env: +# CI_VERIFY_UPLOADINFO: ${{ secrets.CI_VERIFY_UPLOADINFO }} +# KUBECONFIG_SECRET: ${{ secrets.SANDBOX_KUBECONFIG }} +# run: |- +# cd .test/k8s/upload +# echo "$KUBECONFIG_SECRET" > kubeconfig.secret +# echo "$CI_VERIFY_UPLOADINFO" > uploadInfo.yaml - - name: Run Upload Test - env: - RW_PAT: ${{ secrets.RW_PAT }} - run: |- - cd .test/k8s/upload - echo "Task Description:" - yq '.tasks.ci-test-rwl-upload.desc' Taskfile.yaml - task ci-test-rwl-upload +# - name: Run Upload Test +# env: +# RW_PAT: ${{ secrets.RW_PAT }} +# run: |- +# cd .test/k8s/upload +# echo "Task Description:" +# yq '.tasks.ci-test-rwl-upload.desc' Taskfile.yaml +# task ci-test-rwl-upload - notify: - runs-on: ubuntu-latest - needs: - - build-image - - k8s-upload-test - if: always() +# notify: +# runs-on: ubuntu-latest +# needs: +# - build-image +# - k8s-upload-test +# if: always() - steps: - - name: Set Workflow Status - id: status - run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then - echo "status=❌ *Failure*" >> $GITHUB_ENV - else - echo "status=✅ *Success*" >> $GITHUB_ENV - fi +# steps: +# - name: Set Workflow Status +# id: status +# run: | +# if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then +# echo "status=❌ *Failure*" >> $GITHUB_ENV +# else +# echo "status=✅ *Success*" >> $GITHUB_ENV +# fi - - name: Send Slack Notification - uses: slackapi/slack-github-action@v1 - with: - channel-id: "#runwhen-local" - slack-message: "${{ env.status }} - Workflow *${{ github.workflow }}* in repo *${{ github.repository }}* on branch *${{ github.ref_name }}*.\nSee the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} \ No newline at end of file +# - name: Send Slack Notification +# uses: slackapi/slack-github-action@v1 +# with: +# channel-id: "#runwhen-local" +# slack-message: "${{ env.status }} - Workflow *${{ github.workflow }}* in repo *${{ github.repository }}* on branch *${{ github.ref_name }}*.\nSee the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" +# env: +# SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8839cbe19..4faae98a3 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,9 @@ src/rw-cli-codecollection-cache .test/k8s/upload/gitlab_sas.json .test/azure/ado/run_sh_output.log .test/azure/ado/container_logs.log +.test/azure/no-aks-resources/run_sh_output.log +.test/azure/no-aks-resources/container_logs.log +.test/gcp/gcp-and-k8s/run_sh_output.log +.test/gcp/gcp-and-k8s/container_logs.log +# Ephemeral local code collection built by generate-gcpapi-test-codecollection +.test/gcp/gcp-and-k8s/gcpapi-test-codecollection/ diff --git a/.test/aws/assume-role/Taskfile.yaml b/.test/aws/assume-role/Taskfile.yaml index 1cf813283..f8caba9d6 100644 --- a/.test/aws/assume-role/Taskfile.yaml +++ b/.test/aws/assume-role/Taskfile.yaml @@ -111,7 +111,7 @@ tasks: -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ -e AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ -e AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" \ - --name $CONTAINER_NAME -p 8081:8081 -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { + --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } diff --git a/.test/aws/awsapi-baseline/.gitignore b/.test/aws/awsapi-baseline/.gitignore new file mode 100644 index 000000000..543d347a3 --- /dev/null +++ b/.test/aws/awsapi-baseline/.gitignore @@ -0,0 +1,10 @@ +# Never commit live credentials or run artifacts for this fixture. +aws.secret +workspaceInfo.yaml +output/ +run_sh_output.log +container_logs.log +slx_count.txt +# Artifacts the workspace builder writes into the mounted dir during a run. +.gitconfig +.azure-devops/ diff --git a/.test/aws/awsapi-baseline/README.md b/.test/aws/awsapi-baseline/README.md new file mode 100644 index 000000000..c6be7f0d3 --- /dev/null +++ b/.test/aws/awsapi-baseline/README.md @@ -0,0 +1,59 @@ +# awsapi-baseline + +Smoke-test for the **native `awsapi` indexer** (AWS Cloud Control API + boto3), +the AWS counterpart of `.test/gcp/gcp-and-k8s` (gcpapi baseline) and +`.test/azure/no-aks-resources` (azureapi baseline). + +It selects `awsIndexerBackend: awsapi` + `resourceStoreBackend: sqlite`, runs +discovery against a static AWS sandbox account, then asserts directly against +`output/resources.sqlite` (resilient presence/threshold checks, not SLX count). + +## Credentials + +Provide an `aws.secret` file in shell-export format (same shape as +`.test/aws/basic/aws.secret`): + +``` +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export AWS_DEFAULT_REGION=us-west-2 +``` + +`aws.secret` is git-ignored. Only the `AWS_*` credential lines are consumed and +forwarded into the container env; the keys never land in `workspaceInfo.yaml`. +The native indexer resolves them through the boto3 default credential chain. + +## Running locally + +```bash +cd .test/aws/awsapi-baseline +# create aws.secret (see above) +task ci-test-awsapi-baseline +``` + +This runs: `generate-awsapi-baseline-config` -> `build-rwl` -> `run-rwl-discovery` +-> `assert-awsapi-baseline`. Because `src/**` changes ship a new indexer, the +test image is rebuilt as part of the flow. + +## Assertions (`assert-awsapi-baseline`) + +1. The **native `awsapi`** backend ran (not CloudQuery) -- grepped from the + container logs. +2. The **account anchor** (`aws_iam_accounts`, surfaced as `account`) is present + in the resource store. It is synthesized from the resolved credentials, so it + is always present on a successful run. +3. **>= 1** AWS resource overall. +4. The default networking the Cloud Control pass always finds is present + (`aws_ec2_vpcs` and `aws_ec2_security_groups`, >= 1 each). Every AWS + account/region ships a default VPC + default security group, and the bundled + generation rules reference both. + +The sandbox is intentionally minimal and its real resources come and go, so the +assertions are presence/threshold only -- no exact counts and no required typed +collectors (S3 / EC2 instances may legitimately be empty). + +## Cleanup + +```bash +task clean +``` diff --git a/.test/aws/awsapi-baseline/Taskfile.yaml b/.test/aws/awsapi-baseline/Taskfile.yaml new file mode 100644 index 000000000..6d36b7a72 --- /dev/null +++ b/.test/aws/awsapi-baseline/Taskfile.yaml @@ -0,0 +1,271 @@ +version: "3" + +# Smoke-test the native ``awsapi`` indexer end-to-end against a static AWS +# sandbox account. Mirrors .test/gcp/gcp-and-k8s (the gcpapi baseline) which in +# turn mirrors .test/azure/no-aks-resources: generate a workspaceInfo that +# selects the native ``awsapi`` backend + the SQLite resource store, run +# discovery, then assert directly against output/resources.sqlite (not the SLX +# count). +# +# Unlike Azure there is no Terraform: the sandbox account already exists, so the +# job just provides credentials, builds the RWL test image, runs discovery with +# awsIndexerBackend=awsapi + the SQLite resource store, and asserts the +# persisted resources directly. +# +# Credentials are read from ``aws.secret`` (shell-export format, the same shape +# as .test/aws/basic/aws.secret). Only the AWS_* credential lines are consumed; +# the access key / secret never land in workspaceInfo.yaml -- the native indexer +# resolves them through the default credential chain from the container env. +# +# Required CI repo secrets (used by .github/workflows/test-aws-indexer.yaml): +# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY - the sandbox IAM user's keys. + +tasks: + default: + desc: "Generate workspaceInfo and run the native awsapi baseline" + cmds: + - task: ci-test-awsapi-baseline + + clean: + desc: "Run cleanup tasks" + cmds: + - task: clean-rwl-discovery + + build-rwl: + desc: "Build RWL test image" + cmds: + - | + BUILD_DIR=../../../src/ + CONTAINER_NAME="RunWhenLocal" + if docker ps -q --filter "name=$CONTAINER_NAME" | grep -q .; then + echo "Stopping and removing existing container $CONTAINER_NAME..." + docker stop $CONTAINER_NAME && docker rm $CONTAINER_NAME + elif docker ps -a -q --filter "name=$CONTAINER_NAME" | grep -q .; then + echo "Removing existing stopped container $CONTAINER_NAME..." + docker rm $CONTAINER_NAME || echo "Container removal failed, continuing..." + else + echo "No existing container named $CONTAINER_NAME found." + fi + + echo "Cleaning up output directory..." + rm -rf output || { echo "Failed to remove output directory"; exit 1; } + mkdir output && chmod 777 output || { echo "Failed to set permissions"; exit 1; } + ## Building Container Image + docker buildx build --builder mybuilder --platform linux/amd64 --build-arg INCLUDE_CODE_COLLECTION_CACHE=true -t runwhen-local:test -f $BUILD_DIR/Dockerfile $BUILD_DIR --load + silent: true + + generate-awsapi-baseline-config: + desc: "Emit workspaceInfo.yaml selecting the native awsapi backend + SQLite store (defaultLOD=detailed)." + silent: true + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "awsapi-baseline"}}' + cmds: + - | + if [ ! -f aws.secret ]; then + echo "aws.secret not found; expected shell-export credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION)." + exit 1 + fi + # Resolve the region from aws.secret (default us-east-1). The access key / + # secret are NOT written into workspaceInfo.yaml; the native indexer + # picks them up from the container env (default credential chain). + AWS_REGION_RESOLVED=$(grep -E '^export AWS_DEFAULT_REGION=' aws.secret | head -n1 | cut -d= -f2- | tr -d '"' || true) + AWS_REGION_RESOLVED=${AWS_REGION_RESOLVED:-us-east-1} + cat > workspaceInfo.yaml <&1 | tee container_logs.log + exit 1 + fi + if docker exec "$CONTAINER_NAME" curl -sf http://localhost:8000/info/ >/dev/null 2>&1; then + rest_ready=1; break + fi + sleep 2 + done + if [ "$rest_ready" -ne 1 ]; then + echo "REST service never became ready. Container logs:" + docker logs "$CONTAINER_NAME" 2>&1 | tee container_logs.log + exit 1 + fi + + echo "Running workspace builder script in container..." + # pipefail so a run.sh failure is NOT masked by tee's exit code (which + # previously let a failed discovery proceed to a confusing "no sqlite" + # assertion instead of failing here with the real error). + set -o pipefail + docker exec -w /workspace-builder "$CONTAINER_NAME" ./run.sh "$1" --verbose 2>&1 | tee run_sh_output.log || { + echo "Error executing script in container. Container logs:" + docker logs "$CONTAINER_NAME" 2>&1 | tee container_logs.log + exit 1 + } + set +o pipefail + # Always snapshot the container (REST/indexer) logs for the asserts + + # the failure artifact. + docker logs "$CONTAINER_NAME" > container_logs.log 2>&1 || true + + echo "Review generated config files under output/workspaces/" + slx_root=$(find 'output/' -type d -name 'slxs' 2>/dev/null) + if [ -n "$slx_root" ]; then + total_slxs=$(find $slx_root -mindepth 1 -type d | wc -l) + else + total_slxs=0 + fi + echo "Total SLXs: $total_slxs" + echo "$total_slxs" > slx_count.txt + + ci-test-awsapi-baseline: + desc: "End to end: native awsapi discovery. SQLite store must contain the account anchor + discovered AWS resources." + cmds: + - task: generate-awsapi-baseline-config + - task: build-rwl + - task: run-rwl-discovery + - task: assert-awsapi-baseline + + assert-awsapi-baseline: + desc: "Verify the native awsapi backend ran and persisted AWS resources to output/resources.sqlite." + silent: true + cmds: + - | + DB=output/resources.sqlite + if [ ! -f "$DB" ]; then + echo "FAIL: $DB not produced" + exit 1 + fi + + # Snapshot the indexer stdout so we can confirm the *native* backend ran. + # The awsapi backend-selection log lives in the container's main-process + # logs (the run.sh exec only triggers the REST call), so capture both. + docker logs RunWhenLocal > container_logs.log 2>&1 || true + + fail=0 + + # 1. The native awsapi backend must have been selected (NOT cloudquery). + if grep -qE "AWS indexer backend: 'awsapi' \(Cloud Control API" run_sh_output.log container_logs.log 2>/dev/null; then + echo "OK baseline: native awsapi backend ran" + else + echo "FAIL baseline: native awsapi backend did not run (expected awsIndexerBackend=awsapi)" + fail=1 + fi + + # 2. The account anchor must be present. The aws_iam_accounts anchor is + # surfaced to generation rules under its first alias 'account' + # (mirrors GCP's 'project' anchor). The account is synthesized from + # the resolved credentials, so it is always present on a successful + # run regardless of what else exists in the sandbox. + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='aws' AND resource_type IN ('account','aws_iam_accounts','aws_account');" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL baseline: AWS account anchor (aws_iam_accounts) not in resource store" + fail=1 + else + echo "OK baseline: AWS account anchor present (${n} row)" + fi + + # 3. At least one AWS resource overall (the anchor counts). Kept as a + # presence/threshold check -- the sandbox is intentionally minimal and + # its real resources come and go, so we do NOT assert exact counts or + # specific typed collectors (S3/EC2 may legitimately be empty). + total=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='aws';" \ + 2>/dev/null || echo 0) + if [ "$total" -lt 1 ]; then + echo "FAIL baseline: expected >= 1 aws resource, found ${total}" + fail=1 + else + echo "OK baseline: ${total} aws resources in store (>= 1)" + fi + + # 4. The Cloud Control generic pass populates the account's default + # networking, which every AWS account/region has out of the box and + # which the bundled generation rules reference. Assert presence + # (>= 1), not exact counts, so the test stays stable as the sandbox + # changes. (Observed on a live run: 1 default VPC + 1 default SG.) + for rtype in aws_ec2_vpcs aws_ec2_security_groups; do + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='aws' AND resource_type='${rtype}';" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL baseline: expected >= 1 ${rtype}, found ${n}" + fail=1 + else + echo "OK baseline: ${rtype} present (${n} row)" + fi + done + + exit "$fail" + + clean-rwl-discovery: + desc: "Check and clean up RunWhen Local discovery output" + cmds: + - | + CONTAINER_NAME="RunWhenLocal" + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + rm -rf output + rm -f workspaceInfo.yaml + rm -f run_sh_output.log container_logs.log slx_count.txt + silent: true diff --git a/.test/aws/basic/Taskfile.yaml b/.test/aws/basic/Taskfile.yaml index 007ccd1d5..b9323c832 100644 --- a/.test/aws/basic/Taskfile.yaml +++ b/.test/aws/basic/Taskfile.yaml @@ -61,7 +61,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGGING=false --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGGING=false --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } diff --git a/.test/aws/irsa-eks/values.yaml b/.test/aws/irsa-eks/values.yaml index 3a5668aef..5b7470207 100644 --- a/.test/aws/irsa-eks/values.yaml +++ b/.test/aws/irsa-eks/values.yaml @@ -61,7 +61,7 @@ runwhenLocal: rules: [] service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false className: "" @@ -94,8 +94,6 @@ runwhenLocal: terminal: disabled: true debugLogs: false - cheatSheet: - disabled: true extraEnv: [] uploadInfo: secretProvided: diff --git a/.test/aws/pod-identity-eks/values.yaml b/.test/aws/pod-identity-eks/values.yaml index 16f4e66e2..a0bbb3af4 100644 --- a/.test/aws/pod-identity-eks/values.yaml +++ b/.test/aws/pod-identity-eks/values.yaml @@ -63,7 +63,7 @@ runwhenLocal: rules: [] service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false className: "" @@ -96,8 +96,6 @@ runwhenLocal: terminal: disabled: true debugLogs: false - cheatSheet: - disabled: true extraEnv: [] uploadInfo: secretProvided: diff --git a/.test/azure/ado/Taskfile.yaml b/.test/azure/ado/Taskfile.yaml index cd0cbefe9..7c55cde69 100644 --- a/.test/azure/ado/Taskfile.yaml +++ b/.test/azure/ado/Taskfile.yaml @@ -100,7 +100,7 @@ tasks: docker run \ -e DEBUG_LOGGING=true \ -e CQ_DEBUG=true \ - --name $CONTAINER_NAME -p 8081:8081 \ + --name $CONTAINER_NAME -p 8000:8000 \ -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } @@ -146,7 +146,7 @@ tasks: docker run -d \ -e DEBUG_LOGGING=true \ -e CQ_DEBUG=true \ - --name "$CONTAINER_NAME" -p 8081:8081 \ + --name "$CONTAINER_NAME" -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test || { echo "Failed to start container"; exit 1; diff --git a/.test/azure/aks-and-k8s/Taskfile.yaml b/.test/azure/aks-and-k8s/Taskfile.yaml index ce2b1974c..82271550a 100644 --- a/.test/azure/aks-and-k8s/Taskfile.yaml +++ b/.test/azure/aks-and-k8s/Taskfile.yaml @@ -74,6 +74,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: kubeconfigFile: /shared/kubeconfig.secret @@ -157,7 +158,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } @@ -197,10 +198,9 @@ tasks: # 1. Start container in the background docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ --name "$CONTAINER_NAME" \ - -p 8081:8081 \ + -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test @@ -262,6 +262,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: kubeconfigFile: /shared/kubeconfig.secret @@ -338,6 +339,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: kubeconfigFile: /shared/kubeconfig.secret @@ -408,6 +410,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: kubeconfigFile: /shared/kubeconfig.secret @@ -487,6 +490,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: null azure: @@ -556,6 +560,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi cloudConfig: kubernetes: kubeconfigFile: /shared/kubeconfig.secret @@ -618,6 +623,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: none + azureIndexerBackend: azureapi taskTagExclusions: - "access:read-write" cloudConfig: diff --git a/.test/azure/aks-helm-installed-mi/Taskfile.yaml b/.test/azure/aks-helm-installed-mi/Taskfile.yaml index 09fdecbd7..64e8b389a 100644 --- a/.test/azure/aks-helm-installed-mi/Taskfile.yaml +++ b/.test/azure/aks-helm-installed-mi/Taskfile.yaml @@ -58,6 +58,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: detailed + azureIndexerBackend: azureapi cloudConfig: kubernetes: null azure: @@ -104,7 +105,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } diff --git a/.test/azure/aks-helm-installed-mi/values.yaml b/.test/azure/aks-helm-installed-mi/values.yaml index 121498020..ccc7e1ba0 100644 --- a/.test/azure/aks-helm-installed-mi/values.yaml +++ b/.test/azure/aks-helm-installed-mi/values.yaml @@ -1,10 +1,10 @@ # This Helm Chart installs RunWhen Local & the Runner -# RunWhen Local consists of the Workspace Builder and the Troubleshooting Cheat Sheet +# RunWhen Local consists of the Workspace Builder REST service (port 8000) # 1. Workspace Builder scans your clusters or cloud accounts and matches them with # applicable troubleshooting commands found in CodeCollection respositories # Workspace Builder content is used to build a workspace in the RunWen Platform. -# 2. Troubleshooting Cheat Sheet generates live documentation from output of Workspace +# 2. Discovery output is written to /shared/output after each run # Builder, tailoring troubleshooting commands for the specific environment # and providing helpful documentation @@ -103,7 +103,7 @@ runwhenLocal: # verbs: ["get", "watch", "list"] service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false className: "" diff --git a/.test/azure/aks-helm-installed-sp/Taskfile.yaml b/.test/azure/aks-helm-installed-sp/Taskfile.yaml index c542decc0..d3112e955 100644 --- a/.test/azure/aks-helm-installed-sp/Taskfile.yaml +++ b/.test/azure/aks-helm-installed-sp/Taskfile.yaml @@ -58,6 +58,7 @@ tasks: workspaceOwnerEmail: authors@runwhen.com defaultLocation: location-01 defaultLOD: detailed + azureIndexerBackend: azureapi cloudConfig: kubernetes: null azure: @@ -105,7 +106,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGS=true --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } diff --git a/.test/azure/aks-helm-installed-sp/values.yaml b/.test/azure/aks-helm-installed-sp/values.yaml index d329958a4..44a199f72 100644 --- a/.test/azure/aks-helm-installed-sp/values.yaml +++ b/.test/azure/aks-helm-installed-sp/values.yaml @@ -1,10 +1,10 @@ # This Helm Chart installs RunWhen Local & the Runner -# RunWhen Local consists of the Workspace Builder and the Troubleshooting Cheat Sheet +# RunWhen Local consists of the Workspace Builder REST service (port 8000) # 1. Workspace Builder scans your clusters or cloud accounts and matches them with # applicable troubleshooting commands found in CodeCollection respositories # Workspace Builder content is used to build a workspace in the RunWen Platform. -# 2. Troubleshooting Cheat Sheet generates live documentation from output of Workspace +# 2. Discovery output is written to /shared/output after each run # Builder, tailoring troubleshooting commands for the specific environment # and providing helpful documentation @@ -103,7 +103,7 @@ runwhenLocal: # verbs: ["get", "watch", "list"] service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false className: "" diff --git a/.test/azure/multi-subscription-aks/Taskfile.yaml b/.test/azure/multi-subscription-aks/Taskfile.yaml index 83002f875..5cfdfe45b 100644 --- a/.test/azure/multi-subscription-aks/Taskfile.yaml +++ b/.test/azure/multi-subscription-aks/Taskfile.yaml @@ -135,7 +135,7 @@ tasks: # Copy the collision test config to the shared directory cp $CONFIG_FILE workspaceInfo.yaml - docker run -e DEBUG_LOGGING=true -e CQ_DEBUG=true --name $CONTAINER_NAME -p 8082:8081 -v $(pwd):/shared -d runwhen-local:collision-test || { + docker run -e DEBUG_LOGGING=true -e CQ_DEBUG=true --name $CONTAINER_NAME -p 8082:8000 -v $(pwd):/shared -d runwhen-local:collision-test || { echo "Failed to start container"; exit 1; } @@ -272,7 +272,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGGING=true --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGGING=true --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } @@ -388,3 +388,56 @@ tasks: rm -rf output rm workspaceInfo.yaml silent: true + + run-backend-equivalence-test: + desc: "Run discovery once with the legacy CloudQuery backend and once with the native Azure SDK (azureapi) backend, then diff the resource dumps to verify equivalence." + cmds: + - task: generate-rwl-config + - | + BUILD_DIR=/home/runwhen/runwhen-local/src + IMAGE=runwhen-local:azure-backend-test + BASELINE_OUT=output-cloudquery + CANDIDATE_OUT=output-azureapi + + rm -rf "$BASELINE_OUT" "$CANDIDATE_OUT" + mkdir "$BASELINE_OUT" "$CANDIDATE_OUT" + chmod 777 "$BASELINE_OUT" "$CANDIDATE_OUT" + + docker buildx build --builder mybuilder --platform linux/amd64 \ + -t "$IMAGE" -f "$BUILD_DIR/Dockerfile" "$BUILD_DIR" --load + + run_with_backend() { + local backend="$1" + local out_dir="$2" + local container="azure-backend-test-$backend" + + # workspaceInfo.yaml is regenerated to inject azureIndexerBackend. + # Append rather than rewrite so we don't fight generate-rwl-config. + if grep -q "^azureIndexerBackend:" workspaceInfo.yaml; then + sed -i "s/^azureIndexerBackend:.*/azureIndexerBackend: $backend/" workspaceInfo.yaml + else + echo "azureIndexerBackend: $backend" >> workspaceInfo.yaml + fi + + docker rm -f "$container" >/dev/null 2>&1 || true + + # Mount the per-backend output directory at /shared/output so the + # tar extraction in run.py lands the resource-dump.yaml there. + rm -rf output && mkdir output && chmod 777 output + + docker run --name "$container" -p 0:8000 -v "$(pwd):/shared" \ + -d "$IMAGE" + docker exec -w /workspace-builder "$container" ./run.sh --verbose + docker stop "$container" >/dev/null + docker rm "$container" >/dev/null + + cp -r output/* "$out_dir"/ + } + + run_with_backend cloudquery "$BASELINE_OUT" + run_with_backend azureapi "$CANDIDATE_OUT" + + python3 diff_resource_dump.py \ + "$BASELINE_OUT/resource-dump.yaml" \ + "$CANDIDATE_OUT/resource-dump.yaml" + silent: true diff --git a/.test/azure/multi-subscription-aks/diff_resource_dump.py b/.test/azure/multi-subscription-aks/diff_resource_dump.py new file mode 100755 index 000000000..870bd1c11 --- /dev/null +++ b/.test/azure/multi-subscription-aks/diff_resource_dump.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Diff two ``resource-dump.yaml`` files while ignoring CloudQuery metadata. + +Used by the ``run-backend-equivalence-test`` Taskfile task to verify that the +new native Azure SDK indexer (``AZURE_INDEXER_BACKEND=azureapi``) produces an +output that is functionally equivalent to the legacy CloudQuery path +(``AZURE_INDEXER_BACKEND=cloudquery``). + +What we ignore: + +* ``_cq_*`` keys (sync_time, source_name, id, parent_id) - exist only because + CloudQuery wrote those rows; the SDK indexer doesn't and shouldn't + fabricate them. + +* The ``creationDate`` at the top level of the dump - it's the timestamp the + dump itself was written, not data about the resources. + +* ``_cq_sync_time`` fields embedded inside ``resource`` blocks. + +* Extra fields that the Azure SDK ``as_dict()`` returns but CloudQuery does + not (and vice versa). We only fail on disagreement on shared keys, not on + presence/absence of keys, when comparing the ``resource`` raw payload. + Top-level Resource attributes (name, qualified_name, tags, lod, etc.) are + required to match exactly - those are what generation rules consume. + +Exit status is ``0`` when the two dumps are equivalent, ``1`` otherwise. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from typing import Any + +import yaml + +CQ_KEY_RE = re.compile(r"^_cq_") + + +# --------------------------------------------------------------------------- +# Custom YAML tag constructors +# --------------------------------------------------------------------------- +# resource-dump.yaml uses application-specific tags (!Registry, !Platform, +# !ResourceType, !Resource, !LevelOfDetail) registered by the workspace +# builder's `resources.py` and `enrichers/generation_rule_types.py`. This +# script is intentionally standalone (no dependency on the workspace builder +# package) so we register lightweight constructors that just return plain +# dicts / strings - the tags carry no semantics we need beyond "this is a +# mapping" / "this is a scalar". +def _construct_mapping(loader, node): + return loader.construct_mapping(node, deep=True) + + +def _construct_scalar(loader, node): + return loader.construct_scalar(node) + + +for tag in ("!Registry", "!Platform", "!ResourceType", "!Resource"): + yaml.SafeLoader.add_constructor(tag, _construct_mapping) + yaml.UnsafeLoader.add_constructor(tag, _construct_mapping) + +for tag in ("!LevelOfDetail",): + yaml.SafeLoader.add_constructor(tag, _construct_scalar) + yaml.UnsafeLoader.add_constructor(tag, _construct_scalar) + +# Top-level Resource attributes whose values must match exactly between the +# two backends. ``resource`` is handled with looser semantics (see below). +RESOURCE_REQUIRED_ATTRS = ( + "name", + "qualified_name", + "tags", + "subscription_id", + "subscription_name", + "lod", + "auth_type", +) + +# Keys inside the raw ``resource`` payload to compare exactly when present in +# both. Anything else is "informational" and a difference is logged but not +# fatal. +RESOURCE_RAW_REQUIRED_KEYS = ( + "id", + "name", + "type", + "location", + "tags", + "subscription_id", +) + + +class DiffError(Exception): + pass + + +def _strip_cq_keys(node: Any) -> Any: + """Recursively drop ``_cq_*`` keys from any nested mappings.""" + if isinstance(node, dict): + return { + k: _strip_cq_keys(v) + for k, v in node.items() + if not (isinstance(k, str) and CQ_KEY_RE.match(k)) + } + if isinstance(node, list): + return [_strip_cq_keys(v) for v in node] + return node + + +def _load_dump(path: str) -> dict[str, Any]: + with open(path) as f: + # SafeLoader plus the constructors registered above; produces nested + # plain dicts / lists / strings. + return yaml.safe_load(f) + + +def _resource_to_dict(resource) -> dict[str, Any]: + if isinstance(resource, dict): + return resource + return {k: v for k, v in vars(resource).items() if k != "resource_type"} + + +def _get_attr_or_key(node, *names): + """Return the first attribute / key from ``names`` that exists on + ``node``. Supports both attribute-style (Resource objects) and + mapping-style (raw YAML-loaded dicts) access. The dump uses camelCase + keys (``resourceTypes``, ``customAttributes``) that don't match the + Python attribute names; checking both shapes keeps the script working + whether or not the workspace builder package is in scope. + """ + for name in names: + if hasattr(node, name): + value = getattr(node, name) + if value is not None: + return value + if isinstance(node, dict) and name in node: + return node[name] + return None + + +def _walk_registry(registry) -> dict[tuple[str, str, str], dict[str, Any]]: + """Flatten a Registry into ``{(platform, type, qualified_name): attrs}``.""" + out: dict[tuple[str, str, str], dict[str, Any]] = {} + platforms = _get_attr_or_key(registry, "platforms") or {} + for platform_name, platform in platforms.items(): + types = _get_attr_or_key(platform, "resource_types", "resourceTypes") or {} + for type_name, resource_type in types.items(): + instances = _get_attr_or_key(resource_type, "instances") or {} + for qualified_name, resource in instances.items(): + out[(platform_name, type_name, qualified_name)] = _resource_to_dict( + resource + ) + return out + + +def _compare_resource( + key: tuple[str, str, str], + cq_attrs: dict[str, Any], + api_attrs: dict[str, Any], +) -> list[str]: + """Return a list of differences (empty iff equivalent).""" + diffs: list[str] = [] + label = "/".join(key) + + for attr in RESOURCE_REQUIRED_ATTRS: + if attr not in cq_attrs and attr not in api_attrs: + continue + cq_val = cq_attrs.get(attr) + api_val = api_attrs.get(attr) + if cq_val != api_val: + diffs.append( + f"{label}: attribute {attr!r} differs: " + f"cloudquery={cq_val!r} azureapi={api_val!r}" + ) + + cq_raw = _strip_cq_keys(cq_attrs.get("resource", {})) or {} + api_raw = _strip_cq_keys(api_attrs.get("resource", {})) or {} + for k in RESOURCE_RAW_REQUIRED_KEYS: + if k in cq_raw and k in api_raw and cq_raw[k] != api_raw[k]: + diffs.append( + f"{label}: resource.{k} differs: " + f"cloudquery={cq_raw[k]!r} azureapi={api_raw[k]!r}" + ) + + return diffs + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("cloudquery_dump", help="Path to resource-dump.yaml from the cloudquery run") + parser.add_argument("azureapi_dump", help="Path to resource-dump.yaml from the azureapi run") + args = parser.parse_args(argv) + + cq_doc = _load_dump(args.cloudquery_dump) + api_doc = _load_dump(args.azureapi_dump) + + cq_registry = cq_doc.get("registry") if isinstance(cq_doc, dict) else cq_doc + api_registry = api_doc.get("registry") if isinstance(api_doc, dict) else api_doc + + cq_resources = _walk_registry(cq_registry) + api_resources = _walk_registry(api_registry) + + # Limit comparison to the Azure platform only (other platforms are + # untouched by the backend swap). + cq_resources = {k: v for k, v in cq_resources.items() if k[0] == "azure"} + api_resources = {k: v for k, v in api_resources.items() if k[0] == "azure"} + + differences: list[str] = [] + + only_cq = set(cq_resources) - set(api_resources) + only_api = set(api_resources) - set(cq_resources) + for k in sorted(only_cq): + differences.append(f"only in cloudquery: {'/'.join(k)}") + for k in sorted(only_api): + differences.append(f"only in azureapi: {'/'.join(k)}") + + for key in sorted(set(cq_resources) & set(api_resources)): + differences.extend(_compare_resource(key, cq_resources[key], api_resources[key])) + + if differences: + print( + f"FAIL: {len(differences)} difference(s) between cloudquery and " + f"azureapi outputs:", + file=sys.stderr, + ) + for d in differences[:200]: + print(f" {d}", file=sys.stderr) + if len(differences) > 200: + print(f" ... and {len(differences) - 200} more", file=sys.stderr) + return 1 + + print( + f"PASS: cloudquery and azureapi outputs are equivalent " + f"({len(cq_resources)} azure resources compared)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.test/azure/no-aks-resources/.gitconfig b/.test/azure/no-aks-resources/.gitconfig new file mode 100644 index 000000000..baf35f1a7 --- /dev/null +++ b/.test/azure/no-aks-resources/.gitconfig @@ -0,0 +1,10 @@ +[safe] + directory = /opt/runwhen/codecollection-cache + directory = /opt/runwhen/codecollection-cache/aws-c7n-codecollection.git + directory = /opt/runwhen/codecollection-cache/azure-c7n-codecollection.git + directory = /opt/runwhen/codecollection-cache/rw-cli-codecollection.git + directory = /opt/runwhen/codecollection-cache/rw-generic-codecollection.git + directory = /opt/runwhen/codecollection-cache/rw-public-codecollection.git + directory = /opt/runwhen/codecollection-cache/rw-workspace-utils.git + directory = /tmp/* + directory = /tmp/tmp* diff --git a/.test/azure/no-aks-resources/README.md b/.test/azure/no-aks-resources/README.md new file mode 100644 index 000000000..2bafc0455 --- /dev/null +++ b/.test/azure/no-aks-resources/README.md @@ -0,0 +1,109 @@ +# `no-aks-resources` Azure fixture + +End-to-end fixture for validating the native `azureapi` indexer's selective-indexing behavior against live Azure infrastructure, **without** provisioning an AKS cluster. Uses cheap, fast-to-deploy resources (resource groups, storage accounts, vnet, NSG, key vault) so the indexer's filter logic can be exercised without the cost or wait time of a full multi-subscription AKS fixture. + +## What it provisions + +Two resource groups in a single subscription: + +| Resource group | Tags | Purpose | +| --------------------- | --------------------------------- | ------- | +| `rwl-azapi-keep-rg-*` | `project=rwl-azureapi-fixture`, `purpose=in-scope` | Should always be discovered. Holds a storage account, a vnet, a subnet, an NSG, and a key vault. | +| `rwl-azapi-drop-rg-*` | `project=rwl-azureapi-fixture`, `purpose=out-of-scope` | Used to assert the indexer drops out-of-scope resources. Holds a storage account. | + +The `*` suffix is a 6-character random string regenerated each `terraform apply` so concurrent runs don't collide. + +## Prerequisites + +1. **Azure subscription + service principal** with at least Reader on the target subscription. The SP also needs `Microsoft.KeyVault/vaults/write` to provision the key vault; see `terraform/main.tf` for the exact resource set. +2. **`terraform/tf.secret`** must export the credentials. Symlinked to `../multi-subscription-aks/terraform/tf.secret` by default so any updates to those creds flow through automatically. To use different creds for this fixture, replace the symlink with a real file containing: + ```bash + export TF_VAR_subscription_id=... + export AZ_TENANT_ID=... + export TF_VAR_tenant_id=$AZ_TENANT_ID + export AZ_CLIENT_ID=... + export AZ_CLIENT_SECRET=... + export TF_VAR_sp_principal_id=$(az ad sp list --filter "appId eq '$AZ_CLIENT_ID'" --query '[0].id' -o tsv) + ``` +3. **Tooling**: `terraform`, `task`, `docker buildx` with a builder named `mybuilder`, `sqlite3`, `jq`, `python3`. The repo's `src/Dockerfile` is built locally as `runwhen-local:test`. + +## Quick start + +```bash +# Provision infra (once). +task build-infra + +# Run the selective-indexing assertion (default). +task ci-test-azureapi-selective + +# Tear down. +task clean +``` + +## CI tests + +Each test generates a `workspaceInfo.yaml` tailored to its scenario, runs RWL discovery in a container, then queries the on-disk `output/resources.sqlite` and the indexer logs. + +| Task | Scenario | Pass criteria | +| --------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------- | +| `ci-test-azureapi-baseline` | `defaultLOD: detailed`. Indexer uses subscription-wide list endpoints. | Every provisioned resource is in `resources.sqlite`. | +| `ci-test-azureapi-selective` | `defaultLOD: none` + per-RG override whitelisting `keep-rg` (detailed). Triggers selective discovery. | Indexer logs `selective discovery, in-scope RGs=[keep-rg-...]`; `keep-rg` and its resources present; `drop-rg` + its storage absent; `list_by_resource_group` is the only non-RG SDK call. | +| `ci-test-azureapi-tag-filter` | `defaultLOD: detailed`, `excludeTags: { purpose: out-of-scope }`. Subscription-wide list + post-filter. | The drop-rg storage is absent; the keep-rg storage is present; indexer logs show `skipped_tag_filter > 0`. | +| `ci-test-azureapi-equivalence` | Runs the same baseline workspace once with `azureIndexerBackend: cloudquery` and once with `azureapi`, then diffs `resource-dump.yaml`. | Diff is empty (uses `multi-subscription-aks/diff_resource_dump.py` if available). | + +### Discovery modes + +The indexer picks one of two strategies per subscription based on the workspaceInfo it's given: + +* **Selective discovery** — fires when the workspace declares a finite scope: `defaultLOD: none`, no non-NONE wildcard (`*`), and an explicit list of per-RG `detailed`/`basic` overrides. The indexer enumerates resource groups once (subscription-wide list, post-filtered to the in-scope set), then calls `list_by_resource_group(rg_name)` for each whitelisted RG. **Out-of-scope RGs are never enumerated** — the SDK is never asked about them. This is what makes selectivity real: zero spend on resources you don't want. +* **Subscription-wide discovery** — fires when *any* of the escape hatches is non-NONE (workspace `defaultLOD`, per-subscription `defaultLOD`, or wildcard `resourceGroupLevelOfDetails["*"]`). The indexer calls `list_all()` and post-filters via `skipped_lod_filter` for any explicitly NONE-marked RGs. + +The indexer logs which mode it chose for each subscription at startup, e.g.: + +``` +INFO: Azure subscription : selective discovery, in-scope RGs=['rwl-azapi-keep-rg-*']. +INFO: Selective discovery: listing azure_storage_accounts per-RG in subscription (in-scope RGs=['rwl-azapi-keep-rg-*']) +``` + +### Observable counters + +The indexer prints a single summary line per run: + +``` +Azure SDK indexing complete: discovered=N, added=N, +skipped_tag_filter=N, skipped_lod_filter=N, skipped_rg_not_found=N, +skipped_parse_error=N, skipped_collector_error=N +``` + +* `skipped_lod_filter` — resources dropped because effective LOD resolved to NONE. In selective mode this only fires for the resource-group enumeration step (RGs themselves can't be discovered per-RG). For non-RG types in selective mode it stays 0 because we never list out-of-scope RGs. +* `skipped_rg_not_found` — workspace whitelisted an RG that doesn't exist in Azure; the per-RG SDK call returned 404. Non-fatal. +* `skipped_tag_filter` — `excludeTags`/`includeTags` rejections. +* `skipped_collector_error` — SDK call failed with anything that isn't a 404; the type is skipped for that subscription. + +## What the assertions actually check + +* **`resources.sqlite` rows.** Each test queries the `resources` table directly: + ```sql + SELECT count(*) FROM resources WHERE platform='azure' AND name=?; + ``` + This is the persisted store the explorer UI / API consume, so absence here is the strongest signal that the indexer dropped a resource. + +* **Indexer log line.** The Azure SDK indexer prints a single summary line at the end of each run: + ``` + Azure SDK indexing complete: discovered=N, added=N, + skipped_tag_filter=N, skipped_lod_filter=N, + skipped_parse_error=N, skipped_collector_error=N + ``` + The fixture greps `run_sh_output.log` and `container_logs.log` for this line and sanity-checks the relevant counter is non-zero. + +## Adding a new scenario + +1. Add a `generate--config` task that writes a workspaceInfo.yaml variant. +2. Add an `assert-` task that queries `output/resources.sqlite` and the indexer logs. +3. Add a `ci-test-azureapi-` task that wires generate ⇒ build ⇒ run ⇒ assert. + +Keep the assertion logic in shell - readability matters more than DRY when the failure mode is "explain what's wrong with this RWL deployment". + +## Cleanup + +`task clean` runs `terraform destroy` (safe even if no infra is deployed) and removes the local `output/`, `workspaceInfo.yaml`, and run logs. diff --git a/.test/azure/no-aks-resources/Taskfile.yaml b/.test/azure/no-aks-resources/Taskfile.yaml new file mode 100644 index 000000000..ef2d607b0 --- /dev/null +++ b/.test/azure/no-aks-resources/Taskfile.yaml @@ -0,0 +1,457 @@ +version: "3" + +# RunWhen Local - "no AKS" Azure indexer fixture. +# +# Provisions a small zoo of inexpensive Azure resources (resource groups, +# storage accounts, vnet/nsg, key vault) and exercises the native ``azureapi`` +# indexer's scope-filter machinery end to end. +# +# Quick reference: +# task build-infra # terraform apply +# task ci-test-azureapi-baseline # everything in scope +# task ci-test-azureapi-selective # one RG marked LOD=none, must drop +# task ci-test-azureapi-tag-filter # excludeTags drops drop-rg's resources +# task ci-test-azureapi-equivalence # baseline cloudquery vs azureapi diff +# task clean # tear it all down +# +# Prerequisites: +# - terraform/tf.secret must exist and export AZ_TENANT_ID, AZ_CLIENT_ID, +# AZ_CLIENT_SECRET, TF_VAR_subscription_id, TF_VAR_tenant_id, and +# TF_VAR_sp_principal_id. The fixture symlinks tf.secret to the one in +# ../multi-subscription-aks/terraform/ so credentials are shared. +# - docker buildx with a builder named "mybuilder" available. + +tasks: + + default: + desc: "Provision (if needed) and run the selective-indexing test." + cmds: + - task: build-infra + - task: ci-test-azureapi-selective + + clean: + desc: "Destroy infra and remove discovery artifacts." + cmds: + - task: check-and-cleanup-terraform + - task: clean-rwl-discovery + + # === Terraform lifecycle ================================================= + + build-infra: + desc: "Apply terraform (idempotent)." + cmds: + - task: build-terraform-infra + + build-terraform-infra: + desc: "terraform apply" + silent: true + cmds: + - | + source terraform/tf.secret + if [ ! -d terraform ]; then + echo "terraform directory missing"; exit 1 + fi + cd terraform + task format-and-init-terraform + echo "Applying terraform plan for no-aks-resources..." + terraform apply -auto-approve + + check-terraform-infra: + desc: "Report whether terraform state has any deployed resources." + silent: true + cmds: + - | + source terraform/tf.secret + cd terraform + if [ ! -f terraform.tfstate ]; then + echo "No terraform state file found." + exit 0 + fi + resources=$(terraform state list 2>/dev/null || true) + if [ -n "$resources" ]; then + echo "Deployed infrastructure detected." + echo "$resources" + else + echo "No deployed infrastructure found." + fi + + cleanup-terraform-infra: + desc: "terraform destroy" + silent: true + cmds: + - | + source terraform/tf.secret + cd terraform + echo "Destroying no-aks-resources infrastructure..." + terraform destroy -auto-approve + + check-and-cleanup-terraform: + desc: "Destroy any deployed terraform resources." + silent: true + cmds: + - | + out=$(task check-terraform-infra) + echo "$out" + if echo "$out" | grep -q "Deployed infrastructure detected"; then + task cleanup-terraform-infra + else + echo "No deployed infrastructure to clean up." + fi + + # === RWL container lifecycle ============================================= + + build-rwl: + desc: "Build the RunWhen Local test image." + silent: true + cmds: + - | + BUILD_DIR=../../../src + docker buildx build --builder mybuilder --platform linux/amd64 \ + -t runwhen-local:test -f "$BUILD_DIR/Dockerfile" "$BUILD_DIR" --load + + run-rwl-discovery: + desc: "Run RWL discovery against the current workspaceInfo.yaml. Container stays up so the explorer UI is reachable at http://localhost:8001/." + silent: true + cmds: + - | + rm -f run_sh_output.log container_logs.log + CONTAINER_NAME="RunWhenLocal-NoAks" + if docker ps -aq --filter "name=$CONTAINER_NAME" | grep -q .; then + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + rm -rf output && mkdir output && chmod 777 output + chmod 777 . + docker run -d \ + -e DEBUG_LOGGING=true \ + --name "$CONTAINER_NAME" -p 8001:8000 \ + -v "$(pwd):/shared" \ + runwhen-local:test + docker exec -w /workspace-builder "$CONTAINER_NAME" ./run.sh --verbose 2>&1 | tee run_sh_output.log + # Snapshot the FastAPI / indexer stdout into container_logs.log so + # the assert-* tasks (which grep for "selective discovery", etc.) + # have a fresh, deterministic file to read. Without this the + # internal [INFO] indexers.azureapi logs only live in docker logs. + docker logs "$CONTAINER_NAME" > container_logs.log 2>&1 || true + echo + echo "Discovery finished. Container '$CONTAINER_NAME' is still running." + echo " - Explorer UI : http://localhost:8001/explorer/" + echo " - Stop it with: docker rm -f $CONTAINER_NAME (or 'task clean-rwl-discovery')" + + clean-rwl-discovery: + desc: "Remove output/ and any generated workspaceInfo.yaml." + silent: true + cmds: + - | + rm -rf output + rm -f workspaceInfo.yaml run_sh_output.log container_logs.log + + # === workspaceInfo.yaml generation ======================================= + # + # All three configs target the azureapi indexer and the SQLite resource + # store so the assertions can query the persisted store directly. + + generate-baseline-config: + desc: "Emit workspaceInfo.yaml with defaultLOD=detailed (everything in scope)." + silent: true + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "no-aks-baseline"}}' + cmds: + - | + source terraform/tf.secret + cat > workspaceInfo.yaml < workspaceInfo.yaml < workspaceInfo.yaml </dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL baseline: '${name}' not in resource store" + fail=1 + else + echo "OK baseline: '${name}' present (${n} row)" + fi + done + exit "$fail" + + assert-selective: + desc: "Verify keep-rg lands and drop-rg + its resources are absent (per-RG SDK calls only)." + silent: true + cmds: + - | + DB=output/resources.sqlite + if [ ! -f "$DB" ]; then + echo "FAIL: $DB not produced" + exit 1 + fi + TFS_JSON=$(terraform -chdir=terraform show -json terraform.tfstate) + keep_rg=$(echo "$TFS_JSON" | jq -r '.values.outputs.keep_rg_name.value') + drop_rg=$(echo "$TFS_JSON" | jq -r '.values.outputs.drop_rg_name.value') + keep_st=$(echo "$TFS_JSON" | jq -r '.values.outputs.keep_storage_name.value') + drop_st=$(echo "$TFS_JSON" | jq -r '.values.outputs.drop_storage_name.value') + + fail=0 + # keep-rg + its storage MUST be present. + for name in "$keep_rg" "$keep_st"; do + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='azure' AND name='${name}';" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL selective: '${name}' should be present, got 0 rows" + fail=1 + else + echo "OK selective: '${name}' present (${n} row)" + fi + done + # drop-rg + its storage MUST be absent. With selective discovery the + # indexer NEVER calls list_by_resource_group("drop-rg-*"), so these + # resources should never even be enumerated, let alone persisted. + for name in "$drop_rg" "$drop_st"; do + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='azure' AND name='${name}';" \ + 2>/dev/null || echo 0) + if [ "$n" -gt 0 ]; then + echo "FAIL selective: '${name}' should be absent, found ${n} row(s)" + fail=1 + else + echo "OK selective: '${name}' correctly absent" + fi + done + # Verify the indexer logged that selective discovery actually ran. + if grep -qE "selective discovery, in-scope RGs" run_sh_output.log container_logs.log 2>/dev/null; then + echo "OK selective: indexer reported selective discovery mode" + else + echo "FAIL selective: indexer did not log 'selective discovery' for any subscription" + fail=1 + fi + # The drop-rg itself is filtered post-list (only the RG enumeration is + # subscription-wide), so we expect skipped_lod_filter >= 1 for the RG. + if grep -qE 'skipped_lod_filter=[1-9][0-9]*' run_sh_output.log container_logs.log 2>/dev/null; then + echo "OK selective: indexer reported skipped_lod_filter > 0" + else + echo "WARN selective: skipped_lod_filter is 0 - the drop-rg LOD post-filter didn't fire (unexpected if drop-rg exists)" + fi + exit "$fail" + + assert-tag-filter: + desc: "Verify purpose=out-of-scope resources are dropped and skipped_tag_filter > 0." + silent: true + cmds: + - | + DB=output/resources.sqlite + if [ ! -f "$DB" ]; then + echo "FAIL: $DB not produced" + exit 1 + fi + TFS_JSON=$(terraform -chdir=terraform show -json terraform.tfstate) + keep_st=$(echo "$TFS_JSON" | jq -r '.values.outputs.keep_storage_name.value') + drop_st=$(echo "$TFS_JSON" | jq -r '.values.outputs.drop_storage_name.value') + + fail=0 + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='azure' AND name='${keep_st}';" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL tag-filter: '${keep_st}' should be present, got 0 rows" + fail=1 + else + echo "OK tag-filter: '${keep_st}' present (${n} row)" + fi + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='azure' AND name='${drop_st}';" \ + 2>/dev/null || echo 0) + if [ "$n" -gt 0 ]; then + echo "FAIL tag-filter: '${drop_st}' should be absent, found ${n} row(s)" + fail=1 + else + echo "OK tag-filter: '${drop_st}' correctly absent" + fi + # Indexer must report skipped_tag_filter > 0. + if grep -E 'skipped_tag_filter=[1-9][0-9]*' run_sh_output.log container_logs.log >/dev/null 2>&1; then + echo "OK tag-filter: indexer reported skipped_tag_filter > 0" + else + echo "FAIL tag-filter: indexer did not report a positive skipped_tag_filter count" + fail=1 + fi + exit "$fail" + + # === Bonus: cross-backend equivalence ==================================== + + ci-test-azureapi-equivalence: + desc: "Run discovery once with the legacy CloudQuery backend and once with azureapi, then diff resource dumps." + cmds: + - task: generate-baseline-config + - | + BUILD_DIR=../../../src + IMAGE=runwhen-local:no-aks-equiv + BASELINE_OUT=output-cloudquery + CANDIDATE_OUT=output-azureapi + + rm -rf "$BASELINE_OUT" "$CANDIDATE_OUT" + mkdir "$BASELINE_OUT" "$CANDIDATE_OUT" + chmod 777 "$BASELINE_OUT" "$CANDIDATE_OUT" + + docker buildx build --builder mybuilder --platform linux/amd64 \ + -t "$IMAGE" -f "$BUILD_DIR/Dockerfile" "$BUILD_DIR" --load + + run_with_backend() { + local backend="$1" + local out_dir="$2" + local container="no-aks-equiv-$backend" + + if grep -q "^azureIndexerBackend:" workspaceInfo.yaml; then + sed -i "s/^azureIndexerBackend:.*/azureIndexerBackend: $backend/" workspaceInfo.yaml + else + echo "azureIndexerBackend: $backend" >> workspaceInfo.yaml + fi + + docker rm -f "$container" >/dev/null 2>&1 || true + rm -rf output && mkdir output && chmod 777 output + + docker run --name "$container" -p 0:8000 -v "$(pwd):/shared" -d "$IMAGE" + docker exec -w /workspace-builder "$container" ./run.sh --verbose + docker stop "$container" >/dev/null + docker rm "$container" >/dev/null + + cp -r output/* "$out_dir"/ + } + + run_with_backend cloudquery "$BASELINE_OUT" + run_with_backend azureapi "$CANDIDATE_OUT" + + if [ -f ../multi-subscription-aks/diff_resource_dump.py ]; then + python3 ../multi-subscription-aks/diff_resource_dump.py \ + "$BASELINE_OUT/resource-dump.yaml" \ + "$CANDIDATE_OUT/resource-dump.yaml" + else + echo "diff_resource_dump.py not available; compare $BASELINE_OUT/ and $CANDIDATE_OUT/ manually" + fi + silent: true diff --git a/.test/azure/no-aks-resources/terraform/Taskfile.yaml b/.test/azure/no-aks-resources/terraform/Taskfile.yaml new file mode 100644 index 000000000..7716e3c2b --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/Taskfile.yaml @@ -0,0 +1,51 @@ +version: "3" + +# Terraform-only lifecycle for the no-aks-resources fixture. +# Mirrors the pattern in .test/azure/multi-subscription-aks/terraform/Taskfile.yaml. + +env: + TERM: screen-256color + +tasks: + default: + cmds: + - task: test + + test: + desc: "fmt + init + validate (no apply)" + cmds: + - task: test-terraform + + format-and-init-terraform: + desc: "terraform fmt + init" + cmds: + - | + terraform fmt + terraform init + + test-terraform: + desc: "fmt --check, init -backend=false, validate" + silent: true + cmds: + - | + BOLD=$(tput bold || true) + NORM=$(tput sgr0 || true) + echo "${BOLD}$PWD:${NORM}" + if ! terraform fmt -check=true -list=false -recursive=false; then + echo " fmt failed" && exit 1 + fi + echo " fmt ok" + if ! terraform init -backend=false -input=false -get=true -no-color > /dev/null; then + echo " init failed" && exit 1 + fi + echo " init ok" + if ! terraform validate > /dev/null; then + echo " validate failed" && exit 1 + fi + echo " validate ok" + + clean-terraform: + desc: "Remove .terraform / lock file" + cmds: + - find . -type d -name .terraform -exec rm -rf {} + + - find . -type f -name .terraform.lock.hcl -delete diff --git a/.test/azure/no-aks-resources/terraform/backend.tf b/.test/azure/no-aks-resources/terraform/backend.tf new file mode 100644 index 000000000..3c533e6bf --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/backend.tf @@ -0,0 +1,5 @@ +terraform { + backend "local" { + path = "terraform.tfstate" + } +} diff --git a/.test/azure/no-aks-resources/terraform/main.tf b/.test/azure/no-aks-resources/terraform/main.tf new file mode 100644 index 000000000..99b69ccc5 --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/main.tf @@ -0,0 +1,124 @@ +# ============================================================================= +# RunWhen Local - "no AKS" Azure indexer fixture +# +# Deploys two resource groups so the azureapi indexer's per-RG selective +# indexing can be exercised end-to-end: +# +# * keep-rg : in-scope. Contains a storage account, a virtual network, +# a network security group, and a key vault. Tagged with +# purpose=in-scope. +# * drop-rg : intended to be excluded from discovery (via LOD=none or +# excludeTags). Contains a storage account tagged with +# purpose=out-of-scope so the tag-filter assertion has +# something concrete to drop. +# +# Everything is intentionally cheap (no VMs, no clusters) - this fixture +# exists to exercise the indexer's filter logic, not to evaluate any +# specific service. +# ============================================================================= + +resource "random_string" "suffix" { + length = 6 + upper = false + special = false +} + +locals { + suffix = random_string.suffix.result + + keep_tags = { + project = "rwl-azureapi-fixture" + purpose = "in-scope" + } + drop_tags = { + project = "rwl-azureapi-fixture" + purpose = "out-of-scope" + } +} + +# -- Resource groups ---------------------------------------------------------- + +resource "azurerm_resource_group" "keep" { + name = "${var.name_prefix}-keep-rg-${local.suffix}" + location = var.location + tags = local.keep_tags +} + +resource "azurerm_resource_group" "drop" { + name = "${var.name_prefix}-drop-rg-${local.suffix}" + location = var.location + tags = local.drop_tags +} + +# -- Storage (one per RG, both ARM-spec compliant 3-24 lowercase alnum) ------- + +resource "azurerm_storage_account" "keep" { + name = "rwlkeepst${local.suffix}" + resource_group_name = azurerm_resource_group.keep.name + location = azurerm_resource_group.keep.location + account_tier = "Standard" + account_replication_type = "LRS" + tags = local.keep_tags +} + +resource "azurerm_storage_account" "drop" { + name = "rwldropst${local.suffix}" + resource_group_name = azurerm_resource_group.drop.name + location = azurerm_resource_group.drop.location + account_tier = "Standard" + account_replication_type = "LRS" + tags = local.drop_tags +} + +# -- Networking (keep-rg only) ----------------------------------------------- + +resource "azurerm_virtual_network" "keep" { + name = "${var.name_prefix}-vnet-${local.suffix}" + resource_group_name = azurerm_resource_group.keep.name + location = azurerm_resource_group.keep.location + address_space = ["10.10.0.0/16"] + tags = local.keep_tags +} + +resource "azurerm_subnet" "keep" { + name = "default" + resource_group_name = azurerm_resource_group.keep.name + virtual_network_name = azurerm_virtual_network.keep.name + address_prefixes = ["10.10.0.0/24"] +} + +resource "azurerm_network_security_group" "keep" { + name = "${var.name_prefix}-nsg-${local.suffix}" + resource_group_name = azurerm_resource_group.keep.name + location = azurerm_resource_group.keep.location + tags = local.keep_tags +} + +# -- Key Vault (keep-rg only) ------------------------------------------------ + +data "azurerm_client_config" "current" {} + +resource "azurerm_key_vault" "keep" { + name = "rwl-kv-${local.suffix}" + location = azurerm_resource_group.keep.location + resource_group_name = azurerm_resource_group.keep.name + tenant_id = var.tenant_id + sku_name = "standard" + soft_delete_retention_days = 7 + purge_protection_enabled = false + tags = local.keep_tags + + dynamic "access_policy" { + for_each = var.sp_principal_id == "" ? [] : [1] + content { + tenant_id = var.tenant_id + object_id = var.sp_principal_id + key_permissions = [ + "Get", "List", + ] + secret_permissions = [ + "Get", "List", + ] + } + } +} diff --git a/.test/azure/no-aks-resources/terraform/outputs.tf b/.test/azure/no-aks-resources/terraform/outputs.tf new file mode 100644 index 000000000..40b5330a7 --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/outputs.tf @@ -0,0 +1,34 @@ +output "subscription_id" { + value = var.subscription_id + description = "Subscription containing the fixture infrastructure." +} + +output "keep_rg_name" { + value = azurerm_resource_group.keep.name + description = "Resource group that should appear in RWL discovery output." +} + +output "drop_rg_name" { + value = azurerm_resource_group.drop.name + description = "Resource group that selective-indexing tests expect to drop." +} + +output "keep_storage_name" { + value = azurerm_storage_account.keep.name +} + +output "drop_storage_name" { + value = azurerm_storage_account.drop.name +} + +output "vnet_name" { + value = azurerm_virtual_network.keep.name +} + +output "nsg_name" { + value = azurerm_network_security_group.keep.name +} + +output "key_vault_name" { + value = azurerm_key_vault.keep.name +} diff --git a/.test/azure/no-aks-resources/terraform/provider.tf b/.test/azure/no-aks-resources/terraform/provider.tf new file mode 100644 index 000000000..239d9b5fa --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/provider.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.7.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = true + recover_soft_deleted_key_vaults = true + } + } + subscription_id = var.subscription_id + tenant_id = var.tenant_id +} diff --git a/.test/azure/no-aks-resources/terraform/vars.tf b/.test/azure/no-aks-resources/terraform/vars.tf new file mode 100644 index 000000000..2fb1a9673 --- /dev/null +++ b/.test/azure/no-aks-resources/terraform/vars.tf @@ -0,0 +1,27 @@ +variable "subscription_id" { + type = string + description = "Target Azure subscription for the fixture." +} + +variable "tenant_id" { + type = string + description = "Azure AD tenant for the service principal." +} + +variable "sp_principal_id" { + type = string + description = "Object ID of the SP that runs Terraform / RWL discovery. Granted Key Vault data-plane access." + default = "" +} + +variable "location" { + type = string + default = "East US" + description = "Azure region for all resources." +} + +variable "name_prefix" { + type = string + default = "rwl-azapi" + description = "Prefix applied to all resource names so they're easy to identify and delete." +} diff --git a/.test/gcp/gcp-and-k8s/Taskfile.yaml b/.test/gcp/gcp-and-k8s/Taskfile.yaml index 19cd2110c..3f070e9bd 100644 --- a/.test/gcp/gcp-and-k8s/Taskfile.yaml +++ b/.test/gcp/gcp-and-k8s/Taskfile.yaml @@ -132,14 +132,49 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGGING=false --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + # AZURE_DEVOPS_CACHE_DIR points the azure-devops SDK's import-time file + # cache (azure/devops/_file_cache.get_cache_dir) at a writable path. + # Defense-in-depth: even if some module eagerly imports azure.devops, + # the os.makedirs no longer targets the read-only /shared (=$HOME) mount. + docker run -e DEBUG_LOGGING=false -e AZURE_DEVOPS_CACHE_DIR=/tmp/.azure-devops --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } + # Wait for the REST service to actually come up. If uvicorn crashes on + # startup the container exits; dump its logs so the Python traceback is + # visible in CI instead of an opaque "Error executing script". Without + # this, a startup crash produced no artifact at all. + echo "Waiting for workspace builder REST service..." + rest_ready=0 + for i in $(seq 1 30); do + if ! docker ps -q --filter "name=$CONTAINER_NAME" | grep -q .; then + echo "Container $CONTAINER_NAME exited during startup. Logs:" + docker logs $CONTAINER_NAME 2>&1 | tee container_logs.log + exit 1 + fi + if docker exec $CONTAINER_NAME curl -sf http://localhost:8000/info/ >/dev/null 2>&1; then + rest_ready=1; break + fi + sleep 2 + done + if [ "$rest_ready" -ne 1 ]; then + echo "REST service never became ready. Container logs:" + docker logs $CONTAINER_NAME 2>&1 | tee container_logs.log + exit 1 + fi + echo "Running workspace builder script in container..." - docker exec -w /workspace-builder $CONTAINER_NAME ./run.sh $1 --verbose || { - echo "Error executing script in container"; exit 1; + # pipefail so a run.sh failure is NOT masked by tee's exit code. + set -o pipefail + docker exec -w /workspace-builder $CONTAINER_NAME ./run.sh $1 --verbose 2>&1 | tee run_sh_output.log || { + echo "Error executing script in container. Container logs:" + docker logs $CONTAINER_NAME 2>&1 | tee container_logs.log + exit 1 } + set +o pipefail + # Always snapshot the container (REST/indexer) logs for the asserts + + # the failure artifact. + docker logs $CONTAINER_NAME > container_logs.log 2>&1 || true echo "Review generated config files under output/workspaces/" total_slxs=$(find $(find 'output/' -type d -name 'slxs') -mindepth 1 -type d | wc -l) @@ -171,10 +206,9 @@ tasks: echo "User $(whoami)" # 1. Start container in the background docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ --name "$CONTAINER_NAME" \ - -p 8081:8081 \ + -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test @@ -382,12 +416,11 @@ tasks: # Start container with airgap mode explicitly disabled docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ -e CLOUDQUERY_AIRGAP_MODE=false \ -e CLOUDQUERY_PLUGINS_PREINSTALLED=false \ --name "$CONTAINER_NAME" \ - -p 8082:8081 \ + -p 8082:8000 \ -v "$(pwd):/shared" \ runwhen-local:test @@ -470,9 +503,223 @@ tasks: cmds: - | rm -rf output + rm -rf gcpapi-test-codecollection || true rm workspaceInfo.yaml rm airgap_test_output.log || true rm container_logs.log || true + rm run_sh_output.log || true rm exec_output.log || true rm slx_count.txt || true silent: true + + # === Native gcpapi indexer (SQLite resource-store assertions) ============= + # + # Mirrors .test/azure/no-aks-resources Taskfile: generate a workspaceInfo + # that selects the native ``gcpapi`` backend + the SQLite resource store, + # run discovery, then assert directly against output/resources.sqlite (not + # the SLX count). The legacy ci-test-1 / ci-test-2 targets above still + # exercise the CloudQuery GCP backend via SLX-count only. + # + # The project under test is read from gcp.secret's project_id so the same + # target works against whatever sandbox SA the CI secret writes. + + generate-gcpapi-baseline-config: + desc: "Emit workspaceInfo.yaml selecting the native gcpapi backend + SQLite store (defaultLOD=detailed)." + silent: true + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "gcpapi-baseline"}}' + cmds: + - | + GCP_PROJECT=$(python3 -c "import json;print(json.load(open('gcp.secret'))['project_id'])") + if [ -z "$GCP_PROJECT" ]; then + echo "Could not read project_id from gcp.secret"; exit 1 + fi + cat > workspaceInfo.yaml < "${GR_DIR}/gcp-compute-baseline.yaml" <<'EOF' + apiVersion: runwhen.com/v1 + kind: GenerationRules + spec: + platform: gcp + generationRules: + - resourceTypes: + - gcp_compute_networks + - gcp_compute_firewalls + matchRules: + - type: pattern + pattern: ".+" + properties: [name] + mode: substring + slxs: [] + EOF + # Turn it into a git repo on branch 'main' so the code-collection loader + # (git-based) can clone it via the file:// URL from the mounted /shared. + ( cd "${CC_DIR}" \ + && git -c init.defaultBranch=main init -q \ + && git -c user.email=ci@runwhen.com -c user.name=ci add -A \ + && git -c user.email=ci@runwhen.com -c user.name=ci commit -qm "test-only gcp compute fallback gen rule" ) + echo "Built local test code collection at ${CC_DIR} (references gcp_compute_networks, gcp_compute_firewalls)" + + ci-test-gcpapi-baseline: + desc: "End to end: native gcpapi discovery. SQLite store must contain the project anchor + discovered GCP resources." + cmds: + - task: generate-gcpapi-baseline-config + - task: generate-gcpapi-test-codecollection + - task: build-rwl + - task: run-rwl-discovery + - task: assert-gcpapi-baseline + + assert-gcpapi-baseline: + desc: "Verify the native gcpapi backend ran and persisted GCP resources to output/resources.sqlite." + silent: true + cmds: + - | + DB=output/resources.sqlite + if [ ! -f "$DB" ]; then + echo "FAIL: $DB not produced" + exit 1 + fi + + # Snapshot the indexer stdout so we can confirm the *native* backend ran + # (the run-rwl-discovery container stays up after discovery). The + # gcpapi backend-selection log only lives in the container's logs. + docker logs RunWhenLocal > container_logs.log 2>&1 || true + + GCP_PROJECT=$(python3 -c "import json;print(json.load(open('gcp.secret'))['project_id'])") + fail=0 + + # 1. The native gcpapi backend must have been selected (NOT cloudquery). + if grep -qE "indexers.gcpapi: GCP indexer backend: 'gcpapi'" run_sh_output.log container_logs.log 2>/dev/null; then + echo "OK baseline: native gcpapi backend ran" + else + echo "FAIL baseline: native gcpapi backend did not run (expected gcpIndexerBackend=gcpapi)" + fail=1 + fi + + # 1b. Cloud Asset Inventory is an OPTIONAL accelerator that broadens + # coverage to resource types without a typed collector. Its absence + # (API not enabled / no CAI viewer role) is normal and MUST NOT fail + # the baseline -- the per-service typed SDK collectors are the + # functional discovery path and are asserted below. This is purely + # informational so operators can see whether the CAI pass ran. + if grep -q "GCP_CAI_PERMISSION_DENIED" run_sh_output.log container_logs.log 2>/dev/null; then + echo "NOTE baseline: Cloud Asset Inventory was not accessible (optional accelerator)." + echo " -> This is expected and non-fatal. Native GCP discovery is driven by the" + echo " per-service typed collectors (asserted below). Enabling cloudasset.googleapis.com" + echo " with a CAI viewer role is optional and only increases coverage breadth." + else + echo "OK baseline: Cloud Asset Inventory generic pass ran (optional accelerator available)" + fi + + # 2. The project anchor must be present (resource_type 'project'). + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='gcp' AND resource_type='project' AND name='${GCP_PROJECT}';" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL baseline: project anchor '${GCP_PROJECT}' not in resource store" + fail=1 + else + echo "OK baseline: project anchor '${GCP_PROJECT}' present (${n} row)" + fi + + # 3. At least a few GCP resources overall (anchor + real resources). + total=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='gcp';" \ + 2>/dev/null || echo 0) + if [ "$total" -lt 3 ]; then + echo "FAIL baseline: expected >= 3 gcp resources, found ${total}" + fail=1 + else + echo "OK baseline: ${total} gcp resources in store (>= 3)" + fi + + # 4. Typed collectors (storage buckets + GKE clusters) populate the + # persistent sandbox infra. Assert presence (>= 1), not exact counts, + # so the test stays stable as buckets come and go. These types are + # referenced by the bundled rw-cli-codecollection generation rules. + for rtype in gcp_storage_buckets gcp_container_clusters; do + n=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='gcp' AND resource_type='${rtype}';" \ + 2>/dev/null || echo 0) + if [ "$n" -lt 1 ]; then + echo "FAIL baseline: expected >= 1 ${rtype}, found ${n}" + fail=1 + else + echo "OK baseline: ${rtype} present (${n} row)" + fi + done + + # 5. PROOF the typed fallback collectors provide function WITHOUT CAI. + # The test-only code collection (gcpapi-test-codecollection, wired in + # by generate-gcpapi-test-codecollection) references gcp_compute_networks + # and gcp_compute_firewalls. Every GCP project has a default network + + # default firewall rules out of the box (like AWS's default VPC/SG), so + # these fallback collectors must return real sandbox resources even with + # CAI denied. We require the network (always present); the firewall count + # is reported but only required to be >= 1 alongside the network. + net=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='gcp' AND resource_type='gcp_compute_networks';" \ + 2>/dev/null || echo 0) + fw=$(sqlite3 "$DB" \ + "SELECT count(*) FROM resources WHERE platform='gcp' AND resource_type='gcp_compute_firewalls';" \ + 2>/dev/null || echo 0) + if [ "$net" -lt 1 ]; then + echo "FAIL baseline: expected >= 1 gcp_compute_networks from the typed fallback collector, found ${net}" + fail=1 + else + echo "OK baseline: gcp_compute_networks present (${net} row) via typed fallback collector (no CAI)" + fi + echo "INFO baseline: gcp_compute_firewalls present (${fw} row) via typed fallback collector" + + # Per-resource_type counts (report artifact + proof of typed-collector function). + echo "---- gcp resource_type counts (output/resources.sqlite) ----" + sqlite3 "$DB" \ + "SELECT resource_type, count(*) FROM resources WHERE platform='gcp' GROUP BY resource_type ORDER BY resource_type;" \ + 2>/dev/null || true + echo "---- referenced GCP resource types (from the indexer log) ----" + grep -hE "GCP resource types referenced by generation rules:" run_sh_output.log container_logs.log 2>/dev/null | tail -1 || true + + exit "$fail" diff --git a/.test/k8s/basic/Taskfile.yaml b/.test/k8s/basic/Taskfile.yaml index a8a2edc27..45e1f6af3 100644 --- a/.test/k8s/basic/Taskfile.yaml +++ b/.test/k8s/basic/Taskfile.yaml @@ -119,7 +119,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGGING=false --add-host github.com:0.0.0.0 --name $CONTAINER_NAME -p 8081:8081 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGGING=false --add-host github.com:0.0.0.0 --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } @@ -156,7 +156,7 @@ tasks: echo "Starting new container $CONTAINER_NAME..." - docker run -e DEBUG_LOGGING=false --name $CONTAINER_NAME -p 8081:8081 -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { + docker run -e DEBUG_LOGGING=false --name $CONTAINER_NAME -p 8000:8000 -v $(pwd):/shared -d runwhen-local:test || { echo "Failed to start container"; exit 1; } @@ -195,10 +195,9 @@ tasks: echo "User $(whoami)" # 1. Start container in the background docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ --name "$CONTAINER_NAME" \ - -p 8081:8081 \ + -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test @@ -248,12 +247,11 @@ tasks: echo "User $(whoami)" # 1. Start container with github.com/api.github.com blocked via /etc/hosts docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ --add-host github.com:0.0.0.0 \ --add-host api.github.com:0.0.0.0 \ --name "$CONTAINER_NAME" \ - -p 8081:8081 \ + -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test @@ -437,7 +435,7 @@ tasks: FAILURES=0 WARNINGS=0 - # Helper: search both log files (container_logs.log has entrypoint + Django server output, + # Helper: search both log files (container_logs.log has entrypoint + FastAPI server output, # run_sh_output.log has the docker-exec run.sh output) search_logs() { local pattern="$1" @@ -670,3 +668,245 @@ tasks: desc: "Run only YAML validation on existing output (for quick testing)" cmds: - task: validate-k8s-yaml + + unit-test-src: + desc: "Fast no-Docker unit tests for the workspace builder source (indexers, normalizers, SQLite writer). Useful while iterating on src/ - no need to rebuild the image to catch encoder / writer regressions." + cmds: + - | + SRC_DIR=../../../src + if [ ! -d "$SRC_DIR" ]; then + echo "❌ src/ not found at $SRC_DIR"; exit 1 + fi + echo "Running unittest discovery under $SRC_DIR/indexers ..." + cd "$SRC_DIR" && python3 -m unittest \ + indexers.test_sqlite_resource_writer \ + indexers.test_azureapi_normalizers \ + -v + silent: false + + ci-test-sqlite-store: + desc: "Run discovery with resourceStoreBackend=sqlite and verify the SQLite file lands in the output directory with non-empty platform/resource rows." + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "my-workspace-sqlite"}}' + cmds: + - | + cat < workspaceInfo.yaml + workspaceName: "$RW_WORKSPACE" + workspaceOwnerEmail: authors@runwhen.com + defaultLocation: location-01 + defaultLOD: detailed + cloudConfig: + kubernetes: + kubeconfigFile: /shared/kubeconfig.secret + contexts: + sandbox-cluster-1: + defaultNamespaceLOD: basic + codeCollections: [] + custom: + kubernetes_distribution_binary: kubectl + resourceStoreBackend: sqlite + resourceStorePath: resources.sqlite + EOF + + - task: build-rwl + - task: run-rwl-discovery + + - | + echo "" + echo "=== SQLite resource store validation ===" + DB_FILE=$(find output -maxdepth 3 -name 'resources.sqlite' | head -n 1) + if [ -z "$DB_FILE" ]; then + echo "❌ resources.sqlite not produced by discovery" + find output -maxdepth 3 -type f | sed 's/^/ /' + exit 1 + fi + echo "✔ Found SQLite store: $DB_FILE ($(stat -c%s "$DB_FILE") bytes)" + + if ! command -v sqlite3 >/dev/null; then + echo "ℹ sqlite3 CLI not available on host; falling back to python3 inspection" + PY=python3 + $PY - "$DB_FILE" <<'PY' || exit 1 + import sqlite3, sys + conn = sqlite3.connect(sys.argv[1]) + platforms = [r[0] for r in conn.execute("SELECT name FROM platforms")] + types = conn.execute("SELECT COUNT(*) FROM resource_types").fetchone()[0] + rows = conn.execute("SELECT COUNT(*) FROM resources").fetchone()[0] + artifacts = conn.execute("SELECT COUNT(*) FROM workspace_artifacts").fetchone()[0] + slx_rows = conn.execute("SELECT COUNT(*) FROM workspace_artifacts WHERE artifact_kind='slx'").fetchone()[0] + version = conn.execute("SELECT value FROM schema_meta WHERE key='schema_version'").fetchone() + print(f" schema_version : {version[0] if version else 'MISSING'}") + print(f" platforms : {platforms}") + print(f" resource_types : {types}") + print(f" resources : {rows}") + print(f" artifacts : {artifacts}") + print(f" slx files : {slx_rows}") + if not platforms: + print("❌ no platforms in store"); sys.exit(1) + if rows == 0: + print("❌ resources table is empty"); sys.exit(1) + if artifacts == 0: + print("❌ workspace_artifacts table is empty"); sys.exit(1) + if slx_rows == 0: + print("❌ no slx.yaml artifacts persisted"); sys.exit(1) + print("✔ SQLite store is populated") + PY + else + PLATFORMS=$(sqlite3 "$DB_FILE" "SELECT name FROM platforms;" | tr '\n' ' ') + TYPES=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM resource_types;") + ROWS=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM resources;") + ARTIFACTS=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM workspace_artifacts;") + SLX_ROWS=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM workspace_artifacts WHERE artifact_kind='slx';") + VERSION=$(sqlite3 "$DB_FILE" "SELECT value FROM schema_meta WHERE key='schema_version';") + echo " schema_version : ${VERSION:-MISSING}" + echo " platforms : ${PLATFORMS:-(none)}" + echo " resource_types : $TYPES" + echo " resources : $ROWS" + echo " artifacts : $ARTIFACTS" + echo " slx files : $SLX_ROWS" + if [ -z "$PLATFORMS" ] || [ "$ROWS" -eq 0 ]; then + echo "❌ SQLite store is empty" + exit 1 + fi + if [ "$ARTIFACTS" -eq 0 ] || [ "$SLX_ROWS" -eq 0 ]; then + echo "❌ workspace artifacts not persisted" + exit 1 + fi + echo "✔ SQLite store is populated" + fi + silent: false + + ci-test-skill-overlay: + desc: "Run discovery pinned to rw-cli-codecollection feat/skill-overlay and verify the Skill overlay pipeline. The overlay is conditional: only codebundles that ship a Skill markdown file (SKILL.md, Skill.md, or skill.md — matched case-insensitively) at the bundle root produce an overlay file. The test asks GitHub how many Skill markdown files exist on the branch and tightens its assertions accordingly, so it stays green today and tightens automatically once Skill markdown files start landing upstream." + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "my-workspace-skill"}}' + SKILL_BRANCH: '{{.SKILL_BRANCH | default "feat/skill-overlay"}}' + cmds: + - | + cat < workspaceInfo.yaml + workspaceName: "$RW_WORKSPACE" + workspaceOwnerEmail: authors@runwhen.com + defaultLocation: location-01 + defaultLOD: detailed + cloudConfig: + kubernetes: + kubeconfigFile: /shared/kubeconfig.secret + contexts: + sandbox-cluster-1: + defaultNamespaceLOD: basic + namespaces: + - ci-verify-basic + codeCollections: + - repoURL: https://github.com/runwhen-contrib/rw-cli-codecollection.git + branch: ${SKILL_BRANCH} + useLocalGit: false + custom: + kubernetes_distribution_binary: kubectl + resourceStoreBackend: sqlite + resourceStorePath: resources.sqlite + EOF + + - task: build-rwl + - task: run-rwl-discovery + + - | + echo "" + echo "=== Branch override validation ===" + # Confirm the codecollection override actually took effect: every rendered + # slx.yaml records the source ref via common-annotations (sourceGenerationRuleRepoRef). + REF_HITS=$(grep -rl "sourceGenerationRuleRepoRef: ${SKILL_BRANCH}" output/workspaces 2>/dev/null | wc -l | tr -d ' ') + echo " slx.yaml files referencing ref '${SKILL_BRANCH}' : $REF_HITS" + if [ "$REF_HITS" -eq 0 ]; then + echo "❌ codeCollections override did not take effect — no SLX references ref '${SKILL_BRANCH}'." + echo " Sample slx.yaml ref lines we did see:" + grep -rh "sourceGenerationRuleRepoRef:" output/workspaces 2>/dev/null | sort -u | head -5 | sed 's/^/ /' + exit 1 + fi + echo "✔ codeCollections override honored" + + - | + echo "" + echo "=== Skill overlay validation ===" + # `-iname` matches any case (SKILL.md, Skill.md, skill.md, sKiLL.Md, ...). + SLX_COUNT=$(find output -mindepth 4 -type d -path '*/slxs/*' | wc -l | tr -d ' ') + SKILL_COUNT=$(find output -type f -iname 'skill.md' -path '*/slxs/*' | wc -l | tr -d ' ') + echo " rendered SLX directories : $SLX_COUNT" + echo " Skill overlay files (any case) : $SKILL_COUNT" + + if [ "$SLX_COUNT" -eq 0 ]; then + echo "❌ No rendered SLX directories produced" + find output -maxdepth 5 -type d | sed 's/^/ /' + exit 1 + fi + + # Ask GitHub how many Skill markdown files exist on the branch we just + # rendered against, matching the filename case-insensitively (codebundles + # can publish it as SKILL.md, Skill.md, or skill.md). Limit the count to + # entries at codebundles//.md at the bundle root — + # sub-directory matches don't trigger the overlay. + UPSTREAM_SKILL_COUNT=0 + TREE_JSON=$(curl -fsSL "https://api.github.com/repos/runwhen-contrib/rw-cli-codecollection/git/trees/${SKILL_BRANCH}?recursive=1" 2>/dev/null || true) + if [ -n "$TREE_JSON" ]; then + UPSTREAM_SKILL_COUNT=$(printf '%s' "$TREE_JSON" \ + | python3 -c 'import json,sys; t=json.load(sys.stdin); paths=[e["path"] for e in t.get("tree",[]) if e.get("type")=="blob"]; print(sum(1 for p in paths if p.startswith("codebundles/") and p.lower().endswith("/skill.md") and p.count("/")==2))' \ + 2>/dev/null || echo 0) + fi + echo " upstream codebundles with Skill markdown (branch ${SKILL_BRANCH}) : ${UPSTREAM_SKILL_COUNT}" + + if [ "$UPSTREAM_SKILL_COUNT" -eq 0 ]; then + echo "" + echo "ℹ️ No codebundle on branch '${SKILL_BRANCH}' ships a Skill markdown yet." + echo " The overlay code path is exercised but produced 0 files (expected)." + echo " This task will auto-tighten once a SKILL.md / Skill.md is added upstream." + echo " For end-to-end coverage of the overlay code path itself, see" + echo " src/renderers/test_skill_overlay.py (unit tests)." + echo "" + echo "✔ Skill overlay pipeline ran cleanly (no upstream sources to overlay)" + exit 0 + fi + + if [ "$SKILL_COUNT" -eq 0 ]; then + echo "❌ Branch '${SKILL_BRANCH}' has ${UPSTREAM_SKILL_COUNT} codebundle Skill markdown file(s)," + echo " but none were overlaid onto rendered SLXs. Investigate _emit_skill_overlay." + find output -type f -path '*/slxs/*' | head -n 30 | sed 's/^/ /' + exit 1 + fi + + echo "" + echo " sample Skill overlay (first 12 lines):" + SAMPLE=$(find output -type f -iname 'skill.md' -path '*/slxs/*' | head -n 1) + echo " -> $SAMPLE" + sed 's/^/ /' "$SAMPLE" | head -n 12 || true + + echo "" + echo "=== SQLite skill-artifact validation ===" + DB_FILE=$(find output -maxdepth 3 -name 'resources.sqlite' | head -n 1) + if [ -z "$DB_FILE" ]; then + echo "❌ resources.sqlite not produced" + exit 1 + fi + + if command -v sqlite3 >/dev/null; then + SKILL_ROWS=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM workspace_artifacts WHERE artifact_kind='skill';") + SAMPLE_ROW=$(sqlite3 "$DB_FILE" "SELECT relative_path FROM workspace_artifacts WHERE artifact_kind='skill' LIMIT 1;") + echo " skill artifacts : $SKILL_ROWS" + echo " sample row : ${SAMPLE_ROW:-(none)}" + if [ "$SKILL_ROWS" -eq 0 ]; then + echo "❌ Skill.md present on disk but not classified as artifact_kind='skill' in SQLite" + exit 1 + fi + else + python3 - "$DB_FILE" <<'PY' || exit 1 + import sqlite3, sys + conn = sqlite3.connect(sys.argv[1]) + rows = conn.execute("SELECT COUNT(*) FROM workspace_artifacts WHERE artifact_kind='skill'").fetchone()[0] + sample = conn.execute("SELECT relative_path FROM workspace_artifacts WHERE artifact_kind='skill' LIMIT 1").fetchone() + print(f" skill artifacts : {rows}") + print(f" sample row : {sample[0] if sample else '(none)'}") + if rows == 0: + print("❌ Skill.md present on disk but not classified as artifact_kind='skill' in SQLite") + sys.exit(1) + PY + fi + + echo "✔ Skill.md overlay rendered and persisted as artifact_kind='skill'" + silent: false diff --git a/.test/k8s/upload/Taskfile.yaml b/.test/k8s/upload/Taskfile.yaml index cd39ab7af..0872be0ac 100644 --- a/.test/k8s/upload/Taskfile.yaml +++ b/.test/k8s/upload/Taskfile.yaml @@ -308,10 +308,9 @@ tasks: # 1. Start container in the background docker run -d \ - -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" \ -e DEBUG_LOGS=true \ --name "$CONTAINER_NAME" \ - -p 8081:8081 \ + -p 8000:8000 \ -v "$(pwd):/shared" \ runwhen-local:test diff --git a/README.md b/README.md index ae642be73..3ca7a89f0 100644 --- a/README.md +++ b/README.md @@ -2,426 +2,285 @@ [![Join Slack](https://img.shields.io/badge/Join%20Slack-%23E01563.svg?&style=for-the-badge&logo=slack&logoColor=white)](https://runwhen.slack.com/join/shared_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) +[![Python 3.14](https://img.shields.io/badge/python-3.14-blue.svg)](https://www.python.org/downloads/) [![Docker](https://img.shields.io/badge/docker-supported-blue.svg)](https://www.docker.com/) -RunWhen Local is an open-source workspace builder and troubleshooting companion that automatically discovers resources from your Kubernetes clusters and cloud environments. It generates personalized troubleshooting commands, runbooks, and automation tasks tailored specifically to your infrastructure. +RunWhen Local is a **discovery tool** for cloud and Kubernetes +infrastructure that turns the resources you already have into a tailored +set of agentic **Skills** — small, AI-agent-readable runbooks bound to +the specific things in *your* environment. -**RunWhen Local** is like your personal troubleshooter's toolbox - it scans your environment, identifies the perfect troubleshooting commands for your resources, and provides them as easy copy-and-paste CLI commands through a web interface. +Pair it with the [RunWhen Platform](https://www.runwhen.com) to manage +and run those Skills safely in production: scheduled execution, audit, +secrets, and team-level governance. + +> Heads up: RunWhen Local is evolving quickly. The **discovery → +> tailored Skills** workflow described below is the current core +> feature. Additional functionality is on the way and will be folded +> into this README and the docs as it lands. ## Table of Contents -- [Features](#features) -- [What is RunWhen?](#what-is-runwhen) -- [Prerequisites](#prerequisites) -- [Installation](#installation) -- [Quick Start](#quick-start) +- [What it does](#what-it-does) +- [How the pieces fit together](#how-the-pieces-fit-together) +- [Quick start](#quick-start) - [Configuration](#configuration) -- [Usage Examples](#usage-examples) +- [What gets discovered](#what-gets-discovered) +- [Going to production with the RunWhen Platform](#going-to-production-with-the-runwhen-platform) - [Documentation](#documentation) - [Contributing](#contributing) -- [Support](#support) +- [Community and support](#community-and-support) - [License](#license) -## Features - -- **Multi-Cloud Discovery**: Automatically discover and index resources from Kubernetes, Azure, AWS, and GCP -- **Code Collections**: Leverage community-maintained runbooks and SLIs from external repositories -- **Template-Based Generation**: Generate workspace configurations using flexible Jinja2 templates -- **Level of Detail Control**: Configure different levels of detail for resource discovery and SLX generation -- **ConfigProvided Overrides**: Override template variables in runbooks and SLIs without modifying original templates -- **Map Customization**: Apply custom grouping and relationship rules to organize your workspace -- **Multiple Output Formats**: Generate runbooks, SLIs, workflows, and workspace configurations -- **Azure DevOps Integration**: Support for Azure DevOps resource discovery and automation -- **Web Interface**: Searchable web interface for copy-and-paste troubleshooting commands -- **Docker Support**: Run in containerized environments for consistent deployments - -## What is RunWhen? - -RunWhen provides **AI Engineering Assistants** that help with troubleshooting and automation: - -- **RunWhen Platform**: SaaS service that orchestrates AI assistants for alert response, developer self-service, and automated troubleshooting -- **RunWhen Local**: Open-source agent that provides personalized troubleshooting commands and can connect to the RunWhen Platform -- **Code Collections**: Community-maintained libraries of troubleshooting automation and runbooks - -**RunWhen Local** works standalone as a troubleshooting companion, or can be connected to the RunWhen Platform for advanced AI-powered automation and workflows. - -## Prerequisites - -- **Python**: 3.10 or higher (for local installation) -- **Docker**: Latest version (for containerized deployment) -- **Cloud Access**: Appropriate credentials for your target cloud platforms: - - **Kubernetes**: Valid kubeconfig file - - **Azure**: Azure CLI or service principal credentials - - **AWS**: AWS CLI credentials or IAM roles - - **GCP**: Service account key or gcloud credentials +## What it does + +1. **Discovery.** RunWhen Local connects to your Kubernetes clusters and + cloud accounts and indexes the resources you point it at — selectively + (per-namespace, per-resource-group) or broadly. The Azure indexer is + native (no CloudQuery dependency) and indexes the full Azure resource + catalog; AWS, GCP, and Kubernetes are supported. + +2. **Skill tailoring.** A library of CodeBundles (in [contrib + CodeCollections](https://github.com/runwhen-contrib)) ships + generation rules that match against discovered resources. The + workspace builder renders one **SLX** (a tailored Skill instance) + per match: a runbook bound to a specific resource, with its own + `SKILL.md` so an MCP-aware AI agent can read what it does and how to + invoke it. + +3. **Local explorer UI.** A FastAPI-backed UI at + `http://localhost:8000/explorer/` lets you browse the discovered + resources, the rendered SLXs, and their Skill descriptions side by + side. + +4. **Built-in MCP server.** A read-only [Model Context Protocol](https://modelcontextprotocol.io) + server is mounted at `http://localhost:8000/mcp/` so AI agents + (Claude Code, Cursor, Claude Desktop, ...) can search, browse, and + read your generated Skills directly. v1 is read-only — search and + suggestion; execution is the natural follow-on. See the + [MCP server guide](docs/user-guide/features/mcp-server.md). + +5. **Optional Platform pairing.** Push the same SLXs to the RunWhen + Platform to gate execution behind RBAC, schedule them, and route + results into alerts or developer self-service flows. Local is fully + useful standalone; the Platform turns it into a production runtime. + +## How the pieces fit together + +A short glossary; the [authoring/concepts +guide](docs/authoring/concepts.md) covers it in more depth. + +- **CodeBundle** — a versioned, distributable unit of automation in a + CodeCollection git repo. Ships code, optional `SKILL.md`, and one or + more generation rules. +- **Skill** — the abstract capability the CodeBundle implements + (e.g. "diagnose a stuck Pod", "rotate a Key Vault secret"). +- **SLX** — a *rendered instance* of a Skill, bound to a specific + resource you discovered (e.g. "diagnose a stuck Pod *in the + `payments-prod` namespace*"). One SLX = one directory of artifacts. +- **Runbook** — the executable artifact inside an SLX (typically a + Robot Framework `.robot`). What a human or agent actually runs. +- **Generation rule** — the YAML in a CodeBundle that tells the + workspace builder: "match this resource type, render this Skill + template into an SLX with this name." + +```text +Discovered resources Generated SLXs +(Kubernetes pods, Azure Key (one per match — runbook + SKILL.md + Vaults, AWS RDS instances, ...) bound to a specific resource) + + │ │ + └──────── generation rules ─────────────┘ + (live in CodeBundles) +``` -## Installation +## Quick start -### Docker (Recommended) +The fastest path is the published image. You'll need a +`workspaceInfo.yaml` (sample below) and credentials for whatever +platform(s) you want to discover. ```bash -# Pull the latest image docker pull ghcr.io/runwhen-contrib/runwhen-local:latest -# Run with volume mounts for configuration and output -docker run -it --rm \ - -v $(pwd)/workspaceInfo.yaml:/shared/workspaceInfo.yaml \ - -v $(pwd)/kubeconfig:/shared/kubeconfig \ - -v $(pwd)/output:/shared/output \ +docker run --rm -it \ + -p 8000:8000 \ + -v "$(pwd):/shared" \ ghcr.io/runwhen-contrib/runwhen-local:latest ``` -### Local Python Installation - -```bash -# Clone the repository -git clone https://github.com/runwhen-contrib/runwhen-local.git -cd runwhen-local/src - -# Install dependencies using Poetry -pip install poetry -poetry install - -# Make the run script executable -chmod +x run.sh - -# Run the workspace builder -./run.sh -``` - -## Quick Start +Once discovery finishes, the explorer UI is at +`http://localhost:8000/explorer/`. Generated SLXs land in `./output/`. -1. **Create a workspace configuration file** (`workspaceInfo.yaml`): +A minimal `workspaceInfo.yaml` to get started against a single +Kubernetes cluster: ```yaml -# Basic workspaceInfo.yaml structure workspaceName: "my-workspace" -workspaceOwnerEmail: "admin@company.com" +workspaceOwnerEmail: "you@example.com" defaultLocation: "location-01" -defaultLOD: "detailed" # Level of detail: "none", "basic", or "detailed" +defaultLOD: "detailed" # discover everything by default -# Cloud configuration cloudConfig: kubernetes: kubeconfigFile: "/shared/kubeconfig" namespaceLODs: - kube-system: "none" - kube-public: "none" + kube-system: "none" + kube-public: "none" kube-node-lease: "none" - - # Optional: Azure configuration - azure: - subscriptionId: "your-subscription-id" - tenantId: "your-tenant-id" - clientId: "your-client-id" - clientSecret: "your-client-secret" - -# Code collections - external repositories with runbooks/SLIs + codeCollections: - repoURL: "https://github.com/runwhen-contrib/rw-cli-codecollection.git" branch: "main" - -# Custom variables for generation rules -custom: - kubernetes_distribution_binary: "kubectl" - cloud_provider: "none" -``` - -2. **Prepare your kubeconfig** (if using Kubernetes discovery): - -```bash -# Copy your kubeconfig to the workspace -cp ~/.kube/config ./kubeconfig ``` -3. **Run the workspace builder**: - -```bash -# Using Docker -docker run -it --rm \ - -v $(pwd)/workspaceInfo.yaml:/shared/workspaceInfo.yaml \ - -v $(pwd)/kubeconfig:/shared/kubeconfig \ - -v $(pwd)/output:/shared/output \ - ghcr.io/runwhen-contrib/runwhen-local:latest - -# Using local installation -cd src && ./run.sh -``` - -4. **Check the generated output**: - -```bash -ls output/ -# You'll find generated runbooks, SLIs, and workspace configurations -``` +For Azure, AWS, and GCP setup, see the per-platform pages in +[`docs/user-guide/cloud-discovery/`](docs/user-guide/cloud-discovery/). ## Configuration -### workspaceInfo.yaml Structure - -The main configuration file is a simple YAML file with these top-level sections: +`workspaceInfo.yaml` is the single source of truth. Top-level shape: ```yaml -# Basic workspace configuration -workspaceName: "workspace-name" -workspaceOwnerEmail: "admin@company.com" +workspaceName: "..." +workspaceOwnerEmail: "..." defaultLocation: "location-01" -defaultLOD: "detailed" # Level of detail +defaultLOD: "detailed" # "none" | "basic" | "detailed" -# Cloud platform configuration cloudConfig: - kubernetes: # Kubernetes cluster configuration - azure: # Azure subscription configuration - aws: # AWS account configuration - gcp: # GCP project configuration + kubernetes: { ... } + azure: { ... } + aws: { ... } + gcp: { ... } -# External code collections codeCollections: - repoURL: "https://github.com/..." - branch: "main" + branch: "main" -# Custom variables for generation rules -custom: - kubernetes_distribution_binary: "kubectl" +custom: { ... } # values exposed to generation rule templates ``` -### Command Line Options +Full reference and examples: -```bash -./run.sh [OPTIONS] - -Options: - -w, --workspace-info FILE Workspace info file (default: workspaceInfo.yaml) - -k, --kubeconfig FILE Kubeconfig file (default: kubeconfig) - -r, --customization-rules FILE Customization rules file - -o, --output DIRECTORY Output directory (default: output) - --upload Upload to RunWhen platform - -v, --verbose Verbose output - --disable-cloudquery Disable cloudquery component - -h, --help Show help message -``` +- [`workspaceInfo.yaml` reference](docs/user-guide/configuration/workspace-info.md) +- [Discovery level of detail](docs/user-guide/configuration/level-of-detail.md) +- [Helm / proxy / private-registry / file-watching](docs/user-guide/configuration/) +- [`workspaceinfo-overrides-example.yaml`](docs/user-guide/configuration/workspaceinfo-overrides-example.yaml) -## Usage Examples +CLI invocation lives in [`src/run.sh`](src/run.sh); run with `--help` +for the current flag list. -### Example 1: Kubernetes-Only Discovery +## What gets discovered -```yaml -workspaceName: "kubernetes-prod" -workspaceOwnerEmail: "admin@company.com" -defaultLocation: "location-01" -defaultLOD: "detailed" +| Platform | Indexer | Coverage | +| ------------ | -------------- | -------- | +| Azure | native `azureapi` (`azure-mgmt-*` SDKs) | 619 resource types — full parity with the legacy CloudQuery plugin. 25 with rich (typed) payloads, the rest via the ARM-resources catch-all. [Catalog](docs/authoring/indexed-resources/azure-resource-catalog.md). | +| Kubernetes | native (`kubernetes` Python client) | Standard kinds (Deployment, StatefulSet, Service, Pod, Ingress, etc.) plus user-listed CRDs, with per-namespace LOD. | +| AWS | CloudQuery AWS plugin | Every CloudQuery AWS table; gen rules match by table name. | +| GCP | CloudQuery GCP plugin | Every CloudQuery GCP table; gen rules match by table name. | -cloudConfig: - kubernetes: - kubeconfigFile: "/shared/kubeconfig" - namespaceLODs: - production: "detailed" - staging: "basic" - kube-system: "none" - kube-public: "none" - kube-node-lease: "none" +Per-indexer authoring reference: +[`docs/authoring/indexed-resources/`](docs/authoring/indexed-resources/). -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/rw-cli-codecollection.git" - branch: "main" +The Azure CloudQuery backend still exists behind +`azureIndexerBackend: cloudquery` for compatibility but is being phased +out in favor of `azureIndexerBackend: azureapi`. Both backends emit a +single grep-able info-level log line on every run so it's obvious which +one ran. -custom: - kubernetes_distribution_binary: "kubectl" - cloud_provider: "none" -``` +## Going to production with the RunWhen Platform -### Example 2: Multi-Cloud with Azure +Local gives you discovered resources and tailored SLXs on disk. The +[RunWhen Platform](https://www.runwhen.com) is the optional companion +that turns those SLXs into a *managed* production runtime: -```yaml -workspaceName: "multi-cloud-ops" -workspaceOwnerEmail: "admin@company.com" -defaultLocation: "location-01" -defaultLOD: "basic" - -cloudConfig: - kubernetes: - kubeconfigFile: "/shared/kubeconfig" - namespaceLODs: - production: "basic" - kube-system: "none" - - azure: - subscriptionId: "your-subscription-id" - tenantId: "your-tenant-id" - clientId: "your-client-id" - clientSecret: "your-client-secret" - -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/rw-cli-codecollection.git" - branch: "main" - -custom: - kubernetes_distribution_binary: "kubectl" - cloud_provider: "azure" -``` - -### Example 3: Azure DevOps Integration - -```yaml -workspaceName: "azure-devops-workspace" -workspaceOwnerEmail: "admin@company.com" -defaultLocation: "location-01" -defaultLOD: "basic" - -cloudConfig: - azure: - subscriptionId: "your-subscription-id" - tenantId: "your-tenant-id" - clientId: "your-client-id" - clientSecret: "your-client-secret" - - devops: - organizationUrl: "https://dev.azure.com/your-organization" - # Use Kubernetes secret for PAT (recommended) - patSecretName: "azure-devops-pat" - - codeCollections: - - repositoryUrl: "https://dev.azure.com/your-org/your-project/_git/your-repo" - branch: "main" - -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/rw-cli-codecollection.git" - branch: "main" +- **Privately host** the Skills your team can run, with RBAC controls + on who can invoke what. +- **Schedule** SLX runs and ingest the results into alerts / + developer-self-service flows. +- **Manage secrets** centrally so credentials never live next to the + CodeBundle. +- **Audit** every SLX execution for compliance. -custom: - azure_devops: - environment: "production" - organization: "your-organization" - critical_project: "your-main-project" - repository_size_threshold: "1000" - pipeline_type: "infrastructure" - monitoring_level: "detailed" - team: "platform-engineering" -``` - -More examples are available in the [examples directory](examples/). +To upload the workspace generated by RunWhen Local to the Platform, +pass `--upload` to `run.sh` (or the equivalent setting in +`workspaceInfo.yaml`); see +[`docs/user-guide/features/upload-to-runwhen-platform.md`](docs/user-guide/features/upload-to-runwhen-platform.md). ## Documentation -### ConfigProvided Overrides - -The configProvided overrides feature allows you to customize template variables in runbooks and SLIs without modifying the original templates. This is particularly useful for: - -- Environment-specific configurations -- Testing different parameter values -- Customizing default values for your organization +The full doc tree is in [`docs/`](docs/) and is split into three +buckets: -**Quick Example:** - -```yaml -# workspaceInfo.yaml -overrides: - codebundles: - - repoURL: "https://github.com/runwhen-contrib/rw-cli-codecollection.git" - codebundleDirectory: "azure-aks-triage" - type: "runbook" - configProvided: - TIME_PERIOD_MINUTES: "120" - DEBUG_MODE: "true" +``` +docs/ +├── user-guide/ I want to deploy and operate RunWhen Local +├── authoring/ I want to write CodeBundles, Skills, or generation rules +└── architecture/ I want to understand how RunWhen Local works internally ``` -For comprehensive documentation, see: [ConfigProvided Overrides Guide](docs/configProvided-overrides.md) - -### Additional Resources - -- 📖 **[Full Documentation](https://docs.runwhen.com/public/v/runwhen-local/)**: Complete user guide and API reference -- 🎯 **[Generation Rules Guide](generation-rules-guide.md)**: How to create custom generation rules -- 💡 **[Examples](examples/)**: Sample configurations for different scenarios -- 🏗️ **[Architecture Overview](docs/Architecture.md)**: Technical architecture and design -- 🛠️ **[Development Guide](docs/Development.md)**: Development setup and contribution guidelines -- 📚 **[Complete Documentation](docs/)**: All documentation in the docs directory - -## Getting Started - -1. Configure your `workspaceInfo.yaml` with cloud credentials and discovery settings -2. (Optional) Add configProvided overrides for custom template variables -3. Run the workspace builder to generate your workspace configuration -4. Deploy the generated runbooks and SLIs to your RunWhen platform +Highlights: + +- [Getting started](docs/user-guide/getting-started.md) +- [Local Docker / Podman install](docs/user-guide/installation/local-docker.md) · + [Kubernetes standalone](docs/user-guide/installation/kubernetes-standalone.md) · + [Self-hosted runner (Platform-connected)](docs/user-guide/installation/kubernetes-self-hosted/README.md) +- [Cloud discovery: Azure](docs/user-guide/cloud-discovery/azure.md) · + [AWS](docs/user-guide/cloud-discovery/aws.md) · + [GCP](docs/user-guide/cloud-discovery/gcp.md) · + [Kubernetes](docs/user-guide/cloud-discovery/kubernetes.md) +- [CodeBundle / Skill / SLX concepts](docs/authoring/concepts.md) +- [Indexed resources reference](docs/authoring/indexed-resources/README.md) +- [Generation rules: schema, lifecycle, and examples](docs/authoring/generation-rules/README.md) +- [Agentic access: built-in MCP server](docs/user-guide/features/mcp-server.md) +- [Architecture overview](docs/architecture/README.md) + +Hosted docs mirror is at +[docs.runwhen.com/public/v/runwhen-local/](https://docs.runwhen.com/public/v/runwhen-local/). ## Contributing -We welcome contributions from the community! Please see our [Contributing Guide](CONTRIBUTING.md) for details on: - -- How to submit bug reports and feature requests -- Development setup and coding standards -- Pull request process -- Code of conduct - -### Quick Development Setup +We welcome contributions. See [`CONTRIBUTING.md`](CONTRIBUTING.md) for +the bug / feature / PR workflow and code of conduct. -#### Option 1: Using Dev Container (Recommended) - -The easiest way to get started with development is using the provided dev container: +### Dev container (recommended) ```bash -# Fork and clone the repository -git clone https://github.com/YOUR-USERNAME/runwhen-local.git +git clone https://github.com/runwhen-contrib/runwhen-local.git cd runwhen-local - -# Open in VS Code with dev container -code . -# VS Code will prompt to "Reopen in Container" +code . # VS Code will prompt: "Reopen in Container" ``` -The dev container includes: -- Python 3.12 with all dependencies -- Pre-configured VS Code extensions (Robot Framework, Python, Pylint, Black formatter) -- CLI tools: kubectl, aws, gcloud, terraform, helm, yq -- Docker-in-Docker for building and testing -- All necessary development tools pre-installed +The dev container ships Python 3.14, Poetry, kubectl, az, gcloud, +helm, terraform, yq, Docker-in-Docker, and the recommended editor +extensions. -#### Option 2: Local Development +### Local dev ```bash -# Fork and clone the repository -git clone https://github.com/YOUR-USERNAME/runwhen-local.git -cd runwhen-local - -# Set up development environment -cd src -pip install poetry -poetry install +git clone https://github.com/runwhen-contrib/runwhen-local.git +cd runwhen-local/src +pip install poetry && poetry install poetry shell -# Run tests -python tests.py - -# Start developing! +cd .. && python src/tests.py # run the test suite ``` -### Community Contributions - -- 🐛 **[Report bugs or share feedback](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea&labels=runwhen-local&projects=&template=runwhen-local-feedback.md&title=%5Brunwhen-local-feedback%5D+)** -- 💡 **[Contribute an awesome troubleshooting command](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea&labels=runwhen-local%2Cawesome-command-contribution&projects=&template=awesome-command-contribution.yaml&title=%5Bawesome-command-contribution%5D+)** -- 🙋 **[Request a new command](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea&labels=runwhen-local%2Cnew-command-request&projects=&template=commands-wanted.yaml&title=%5Bnew-command-request%5D+)** -- 💬 **[GitHub Discussions](https://github.com/orgs/runwhen-contrib/discussions)** +For deeper dives, see +[`docs/architecture/development.md`](docs/architecture/development.md). -## Support +## Community and support -- 📚 **[Documentation](https://docs.runwhen.com/public/v/runwhen-local/)**: Complete user guide and API reference -- 🐛 **[GitHub Issues](https://github.com/runwhen-contrib/runwhen-local/issues)**: Bug reports and feature requests -- 💬 **[Slack Community](https://runwhen.slack.com/join/shared_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A)**: Join our community discussions -- 🎮 **[Discord](https://discord.com/invite/Ut7Ws4rm8Q)**: Alternative community chat -- 🎥 **[YouTube Channel](https://www.youtube.com/@whatdoirunwhen)**: Demo videos and tutorials -- 🌐 **[Live Demo](https://runwhen-local.sandbox.runwhen.com/)**: Try it out in our sandbox environment +- [Slack](https://runwhen.slack.com/join/shared_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A) +- [GitHub issues](https://github.com/runwhen-contrib/runwhen-local/issues) +- [GitHub discussions](https://github.com/orgs/runwhen-contrib/discussions) +- [YouTube channel (demos)](https://www.youtube.com/@whatdoirunwhen) +- [Live sandbox](https://runwhen-local.sandbox.runwhen.com/) -When reporting issues, please include: -- RunWhen Local version -- Operating system and version -- Cloud platform details (Kubernetes version, cloud provider, etc.) -- Complete error messages and logs -- Steps to reproduce the issue +When opening an issue, please include the RunWhen Local version, +target platform (Kubernetes version, cloud provider, etc.), and a +sanitized log excerpt. ## License -This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. - ---- - -**Made with ❤️ by the RunWhen community** \ No newline at end of file +Apache License 2.0 — see [`LICENSE`](LICENSE). diff --git a/assets/high-level.drawio b/assets/high-level.drawio index 3f0430022..e94fb1829 100644 --- a/assets/high-level.drawio +++ b/assets/high-level.drawio @@ -22,7 +22,7 @@ - + @@ -80,10 +80,10 @@ - + - + @@ -190,7 +190,7 @@ - + diff --git a/deploy/scripts/dev/rebuild.sh b/deploy/scripts/dev/rebuild.sh index 7fc09a9d6..55b02031c 100644 --- a/deploy/scripts/dev/rebuild.sh +++ b/deploy/scripts/dev/rebuild.sh @@ -22,8 +22,8 @@ if [[ "$workdir" ]];then echo "rebuild image" docker build -t runwhen-local:test -f ../runwhen-local/src/Dockerfile ../runwhen-local/src/ echo "Running RunWhenLocal container" - #docker run --name RunWhenLocal -p 8081:8081 -e AUTORUN_WORKSPACE_BUILDER_INTERVAL=300 -e RW_LOCAL_UPLOAD_ENABLED=true -e RW_LOCAL_UPLOAD_MERGE_MODE="keep-uploaded" -e RW_LOCAL_TERMINAL_DISABLED=false -v $workdir/shared:/shared -d runwhen-local:test - docker run --name RunWhenLocal -p 8081:8081 -e RW_LOCAL_TERMINAL_DISABLED=false -v $workdir/shared:/shared -d runwhen-local:test + #docker run --name RunWhenLocal -p 8000:8000 -e AUTORUN_WORKSPACE_BUILDER_INTERVAL=300 -e RW_LOCAL_UPLOAD_ENABLED=true -e RW_LOCAL_UPLOAD_MERGE_MODE="keep-uploaded" -e RW_LOCAL_TERMINAL_DISABLED=false -v $workdir/shared:/shared -d runwhen-local:test + docker run --name RunWhenLocal -p 8000:8000 -e RW_LOCAL_TERMINAL_DISABLED=false -v $workdir/shared:/shared -d runwhen-local:test sleep 5 echo "Running discovery" docker exec -w /workspace-builder -- RunWhenLocal ./run.sh $1 --verbose diff --git a/deploy/scripts/registry-sync/sample_values.orig b/deploy/scripts/registry-sync/sample_values.orig index dd7c6d17c..3a1e40443 100644 --- a/deploy/scripts/registry-sync/sample_values.orig +++ b/deploy/scripts/registry-sync/sample_values.orig @@ -1,10 +1,11 @@ # This Helm Chart installs RunWhen Local & the Runner -# RunWhen Local consists of the Workspace Builder and the Troubleshooting Cheat Sheet +# RunWhen Local consists of the Workspace Builder REST service (port 8000) +# and discovery output written to /shared/output. # 1. Workspace Builder scans your clusters or cloud accounts and matches them with # applicable troubleshooting commands found in CodeCollection respositories # Workspace Builder content is used to build a workspace in the RunWen Platform. -# 2. Troubleshooting Cheat Sheet generates live documentation from output of Workspace +# 2. Discovery output is written to /shared/output after each run # Builder, tailoring troubleshooting commands for the specific environment # and providing helpful documentation @@ -117,7 +118,7 @@ runwhenLocal: service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false diff --git a/deploy/scripts/registry-sync/sample_values.yaml b/deploy/scripts/registry-sync/sample_values.yaml index dd7c6d17c..3a1e40443 100644 --- a/deploy/scripts/registry-sync/sample_values.yaml +++ b/deploy/scripts/registry-sync/sample_values.yaml @@ -1,10 +1,11 @@ # This Helm Chart installs RunWhen Local & the Runner -# RunWhen Local consists of the Workspace Builder and the Troubleshooting Cheat Sheet +# RunWhen Local consists of the Workspace Builder REST service (port 8000) +# and discovery output written to /shared/output. # 1. Workspace Builder scans your clusters or cloud accounts and matches them with # applicable troubleshooting commands found in CodeCollection respositories # Workspace Builder content is used to build a workspace in the RunWen Platform. -# 2. Troubleshooting Cheat Sheet generates live documentation from output of Workspace +# 2. Discovery output is written to /shared/output after each run # Builder, tailoring troubleshooting commands for the specific environment # and providing helpful documentation @@ -117,7 +118,7 @@ runwhenLocal: service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false diff --git a/deploy/scripts/registry-sync/updated_values.yaml b/deploy/scripts/registry-sync/updated_values.yaml index e229a8034..ad71e43da 100644 --- a/deploy/scripts/registry-sync/updated_values.yaml +++ b/deploy/scripts/registry-sync/updated_values.yaml @@ -1,10 +1,11 @@ # This Helm Chart installs RunWhen Local & the Runner -# RunWhen Local consists of the Workspace Builder and the Troubleshooting Cheat Sheet +# RunWhen Local consists of the Workspace Builder REST service (port 8000) +# and discovery output written to /shared/output. # 1. Workspace Builder scans your clusters or cloud accounts and matches them with # applicable troubleshooting commands found in CodeCollection respositories # Workspace Builder content is used to build a workspace in the RunWen Platform. -# 2. Troubleshooting Cheat Sheet generates live documentation from output of Workspace +# 2. Discovery output is written to /shared/output after each run # Builder, tailoring troubleshooting commands for the specific environment # and providing helpful documentation @@ -98,7 +99,7 @@ runwhenLocal: # verbs: ["get", "watch", "list"] service: type: ClusterIP - port: 8081 + port: 8000 ingress: enabled: false className: "" diff --git a/docs/.gitbook/assets/Kubernetes_logo_without_workmark.svg.png b/docs/.gitbook/assets/Kubernetes_logo_without_workmark.svg.png deleted file mode 100644 index d761bf187..000000000 Binary files a/docs/.gitbook/assets/Kubernetes_logo_without_workmark.svg.png and /dev/null differ diff --git a/docs/.gitbook/assets/PoC Flow Concepts-Automatic Workspace Configuration.drawio.png b/docs/.gitbook/assets/PoC Flow Concepts-Automatic Workspace Configuration.drawio.png deleted file mode 100644 index 25cf7d569..000000000 Binary files a/docs/.gitbook/assets/PoC Flow Concepts-Automatic Workspace Configuration.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/PoC Flow Concepts-Private Task Execution.drawio.png b/docs/.gitbook/assets/PoC Flow Concepts-Private Task Execution.drawio.png deleted file mode 100644 index cbf6a4b66..000000000 Binary files a/docs/.gitbook/assets/PoC Flow Concepts-Private Task Execution.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (1).png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (1).png deleted file mode 100644 index 1ade49087..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (2).png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (2).png deleted file mode 100644 index 1ade49087..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1) (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1).png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1).png deleted file mode 100644 index 37d14d9d2..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (2).png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (2).png deleted file mode 100644 index 37d14d9d2..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (3).png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (3).png deleted file mode 100644 index 37d14d9d2..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio (3).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio.png b/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio.png deleted file mode 100644 index 37d14d9d2..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-2.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (1).png b/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (1).png deleted file mode 100644 index ec6de4725..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio (1).png b/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio (1).png deleted file mode 100644 index 6b5dd4258..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio.png b/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio.png deleted file mode 100644 index e8a4fd5c4..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2) (1)-Page-4.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2).png b/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2).png deleted file mode 100644 index aee6773ba..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio.png b/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio.png deleted file mode 100644 index 9cb8b875d..000000000 Binary files a/docs/.gitbook/assets/Untitled Diagram-Page-3.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/flow (1).png b/docs/.gitbook/assets/flow (1).png deleted file mode 100644 index dd31c7765..000000000 Binary files a/docs/.gitbook/assets/flow (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/flow (2).png b/docs/.gitbook/assets/flow (2).png deleted file mode 100644 index 89ac1952b..000000000 Binary files a/docs/.gitbook/assets/flow (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/flow.png b/docs/.gitbook/assets/flow.png deleted file mode 100644 index c78597cc0..000000000 Binary files a/docs/.gitbook/assets/flow.png and /dev/null differ diff --git a/docs/.gitbook/assets/high-level.drawio.svg b/docs/.gitbook/assets/high-level.drawio.svg deleted file mode 100644 index f9e4a35ea..000000000 --- a/docs/.gitbook/assets/high-level.drawio.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Workspace Builder
(Indexers/CloudQuery/Enrichers)
Workspace Builder...
Cheat Sheet
Cheat Sheet
Open Source Troubleshooting Commands/Scripts
Open Source Troubleshooting Commands/Scripts
RunWhen  CodeCollections
RunWhen  CodeCollect...
Community  CodeCollections
Community  CodeColle...
User Clusters
User Clusters
User Clusters
User Clusters
cloudConfig
cloudConfig
workspaceInfo.yaml
workspaceInfo.ya...
User Provided Configuration
User Provided Configuration
RunWhen Local Container
RunWhen Local Container
mkdocs
mkdocs
markdown documentation
markdown documentati...
Rendered Content
Rendered Content
Customization Rules
Customization Ru...
Matching Rules
Matching Rules
3
3
1
1
2
2
Shared Volume / Folder 
Shared Volume / Folder 
4
4
5
5




In memory data model
In memory data model...
Output
Output
Configuration Files
Configuration Files
Repository References
Repository References
7
7
http://localhost:8081
http://l...
Kubernetes
Kubernetes
Cloud Resources
Cloud Resources
Cloud Resources
Cloud Resources
Microsoft Azure
Microsoft Azure
Google Cloud Platform
Google Cloud Platform
Generation Rules
Generation Rules
Generation Rules
Generation Rules
Generation Rule -> Resource Matching
Generation Rule...
6
6
Text is not SVG - cannot display
\ No newline at end of file diff --git a/docs/.gitbook/assets/image (1) (1) (1).png b/docs/.gitbook/assets/image (1) (1) (1).png deleted file mode 100644 index 689ab2dc3..000000000 Binary files a/docs/.gitbook/assets/image (1) (1) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (1) (1).png b/docs/.gitbook/assets/image (1) (1).png deleted file mode 100644 index 74aca7da6..000000000 Binary files a/docs/.gitbook/assets/image (1) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (1).png b/docs/.gitbook/assets/image (1).png deleted file mode 100644 index 988c6fd2c..000000000 Binary files a/docs/.gitbook/assets/image (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (10).png b/docs/.gitbook/assets/image (10).png deleted file mode 100644 index 29b3b4acb..000000000 Binary files a/docs/.gitbook/assets/image (10).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (11).png b/docs/.gitbook/assets/image (11).png deleted file mode 100644 index 45b313652..000000000 Binary files a/docs/.gitbook/assets/image (11).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (12).png b/docs/.gitbook/assets/image (12).png deleted file mode 100644 index 21560cd75..000000000 Binary files a/docs/.gitbook/assets/image (12).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (13).png b/docs/.gitbook/assets/image (13).png deleted file mode 100644 index 90e335026..000000000 Binary files a/docs/.gitbook/assets/image (13).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (14).png b/docs/.gitbook/assets/image (14).png deleted file mode 100644 index e3b5cf061..000000000 Binary files a/docs/.gitbook/assets/image (14).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (15).png b/docs/.gitbook/assets/image (15).png deleted file mode 100644 index 33c1476a0..000000000 Binary files a/docs/.gitbook/assets/image (15).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (16).png b/docs/.gitbook/assets/image (16).png deleted file mode 100644 index 65d4a39d6..000000000 Binary files a/docs/.gitbook/assets/image (16).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (17).png b/docs/.gitbook/assets/image (17).png deleted file mode 100644 index 7f983e8af..000000000 Binary files a/docs/.gitbook/assets/image (17).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (18).png b/docs/.gitbook/assets/image (18).png deleted file mode 100644 index 7f983e8af..000000000 Binary files a/docs/.gitbook/assets/image (18).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (19).png b/docs/.gitbook/assets/image (19).png deleted file mode 100644 index e6ad0b1c0..000000000 Binary files a/docs/.gitbook/assets/image (19).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (2) (1) (1).png b/docs/.gitbook/assets/image (2) (1) (1).png deleted file mode 100644 index ea6a43bae..000000000 Binary files a/docs/.gitbook/assets/image (2) (1) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (2) (1).png b/docs/.gitbook/assets/image (2) (1).png deleted file mode 100644 index 3dbbdc0bd..000000000 Binary files a/docs/.gitbook/assets/image (2) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (2).png b/docs/.gitbook/assets/image (2).png deleted file mode 100644 index 988c6fd2c..000000000 Binary files a/docs/.gitbook/assets/image (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (3) (1).png b/docs/.gitbook/assets/image (3) (1).png deleted file mode 100644 index 9ccc0f911..000000000 Binary files a/docs/.gitbook/assets/image (3) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (3).png b/docs/.gitbook/assets/image (3).png deleted file mode 100644 index 97b7ace94..000000000 Binary files a/docs/.gitbook/assets/image (3).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (4).png b/docs/.gitbook/assets/image (4).png deleted file mode 100644 index 5c271b7fc..000000000 Binary files a/docs/.gitbook/assets/image (4).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (5).png b/docs/.gitbook/assets/image (5).png deleted file mode 100644 index ed5b746e8..000000000 Binary files a/docs/.gitbook/assets/image (5).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (6).png b/docs/.gitbook/assets/image (6).png deleted file mode 100644 index 3f1f37131..000000000 Binary files a/docs/.gitbook/assets/image (6).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (7).png b/docs/.gitbook/assets/image (7).png deleted file mode 100644 index 73ae8ffdb..000000000 Binary files a/docs/.gitbook/assets/image (7).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (8).png b/docs/.gitbook/assets/image (8).png deleted file mode 100644 index 84f77fd91..000000000 Binary files a/docs/.gitbook/assets/image (8).png and /dev/null differ diff --git a/docs/.gitbook/assets/image (9).png b/docs/.gitbook/assets/image (9).png deleted file mode 100644 index 480b2ef91..000000000 Binary files a/docs/.gitbook/assets/image (9).png and /dev/null differ diff --git a/docs/.gitbook/assets/image.png b/docs/.gitbook/assets/image.png deleted file mode 100644 index a69cebcb6..000000000 Binary files a/docs/.gitbook/assets/image.png and /dev/null differ diff --git a/docs/.gitbook/assets/login.drawio (1) (1).png b/docs/.gitbook/assets/login.drawio (1) (1).png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login.drawio (1) (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/login.drawio (1).png b/docs/.gitbook/assets/login.drawio (1).png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login.drawio (1).png and /dev/null differ diff --git a/docs/.gitbook/assets/login.drawio (2).png b/docs/.gitbook/assets/login.drawio (2).png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login.drawio (2).png and /dev/null differ diff --git a/docs/.gitbook/assets/login.drawio (3).png b/docs/.gitbook/assets/login.drawio (3).png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login.drawio (3).png and /dev/null differ diff --git a/docs/.gitbook/assets/login.drawio.png b/docs/.gitbook/assets/login.drawio.png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login.drawio.png and /dev/null differ diff --git a/docs/.gitbook/assets/login_create_workspace.png b/docs/.gitbook/assets/login_create_workspace.png deleted file mode 100644 index 8025562b1..000000000 Binary files a/docs/.gitbook/assets/login_create_workspace.png and /dev/null differ diff --git a/docs/.gitbook/assets/terminal (1).gif b/docs/.gitbook/assets/terminal (1).gif deleted file mode 100644 index 0fd8597e3..000000000 Binary files a/docs/.gitbook/assets/terminal (1).gif and /dev/null differ diff --git a/docs/.gitbook/assets/terminal (2).gif b/docs/.gitbook/assets/terminal (2).gif deleted file mode 100644 index 0fd8597e3..000000000 Binary files a/docs/.gitbook/assets/terminal (2).gif and /dev/null differ diff --git a/docs/.gitbook/assets/terminal.gif b/docs/.gitbook/assets/terminal.gif deleted file mode 100644 index 0fd8597e3..000000000 Binary files a/docs/.gitbook/assets/terminal.gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload.gif b/docs/.gitbook/assets/upload.gif deleted file mode 100644 index bd225abe3..000000000 Binary files a/docs/.gitbook/assets/upload.gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload_1 (1).gif b/docs/.gitbook/assets/upload_1 (1).gif deleted file mode 100644 index 5301fc581..000000000 Binary files a/docs/.gitbook/assets/upload_1 (1).gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload_1 (2).gif b/docs/.gitbook/assets/upload_1 (2).gif deleted file mode 100644 index 5301fc581..000000000 Binary files a/docs/.gitbook/assets/upload_1 (2).gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload_1 (3).gif b/docs/.gitbook/assets/upload_1 (3).gif deleted file mode 100644 index 5301fc581..000000000 Binary files a/docs/.gitbook/assets/upload_1 (3).gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload_1.gif b/docs/.gitbook/assets/upload_1.gif deleted file mode 100644 index 5301fc581..000000000 Binary files a/docs/.gitbook/assets/upload_1.gif and /dev/null differ diff --git a/docs/.gitbook/assets/upload_1.webm b/docs/.gitbook/assets/upload_1.webm deleted file mode 100644 index e9cfb146a..000000000 Binary files a/docs/.gitbook/assets/upload_1.webm and /dev/null differ diff --git a/docs/README.md b/docs/README.md index d48cc0908..25eb25d02 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,126 +1,56 @@ -# Introduction - -[![Join Slack](https://img.shields.io/badge/Join%20Slack-%23E01563.svg?\&style=for-the-badge\&logo=slack\&logoColor=white)](https://runwhen.slack.com/join/shared\_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A)\ -![](https://github.com/runwhen-contrib/runwhen-local/actions/workflows/merge\_to\_main.yaml/badge.svg) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/runwhen-contrib)](https://artifacthub.io/packages/search?repo=runwhen-contrib) - -![RunWhen Local Overview](../assets/rw-local-product.png) - -* [Welcome to RunWhen Local!](./#welcome-to-runwhen-local) -* [Who Can Benefit?](./#who-can-benefit) -* [Where Can You See It in Action?](./#where-can-you-see-it-in-action) -* [How Can I Get It?](./#how-can-i-get-it) -* [How You Can Contribute](./#how-you-can-contribute) - * [Expanding the Troubleshooting Library](./#expanding-the-troubleshooting-library) - * [Contribute to Existing Libraries](./#contribute-to-existing-libraries) - * [Create Your Own Library](./#create-your-own-library) - * [Share New Commands or Enhance Existing Ones](./#share-new-commands-or-enhance-existing-ones) - * [Improving RunWhen Local](./#improving-runwhen-local) -* [Connect with the RunWhen Community](./#connect-with-the-runwhen-community) -* [Check Out Our Documentation](./#check-out-our-documentation) -* [Stay Updated with Release Notes](./#stay-updated-with-release-notes) - -### Welcome to RunWhen Local! - -Are you tired of searching through files and wikis for those elusive CLI commands that come in handy but always need tweaking? - -**RunWhen Local** is like your personal troubleshooter's toolbox. It's a friendly container that offers an easy-to-use web interface, filled with helpful copy & paste CLI commands specifically designed to troubleshoot applications in your Kubernetes environment. And guess what? It's open-source! Here's how it works: - -1. You launch the container. -2. It scans your clusters. -3. It identifies the perfect troubleshooting commands tailored to your resources. -4. You simply copy and paste the commands to help solve your issues! - -![](../assets/trouble-town-ingress.gif) - -### Can it Automatically Run These Commands? - -Yes, but only when connected to the RunWhen Platform SaaS service. RunWhen Local can be installed in Kubernetes, connected to the RunWhen Platform, with self-hosted private runners. In this configuration, your RunWhen Workspace is automatically updated with: - -* newly discovered cloud resources -* automation tasks -* troubleshooting tasks -* health checks -* workflows -* SLI and SLO alerting -* and much more! - -See [www.runwhen.com](https://www.runwhen.com) or [docs.runwhen.com](https://docs.runwhen.com) for more details, or jump to [kubernetes\_self\_hosted\_runner](installation/kubernetes\_self\_hosted\_runner/ "mention") for more details on this installation option. - -### Who Can Benefit? - -If you're involved with **Kubernetes** or **Public Cloud** environments, RunWhen Local could be for you. It's designed for: - -* Kubernetes administrators -* Kubernetes application developers -* Support teams working with Kubernetes -* Cloud infrastructure teams in GCP, AWS, or Azure -* Public cloud administrators managing workloads or services -* Anyone needing help troubleshooting cloud-based services - -### Where Can You See It in Action? - -Curious to see it in action? We've got a few options for you: - -* Check out our live demo instance [here](https://runwhen-local.sandbox.runwhen.com/). Please note that it's linked to our sandbox cluster, so the commands are suited for that environment. -* Explore our [YouTube Channel](https://www.youtube.com/@whatdoirunwhen) with short demo videos in this [playlist](https://www.youtube.com/playlist?list=PLq37As8dgg\_C0wFaPQLVUFQ79YiQjzHGU) -* Want to check out the RunWhen Platform? We have tutorials that you can play with right in our sandbox cluster. Click [here](https://docs.runwhen.com/public/runwhen-platform/tutorials). Sign in. Play. - -### How Can I Get It? - -Ready to dive in? [Run it yourself](https://docs.runwhen.com/public/v/runwhen-local/user-guide/getting-started). - -### How You Can Contribute - -We love when the community gets involved! There are two main ways you can contribute: - -#### Expanding the Troubleshooting Library - -**Contribute to Existing Libraries** - -Our troubleshooting library is fully open-source and welcomes contributions. You can contribute directly to any of the CodeCollection libraries hosted in our [Registry](https://registry.runwhen.com). - - - -For more details, check out the contribution guides within each repo. - -**Create Your Own Library** - -Interested in maintaining your own code collection and being rewarded for your efforts? Learn more about the [RunWhen Author Program](https://docs.runwhen.com/public/runwhen-authors/getting-started-with-codecollection-development). - - - -**Share New Commands or Enhance Existing Ones** - -Join the community by: - -* [Contributing an awesome troubleshooting command](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local%2Cawesome-command-contribution\&projects=\&template=awesome-command-contribution.yaml\&title=%5Bawesome-command-contribution%5D+) -* [Requesting a new command](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local%2Cnew-command-request\&projects=\&template=commands-wanted.yaml\&title=%5Bnew-command-request%5D+) -* Engaging in discussions on [GitHub Discussions](https://github.com/orgs/runwhen-contrib/discussions). - -#### Improving RunWhen Local - -Your ideas matter! Help us enhance the tool: - -* [Report bugs or share feedback](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local\&projects=\&template=runwhen-local-feedback.md\&title=%5Brunwhen-local-feedback%5D+) -* Want to make your own changes? [Read the CONTRIBUTING documentation](../CONTRIBUTING.md), [fork the repo](https://github.com/runwhen-contrib/runwhen-local/fork), and [explore the DEVELOPMENT documentation](DEVELOPMENT.md). - -### Connect with the RunWhen Community - -We're a friendly bunch! Connect with us on: - -* [Slack](https://runwhen.slack.com/join/shared\_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A) - -### Check Out Our Documentation - -All documentation is stored in [/docs](https://github.com/runwhen-contrib/runwhen-local/tree/main/docs), but is also rendered by GitBook [here](https://docs.runwhen.com/public/v/runwhen-local/). - -* [User Guide](https://docs.runwhen.com/public/v/runwhen-local/user-guide/) -* [Architecture](https://docs.runwhen.com/public/v/runwhen-local/architecture) -* [Development](https://docs.runwhen.com/public/v/runwhen-local/development/) -* [Kubernetes LOD Configuration](./kubernetes-lod-index.md) - Configure resource discovery across multiple clusters - -### Stay Updated with Release Notes - -Catch up on our latest updates in the [release notes](https://github.com/runwhen-contrib/runwhen-local/releases). - -Welcome to RunWhen Local – your go-to troubleshooter's companion! 🚀 +# RunWhen Local Documentation + +RunWhen Local is a self-hosted container that scans the cloud and Kubernetes +resources you care about and turns them into a curated list of copy-paste +runbooks (called **SLXs**) that draw from RunWhen's open-source CodeCollections. + +This `docs/` tree is split into three top-level buckets so you can find the +right kind of help quickly: + +``` +docs/ +├── user-guide/ # I want to deploy and operate RunWhen Local +├── authoring/ # I want to write CodeBundles, Skills, or generation rules +└── architecture/ # I want to understand how RunWhen Local works internally +``` + +## Quick links + +### Run RunWhen Local + +* [Getting started](./user-guide/getting-started.md) - 5-minute quickstart +* [Local Docker / Podman install](./user-guide/installation/local-docker.md) +* [Kubernetes standalone install](./user-guide/installation/kubernetes-standalone.md) +* [`workspaceInfo.yaml` reference](./user-guide/configuration/workspace-info.md) +* [Cloud discovery: Azure](./user-guide/cloud-discovery/azure.md) / + [AWS](./user-guide/cloud-discovery/aws.md) / + [GCP](./user-guide/cloud-discovery/gcp.md) / + [Kubernetes](./user-guide/cloud-discovery/kubernetes.md) +* [Agentic access: built-in MCP server](./user-guide/features/mcp-server.md) - + point Claude Code, Cursor, or Claude Desktop at your indexed Skills +* [Troubleshooting](./user-guide/troubleshooting/stuck.md) + +### Author CodeBundles, Skills, and generation rules + +* [Concepts: CodeBundle / Skill / SLX / Runbook](./authoring/concepts.md) +* [Indexed resources reference](./authoring/indexed-resources/README.md) - + what RunWhen Local can discover, and the shape of the data your generation + rules will see +* [Generation rules: schema, lifecycle, and examples](./authoring/generation-rules/README.md) + +### Internals + +* [High-level architecture](./architecture/overview.md) +* [Self-hosted runner architecture](./architecture/high-level-architecture-self-hosted-runner-connected.md) +* [Resource store query API](./architecture/resource-store-query-api.md) +* [ResourceWriter](./architecture/resource-writer.md) +* [MCP server design](./architecture/mcp-server.md) +* [Kubernetes Level-of-Detail internals](./architecture/kubernetes-lod/README.md) +* [Workspace generation statistics](./architecture/workspace-generation-statistics.md) +* [Container development guide](./architecture/development.md) + +## Project links + +* GitHub: [runwhen-contrib/runwhen-local](https://github.com/runwhen-contrib/runwhen-local) +* Slack: [join the RunWhen workspace](https://runwhen.slack.com/join/shared_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A) +* Hosted platform docs: [docs.runwhen.com](https://docs.runwhen.com) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md deleted file mode 100644 index 190109139..000000000 --- a/docs/SUMMARY.md +++ /dev/null @@ -1,60 +0,0 @@ -# Table of contents - -* [Introduction](README.md) - -## User Guide - -* [Getting Started](user-guide/getting-started.md) -* [Features](user-guide/features/README.md) - * [Workspace Builder](user-guide/features/workspace-builder.md) - * [Troubleshooting Cheat Sheet](user-guide/features/user\_guide-feature\_overview.md) - * [Upload to RunWhen Platform](user-guide/features/upload-to-runwhen-platform.md) -* [Stuck?Read This](User\_Guide-Stuck\_Read\_This.md) -* [Privacy & Security](User\_Guide-Privacy\_and\_Security.md) -* [Release Notes](User\_Guide-Release\_Notes.md) - -## Installation - -* [Kubernetes Self-Hosted Runner (Connected)](https://docs.runwhen.com/public/getting-started/getting-started-with-runwhen-platform) - * [Runner Network Requirements](https://docs.runwhen.com/public/getting-started/getting-started-with-runwhen-platform#detailed-network-requirements) -* [Kubernetes Installation (Disconnected)](installation/kubernetes\_standalone.md) -* [Docker/Podman Installation (Disconnected)](installation/getting\_started-running\_locally.md) - -## Cloud Discovery Configuration - -* [Amazon Web Services](cloud-discovery-configuration/google-cloud-platform.md) -* [Microsoft Azure](cloud-discovery-configuration/microsoft-azure.md) -* [Google Cloud Platform](cloud-discovery-configuration/google-cloud-platform-1.md) -* [Kubernetes Configuration](cloud-discovery-configuration/kubernetes-configuration.md) - -## Configuration - -* [WorkspaceInfo Customization](configuration/workspaceinfo-customization.md) -* [File Watching Configuration](configuration/file-watching-configuration.md) -* [CodeCollection Configuration](configuration/codecollection-configuration.md) -* [Group / Map Customizations](configuration/user\_reference.md) -* [Cheat Sheet Features](configuration/cheat-sheet-features/README.md) - * [Terminal Configuration](configuration/cheat-sheet-features/terminal-configuration.md) -* [Helm Configuration](configuration/helm-configuration.md) -* [Discovery Level of Detail](configuration/level-of-detail.md) -* [Proxy Configuration & Outbound Connections](configuration/proxy-configuration-and-outbound-connections.md) -* [Using a Private Container Registry](configuration/using-a-private-container-registry.md) - -## Architecture​ - -* [High Level Architecture (Disconnected)](Architecture.md) -* [High Level Architecture - Self-Hosted Runner (Connected)](architecture/high-level-architecture-self-hosted-runner-connected.md) - -## Development - -* [Container Development](Development.md) - -## Community - -* [Slack](community/discord-slack-chat.md) -* [GitHub Discussions](community/github-discussions.md) - -*** - -* [GitHub Issues - Request or Share Commands, Report Bugs, Request Features](github-issues-request-or-share-commands-report-bugs-request-features.md) -* [Roadmap](roadmap.md) diff --git a/docs/User_Guide-Introduction.md b/docs/User_Guide-Introduction.md deleted file mode 100644 index 28d1df6f3..000000000 --- a/docs/User_Guide-Introduction.md +++ /dev/null @@ -1,58 +0,0 @@ -# Introduction - -

- -## What Is it? - -We all have complex CLI commands saved in a file or wiki somewhere that are helpful, but constantly need to be adapted, or are difficult to find when we need them. - -RunWhen Local is a container that provides a searchable web interface that provides helpful copy & paste CLI commands for troubleshooting apps deployed to **your** Kubernetes environment from an open source community of DevOps/Platform/SRE engineers. Oh yeah, and it's [FOSS](https://en.wikipedia.org/wiki/Free\_and\_open-source\_software). - -* You run the container -* It scans your clusters -* It finds the right troubleshooting commands that match your resources -* And delivers troubleshooting commands that you can copy and paste - -## Who is it for? - -Anyone that is working on, or in, Kubernetes environments might find RunWhen Local helpful. - -* Kubernetes administrators -* Kubernetes application developers -* L1/L2/L3 Support teams that manage Kubernetes environments - -## Where can I see it? - -There are a few ways to see if this is interesting to you: - -* Checkout out our _live demo instance_ here: [https://runwhen-local.sandbox.runwhen.com/](https://runwhen-local.sandbox.runwhen.com/) - * Note: It's pointed at our sandbox cluster, and so the commands are tailored for that environment. -* Check out our [YouTube Channel](https://www.youtube.com/@whatdoirunwhen) with **short demo videos** -* Run it yourself :point\_down: - -## How can I run my own version? - -Jump right over to [running-locally.md](getting-started/running-locally.md "mention") or [running-in-kubernetes.md](getting-started/running-in-kubernetes.md "mention") to deploy your own instance of RunWhen Local. - - - -### Want to Contribute? Have an awesome troubleshooting command to share? - -If you want to get involved in the community: - -* [Report bugs or share feedback](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local\&projects=\&template=runwhen-local-feedback.md\&title=%5Brunwhen-local-feedback%5D+) -* [Contribute and awesome troubleshooting command](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local%2Cawesome-command-contribution\&projects=\&template=awesome-command-contribution.yaml\&title=%5Bawesome-command-contribution%5D+) -* [Need a new command? Ask the community for help](https://github.com/runwhen-contrib/runwhen-local/issues/new?assignees=stewartshea\&labels=runwhen-local%2Cnew-command-request\&projects=\&template=commands-wanted.yaml\&title=%5Bnew-command-request%5D+) -* [Get involved in GitHub Discussion](https://github.com/orgs/runwhen-contrib/discussions)s - - - -### Connect with Us - -Want to connect with the RunWhen community, [join us on slack](https://runwhen.slack.com/join/shared\_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A) or [Discord](https://discord.com/invite/Ut7Ws4rm8Q). - - - -## What is the future of RunWhen Local? - -RunWhen Local will be officially FOSS in the next few sprints (e.g. open source licened, better contributor guides, and the main code base added to the repo). It will continue to become more useful as the open source community of troubleshooting contributions grows and as we add more indexing capabilities (such as discovering more Kubernetes resources, or cloud resources from AWS, GCP, and Azure). \ No newline at end of file diff --git a/docs/architecture/README.md b/docs/architecture/README.md new file mode 100644 index 000000000..8956631f4 --- /dev/null +++ b/docs/architecture/README.md @@ -0,0 +1,49 @@ +# Architecture + +Engineering-facing internals of RunWhen Local: how the workspace-builder +service is put together, how each indexer pulls cloud state, how the +resource store works, and how the rendered workspace is assembled. + +If you want to *use* RunWhen Local, see the [user guide](../user-guide/README.md). +If you want to *extend* it (CodeBundles, Skills, generation rules), see the +[authoring guide](../authoring/README.md). + +## Overall shape + +* [High-level architecture (disconnected)](./overview.md) +* [Self-hosted runner architecture (connected)](./high-level-architecture-self-hosted-runner-connected.md) + +## Resource store + +The intermediate datastore between indexers and renderers. + +* [ResourceWriter](./resource-writer.md) - the indexer-facing write API +* [Resource store query API](./resource-store-query-api.md) - the + enricher / generation-rule-facing read API + +## Indexers + +Per-platform deep dives. + +* [Azure indexer internals](./azure-indexer-internals.md) +* [GCP indexer internals](./gcp-indexer-internals.md) +* [AWS indexer internals](./aws-indexer-internals.md) +* [Kubernetes Level-of-Detail internals](./kubernetes-lod/README.md) + * [Configuration internals](./kubernetes-lod/configuration.md) + * [Decision flowchart](./kubernetes-lod/flowchart.md) + * [Quick reference](./kubernetes-lod/quick-reference.md) + * [AKS namespace LODs (Azure-specific addendum)](./kubernetes-lod/aks-namespace-lods.md) + +## Workspace assembly + +* [Workspace generation statistics](./workspace-generation-statistics.md) + +## Agentic surface + +* [MCP server design](./mcp-server.md) - the read-only Model Context + Protocol server that exposes the indexed resources and rendered + Skills to AI agents + +## Development + +* [Container development guide](./development.md) diff --git a/docs/architecture/aws-indexer-internals.md b/docs/architecture/aws-indexer-internals.md new file mode 100644 index 000000000..4beeb1075 --- /dev/null +++ b/docs/architecture/aws-indexer-internals.md @@ -0,0 +1,177 @@ +# AWS indexer internals + +Engineering-level reference for the native AWS SDK indexer (`awsapi`). For the +user-facing list of supported types and data shapes, see +[indexed-resources/aws.md](../authoring/indexed-resources/aws.md). + +## Why a second indexer? + +Historically RunWhen Local discovered AWS resources via the CloudQuery AWS +plugin (`indexers/cloudquery.py`). That works, but it pulls a heavy Go binary, +writes to an intermediate SQLite, and bundles its own AWS API coverage. The +native indexer (`indexers/awsapi.py`) was added so RunWhen Local can: + +* Discover only what `workspaceInfo.yaml` actually asks for (per-account scope + + generation-rule-driven type selection). +* Use first-party AWS Cloud Control API + `boto3` SDKs and skip the CloudQuery + intermediate. +* Add new resource types incrementally without bumping a separate CloudQuery + plugin version. + +The end goal is to remove CloudQuery entirely once Azure, GCP, and AWS all have +native indexers at parity (AWS is the last of the three). Both indexers live +in-tree and share the same downstream pipeline; the toggle is +`awsIndexerBackend: cloudquery|awsapi` in `workspaceInfo.yaml` (or +`WB_AWS_INDEXER_BACKEND`). + +## Cloud Control is the parity workhorse + +Like GCP's Cloud Asset Inventory, AWS's **Cloud Control API** (`cloudcontrol` +boto3 client, `list_resources` / `get_resource`) is a single broad API that +enumerates hundreds of resource types by their CloudFormation type name. A +`list_resources` call returns each resource's full CFN-schema `Properties` +(a JSON blob), so one call per (account, region, CFN type) covers every +registry type that has a CFN type, with rich payloads. Typed `boto3` service +collectors are a thin enrichment layer for a handful of high-value resources +plus the synthesized `account` anchor. + +The join key is the **CloudFormation resource type name** +(`AWS::::`, e.g. `AWS::EC2::Instance`, `AWS::S3::Bucket`) — the +AWS analogue of Azure's ARM type and GCP's CAI asset type. + +> **Coverage note.** Cloud Control does not cover every CloudQuery table. Tables +> with no Cloud Control type (cost/usage reports, metric rollups, inventory +> sub-resources, the synthesized account anchor) are pinned to `null` in the +> overrides; the generic pass skips them, exactly like GCP's `null` CAI types. +> They still resolve via `find_spec` for gen-rule parity and would need a +> dedicated typed collector if ever referenced. + +## Pieces + +``` +src/indexers/ +├── awsapi.py # the orchestration loop +├── awsapi_resource_types.py # Cloud Control generic collector + typed SDK collectors + specs +├── awsapi_normalizers.py # Cloud Control resource / SDK payload -> CQ-shaped dict +├── aws_common.py # credential + account/region resolution, tag filters +├── aws_resource_type_registry.py # registry loader (data class) +├── aws_resource_type_registry.yaml # GENERATED catalog of all CQ tables +└── test_aws*.py / test_awsapi*.py # unit tests + +scripts/aws/ +├── sync_aws_resource_type_registry.py # registry generator +├── aws_resource_type_overrides.yaml # hand-edited overrides +└── aws_cloudquery_tables.txt # parity source (CloudQuery hub table list) +``` + +### `awsapi.py` - the orchestration loop + +`index(ctx)` runs in phases: + +1. **Bootstrap**: resolves a `boto3.Session` + scope (account + regions) via + `aws_common.aws_get_session_and_scope` (explicit keys, K8s secret, IRSA / Pod + Identity, assume-role, or the default credential chain), mirrors the + credentials into `enrichers.aws`, and publishes the `{account_id: + account_name}` map the handler reads. The account's effective LOD + (`accountLevelOfDetails[]` -> workspace default) gates discovery: + an account whose LOD is `NONE` is skipped entirely. AWS LOD is + **per-account**; the region list is the second scope dimension. +2. **Phase 0 - Account anchor**: the `account` resource (`aws_iam_accounts`) is + synthesized from the resolved credentials (no API call needed; it gets an + `arn:aws:iam:::root` ARN and `global` region) and written *first* + so child resources are scoped under it. +3. **Phase 1 - Typed pass**: for each accessed type that has a hand-written + `boto3` collector (EC2 instances, S3 buckets), call + `spec.collector(session, account_id, region)`. Regional collectors run once + per region; global collectors (S3) run once. Rich SDK payloads land in the + resource store. +4. **Phase 2 - Cloud Control generic pass**: one `list_resources` call per + (region, CFN type) referenced by accessed *generic* types (typed types are + excluded so nothing is written twice). Each returned resource is routed by + its CFN type through `find_spec_by_cfn_type` back to the registry-mapped + `resource_type_name`. + +Every resource flows through: normalize -> tag filter -> handler +`parse_resource_data` -> `writer.add_resource`. A global resource listed from +multiple regional endpoints is de-duplicated by `(resource_type_name, arn)`. The +indexer is generation-rule-driven: only types referenced by loaded gen rules are +collected, plus the mandatory `aws_iam_accounts` anchor. + +### Normalization (CloudQuery shape parity) + +`AWSPlatformHandler.parse_resource_data` (in `enrichers/aws.py`, shared with the +CloudQuery path) requires an **ARN** and reads `name`, `account_id`, `region`, +and `tags`. `awsapi_normalizers` produces exactly that shape from either source: + +* `normalize_cloudcontrol_resource` parses the JSON `Properties` blob of a + Cloud Control `ResourceDescription`, hoists it to the top level, and stamps + the handler-read fields. +* `normalize_aws_resource` does the same for a typed boto3 payload. +* AWS tags (a list of `{"Key","Value"}` pairs, or a dict) collapse to a flat + `{key: value}` dict so cross-cloud include/exclude tag matchers work + unchanged. +* When a payload carries no ARN, a structurally-valid one is synthesized from + the CFN type + scope (`arn:aws::::/`) + because the handler parses the ARN for account/region/service. + +Because both backends normalize into the same dict, generation rules reference +the **CloudQuery table name** as `resource_type` and don't change when the +backend flips. + +### The registry (`aws_resource_type_registry.yaml`) + +Generated by `scripts/aws/sync_aws_resource_type_registry.py` from three inputs: + +1. The CloudQuery AWS table list (`scripts/aws/aws_cloudquery_tables.txt`, + currently the v33.26.0 hub snapshot, 1119 tables) — the parity source. +2. A heuristic mapping `aws__` -> + `AWS::::` (singularises + PascalCases the + entity; maps service tokens to CFN service segments, e.g. `ec2` -> `EC2`, + `s3` -> `S3`). +3. Hand-curated overrides in `scripts/aws/aws_resource_type_overrides.yaml` + (service-segment remaps, pinned CFN types for irregular tables — e.g. + `aws_rds_instances` -> `AWS::RDS::DBInstance` — aliases, typed-collector + flags, the mandatory `aws_iam_accounts` anchor, and `null` CFN types for + tables with no Cloud Control equivalent so the generic pass skips them). + +Never hand-edit the YAML; edit the overrides and re-run the sync script. +`python scripts/aws/sync_aws_resource_type_registry.py --dry-run` reports drift. + +### Coverage parity vs. CloudQuery + +Every one of the 1119 CloudQuery tables resolves via `find_spec` (by canonical +table name or alias). Tables with a CFN type are discoverable via the Cloud +Control pass the moment a generation rule references them; the typed tables get +richer SDK payloads. Tables with no Cloud Control equivalent map to `null` and +would need a dedicated typed collector if ever referenced — the same +incremental model the Azure and GCP indexers use. + +## Authentication + +`aws_common.aws_get_session_and_scope` reuses `aws_utils.get_aws_credential`, +which resolves, in order: explicit access keys in `workspaceInfo.yaml`, +credentials from a Kubernetes secret, IRSA / Pod Identity, an assumed role, or +the default credential chain (instance profile / environment). It returns a +`boto3.Session` for the Cloud Control + typed service clients, the resolved +account id / alias / human-readable name, and the region list (from `regions`, +`region`/`defaultRegion`, the credential's region, `AWS_REGION` / +`AWS_DEFAULT_REGION`, or `us-east-1` as a last resort). + +## Adding a typed collector + +1. Ensure the table is in the registry (regenerate if needed) and flag it as a + `typed_collector` in the overrides YAML. +2. Implement `_collect_(session, account_id, region)` in + `awsapi_resource_types.py` (lazy-construct the `boto3` service client from + the passed-in `session`) and register it in `_TYPED_COLLECTORS` keyed by + canonical table name, with its regional/global flag. +3. Its CFN type is automatically excluded from the generic pass so the resource + is written once, from the richer SDK payload. + +## Tests + +* `test_aws_resource_type_registry.py` — loader contract + spec materialization. +* `test_awsapi_normalizers.py` — Cloud Control / SDK normalization + handler + round-trip. +* `test_awsapi_selective.py` — per-account selective discovery + Cloud Control + filter dispatch. diff --git a/docs/architecture/azure-indexer-internals.md b/docs/architecture/azure-indexer-internals.md new file mode 100644 index 000000000..c037639f6 --- /dev/null +++ b/docs/architecture/azure-indexer-internals.md @@ -0,0 +1,263 @@ +# Azure indexer internals + +Engineering-level reference for the native Azure SDK indexer (`azureapi`). +For the user-facing list of supported types and data shapes, see +[indexed-resources/azure.md](../authoring/indexed-resources/azure.md). + +## Why a second indexer? + +Historically RunWhen Local discovered Azure resources via the CloudQuery +Azure plugin (`indexers/cloudquery.py`). That works, but it pulls a +heavy Go binary, writes to an intermediate SQLite, and has a fixed scope +(everything-or-nothing per subscription). The native indexer +(`indexers/azureapi.py`) was added so RunWhen Local can: + +* Discover only what `workspaceInfo.yaml` actually asks for (selective + per-RG enumeration), saving Azure throttling budget. +* Use first-party `azure-mgmt-*` SDKs and skip the CloudQuery + intermediate. +* Add new resource types incrementally without bumping a separate + CloudQuery plugin version. + +Both indexers live in-tree and share the same downstream pipeline; the +toggle is `azureIndexerBackend: cloudquery|azureapi` in +`workspaceInfo.yaml`. + +## Pieces + +``` +src/indexers/ +├── azureapi.py # the orchestration loop +├── azureapi_resource_types.py # 25 typed collectors + dispatch dict +├── azureapi_normalizers.py # SDK model -> CQ-shaped dict +├── azure_common.py # credential resolution +├── azure_resource_type_registry.py # registry loader (data class) +├── azure_resource_type_registry.yaml # GENERATED catalog of all CQ tables +└── test_azureapi_*.py # unit tests + +scripts/azure/ +├── sync_azure_resource_type_registry.py # registry generator +└── azure_resource_type_overrides.yaml # hand-edited overrides +``` + +### `azureapi.py` - the orchestration loop + +`index(ctx)` runs in three phases: + +1. **Bootstrap**: resolves credentials via + `azure_common.az_get_credentials_and_subscription_id` (Service + Principal or `DefaultAzureCredential`), builds the per-RG LOD map, + decides per-subscription whether discovery is **selective** (finite + RG list) or **subscription-wide** (`None`). +2. **Phase 1 - Resource groups**: enumerates RGs subscription-wide via + the typed `_collect_resource_groups_all` collector. RGs are mandatory + because every other phase keys off the RG list to compute scope. +3. **Phase 2 - Typed pass**: for every typed spec referenced by a loaded + generation rule (or otherwise mandatory), calls either + `spec.collector_in_rg(...)` (selective) or `spec.collector_all(...)` + (unbounded). Rich SDK payloads land in the resource store. +4. **Phase 3 - Generic pass**: one + `ResourceManagementClient.resources.list[_by_resource_group]` call + per subscription / in-scope RG returns every top-level ARM resource + the credential can see. Each `GenericResource` is routed through the + registry by ARM type to the spec that owns its `resource_type_name`, + filtered against: + * ARM types already owned by the typed pass (skip - typed wins). + * ARM types not referenced by any loaded generation rule (skip - + the indexer is gen-rule-driven by design). + +Every resource (typed or generic) flows through `_process_models`: + +* `normalize_azure_resource(model, subscription_id)` flattens to a + CloudQuery-shaped dict. +* `_resource_is_in_scope` drops out-of-scope rows (effective LOD = + NONE) before they reach the writer. +* On the generic pass only, the dispatcher consults a `typed_arm_ids` + set so a typed-emitted resource never gets overwritten by a sparser + generic copy. +* The ResourceWriter writes the normalized dict + parsed metadata. + +### `azureapi_resource_types.py` - typed + generic collectors + +This module owns three things: + +1. **Typed collectors** - one pair per rich-payload type: + + ```python + def _collect__all(credential, subscription_id): + client = SomeMgmtClient(credential, subscription_id) + return client..list() # subscription-wide + + def _collect__in_rg(credential, subscription_id, rg_name): + client = SomeMgmtClient(credential, subscription_id) + return client..list_by_resource_group(resource_group_name=rg_name) + ``` + + The pair is registered in `_TYPED_COLLECTORS` keyed by the canonical + CQ table name. The result is a typed `AzureResourceTypeSpec` + (`spec.typed = True`). + +2. **Generic catch-all collectors** - + `_collect_generic_resources_all` and `_in_rg`. These wrap + `ResourceManagementClient.resources.list[_by_resource_group]` and + are reused by the synthetic generic spec the registry materializes + for every untyped entry (`spec.typed = False`). The indexer's + Phase 3 routes by ARM type rather than calling these per-spec, so + only one generic call fires per subscription / RG. + +3. **Spec materialization** - `_build_specs()` walks the registry and + produces: + * Typed specs first (resource groups, then the rest of the + `_TYPED_COLLECTORS` dictionary). + * One generic spec per remaining registry entry, so every + registered type is addressable by `find_spec`. + +Lookups: `find_spec(name)` returns the spec for any registry name or +alias (typed or generic). `find_spec_by_arm_type(arm_type)` is what +Phase 3 uses to route each `GenericResource.type` back to the owning +spec. + +The same module hosts a small helper, `_rg_from_arm_id`, used by the +two **child-resource collectors** (`azure_postgresql_databases`, +`azure_cosmos_sql_databases`) that have to walk parent → child: + +```python +def _collect_postgresql_databases_all(credential, subscription_id): + client = PostgreSQLManagementClient(credential, subscription_id) + for server in client.servers.list(): + rg = _rg_from_arm_id(server.id) + try: + yield from client.databases.list_by_server(rg, server.name) + except Exception: + continue +``` + +Per-server failures are swallowed so a single misconfigured server +doesn't abort the whole subscription. + +### `azureapi_normalizers.py` - shape parity with CloudQuery + +The downstream pipeline (parsers, generation rules, the resource store) +was originally written against CloudQuery's table-row shape. To avoid +bifurcating that code, the native indexer normalizes every SDK model +into the same shape via `normalize_azure_resource`: + +* Top-level keys are snake-cased (`resource_id`, `subscription_id`, ...). +* `properties` / `sku` / `identity` are preserved verbatim. +* `tags` is always a dict (defaulting to `{}` when absent). +* `subscription_id` is always set. + +The contract is pinned by `test_azureapi_normalizers.py`, which feeds +SDK-shaped fakes through the normalizer and asserts that +`AzurePlatformHandler.parse_resource_data` returns the same +`(name, qualified_name, attributes)` triple it would have for the +CloudQuery row. + +### Registry + overrides + +`azure_resource_type_registry.yaml` is a GENERATED catalog of every +CloudQuery Azure plugin table (619 entries today). Each entry has: + +```yaml +azure_keyvault_keyvaults: + arm_type: Microsoft.KeyVault/vaults + arm_type_source: override # heuristic | override + category: keyvault + aliases: [azure_keyvault_vaults, azure_keyvault_keyvault] + typed_collector: true # set if listed in overrides.typed_collectors + mandatory: false +``` + +`scripts/azure/azure_resource_type_overrides.yaml` is the hand-edited +input. The sync script (`sync_azure_resource_type_registry.py`) merges: + +1. The current list of CQ table names (defaults to round-tripping the + existing registry; can also fetch from CloudQuery's hub). +2. A heuristic that converts `azure__` into + `Microsoft./`. +3. Manual overrides (`arm_type_overrides`, `aliases`, + `typed_collectors`, `mandatory`). + +**Always edit the overrides YAML, then re-run the sync script.** Never +hand-edit `azure_resource_type_registry.yaml`. + +## Adding a new typed collector + +The mechanical recipe (the same one used to add the 18 collectors in +the Bucket A/B/C/D pass): + +1. **Implement the two functions** in `azureapi_resource_types.py`: + + ```python + def _collect__all(credential, subscription_id): + from azure.mgmt. import + client = (credential, subscription_id) + return client..list() + + def _collect__in_rg(credential, subscription_id, rg_name): + from azure.mgmt. import + client = (credential, subscription_id) + return client..list_by_resource_group(resource_group_name=rg_name) + ``` + +2. **Register in `_TYPED_COLLECTORS`** keyed by the canonical CQ table + name. If no per-RG variant exists, pass `None` for the second slot; + the orchestrator will fall back to `collector_all` and warn. + +3. **Edit `scripts/azure/azure_resource_type_overrides.yaml`**: + * Append the table name to `typed_collectors:`. + * If the heuristic ARM type is wrong, add an entry to + `arm_type_overrides:`. + +4. **Regenerate the registry**: + ```bash + python scripts/azure/sync_azure_resource_type_registry.py + ``` + +5. **Update the registry test set** in + `src/indexers/test_azure_resource_type_registry.py::test_typed_collectors_present`. + +6. **Bump dependencies if needed**: if the SDK isn't already a project + dep, add it to `src/pyproject.toml` and regenerate `src/poetry.lock` + inside a `python:3.14-slim` container: + ```bash + docker run --rm -v "$PWD/src:/app" -w /app python:3.14-slim \ + bash -lc "pip install poetry && poetry lock" + ``` + +7. **Run the unit tests**: + ```bash + cd src && python -m pytest indexers/test_azureapi_*.py \ + indexers/test_azure_resource_type_registry.py + ``` + +8. **Document the new type** in + [`docs/authoring/indexed-resources/azure.md`](../authoring/indexed-resources/azure.md). + +## Selective vs unbounded discovery (in code) + +`_rgs_in_scope_from_config` returns: + +* `None` -> unbounded discovery for this subscription. The orchestrator + uses `collector_all`. +* A list of RG names -> selective. Empty list means "discover nothing + for this subscription except the RG list itself" (zero SDK calls + beyond the RG enumeration). + +The decision is made once per subscription, at the top of the indexer's +inner loop. Selective mode is triggered iff *every* escape hatch +resolves to `LevelOfDetail.NONE`: + +* Workspace-wide `defaultLOD` is `none`. +* Per-subscription `defaultLOD` is `none`. +* No `*` wildcard override under + `subscriptions[].resourceGroupLevelOfDetails`. + +Anything else (a single non-`none` default anywhere) flips to unbounded +mode for safety. + +## See also + +* [User guide: Azure cloud discovery](../user-guide/cloud-discovery/azure.md) +* [Indexed resources: Azure reference](../authoring/indexed-resources/azure.md) +* [Resource store / ResourceWriter](./resource-writer.md) diff --git a/docs/Development.md b/docs/architecture/development.md similarity index 94% rename from docs/Development.md rename to docs/architecture/development.md index 0867c3b83..891dd21e3 100644 --- a/docs/Development.md +++ b/docs/architecture/development.md @@ -29,7 +29,7 @@ In order to test the image, a valid `kubeconfig` and `workspaceInfo.yaml` file m * Testing with built in generation and customization rules ``` -docker run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared -d runwhen-local:test +docker run --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared -d runwhen-local:test docker exec -w /workspace-builder -- RunWhenLocal ./run.sh ``` @@ -37,7 +37,7 @@ docker exec -w /workspace-builder -- RunWhenLocal ./run.sh ``` git_repo_path=[FULL PATH TO GIT REPO BASE] -cd $workdir; docker run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared -v $git_repo_path/src/generation-rules:/workspace-builder/generation-rules -v $git_repo_path/src/templates:/workspace-builder/templates -v $git_repo_path/src/map-customization-rules:/workspace-builder/map-customization-rules -d runwhen-local:test +cd $workdir; docker run --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared -v $git_repo_path/src/generation-rules:/workspace-builder/generation-rules -v $git_repo_path/src/templates:/workspace-builder/templates -v $git_repo_path/src/map-customization-rules:/workspace-builder/map-customization-rules -d runwhen-local:test ``` ### Quick test script with verbose output @@ -59,7 +59,7 @@ if [[ "$workdir" ]];then echo "rebuild image" docker build -t runwhen-local:test -f ../runwhen-local/src/Dockerfile ../runwhen-local/src/ echo "Running RunWhenLocal container" - docker run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared -d runwhen-local:test + docker run --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared -d runwhen-local:test sleep 5 echo "Running discovery" docker exec -w /workspace-builder -- RunWhenLocal ./run.sh --verbose diff --git a/docs/architecture/gcp-indexer-internals.md b/docs/architecture/gcp-indexer-internals.md new file mode 100644 index 000000000..2d86afab9 --- /dev/null +++ b/docs/architecture/gcp-indexer-internals.md @@ -0,0 +1,178 @@ +# GCP indexer internals + +Engineering-level reference for the native GCP SDK indexer (`gcpapi`). For the +user-facing list of supported types and data shapes, see +[indexed-resources/gcp.md](../authoring/indexed-resources/gcp.md). + +## Why a second indexer? + +Historically RunWhen Local discovered GCP resources via the CloudQuery GCP +plugin (`indexers/cloudquery.py`). That works, but it pulls a heavy Go binary, +writes to an intermediate SQLite, and shells out to `gcloud`. The native +indexer (`indexers/gcpapi.py`) was added so RunWhen Local can: + +* Discover only what `workspaceInfo.yaml` actually asks for (per-project scope + + generation-rule-driven type selection). +* Use first-party Cloud Asset Inventory + `google-cloud-*` SDKs and skip the + CloudQuery intermediate. +* Add new resource types incrementally without bumping a separate CloudQuery + plugin version. + +The end goal is to remove CloudQuery entirely once Azure, GCP, and AWS all have +native indexers at parity. Both indexers live in-tree and share the same +downstream pipeline; the toggle is `gcpIndexerBackend: cloudquery|gcpapi` in +`workspaceInfo.yaml` (or `WB_GCP_INDEXER_BACKEND`). + +## Typed collectors are the baseline; CAI is an optional accelerator + +The native GCP indexer's supported **functional baseline** is the per-service +typed `google-cloud-*` SDK collectors (plus the synthesized `project` anchor). +Any type with a typed collector is discovered using only its per-service viewer +role, whether or not Cloud Asset Inventory is available. + +**Cloud Asset Inventory** (CAI) is an **optional accelerator** layered on top. +The key difference from Azure: Azure's generic `resources.list()` returns only a +sparse envelope (no `properties`), so Azure needs many hand-written typed +collectors for rich payloads. GCP's CAI +`list_assets(..., content_type=RESOURCE)` returns the *full* API representation +of each asset under `resource.data`, so a single CAI call per project broadens +coverage to the long tail of registry types that have a CAI asset type but no +typed collector. CAI is **not a hard dependency**: if it is not enabled or not +permitted, the typed baseline still discovers normally and the CAI-only types +are simply skipped. + +The join key is the **CAI asset type** (`.googleapis.com/`, +e.g. `compute.googleapis.com/Instance`) — the GCP analogue of Azure's ARM type. + +## Pieces + +``` +src/indexers/ +├── gcpapi.py # the orchestration loop +├── gcpapi_resource_types.py # CAI generic collector + typed SDK collectors + specs +├── gcpapi_normalizers.py # CAI asset / SDK model -> CQ-shaped dict +├── gcp_common.py # credential + project resolution, label filters +├── gcp_resource_type_registry.py # registry loader (data class) +├── gcp_resource_type_registry.yaml # GENERATED catalog of all CQ tables +└── test_gcp*.py # unit tests + +scripts/gcp/ +├── sync_gcp_resource_type_registry.py # registry generator +├── gcp_resource_type_overrides.yaml # hand-edited overrides +└── gcp_cloudquery_tables.txt # parity source (CloudQuery hub table list) +``` + +### `gcpapi.py` - the orchestration loop + +`index(ctx)` runs in phases: + +1. **Bootstrap**: resolves credentials + project list via + `gcp_common.gcp_get_credentials_and_projects` (service-account key, K8s + secret, or Application Default Credentials), mirrors them into + `enrichers.gcp`, and resolves which projects are in scope. A project whose + effective LOD (`projectLevelOfDetails[]` -> workspace default) is `NONE` + is skipped entirely — no anchor, no typed pass, no CAI pass. GCP LOD is + **per-project**; there is no Azure-style resource-group dimension. +2. **Phase 0 - Project anchors**: the `project` resource is synthesized from + config (no API call needed) and written *first* for every in-scope project, + so child resources can link to their parent project at parse time (the + handler does an immediate registry lookup). +3. **Phase 1 - Typed pass**: for each accessed type that has a hand-written + `google-cloud-*` collector (the 12 typed tables — compute instances, disks, + snapshots, networks, subnetworks, firewalls, addresses; storage buckets; GKE + clusters; Pub/Sub topics and subscriptions; IAM service accounts), call + `spec.collector(credentials, project_id)` per in-scope project. Rich SDK + payloads land in the resource store, and these types are excluded from the + Phase 2 CAI filter so they survive even when CAI is denied. +4. **Phase 2 - CAI generic pass**: one `list_assets` call per in-scope project, + scoped to the CAI asset types of accessed *generic* types (typed types are + excluded so nothing is written twice). Each returned asset is routed by + `asset_type` through `find_spec_by_cai_type` back to the registry-mapped + `resource_type_name`. + +Every resource flows through: normalize -> label/tag filter -> handler +`parse_resource_data` -> `writer.add_resource`. The indexer is +generation-rule-driven: only types referenced by loaded gen rules are +collected, plus the mandatory `gcp_projects` anchor. + +### The registry (`gcp_resource_type_registry.yaml`) + +Generated by `scripts/gcp/sync_gcp_resource_type_registry.py` from three inputs: + +1. The CloudQuery GCP table list (`scripts/gcp/gcp_cloudquery_tables.txt`, + currently the v22.1.0 hub snapshot, 404 tables) — the parity source. +2. A heuristic mapping `gcp__` -> + `.googleapis.com/` (singularises + PascalCases + the entity; remaps service tokens to API hosts, e.g. `sql` -> `sqladmin`). +3. Hand-curated overrides in `scripts/gcp/gcp_resource_type_overrides.yaml` + (service-host remaps, pinned CAI types for high-value/irregular tables, + aliases, typed-collector flags, the mandatory `gcp_projects` anchor, and + `null` CAI types for tables with no CAI equivalent — IAM bindings, billing + rollups — so the generic pass skips them). + +Never hand-edit the YAML; edit the overrides and re-run the sync script. + +### Coverage parity vs. CloudQuery + +Every one of the 404 CloudQuery tables resolves via `find_spec` (by canonical +table name or alias). Tables with a CAI asset type are discoverable via the CAI +pass the moment a generation rule references them; the 12 typed tables get +richer SDK payloads **and** survive without CAI. Of the 403 tables with a CAI +asset type, 390 are served solely by the CAI pass (down from 399 before the +typed fallback tier was expanded); the remaining high-value types have typed +collectors. The handful of tables with no CAI equivalent map to `null` and +would need a dedicated typed collector if ever referenced — the same +incremental model the Azure indexer uses. + +#### What the baseline discovers when CAI is unavailable (normal, non-fatal) + +If the optional Phase 2 `list_assets` call is permission-denied (missing +`roles/cloudasset.viewer` / `cloudasset.assets.listResource`, or the API is not +enabled), `index()` logs an **informational** `GCP_CAI_PERMISSION_DENIED` note +(at INFO — no error, no banner, no warning), increments `cai_permission_denied`, +and **continues** — it does not abort the run. The discovery that remains is the +functional baseline: the **12 typed types + the synthesized `gcp_projects` +anchor** (and only the subset the loaded generation rules reference). The 390 +CAI-only types are simply skipped. This matches the live run against the sandbox +project, which does not have CAI enabled: the typed `gcp_storage_buckets`, +`gcp_container_clusters`, `gcp_compute_networks`, and `gcp_compute_firewalls` +collectors ran cleanly and populated the store while the optional generic pass +was skipped. The `assert-gcpapi-baseline` CI step treats the note as +**informational** and **passes** on the typed-baseline results — CAI's absence +is expected and never gates CI. + +#### Deferred typed collectors + +- `gcp_sql_instances`: the Cloud SQL Admin API has no idiomatic + `google-cloud-*` client; listing instances requires the + `google-api-python-client` discovery layer, which is a heavy non-idiomatic + dependency not otherwise used. Deferred — stays CAI-served. +- `gcp_run_services`: the Cloud Run Admin v2 `ListServices` call has no + aggregated/`-` cross-region wildcard, so a clean single-call fallback would + require enumerating regions. Deferred — stays CAI-served. + +## Authentication + +`gcp_get_credentials_and_projects` resolves, in order: a K8s secret +(`saSecretName` -> `serviceAccountKey`), an inline `serviceAccountKey` or a +decoded `applicationCredentialsFile`, or Application Default Credentials. It +returns a `google.auth` credentials object suitable for the CAI and +`google-cloud-*` clients, plus the project list (from `projects`, `projectId`, +or `GOOGLE_CLOUD_PROJECT` / `GCP_PROJECT`). + +## Adding a typed collector + +1. Ensure the table is in the registry (regenerate if needed) and flag it as a + `typed_collector` in the overrides YAML. +2. Implement `_collect_(credentials, project_id)` in + `gcpapi_resource_types.py` (lazy-import the `google-cloud-*` client) and + register it in `_TYPED_COLLECTORS` keyed by canonical table name. +3. Its CAI asset type is automatically excluded from the generic pass so the + resource is written once, from the richer SDK payload. + +## Tests + +* `test_gcp_resource_type_registry.py` — loader contract + spec materialization. +* `test_gcpapi_normalizers.py` — CAI/SDK normalization + handler round-trip. +* `test_gcpapi_selective.py` — per-project selective discovery + CAI filter + dispatch. diff --git a/docs/kubernetes-lod-index.md b/docs/architecture/kubernetes-lod/README.md similarity index 100% rename from docs/kubernetes-lod-index.md rename to docs/architecture/kubernetes-lod/README.md diff --git a/docs/aks-namespacelods-support.md b/docs/architecture/kubernetes-lod/aks-namespace-lods.md similarity index 100% rename from docs/aks-namespacelods-support.md rename to docs/architecture/kubernetes-lod/aks-namespace-lods.md diff --git a/docs/kubernetes-lod-configuration.md b/docs/architecture/kubernetes-lod/configuration.md similarity index 100% rename from docs/kubernetes-lod-configuration.md rename to docs/architecture/kubernetes-lod/configuration.md diff --git a/docs/kubernetes-lod-flowchart.md b/docs/architecture/kubernetes-lod/flowchart.md similarity index 100% rename from docs/kubernetes-lod-flowchart.md rename to docs/architecture/kubernetes-lod/flowchart.md diff --git a/docs/kubernetes-lod-quick-reference.md b/docs/architecture/kubernetes-lod/quick-reference.md similarity index 100% rename from docs/kubernetes-lod-quick-reference.md rename to docs/architecture/kubernetes-lod/quick-reference.md diff --git a/docs/architecture/mcp-server.md b/docs/architecture/mcp-server.md new file mode 100644 index 000000000..3bb03443a --- /dev/null +++ b/docs/architecture/mcp-server.md @@ -0,0 +1,177 @@ +# MCP server: design notes + +This page covers the engineering shape of the MCP server. For the +user-facing "how do I connect Claude Code to this" guide, see +[user-guide/features/mcp-server.md](../user-guide/features/mcp-server.md). + +## Goal + +Give OSS users of `runwhen-local` (i.e. people who are not on the +[RunWhen Platform](https://www.runwhen.com)) a small, dependency-free +way to plug their indexed environment into agentic clients. The +workspace-builder already discovers resources and renders Skills; this +just exposes those over a protocol agents already speak. + +We deliberately scope v1 to **read-only** so the MCP layer is a thin +wrapper over the existing SQLite resource store. Execution lives on +the Platform, and a sandboxed local runtime is a future iteration. + +## Layout + +``` +src/workspace_builder/mcp/ +├── __init__.py # package docstring +├── tools.py # data-access layer; one function per MCP tool +├── search.py # token-overlap ranking helpers (no third-party deps) +└── server.py # FastMCP server + tool registrations + ASGI app builder +``` + +`server.py` is intentionally thin: each `@mcp.tool()` decorator does +nothing but forward to a function in `tools.py`. That split lets the +MCP-side surface (schemas, descriptions, capability negotiation) move +independently of the data plane. + +## Wiring into FastAPI + +The MCP server is a Starlette ASGI sub-app mounted into the existing +FastAPI app at `/mcp`. The lifecycle wiring is the only subtle bit: + +```python +# src/workspace_builder/api.py +_mcp_lifespan = build_mcp_lifespan() if is_mcp_enabled() else None +app = FastAPI(title="...", lifespan=_mcp_lifespan) +if _mcp_lifespan is not None: + app.mount("/mcp", build_streamable_http_app()) +``` + +`build_mcp_lifespan()` returns an async context manager that drives +the FastMCP server's `session_manager.run()`. Without it, every +JSON-RPC call to `/mcp/` fails with +`FastMCP's StreamableHTTPSessionManager task group was not initialized`, +because FastAPI does not propagate nested-mount lifespans automatically. + +`is_mcp_enabled()` is a small env-var gate (`RW_MCP_DISABLED=true` +turns the route off) so operators who only want the discovery side of +the server can opt out. + +## Tool surface + +Twelve tools, all read-only. Every tool is side-effect-free. Content +is clipped per artifact at `MAX_CONTENT_CHARS = 32_000` to stay within +agent context budgets; the explorer REST API is the escape hatch for +full content. + +### v1: ground-truth resource and Skill access + +| Tool | Backed by | +| --- | --- | +| `get_workspace_summary` | `resource_store_reader.get_store_summary` + `list_slx_bundles` (count) + per-platform `count_resources` | +| `search_skills` | `list_slx_bundles` (candidate pool) + `search.score_candidate` re-rank | +| `list_skills` | `list_slx_bundles` | +| `get_skill` | `list_slx_bundles` (lookup) + `get_workspace_artifact` (full content per file) | +| `search_resources` | `count_resources` + `search_resources` | +| `get_resource` | `sqlite_resource_writer.get_resource` | + +### v1.1: joins, recommendations, previews + +| Tool | Backed by | +| --- | --- | +| `get_skills_for_resource` | Best-effort match: pull the resource, derive a small set of match terms (qualified name, short name, slugified variants), `list_slx_bundles(q=term)` for each, dedupe and rank by number of matched terms. The renderer doesn't store an explicit resource→SLX binding today; this scan is the contract that lets us later swap in a stored binding without changing the tool surface. | +| `get_workspace_health` | `workspace_builder.health.get_health_tracker()` — the same source the existing `/health/` endpoint uses. | +| `list_codebundles` | Iterates `enrichers.code_collection.code_collection_cache`, returning each loaded `CodeCollection`'s repo URL + on-disk path + load state. | +| `get_resource_neighbors` | Forward refs: walk the decoded attributes for `$ref` markers (the `_REF_KEY` produced by `sqlite_resource_writer.encode_attributes`) and resolve each via `get_resource`. Reverse refs: SQL `LIKE` against `attributes_json` for the resource's qualified name with proper `_` / `%` escaping. Both bounded by `limit`. | +| `recommend_skills` | Same scoring as `search_skills`, but skips the `LIKE` pre-filter and ranks every bundle in the candidate pool. Tuned for longer free-text input (error traces, user messages). | +| `preview_skill_invocation` | Reuses `get_skill` to fetch the bundle, then formats a templated `runwhen-cli` invocation. Read-only by design; the future micro-runtime tool replaces this with sandboxed execution. | + +## Prompts + +Four canned investigation prompts registered with `@mcp.prompt()`. +Each one returns a templated string that the MCP client surfaces as a +slash-menu entry. The agent receives the rendered prompt as the first +turn and orchestrates calls to the tools above. + +| Prompt | Args | Tools it orchestrates | +| --- | --- | --- | +| `kickoff_investigation` | — | `get_workspace_summary`, `get_workspace_health`, `list_codebundles`, `list_skills` | +| `triage_kubernetes_namespace` | `namespace` | `search_resources`, `get_skills_for_resource`, `get_skill` | +| `diagnose_failing_deployment` | `namespace`, `deployment` | `get_resource`, `get_resource_neighbors`, `get_skills_for_resource`, `preview_skill_invocation`, `recommend_skills` | +| `audit_azure_keyvaults` | — | `search_resources`, `search_skills`, `get_skills_for_resource` | + +Prompts are intentionally thin — they are *teaching* prompts, not +behaviour. The agent is free to skip steps if a tool returns enough +context, and the prompts are explicit about uncertainty (e.g. flag +gaps where Skills don't exist rather than inventing investigation +steps). + +## Search ranking + +`search_skills` is the only tool with non-trivial behaviour. The flow: + +1. Pull a candidate pool of up to `_SEARCH_CANDIDATE_POOL = 100` + bundles from `list_slx_bundles`. The query string is **not** used as + a SQL `LIKE` filter for free-text queries because that would force + all words to appear contiguously ("rotate key vault secrets" almost + never appears verbatim). Only explicit `platform` / `resource_type` + filters are applied as `LIKE`. +2. For each candidate, concatenate the `skill` + `runbook` + `sli` + + `slx` artifact contents into one string and run token-overlap + scoring against the query. +3. Field weights: `name × 4`, `path × 2`, `kinds × 1.5`, `content × 1`. + Stop-words filtered, lowercased, distinct-token counts (so a long + runbook can't drown out a tight name match). +4. Sort descending, return top-N with a snippet of `2 × _SNIPPET_RADIUS` + characters around the first matching token. + +This is dependency-free and good enough for typical workspace sizes +(O(few hundred) bundles). The contract is intentionally a `score` + +`snippet` pair attached to each candidate, so a future iteration can +swap in embeddings (sqlite-vec, Chroma sidecar, ...) without changing +the tool signature. + +## Tests + +`src/workspace_builder/test_mcp.py`: + +* `SearchRankingTests` - `tokenize`, `score_candidate`, `make_snippet` + unit tests with no DB. +* `WorkspaceSummaryTests` - empty-DB fallback + seeded-DB summary. +* `SkillToolTests` - `list`, ranked search (deployment vs key vault + queries route to the right bundle), `get_skill`, lookup error. +* `ResourceToolTests` - `search_resources` filtering, `get_resource` + attribute round-trip. +* `GetSkillsForResourceTests` - resource-bound Skill match via fan-out + on qualified name / short name / slugified variants. +* `GetResourceNeighborsTests` - forward-ref resolution (Deployment → + Namespace), reverse-ref discovery (Namespace ← Deployment), unknown- + resource error. +* `RecommendSkillsTests` - long natural-language context surfaces a + Deployment-related Skill ahead of unrelated Key Vault Skill. +* `PreviewSkillInvocationTests` - returns runbook content + templated + `runwhen-cli` example. +* `GetWorkspaceHealthTests` / `ListCodebundlesTests` - shape contracts + (the underlying state can vary in tests). +* `McpServerSmokeTests` - all 12 tools registered, all 4 prompts + registered, ASGI app shape, env-var gate. + +`MAX_CONTENT_CHARS` and `MAX_LIMIT` are exported from `tools.py` so +tests and future telemetry can read them without reaching into the +server module. + +## Future work + +| Theme | Sketch | +| --- | --- | +| **Auth** | Optional `RW_MCP_AUTH_TOKEN` bearer-token guard for non-localhost deployments. | +| **Explicit resource → SLX binding** | Today `get_skills_for_resource` falls back to scanning rendered content because the renderer doesn't record the binding. Storing it at render time would let this tool be exact rather than best-effort. | +| **Semantic search** | Optional embedding-based ranking. Likely sqlite-vec for the OSS path; the tool contract stays the same. Drops cleanly into `recommend_skills` first, then `search_skills`. | +| **Micro-runtime** | A sandboxed `run_skill(slx_name, parameters)` tool that executes the codebundle in a constrained subprocess. This is where the v1 read-only contract ends. `preview_skill_invocation` is the placeholder until then. | +| **More prompts** | The four shipped prompts cover Kubernetes triage, Deployment diagnosis, Key Vault audit, and a generic kickoff. AWS- and GCP-specific prompts (RDS audit, IAM key audit, GKE namespace triage) are obvious next additions. | + +## See also + +* [Resource store query API](./resource-store-query-api.md) - the + lower-level read API the MCP tools delegate to. +* [ResourceWriter](./resource-writer.md) - how the SQLite store gets + populated in the first place. +* [user-guide/features/mcp-server.md](../user-guide/features/mcp-server.md) - + user-facing connection guide. diff --git a/docs/Architecture.md b/docs/architecture/overview.md similarity index 75% rename from docs/Architecture.md rename to docs/architecture/overview.md index a5e4fdf35..4565ff315 100644 --- a/docs/Architecture.md +++ b/docs/architecture/overview.md @@ -6,10 +6,7 @@ The current design and packaging of RunWhen Local is such that it can be deploye

High Level Architecture

-In the diagram above, we can see that the RunWhen Local container image has two main components: - -* **Workspace Builder**: Performs discovery of Kubernetes and Public Cloud based resources, scans CodeCollection repositories for troubleshooting commands, and uses specific rules to generate RunWhen configuration data -* **Cheat Sheet**: This component reads in all of RunWhen specific configuration data generated by Workspace builder and creates tailored troubleshooting commands that are displayed by an MkDocs instance +In the diagram above, the RunWhen Local container image is centered on the **Workspace Builder**: it performs discovery of Kubernetes and public-cloud resources, scans CodeCollection repositories for troubleshooting commands, and uses generation rules to produce RunWhen configuration data. The following high-level flow is shown: @@ -19,12 +16,6 @@ The following high-level flow is shown: 2. The Workspace Builder performs a discovery task and stores all discovered resources in memory 3. CodeCollections are then indexed, which include Generation Rules, to match CodeBundles (which contain troubleshooting commands) with the resources discovered in step 2. Customization Rules are also applied to the discovered resources to determine how to best group the discovered resources. 4. Configuration files are generated and stored in the output directory that are specific to RunWhen - these files contain links to the applicable open source troubleshooting code, along with the specific configurations needed to make the commands work for the specific users environment. -5. The Cheat Sheet process then reads in the configuration files. -6. With the configuration files read, it - * Downloads the open source troubleshooting code - * Combines the configuration parameters with the open source troubleshooting code - * Creates markdown documentation with each applicable troubleshooting command -7. Finally all markdown content is copied to a directory that MkDocs is actively watching, presenting this to the user. ## Qualified Name vs. SLX Qualifiers diff --git a/docs/architecture/resource-store-query-api.md b/docs/architecture/resource-store-query-api.md new file mode 100644 index 000000000..9a20f7d5c --- /dev/null +++ b/docs/architecture/resource-store-query-api.md @@ -0,0 +1,362 @@ +# Resource store query API + +When discovery runs with `resourceStoreBackend: sqlite`, RunWhen Local writes a single SQLite file (default `output/resources.sqlite`) containing: + +1. **Discovered resources** — Kubernetes and cloud objects from indexers (`resources` table). +2. **Rendered workspace files** — SLX, SLI, runbook, and workspace YAML from `render_output_items` (`workspace_artifacts` table). + +The workspace-builder FastAPI service exposes read-only JSON endpoints under `/explorer/api/*`, and the same data can be queried directly with SQL or the Python helpers in `indexers/sqlite_resource_writer.py`. + +## Prerequisites + +Add to `workspaceInfo.yaml` (or pass equivalent fields via the `/run/` request): + +```yaml +resourceStoreBackend: sqlite +resourceStorePath: resources.sqlite # optional; relative to output directory +``` + +Run discovery (`./run.sh` or `task run-rwl-discovery`). The database appears under the workspace output directory, typically: + +- Container: `/shared/output/resources.sqlite` +- Host (bind mount): `$workdir/shared/output/resources.sqlite` + +Optional environment overrides when reading from a running container: + +| Variable | Purpose | +|----------|---------| +| `RW_RESOURCE_STORE_PATH` | Absolute path to the `.sqlite` file | +| `RW_OUTPUT_DIR` | Output directory if the DB path is relative | + +Interactive browser UI: [http://localhost:8000/explorer/](http://localhost:8000/explorer/) (use the **Rendered workspace** tab for SLXs). + +## Schema (version 2) + +### Discovered resources + +| Table | Role | +|-------|------| +| `platforms` | `kubernetes`, `azure`, `aws`, … | +| `resource_types` | Types per platform + JSON list of custom attribute names | +| `resources` | One row per resource; `attributes_json` holds encoded indexer attributes | + +Primary key: `(platform, resource_type, qualified_name)`. + +### Rendered workspace artifacts (SLX / SLI / runbook) + +| Column | Description | +|--------|-------------| +| `workspace_name` | From `workspaceName` in workspaceInfo | +| `relative_path` | Path under output, e.g. `workspaces/my-ws/slxs/backend-api/slx.yaml` | +| `artifact_kind` | `slx`, `sli`, `runbook`, `workspace`, `slx_bundle`, `other` | +| `media_type` | Usually `yaml` | +| `slx_directory` | Parent folder for SLX bundles, e.g. `workspaces/my-ws/slxs/backend-api` | +| `content` | Full rendered file text | + +Primary key: `(workspace_name, relative_path)`. + +**Listing all SLXs** means filtering `artifact_kind = 'slx'`. Each SLX directory normally also has sibling rows with `artifact_kind` of `sli` and `runbook` sharing the same `slx_directory`. + +## HTTP API + +Base URL: `http://localhost:8000` (workspace-builder service). + +### Store summary + +```http +GET /explorer/api/summary +``` + +Returns resource counts, artifact counts, platform list, navigation tree, and `db_path`. + +```bash +curl -s http://localhost:8000/explorer/api/summary | jq . +``` + +### List all SLXs + +```http +GET /explorer/api/artifacts?artifact_kind=slx&workspace_name={name}&limit=500&offset=0 +``` + +| Query param | Description | +|-------------|-------------| +| `workspace_name` | Filter to one workspace (recommended) | +| `artifact_kind` | Use `slx` to list SLX definitions only | +| `q` | Substring search on `relative_path` or `content` | +| `limit` | Page size (1–500, default 100) | +| `offset` | Pagination offset | + +Example: + +```bash +WORKSPACE=my-workspace-sqlite + +curl -s "http://localhost:8000/explorer/api/artifacts?workspace_name=${WORKSPACE}&artifact_kind=slx&limit=500" \ + | jq '.items[] | {path: .relative_path, dir: .slx_directory, preview: .content_preview}' +``` + +Response shape: + +```json +{ + "total": 75, + "limit": 500, + "offset": 0, + "items": [ + { + "workspace_name": "my-workspace-sqlite", + "relative_path": "workspaces/my-workspace-sqlite/slxs/backend-api/slx.yaml", + "artifact_kind": "slx", + "media_type": "yaml", + "slx_directory": "workspaces/my-workspace-sqlite/slxs/backend-api", + "content_preview": "kind: ServiceLevelX\nmetadata:\n name: backend-api\n..." + } + ] +} +``` + +List responses omit full `content`; use the detail endpoint for the YAML body. + +### Get one SLX (full YAML) + +```http +GET /explorer/api/artifact?workspace_name={name}&relative_path={path} +``` + +`relative_path` must match the stored path exactly (URL-encode if needed). + +```bash +curl -s --get "http://localhost:8000/explorer/api/artifact" \ + --data-urlencode "workspace_name=${WORKSPACE}" \ + --data-urlencode "relative_path=workspaces/${WORKSPACE}/slxs/backend-api/slx.yaml" \ + | jq -r .content +``` + +### List SLI and runbook files for the same SLX + +Use the shared `slx_directory` from an SLX row: + +```bash +SLX_DIR="workspaces/${WORKSPACE}/slxs/backend-api" + +curl -s "http://localhost:8000/explorer/api/artifacts?workspace_name=${WORKSPACE}&q=${SLX_DIR}" \ + | jq '.items[] | select(.slx_directory == "'"${SLX_DIR}"'") | {kind: .artifact_kind, path: .relative_path}' +``` + +Or fetch each kind explicitly: + +```bash +for kind in slx sli runbook; do + curl -s "http://localhost:8000/explorer/api/artifacts?workspace_name=${WORKSPACE}&artifact_kind=${kind}&q=${SLX_DIR}" +done +``` + +### List SLX bundles (SLX + SLI + runbook + Skill grouped together) + +```http +GET /explorer/api/slx-bundles?workspace_name={name}&q={substring}&limit=100&offset=0 +``` + +Each bundle returned in `items` groups the rendered files that share the same +`slx_directory`. The response shape is: + +```json +{ + "total": 17, + "limit": 100, + "offset": 0, + "items": [ + { + "workspace_name": "demo-ws", + "slx_directory": "workspaces/demo-ws/slxs/backend-api", + "slx_name": "backend-api", + "file_count": 4, + "kinds": ["runbook", "skill", "sli", "slx"], + "has_slx": true, + "has_sli": true, + "has_runbook": true, + "has_skill": true, + "files": [ + {"relative_path": "...slx.yaml", "artifact_kind": "slx", "media_type": "yaml", "size_bytes": 412}, + {"relative_path": "...sli.yaml", "artifact_kind": "sli", "media_type": "yaml", "size_bytes": 198}, + {"relative_path": "...runbook.yaml", "artifact_kind": "runbook", "media_type": "yaml", "size_bytes": 1031}, + {"relative_path": "...Skill.md", "artifact_kind": "skill", "media_type": "markdown", "size_bytes": 624} + ] + } + ] +} +``` + +This endpoint backs the **SLX Bundles** tab in the explorer UI and is the most +convenient way to enumerate the rendered SLX surface without reassembling files +on the client. + +The `skill` artifact is a verbatim copy of the source CodeBundle's Skill +markdown (when present, matched case-insensitively against `SKILL.md` / +`Skill.md` / `skill.md` at the bundle root). An MCP/agent can read the Skill +alongside the SLX to understand what the skill does and decide when to invoke +it — see [resource-writer.md](resource-writer.md#skill-overlay) for the overlay +mechanism. + +### Discovered resources (indexer graph) + +```http +GET /explorer/api/resources?platform=kubernetes&resource_type=Namespace&q=default&limit=100 +GET /explorer/api/resource?platform=kubernetes&resource_type=Namespace&qualified_name=default +``` + +## SQL queries + +Open the database: + +```bash +sqlite3 output/resources.sqlite +``` + +### List all SLXs + +```sql +SELECT + workspace_name, + relative_path, + slx_directory, + length(content) AS bytes +FROM workspace_artifacts +WHERE artifact_kind = 'slx' +ORDER BY workspace_name, slx_directory; +``` + +Filter to one workspace: + +```sql +SELECT relative_path, slx_directory +FROM workspace_artifacts +WHERE workspace_name = 'my-workspace-sqlite' + AND artifact_kind = 'slx'; +``` + +### Count SLXs per workspace + +```sql +SELECT workspace_name, COUNT(*) AS slx_count +FROM workspace_artifacts +WHERE artifact_kind = 'slx' +GROUP BY workspace_name; +``` + +### Full SLX bundle (SLX + SLI + runbook) per directory + +```sql +SELECT artifact_kind, relative_path +FROM workspace_artifacts +WHERE slx_directory = 'workspaces/my-workspace-sqlite/slxs/backend-api' +ORDER BY artifact_kind; +``` + +### SLXs missing a runbook or SLI + +```sql +WITH dirs AS ( + SELECT DISTINCT slx_directory, workspace_name + FROM workspace_artifacts + WHERE artifact_kind = 'slx' AND slx_directory IS NOT NULL +) +SELECT d.slx_directory, + MAX(CASE WHEN a.artifact_kind = 'sli' THEN 1 ELSE 0 END) AS has_sli, + MAX(CASE WHEN a.artifact_kind = 'runbook' THEN 1 ELSE 0 END) AS has_runbook +FROM dirs d +LEFT JOIN workspace_artifacts a + ON a.workspace_name = d.workspace_name + AND a.slx_directory = d.slx_directory +GROUP BY d.workspace_name, d.slx_directory +HAVING has_sli = 0 OR has_runbook = 0; +``` + +### Search SLX content by substring + +```sql +SELECT relative_path, slx_directory +FROM workspace_artifacts +WHERE artifact_kind = 'slx' + AND content LIKE '%namespace%' +ORDER BY relative_path; +``` + +### List Kubernetes namespaces (discovered resources) + +```sql +SELECT name, qualified_name, attributes_json +FROM resources +WHERE platform = 'kubernetes' + AND resource_type = 'Namespace' +ORDER BY name; +``` + +Decode `attributes_json` in application code using `decode_attributes()` from `indexers.sqlite_resource_writer`, or inspect raw JSON for simple fields. + +## Python read helpers + +```python +from indexers.sqlite_resource_writer import ( + open_database, + search_workspace_artifacts, + count_workspace_artifacts, + get_workspace_artifact, + list_resources, +) + +conn = open_database("output/resources.sqlite") + +# All SLXs in a workspace +slxs = search_workspace_artifacts( + conn, + workspace_name="my-workspace-sqlite", + artifact_kind="slx", + limit=500, +) +print(f"{len(slxs)} SLXs (page); total={count_workspace_artifacts(conn, workspace_name='my-workspace-sqlite', artifact_kind='slx')}") + +for row in slxs: + print(row["relative_path"], row["slx_directory"]) + +# Full YAML for one SLX +doc = get_workspace_artifact( + conn, + "my-workspace-sqlite", + "workspaces/my-workspace-sqlite/slxs/backend-api/slx.yaml", +) +print(doc["content"]) + +conn.close() +``` + +## Relating SLXs to discovered resources + +SLX YAML carries resource context under `spec.additionalContext` (for example `resourcePath`, `hierarchy`, and tags). The store does not yet foreign-key SLX rows to `resources` rows; correlate in application logic by: + +1. Parsing SLX `content` as YAML and reading `spec.additionalContext.resourcePath` or tag values. +2. Matching those values to `resources.qualified_name` or attributes in `attributes_json`. + +Example SQL to pair by path substring (illustrative only): + +```sql +SELECT + a.relative_path AS slx_path, + r.platform, + r.resource_type, + r.qualified_name +FROM workspace_artifacts a +JOIN resources r + ON a.artifact_kind = 'slx' + AND a.content LIKE '%' || r.qualified_name || '%' +WHERE a.workspace_name = 'my-workspace-sqlite' +LIMIT 20; +``` + +Prefer explicit fields from parsed SLX YAML over fuzzy `LIKE` joins for production queries. + +## Related documentation + +- [ResourceWriter](resource-writer.md) — how data is written and schema versioning +- [Discovery Output](../user-guide/features/user_guide-feature_overview.md) — enabling the SQLite backend and explorer UI diff --git a/docs/architecture/resource-writer.md b/docs/architecture/resource-writer.md new file mode 100644 index 000000000..d8e7bd0da --- /dev/null +++ b/docs/architecture/resource-writer.md @@ -0,0 +1,170 @@ +# ResourceWriter — the indexer / registry seam + +## Why this exists + +Until now, every indexer in `runwhen-local` mutates the in-memory `Registry` directly via `Registry.add_resource(...)`. That works for the current single-request pipeline (FastAPI REST shell → indexers → enrichers → tar response), but it tightly couples each indexer to one specific storage backend. + +The forward-looking design for `runwhen-local` is: + +1. A small **local resource DB** (e.g. SQLite) that persists discovered resources between runs. +2. A **fast, read-only REST API** sitting on top of that DB, extending the workspace-builder service as the public surface. +3. Indexers continue to discover; the registry / DB and the REST layer are independent concerns. + +To make that swap a plug-in instead of a rewrite, every *new* indexer funnels writes through the [`ResourceWriter`](../../src/indexers/resource_writer.py) protocol. The seam is small (two methods) and the contract is fixed by the existing `parse_resource_data` output shape, so when the DB / REST substrate lands, only the writer implementation changes. + +## Contract + +```python +class ResourceWriter(Protocol): + def add_resource( + self, + platform: str, + resource_type: str, + name: str, + qualified_name: str, + attributes: dict[str, Any], + ) -> "Resource": ... + + def finalize(self) -> None: ... +``` + +* `add_resource(...)` is the canonical write. The shape of `attributes` is whatever the platform handler's `parse_resource_data(...)` produces, plus the indexer-supplied `resource` (the raw cloud payload), `auth_type`, and `auth_secret` keys. This matches what the legacy CloudQuery indexer passes to `registry.add_resource(...)` today, so the contract is fixed by current behavior. + +* `finalize()` runs once after all resources have been written. The in-memory implementation runs Azure deferred-RG resolution (so child resources written before their RG still get linked correctly). DB-backed implementations would commit their transaction here. + +## Implementations + +### `InMemoryRegistryWriter` (default) + +Wraps the `Registry` carried on `Context` under the `registry` property. Delegates to `Registry.add_resource(...)` and runs `resolve_deferred_azure_relationships(...)` on `finalize()`. This is the default writer. + +The legacy `cloudquery` indexer still writes to the registry directly (intentional — we don't want to perturb the existing path during the Azure SDK migration). It will migrate in a follow-up once AWS / GCP indexers are also native, at which point all indexers go through `ResourceWriter`. + +### `SqliteResourceWriter` (opt-in) + +A *dual* writer: it composes `InMemoryRegistryWriter` and forwards every `add_resource` / `finalize` call to it (so enrichers / renderers keep reading from the in-memory `Registry` unchanged), and on `finalize()` it snapshots the **full** registry into a local SQLite database. The DB lands wherever non-SQL artefacts go (filesystem dir or tar archive) via the active `Outputter`. + +Selected by setting `resourceStoreBackend: sqlite` (see [Selecting a writer](#selecting-a-writer)). Lives in [`src/indexers/sqlite_resource_writer.py`](../../src/indexers/sqlite_resource_writer.py). + +#### Schema + +Three tables form a normalised resource graph: + +```sql +CREATE TABLE platforms ( + name TEXT PRIMARY KEY +); + +CREATE TABLE resource_types ( + platform TEXT NOT NULL, + name TEXT NOT NULL, + custom_attributes TEXT NOT NULL, -- JSON list + PRIMARY KEY (platform, name), + FOREIGN KEY (platform) REFERENCES platforms(name) ON DELETE CASCADE +); + +CREATE TABLE resources ( + platform TEXT NOT NULL, + resource_type TEXT NOT NULL, + qualified_name TEXT NOT NULL, + name TEXT NOT NULL, + attributes_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (platform, resource_type, qualified_name), + FOREIGN KEY (platform, resource_type) + REFERENCES resource_types(platform, name) ON DELETE CASCADE +); + +CREATE INDEX idx_resources_name ON resources (platform, resource_type, name); +``` + +A small `schema_meta` table carries the schema version (currently `2`) so future migrations can detect old DBs. + +A `workspace_artifacts` table stores rendered SLX, SLI, runbook, workspace YAML, and Skill overlays written by `render_output_items`. Rows are keyed by `(workspace_name, relative_path)` with `artifact_kind`, `media_type`, `slx_directory`, and full `content` text. The DB is written once at the end of the pipeline via `persist_sqlite_store` in `dump_resources`. + +`artifact_kind` values today: `slx`, `sli`, `runbook`, `workspace`, `skill`, `slx_bundle` (any other file under `/slxs/`), and `other`. The `skill` kind corresponds to a `Skill.md` overlaid from the source CodeBundle — see [Skill overlay](#skill-overlay) below. + +### Skill overlay + +A CodeBundle defines a Skill (Skill Template); each rendered SLX is an instance of that Skill. When a CodeBundle ships a Skill markdown file at its root, `enrichers/generation_rules._emit_skill_overlay` adds a non-templated `RendererOutputItem` (`raw_content`) for every SLX rendered from that bundle. The renderer writes it verbatim and `record_rendered_artifact` classifies it as `artifact_kind='skill'` so it lands in `workspace_artifacts` alongside the slx/sli/runbook YAML for the same SLX directory. The lookup is cached per `(repo_url, ref, code_bundle_name)` on the context to avoid re-walking the git tree once per SLX. + +The filename is matched **case-insensitively** at the codebundle root — common variants are `SKILL.md`, `Skill.md`, and `skill.md`. The upstream casing is preserved when writing into the SLX directory (so a codebundle that publishes `SKILL.md` produces `SKILL.md` in every rendered SLX, not a normalized `Skill.md`). Files named `skill.md` deeper than the bundle root are ignored. + +#### Encoding + +`attributes_json` holds a deterministic JSON encoding (`sort_keys=True`) of every non-structural attribute on the `Resource` object. Rich Python types are preserved with reserved markers so a future REST service can decode them faithfully: + +| Python type | JSON shape | +| -------------------------------- | -------------------------------------------------------------- | +| Primitives, dicts, lists | as-is | +| `set` / `frozenset` / `tuple` | list | +| `datetime.datetime` | `{"$datetime": ""}` | +| `datetime.date` | `{"$date": ""}` | +| `LevelOfDetail` | `{"$lod": "BASIC" \| "DETAILED" \| "NONE"}` | +| `enum.Enum` (other) | `{"$enum": {"class": "", "name": ""}}` | +| `Resource` (cross-resource link) | `{"$ref": {"platform": ..., "resource_type": ..., "qualified_name": ..., "name": ...}}` | +| Anything else | `str(value)` (with a debug log; we shouldn't hit this) | + +Cross-resource references (e.g. an Azure storage account's `resource_group`) are serialised as `$ref` markers rather than embedded objects, so each resource appears exactly once in the DB. The decoder leaves `$ref` entries as plain dicts; resolution is the caller's choice. + +#### Snapshot semantics + +`SqliteResourceWriter.finalize()`: + +1. Calls the in-memory writer's `finalize()` first, so `resolve_deferred_azure_relationships(...)` settles all `_deferred_rg_lookup` markers and re-keys child resources by their final `/` qualified names. +2. Walks the **full** in-memory `Registry` (not just resources written through this writer) and snapshots it to SQLite in a single transaction. Today only the `azureapi` indexer goes through `ResourceWriter`, but `kubeapi` / `cloudquery` mutate the same `Registry`, so their resources land in the snapshot too. +3. Replaces existing rows on each finalize so the DB is always a fresh, idempotent snapshot. + +This means the SQLite store reflects the post-indexing, post-deferred-resolution state — i.e. exactly what `dump_resources.py` writes to `resource-dump.yaml`, but normalised into queryable rows. + +For HTTP endpoints, SQL examples, and **listing all SLXs**, see [Resource store query API](resource-store-query-api.md). + +### Future: `RestApiResourceWriter` + +POST resources to a separate fast REST service if we want to fully decouple workspace-builder from the storage layer (e.g. multiple indexer workers, central resource graph). Same protocol, different transport. + +## Selecting a writer + +Indexers obtain their writer via: + +```python +from indexers.resource_writer import get_resource_writer +writer = get_resource_writer(context) +``` + +`get_resource_writer(context)` is the single place that decides which implementation to construct. The selection is driven by two settings, exposed on the `azureapi` component: + +| Setting (JSON name) | Type | Default | Description | +| ------------------- | ------ | ------------------ | ------------------------------------------------------------------------------------------------------------ | +| `resourceStoreBackend` | string (`memory` \| `sqlite`) | `memory` | Selects the writer implementation. `memory` keeps the in-memory `Registry` only; `sqlite` additionally snapshots the registry to a local SQLite DB on `finalize()`. | +| `resourceStorePath` | string | `resources.sqlite` | Output path (relative to the workspace output directory) for the SQLite file. Only used when the backend is `sqlite`. | + +Unknown values fall back to `memory` with a warning so a typo in `workspaceInfo.yaml` doesn't silently break indexing. + +Example `workspaceInfo.yaml` snippet: + +```yaml +resourceStoreBackend: sqlite +resourceStorePath: db/resources.sqlite +``` + +The resulting SQLite file lands next to `resource-dump.yaml` in the workspace output. You can poke at it with the standard `sqlite3` CLI or via the `list_platforms` / `list_resource_types` / `list_resources` / `get_resource` helpers in `indexers.sqlite_resource_writer`. + +## Migration roadmap + +1. **Done:** `InMemoryRegistryWriter` is the default. `azureapi` indexer writes through it. Legacy `cloudquery` still writes directly. +2. **Done:** `SqliteResourceWriter` ships as an opt-in dual writer (`resourceStoreBackend: sqlite`). It composes the in-memory writer and snapshots the registry to SQLite on `finalize()`. +3. **Next:** AWS / GCP native indexers (`awsapi`, `gcpapi`) — also write through `ResourceWriter`. +4. **Then:** Delete the CloudQuery code path entirely; every indexer goes through `ResourceWriter`. At that point, `SqliteResourceWriter`'s snapshot is the **complete** resource graph for every active backend. +5. **Then:** Make `SqliteResourceWriter` the default and treat the DB as the source of truth; the `Registry` becomes a hydrated cache in front of the DB. +6. **Then:** Extend the FastAPI REST service in front of the DB for read-only resource queries. The `/run/` endpoint continues to call `run_components(...)` via the SDK. + +Throughout, the `Resource` shape that `parse_resource_data` produces — and that generation rules consume — does not change. + +## See also + +* [`src/indexers/resource_writer.py`](../../src/indexers/resource_writer.py) — protocol + in-memory implementation + selector. +* [`src/indexers/sqlite_resource_writer.py`](../../src/indexers/sqlite_resource_writer.py) — SQLite implementation, encoder, schema, and read helpers. +* [`src/indexers/azureapi.py`](../../src/indexers/azureapi.py) — the first indexer to write via this seam. +* [`src/resources.py`](../../src/resources.py) — the `Resource` / `Registry` shape every backend must speak. diff --git a/docs/workspace-generation-statistics.md b/docs/architecture/workspace-generation-statistics.md similarity index 100% rename from docs/workspace-generation-statistics.md rename to docs/architecture/workspace-generation-statistics.md diff --git a/docs/authoring/README.md b/docs/authoring/README.md new file mode 100644 index 000000000..055768533 --- /dev/null +++ b/docs/authoring/README.md @@ -0,0 +1,45 @@ +# Authoring Guide + +Everything you need to extend RunWhen Local: write CodeBundles, ship Skills, +and teach the workspace builder how to wire them up via generation rules. + +If you only want to *use* RunWhen Local against your own infrastructure, see +the [user guide](../user-guide/README.md) instead. + +## Concepts + +* [CodeBundle / Skill / SLX / Runbook terminology](./concepts.md) - read this + first; the rest of the authoring docs assume you know what these mean. + +## Indexed resources + +Generation rules match against resources discovered by RunWhen Local's +indexers. Before you can write a rule that targets, say, an Azure App Service +or a Kubernetes Deployment, you need to know: + +* Whether the indexer actually discovers that resource type today. +* What the data looks like once it's been normalized. +* Which fields are stable enough to match against. + +Reference docs per platform: + +* [Indexed resources overview](./indexed-resources/README.md) +* [Azure indexer](./indexed-resources/azure.md) - 25 typed resource types, + with the data shape your generation rules will see for each. +* [Kubernetes indexer](./indexed-resources/kubernetes.md) +* [AWS indexer](./indexed-resources/aws.md) +* [GCP indexer](./indexed-resources/gcp.md) + +## Generation rules + +Generation rules are the bridge between an indexed resource and a rendered +SLX. They live in CodeBundles under `.runwhen/generation-rules/`. + +* [Generation rules: schema, lifecycle, and how-to](./generation-rules/README.md) +* [Tag-hierarchy contract](./generation-rules/tag-hierarchy-contract.md) - + how SLX names are composed from the resource graph. +* Worked examples: + * [Azure VM + disk runbook](./generation-rules/examples/azure-vm-disk-runbook.md) + * [Azure Key Vault SLX](./generation-rules/examples/azure-keyvault-slx.md) + * [Kubernetes Deployment SLX](./generation-rules/examples/kubernetes-deployment-slx.md) + * [Multi-resource runbook](./generation-rules/examples/multi-resource-runbook.md) diff --git a/docs/authoring/concepts.md b/docs/authoring/concepts.md new file mode 100644 index 000000000..143a99566 --- /dev/null +++ b/docs/authoring/concepts.md @@ -0,0 +1,94 @@ +# Concepts: CodeBundle, Skill, SLX, Runbook + +RunWhen has a small but specific vocabulary. Knowing exactly what each term +means makes the rest of the authoring guide make sense. + +## CodeBundle + +A **CodeBundle** is a versioned, distributable bundle of automation. Today +that's a Git repo (or sub-directory of one) inside a [CodeCollection][cc] +such as [`rw-cli-codecollection`][rw-cli] or +[`rw-public-codecollection`][rw-public]. A CodeBundle ships: + +* The actual code (Robot Framework files for `rw-cli-codecollection`, + Bash/Python helpers, etc.). +* Optional **Skill metadata**: a `SKILL.md` file describing what the bundle + does and what an AI agent can do with it. +* Generation rules in `.runwhen/generation-rules/*.yaml` that tell RunWhen + Local *when* to render an SLX from this CodeBundle. + +[cc]: https://github.com/runwhen-contrib/codecollection-registry +[rw-cli]: https://github.com/runwhen-contrib/rw-cli-codecollection +[rw-public]: https://github.com/runwhen-contrib/rw-public-codecollection + +## Skill + +A **Skill** (or *Skill template*) is the abstract capability defined by a +CodeBundle: "diagnose a stuck Pod", "rotate an Azure storage account key", +"check whether a database has free disk". When the user says "the CodeBundle +contains a Skill", they mean the CodeBundle exposes runnable code plus the +metadata an agent or human needs to invoke it. + +The on-disk marker is a `SKILL.md` file at the root of the CodeBundle (any +casing - RunWhen Local matches case-insensitively). When the workspace +builder renders an SLX from a CodeBundle that has a `SKILL.md`, it copies +that file alongside the rendered artifacts so downstream tools (the explorer +UI, MCP agents) can read it. + +## SLX + +An **SLX** (Service-Level X-objective) is a *rendered instance* of a Skill, +bound to a specific resource (or set of resources) in your environment. +"Diagnose a stuck Pod" is a Skill; "Diagnose a stuck Pod **in the +`payments-prod` namespace of cluster `west-2`**" is an SLX. + +RunWhen Local generates SLXs by walking discovered resources and matching +them against generation rules. Each SLX gets a directory under `output/` +containing: + +* The Skill template (`SKILL.md`, copied from the CodeBundle) +* A rendered `runbook.robot` (or equivalent), with placeholders bound to + the matched resource +* SLI / SLO metadata (`sli.yaml`, `slx.yaml`) for the RunWhen Platform +* Anything else the CodeBundle's templates emit + +## Runbook + +A **Runbook** is the executable artifact inside an SLX - typically a +Robot Framework `.robot` file. It's what a human (or agent) actually runs +when the SLX fires. The runbook is one output of the rendering pipeline; +the SLX is the wrapper around it. + +## Generation rule + +A small YAML document inside a CodeBundle that says: + +* "Match this resource type" (e.g. `azure_keyvault_keyvaults`) +* "When the resource looks like THIS" (predicates over its attributes) +* "Render an SLX from THIS template" (the path to the runbook template) + +See [generation-rules/README.md](./generation-rules/README.md) for the full +schema. + +## Putting it together + +```text +CodeBundle (in a CodeCollection git repo) +├── SKILL.md # describes the Skill +├── runbook.robot.template # the rendered runbook template +└── .runwhen/generation-rules/ + └── keyvault-rotation.yaml # generation rule + + ↓ workspace builder runs ↓ + +Discovered resource: azure_keyvault_keyvaults / kv-prod-001 + matched by keyvault-rotation.yaml + ↓ +output/slx/azure-keyvault-kv-prod-001-rotation/ +├── SKILL.md # copied from the CodeBundle +├── runbook.robot # rendered with kv-prod-001's id, name, etc. +├── sli.yaml +└── slx.yaml +``` + +The rest of the authoring guide is about each of those layers in detail. diff --git a/docs/authoring/generation-rules/README.md b/docs/authoring/generation-rules/README.md new file mode 100644 index 000000000..6130d6f03 --- /dev/null +++ b/docs/authoring/generation-rules/README.md @@ -0,0 +1,138 @@ +# Generation rules + +Generation rules are the bridge between an indexed resource and a rendered +SLX. Each rule is a YAML document that lives inside a CodeBundle under +`.runwhen/generation-rules/.yaml` and tells RunWhen Local: + +1. Which resource type to match (e.g. `azure_keyvault_keyvaults`). +2. Which subset of those resources to match (predicates over their + attributes / tags / hierarchy). +3. Which template files to render into the SLX (runbook, SLI, SLO, etc.). +4. How to name the resulting SLX. + +This page is the reference; for end-to-end examples see +[examples/](./examples/). + +## Schema + +```yaml +# .runwhen/generation-rules/.yaml +apiVersion: runwhen.com/v1 +kind: GenerationRule +spec: + match: + resource_type: azure_keyvault_keyvaults # required + predicates: # optional, ALL must pass + - jsonpath: $.tags.environment + in: ["prod", "staging"] + - jsonpath: $.properties.publicNetworkAccess + equals: "Disabled" + + slxName: + template: "azure-keyvault-{{ resource.name }}-rotation" + # or: + # tagHierarchy: env/region/{resource.name} + # see ./tag-hierarchy-contract.md for the full hierarchy contract. + + templates: + runbook: runbook.robot.j2 + sli: sli.yaml.j2 + slo: slo.yaml.j2 + skill: SKILL.md # optional, copied verbatim if present + + context: # values exposed to the templates as `{{ ... }}` + keyVaultId: "{{ resource.id }}" + keyVaultName: "{{ resource.name }}" + resourceGroup: "{{ resource.resource_group }}" + subscription: "{{ resource.subscription_id }}" + rotationDays: 90 +``` + +### `match.resource_type` + +Must be the canonical name (or an alias) of an indexed resource type. See +[indexed-resources/](../indexed-resources/README.md) for the per-platform +catalog. Aliases are resolved by the registry, so all of +`virtual_machine`, `azure_compute_virtual_machines`, and +`azure_keyvault_vaults` resolve to the same underlying type. + +### `match.predicates` + +Each predicate is a `(jsonpath, op, value)` triple. Supported ops: + +| Op | Semantics | +| --- | --- | +| `equals` | Exact match (case-sensitive). | +| `not_equals` | Negation of `equals`. | +| `in` | Value is in a list. | +| `not_in` | Value is not in a list. | +| `matches` | Regex match (Python `re.search`). | +| `exists` | The path resolves to a non-`None` value. | +| `greater_than` / `less_than` | Numeric comparison. | +| `present` / `absent` | Tag-shape predicate (`tags.` is/isn't set). | + +All predicates must pass for the rule to fire (logical AND). For OR / +NOT logic, write multiple rules or combine with `not_in` etc. + +### `slxName` + +The SLX name has to be globally unique per workspace. Two strategies: + +* `template` - a Jinja-style string referencing the matched + `resource`. Must produce a stable, DNS-friendly slug. +* `tagHierarchy` - delegate naming to the + [tag-hierarchy contract](./tag-hierarchy-contract.md). Useful when you + want SLXs grouped by `env/team/cluster/...`. + +### `templates` + +Paths are resolved **relative to the CodeBundle root**, not the rule +file. `runbook` is required; `sli`, `slo`, `skill` are optional. Any +template marked here is rendered with the rule's `context` plus the +matched `resource` dict. + +If a `SKILL.md` (any case) sits at the CodeBundle root, the workspace +builder copies it next to the rendered SLX even if you don't list it +under `templates`. This is what makes the AI-agent-readable Skill +overlay automatic. + +### `context` + +A flat dict of values made available to the rendered templates. Values +can reference the matched `resource` via `{{ resource. }}`. +Good practice is to expose every ID / name / region your runbook needs +via `context`, so the runbook itself stays free of resource-store-specific +plumbing. + +## Lifecycle + +1. The workspace builder runs the indexer for each configured platform. +2. For every `(CodeBundle, generation-rule)` pair it walks the matching + resources in the resource store. +3. For each match it renders the templates into + `output/slx//`, copies any `SKILL.md`, and emits an entry + in the SLX manifest. +4. The explorer UI reads the manifest at runtime; the platform upload + path can ship the same artifacts to the connected RunWhen Platform. + +## Authoring tips + +* **Match against stable fields.** ARM IDs and `resource_type` don't + change; tags and statuses do. Predicates over `properties.*` are fine, + but expect occasional flakiness if the cloud provider mutates a + status field unexpectedly. +* **Keep `context` flat.** Templates rendered with deeply nested + contexts are harder to debug than ones that read a flat keyspace. +* **Test with dry-run.** Run the workspace builder with + `--verbose` and inspect `output/` before shipping the CodeBundle. +* **One rule per "shape" of SLX.** If an SLX template only makes sense + for production resources, write a `prod-only` rule rather than a + `template if env == prod`. It's easier to read and easier to disable. + +## See also + +* [Worked examples](./examples/) - four end-to-end rules. +* [Tag-hierarchy contract](./tag-hierarchy-contract.md). +* [Indexed resources](../indexed-resources/README.md) - the catalog of + matchable types. +* [Concepts](../concepts.md) - CodeBundle / Skill / SLX terminology. diff --git a/docs/authoring/generation-rules/examples/README.md b/docs/authoring/generation-rules/examples/README.md new file mode 100644 index 000000000..53c161a77 --- /dev/null +++ b/docs/authoring/generation-rules/examples/README.md @@ -0,0 +1,14 @@ +# Generation rule examples + +Four end-to-end examples that exercise different patterns. Each example +shows the matching resource, the rule YAML, and a sketch of the rendered +output. Use them as starting points for your own CodeBundles. + +| Example | Pattern | +| --- | --- | +| [Azure VM + disk runbook](./azure-vm-disk-runbook.md) | Single resource type, predicate over tags. | +| [Azure Key Vault SLX](./azure-keyvault-slx.md) | Single resource type with a `SKILL.md` overlay. | +| [Kubernetes Deployment SLX](./kubernetes-deployment-slx.md) | Predicates over `metadata` + `spec`, namespace-scoped. | +| [Multi-resource runbook](./multi-resource-runbook.md) | A rule that bundles several related resources into one SLX. | + +For the schema reference, see [../README.md](../README.md). diff --git a/docs/authoring/generation-rules/examples/azure-keyvault-slx.md b/docs/authoring/generation-rules/examples/azure-keyvault-slx.md new file mode 100644 index 000000000..ed0b91ed5 --- /dev/null +++ b/docs/authoring/generation-rules/examples/azure-keyvault-slx.md @@ -0,0 +1,117 @@ +# Example: Azure Key Vault SLX with Skill overlay + +Generate an SLX for every Azure Key Vault that has public network access +disabled, and ship a `SKILL.md` so the SLX is invokable by an MCP-aware +AI agent. + +## Matched resource + +`azure_keyvault_keyvaults` with `properties.publicNetworkAccess == +"Disabled"`: + +```yaml +id: /subscriptions/abc/resourceGroups/rg-prod/providers/Microsoft.KeyVault/vaults/kv-prod-001 +name: kv-prod-001 +resource_type: azure_keyvault_keyvaults +resource_group: rg-prod +subscription_id: abc +location: eastus +tags: + environment: prod +properties: + vaultUri: https://kv-prod-001.vault.azure.net/ + publicNetworkAccess: Disabled + enableRbacAuthorization: true + sku: + name: standard + family: A +``` + +## CodeBundle layout + +``` +codebundles/azure-keyvault-rotation/ +├── SKILL.md # AI-agent-readable Skill description +├── runbook.robot.j2 # rendered into each SLX +├── sli.yaml.j2 +├── slo.yaml.j2 +└── .runwhen/generation-rules/ + └── private-keyvault-rotation.yaml # the rule below +``` + +## Generation rule + +```yaml +apiVersion: runwhen.com/v1 +kind: GenerationRule +spec: + match: + resource_type: azure_keyvault_keyvaults + predicates: + - jsonpath: $.properties.publicNetworkAccess + equals: "Disabled" + + slxName: + template: "azure-keyvault-{{ resource.name }}-rotation" + + templates: + runbook: runbook.robot.j2 + sli: sli.yaml.j2 + slo: slo.yaml.j2 + # No 'skill:' line is needed - the workspace builder auto-copies + # SKILL.md from the CodeBundle root into every rendered SLX. + + context: + keyVaultName: "{{ resource.name }}" + keyVaultId: "{{ resource.id }}" + vaultUri: "{{ resource.properties.vaultUri }}" + resourceGroup: "{{ resource.resource_group }}" + subscription: "{{ resource.subscription_id }}" + rotationDays: 90 +``` + +## SKILL.md (excerpt) + +```markdown +# Azure Key Vault rotation + +This Skill rotates secrets older than the configured threshold in a +single Azure Key Vault. It assumes the running identity has +`Microsoft.KeyVault/vaults/secrets/setSecret/action` permission on the +target vault. + +## Inputs +- `keyVaultName` (string, required) +- `vaultUri` (URL, required) +- `rotationDays` (int, default 90) + +## Side effects +- Generates new secret versions for any secret whose current version is + older than `rotationDays`. +- Old versions are kept (not purged) so a rollback path remains. +``` + +When the workspace builder fires this rule for `kv-prod-001` it +renders: + +``` +output/slx/azure-keyvault-kv-prod-001-rotation/ +├── SKILL.md # auto-copied from CodeBundle root +├── runbook.robot +├── sli.yaml +└── slo.yaml +``` + +The explorer UI shows the `SKILL.md` next to the runbook; an MCP-aware +agent can read it as the canonical description of what the SLX *does*. + +## Notes + +* The `publicNetworkAccess` predicate scopes the rule to private vaults + only; vaults that allow public network access get a different rule (or + no SLX at all). +* `vaultUri` is exposed in the context because the runbook uses the + data-plane URL, not the ARM ID, when calling the Key Vault REST API. +* If you also want a separate SLX for *public* vaults, write a sibling + rule with `predicates: - jsonpath: $.properties.publicNetworkAccess + not_equals: "Disabled"`. diff --git a/docs/authoring/generation-rules/examples/azure-vm-disk-runbook.md b/docs/authoring/generation-rules/examples/azure-vm-disk-runbook.md new file mode 100644 index 000000000..13d7f76a6 --- /dev/null +++ b/docs/authoring/generation-rules/examples/azure-vm-disk-runbook.md @@ -0,0 +1,99 @@ +# Example: Azure VM disk runbook + +Generate an SLX for every production-tagged Azure VM that runs a "diagnose +slow disk" runbook against its OS disk. + +## Matched resource + +A `azure_compute_virtual_machines` resource with `tags.environment == +"prod"`. Example payload (trimmed): + +```yaml +id: /subscriptions/abc/resourceGroups/rg-prod/providers/Microsoft.Compute/virtualMachines/web-01 +name: web-01 +resource_type: azure_compute_virtual_machines +subscription_id: abc +resource_group: rg-prod +location: eastus +tags: + environment: prod + team: web +properties: + storageProfile: + osDisk: + managedDisk: + id: /subscriptions/abc/resourceGroups/rg-prod/providers/Microsoft.Compute/disks/web-01-os + name: web-01-os +``` + +## Generation rule + +`.runwhen/generation-rules/azure-vm-disk-diagnose.yaml` inside your +CodeBundle: + +```yaml +apiVersion: runwhen.com/v1 +kind: GenerationRule +spec: + match: + resource_type: azure_compute_virtual_machines + predicates: + - jsonpath: $.tags.environment + equals: "prod" + - jsonpath: $.properties.storageProfile.osDisk.managedDisk.id + exists: true + + slxName: + template: "azure-vm-{{ resource.name }}-disk-health" + + templates: + runbook: runbook.robot.j2 + sli: sli.yaml.j2 + skill: SKILL.md + + context: + vmName: "{{ resource.name }}" + vmId: "{{ resource.id }}" + osDiskId: "{{ resource.properties.storageProfile.osDisk.managedDisk.id }}" + osDiskName: "{{ resource.properties.storageProfile.osDisk.name }}" + resourceGroup: "{{ resource.resource_group }}" + subscription: "{{ resource.subscription_id }}" +``` + +## Rendered output + +For each matching VM the workspace builder produces a directory like: + +``` +output/slx/azure-vm-web-01-disk-health/ +├── SKILL.md # copied verbatim from the CodeBundle root +├── runbook.robot # rendered with vmName="web-01", osDiskName="web-01-os", ... +└── sli.yaml +``` + +`runbook.robot.j2` can reference the context directly: + +```robot +*** Settings *** +Documentation Diagnose slow disk on Azure VM ${vmName}. + +*** Variables *** +${VM_ID} {{ vmId }} +${DISK_ID} {{ osDiskId }} +${RESOURCE_GROUP} {{ resourceGroup }} +${SUBSCRIPTION} {{ subscription }} + +*** Tasks *** +Check IOPS on ${osDiskName} + [Documentation] Pulls 24h of disk metrics from Azure Monitor. + ... +``` + +## Notes + +* `predicates` here include an `exists` check so we skip VMs that don't + expose a managed-disk OS disk (e.g. ephemeral OS disks). +* The `osDiskId` is enough on its own; we also expose `osDiskName` purely + for human-readable runbook headers. +* `tags.environment` is user-supplied data; if it's missing the predicate + evaluates to false and the SLX isn't generated. diff --git a/docs/authoring/generation-rules/examples/kubernetes-deployment-slx.md b/docs/authoring/generation-rules/examples/kubernetes-deployment-slx.md new file mode 100644 index 000000000..80ef5f4e6 --- /dev/null +++ b/docs/authoring/generation-rules/examples/kubernetes-deployment-slx.md @@ -0,0 +1,88 @@ +# Example: Kubernetes Deployment SLX + +Generate a "diagnose Deployment rollout" SLX for every multi-replica +Deployment in namespaces tagged for production. + +## Matched resource + +A `deployment` resource: + +```yaml +id: /apis/apps/v1/namespaces/payments/deployments/checkout-api +name: checkout-api +resource_type: deployment +subscription_id: prod-west # cluster name +metadata: + namespace: payments + labels: + team: payments + tier: backend +spec: + replicas: 5 + selector: + matchLabels: + app: checkout-api +status: + availableReplicas: 5 + conditions: + - type: Available + status: "True" +``` + +## Generation rule + +```yaml +apiVersion: runwhen.com/v1 +kind: GenerationRule +spec: + match: + resource_type: deployment + predicates: + - jsonpath: $.spec.replicas + greater_than: 1 + - jsonpath: $.metadata.labels.team + exists: true + - jsonpath: $.metadata.namespace + not_equals: "kube-system" + + slxName: + template: >- + k8s-{{ resource.subscription_id }}-{{ resource.metadata.namespace }}-{{ resource.name }}-rollout + + templates: + runbook: runbook.robot.j2 + sli: sli.yaml.j2 + + context: + cluster: "{{ resource.subscription_id }}" + namespace: "{{ resource.metadata.namespace }}" + deployment: "{{ resource.name }}" + team: "{{ resource.metadata.labels.team }}" + replicas: "{{ resource.spec.replicas }}" +``` + +## Rendered output + +For `checkout-api` in cluster `prod-west` you get: + +``` +output/slx/k8s-prod-west-payments-checkout-api-rollout/ +├── runbook.robot +└── sli.yaml +``` + +`runbook.robot.j2` then has access to `${cluster}`, `${namespace}`, +`${deployment}`, `${team}`, `${replicas}` and can shell out to `kubectl` +or call the Kubernetes API directly. + +## Notes + +* The `subscription_id` field on Kubernetes resources is the cluster + name; the SLX naming template includes it so you don't get name + collisions across multiple clusters. +* The `metadata.labels.team` predicate uses `exists`, which is enough + to require *any* team label without pinning to a specific value. +* A namespace's effective Level of Detail must be `BASIC` or `DETAILED` + for its Deployments to even arrive at this rule. Configure that under + `cloudConfig.kubernetes.contexts[].namespaceLevelOfDetails` - + see [Kubernetes-LOD configuration](../../../architecture/kubernetes-lod/configuration.md). diff --git a/docs/authoring/generation-rules/examples/multi-resource-runbook.md b/docs/authoring/generation-rules/examples/multi-resource-runbook.md new file mode 100644 index 000000000..c5fc057db --- /dev/null +++ b/docs/authoring/generation-rules/examples/multi-resource-runbook.md @@ -0,0 +1,109 @@ +# Example: Multi-resource runbook + +Bundle related resources into a single SLX. This pattern is useful when +the natural unit of troubleshooting spans more than one resource type. + +The setup: an Azure App Service web app + its server farm (App Service +plan) + the Application Gateway in front of it. We want one SLX per web +app that pulls all three IDs into the runbook context. + +## Matched resource + +The "primary" resource for the rule is `azure_appservice_web_apps`. The +related App Service plan and Application Gateway are looked up via +`relatedResources` (resolved at generation time): + +```yaml +id: /subscriptions/abc/resourceGroups/rg-prod/providers/Microsoft.Web/sites/checkout-api +name: checkout-api +resource_type: azure_appservice_web_apps +resource_group: rg-prod +subscription_id: abc +properties: + serverFarmId: /subscriptions/abc/resourceGroups/rg-prod/providers/Microsoft.Web/serverFarms/asp-prod + hostNames: + - checkout-api.example.com +tags: + environment: prod + appgw: appgw-prod +``` + +## Generation rule + +```yaml +apiVersion: runwhen.com/v1 +kind: GenerationRule +spec: + match: + resource_type: azure_appservice_web_apps + predicates: + - jsonpath: $.tags.environment + equals: "prod" + + relatedResources: + plan: + resource_type: azure_appservice_plans + where: + # Match the plan whose ARM ID equals this web app's serverFarmId. + idEquals: "{{ resource.properties.serverFarmId }}" + appGateway: + resource_type: azure_network_application_gateways + where: + # Match the appgw whose name matches the 'appgw' tag on the web app. + nameEquals: "{{ resource.tags.appgw }}" + + slxName: + template: "azure-webapp-{{ resource.name }}-end-to-end" + + templates: + runbook: runbook.robot.j2 + sli: sli.yaml.j2 + + context: + webAppName: "{{ resource.name }}" + webAppId: "{{ resource.id }}" + hostName: "{{ resource.properties.hostNames[0] }}" + serverFarmId: "{{ resource.properties.serverFarmId }}" + serverFarmName: "{{ related.plan.name }}" + appGatewayId: "{{ related.appGateway.id }}" + appGatewayName: "{{ related.appGateway.name }}" + resourceGroup: "{{ resource.resource_group }}" + subscription: "{{ resource.subscription_id }}" +``` + +## Rendered output + +``` +output/slx/azure-webapp-checkout-api-end-to-end/ +├── runbook.robot +└── sli.yaml +``` + +The runbook can now perform multi-step diagnosis: + +```robot +*** Tasks *** +Check Application Gateway Health + Run az network application-gateway show-backend-health + ... --ids ${appGatewayId} + +Check Web App Availability + Run curl -sS https://${hostName}/health + +Check Server Farm Capacity + Run az appservice plan show --ids ${serverFarmId} +``` + +## Notes + +* `relatedResources` is a separate top-level field from `match`. The + rule fires once per *matched primary*, with each related resource + bound under `related.` for the templates. +* If a related resource isn't found (no matching plan, no matching + appgw), the rule still fires, but the corresponding `related.*` + values are empty. Templates should defensively handle missing related + resources or the rule should add a stricter `predicates` block. +* Cross-resource predicates (e.g. "fire only when *both* the web app + and its appgw have `environment: prod`") are best expressed by + predicates on the primary plus a `where` clause that filters the + related lookup. diff --git a/docs/tag-hierarchy-contract.md b/docs/authoring/generation-rules/tag-hierarchy-contract.md similarity index 100% rename from docs/tag-hierarchy-contract.md rename to docs/authoring/generation-rules/tag-hierarchy-contract.md diff --git a/docs/authoring/indexed-resources/README.md b/docs/authoring/indexed-resources/README.md new file mode 100644 index 000000000..9868edc87 --- /dev/null +++ b/docs/authoring/indexed-resources/README.md @@ -0,0 +1,55 @@ +# Indexed resources + +Generation rules can only match against resources that an indexer has +actually written to the resource store. This section is the authoritative +reference for what each indexer discovers and the data shape that arrives +in your generation rules. + +## Per-platform reference + +* [Azure](./azure.md) - native `azure-mgmt-*` SDK indexer (`azureapi`). + Full parity with the legacy CloudQuery Azure plugin: 619 indexable + resource types, 25 typed (rich-payload) plus 594 generic (basic + envelope). Sortable catalog at + [azure-resource-catalog.md](./azure-resource-catalog.md). +* [Kubernetes](./kubernetes.md) - in-cluster scan via the Kubernetes Python + client, plus per-namespace LOD support. +* [AWS](./aws.md) - CloudQuery-backed indexer. +* [GCP](./gcp.md) - CloudQuery-backed indexer. + +## Common resource shape + +Every discovered resource lands in the resource store as a normalized dict +with at least: + +| Field | Type | Notes | +| --- | --- | --- | +| `id` | string | The platform's canonical resource ID (ARM ID for Azure, ARN for AWS, `/api/v1/.../` for Kubernetes, etc.). | +| `name` | string | Human-friendly name. | +| `resource_type` | string | RunWhen Local's canonical type name. Use this in generation rule `match.resource_type`. | +| `subscription_id` / `account_id` / `project_id` | string | Cloud account scope. Always set for cloud platforms. | +| `tags` | dict | Always a dict, may be empty. Cloud-platform user tags. | +| `properties` | dict | Platform-specific payload (preserved verbatim). | + +Indexer-specific fields layer on top of those common ones; per-platform +docs spell out the extras and give example payloads. + +## Selective vs unbounded discovery + +All indexers honor the `defaultLOD` / `resourceGroupLevelOfDetails` +contract from `workspaceInfo.yaml`: + +* `defaultLOD: detailed` (and any non-`none` value) means **unbounded + discovery** - the indexer enumerates the whole subscription / account / + cluster. +* `defaultLOD: none` plus a finite list of "keep this" entries means + **selective discovery** - the indexer scopes its API calls to exactly + those resource groups / namespaces. + +When you're authoring generation rules in a CodeBundle, you don't usually +have to think about this; you just match resources. But it does affect +*which* resources show up at runtime. See the user-facing +[level-of-detail guide](../../user-guide/configuration/level-of-detail.md) +and the architecture-level +[Kubernetes-LOD internals](../../architecture/kubernetes-lod/README.md) for +the full mechanics. diff --git a/docs/authoring/indexed-resources/aws.md b/docs/authoring/indexed-resources/aws.md new file mode 100644 index 000000000..e15738c66 --- /dev/null +++ b/docs/authoring/indexed-resources/aws.md @@ -0,0 +1,92 @@ +# AWS indexer + +RunWhen Local can discover AWS resources two ways, selected by +`awsIndexerBackend` in `workspaceInfo.yaml`: + +* **`cloudquery`** (default): invokes the + [CloudQuery AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws) + against the account(s) you've configured and reads the resulting SQLite + intermediate. +* **`awsapi`**: the native indexer — uses the AWS Cloud Control API plus + first-party `boto3` SDKs, no CloudQuery binary. It discovers only the resource + types your generation rules reference, per account/region, and respects + per-account `accountLevelOfDetails` (an account with LOD `none` is skipped). + See [AWS indexer internals](../../architecture/aws-indexer-internals.md) for + the design. + +```yaml +# workspaceInfo.yaml +awsIndexerBackend: awsapi +``` + +Either way the CodeBundle-facing contract is the same: generation rules +reference the **CloudQuery table name** as `resource_type` (e.g. +`aws_ec2_instances`), and field shapes follow the CloudQuery AWS plugin output. +The native backend normalizes Cloud Control payloads into that same shape, so +rules don't change when you flip the backend. Per-table schemas live in the +[plugin's table reference](https://hub.cloudquery.io/plugins/source/cloudquery/aws/latest/tables). + +For credential setup, IAM permissions, and `workspaceInfo.yaml` snippets, see +the user guide's [AWS cloud-discovery page](../../user-guide/cloud-discovery/aws.md) +and the [IAM key reference](../../user-guide/cloud-discovery/aws-iam-keys.md). + +## Common matchable types + +Generation rules in the contrib CodeBundles most often target: + +* `aws_ec2_instances` (`AWS::EC2::Instance`) +* `aws_ec2_volumes`, `aws_ec2_snapshots` +* `aws_s3_buckets` (`AWS::S3::Bucket`) +* `aws_rds_instances` (`AWS::RDS::DBInstance`), `aws_rds_clusters` +* `aws_eks_clusters` (`AWS::EKS::Cluster`) +* `aws_lambda_functions` +* `aws_elbv2_load_balancers` +* `aws_ecs_clusters`, `aws_ecs_services` +* `aws_iam_users`, `aws_iam_roles` + +Use the CloudQuery table name as `resource_type` in your generation rule. Field +shapes follow the CloudQuery AWS plugin output, so the easiest reference is the +plugin's per-table schema page. + +### Typed vs. generic types + +The native `awsapi` backend ships hand-written `boto3` collectors for +`aws_ec2_instances` and `aws_s3_buckets` (richer payloads); every other table +with a CloudFormation type is served by the Cloud Control API generic pass. The +mandatory `aws_iam_accounts` anchor (alias `account`) is synthesized from your +credentials and emitted first so every other resource is scoped under its +account. The full mapping of CloudQuery table -> CloudFormation type lives in +the generated registry (`src/indexers/aws_resource_type_registry.yaml`). + +Some CloudQuery tables (cost/usage reports, metric rollups, certain inventory +sub-resources) have no Cloud Control type; they map to `null` in the registry +and are skipped by the generic pass. They still resolve by name for gen-rule +compatibility, and could get a dedicated typed collector if a rule needs them. + +## Running native AWS discovery locally + +1. Set the toggle in `workspaceInfo.yaml`: + +```yaml +awsIndexerBackend: awsapi +cloudConfig: + aws: + # credentials resolve via aws_utils: explicit keys, a K8s secret, + # IRSA / Pod Identity, an assumed role, or the default credential chain + regions: + - us-east-1 + - us-west-2 +``` + + (Or export `WB_AWS_INDEXER_BACKEND=awsapi`.) + +2. Run discovery the usual way (e.g. `./run.sh` / the documented CLI). When + `awsapi` is selected, the CloudQuery indexer skips the AWS block and the + native indexer handles it; the two are mutually exclusive per platform. + +## Roadmap + +The native `awsapi` indexer is the path toward removing the CloudQuery +dependency entirely (alongside `azureapi` and `gcpapi`). `cloudquery` remains +the default backend until the native path has been validated across the contrib +CodeBundles. diff --git a/docs/authoring/indexed-resources/azure-resource-catalog.md b/docs/authoring/indexed-resources/azure-resource-catalog.md new file mode 100644 index 000000000..7b89ecd8d --- /dev/null +++ b/docs/authoring/indexed-resources/azure-resource-catalog.md @@ -0,0 +1,633 @@ +# Azure resource catalog + +Every Azure resource type the native `azureapi` indexer can discover. This page is the companion catalog for [`azure.md`](./azure.md); see that page for how to enable the indexer, what data each row carries, and the typed/generic distinction. + +_619 resource types - 25 typed (rich-payload), 594 generic (basic envelope). Generated 2026-05-28 from `src/indexers/azure_resource_type_registry.yaml`._ + +_Regenerate with `python scripts/azure/dump_azure_resource_catalog.py` after touching the registry or overrides; do not hand-edit this file._ + +* `typed` - hand-written `azure-mgmt-*` collector returns the full SDK payload (rich `properties`). +* `generic` - covered by the ARM-resources catch-all (`ResourceManagementClient.resources.list[_by_resource_group]`); row carries the basic envelope (`id`, `name`, `type`, `location`, `tags`, `sku`, `kind`, `identity`, `managed_by`) but **no** `properties` (an ARM API limitation, not a workspace-builder one). + +| Service | CloudQuery table name | ARM type | Tier | +| --- | --- | --- | --- | +| advisor | `azure_advisor_recommendation_metadata` | `Microsoft.Advisor/metadata` | generic | +| advisor | `azure_advisor_recommendations` | `Microsoft.Advisor/recommendations` | generic | +| advisor | `azure_advisor_suppressions` | `Microsoft.Advisor/suppressions` | generic | +| analysisservices | `azure_analysisservices_servers` | `Microsoft.AnalysisServices/servers` | generic | +| apimanagement | `azure_apimanagement_service` | `Microsoft.ApiManagement/service` | typed | +| appcomplianceautomation | `azure_appcomplianceautomation_reports` | `Microsoft.AppComplianceAutomation/reports` | generic | +| appconfiguration | `azure_appconfiguration_configuration_stores` | `Microsoft.AppConfiguration/configurationStores` | generic | +| applicationinsights | `azure_applicationinsights_components` | `Microsoft.Insights/components` | generic | +| applicationinsights | `azure_applicationinsights_web_tests` | `Microsoft.Insights/webtests` | generic | +| appservice | `azure_appservice_certificate_orders` | `Microsoft.CertificateRegistration/certificateOrders` | generic | +| appservice | `azure_appservice_certificates` | `Microsoft.Web/certificates` | generic | +| appservice | `azure_appservice_deleted_web_apps` | `Microsoft.Web/deletedSites` | generic | +| appservice | `azure_appservice_domains` | `Microsoft.DomainRegistration/domains` | generic | +| appservice | `azure_appservice_environments` | `Microsoft.Web/hostingEnvironments` | generic | +| appservice | `azure_appservice_plans` | `Microsoft.Web/serverFarms` | typed | +| appservice | `azure_appservice_recommendations` | `Microsoft.Web/recommendations` | generic | +| appservice | `azure_appservice_resource_health_metadata` | `Microsoft.Web/sites/resourceHealthMetadata` | generic | +| appservice | `azure_appservice_static_sites` | `Microsoft.Web/staticSites` | generic | +| appservice | `azure_appservice_top_level_domains` | `Microsoft.DomainRegistration/topLevelDomains` | generic | +| appservice | `azure_appservice_web_app_auth_settings` | `Microsoft.Web/webAppAuthSettings` | generic | +| appservice | `azure_appservice_web_app_configurations` | `Microsoft.Web/webAppConfigurations` | generic | +| appservice | `azure_appservice_web_app_functions` | `Microsoft.Web/webAppFunctions` | generic | +| appservice | `azure_appservice_web_app_vnet_connections` | `Microsoft.Web/webAppVnetConnections` | generic | +| appservice | `azure_appservice_web_apps` | `Microsoft.Web/sites` | typed | +| authorization | `azure_authorization_classic_administrators` | `Microsoft.Authorization/classicAdministrators` | generic | +| authorization | `azure_authorization_provider_operations_metadata` | `Microsoft.Authorization/providerOperations` | generic | +| authorization | `azure_authorization_role_assignments` | `Microsoft.Authorization/roleAssignments` | generic | +| authorization | `azure_authorization_role_definitions` | `Microsoft.Authorization/roleDefinitions` | generic | +| automation | `azure_automation_account` | `Microsoft.Automation/account` | generic | +| azurearcdata | `azure_azurearcdata_postgres_instances` | `Microsoft.AzureArcData/postgresInstances` | generic | +| azurearcdata | `azure_azurearcdata_sql_managed_instances` | `Microsoft.AzureArcData/sqlManagedInstances` | generic | +| azurearcdata | `azure_azurearcdata_sql_server_instances` | `Microsoft.AzureArcData/sqlServerInstances` | typed | +| batch | `azure_batch_account` | `Microsoft.Batch/account` | generic | +| billing | `azure_billing_accounts` | `Microsoft.Billing/billingAccounts` | generic | +| billing | `azure_billing_enrollment_accounts` | `Microsoft.Billing/enrollmentAccounts` | generic | +| billing | `azure_billing_periods` | `Microsoft.Billing/billingPeriods` | generic | +| botservice | `azure_botservice_bots` | `Microsoft.Botservice/bots` | generic | +| cdn | `azure_cdn_edge_nodes` | `Microsoft.Cdn/edgeNodes` | generic | +| cdn | `azure_cdn_endpoints` | `Microsoft.Cdn/profiles/endpoints` | generic | +| cdn | `azure_cdn_managed_rule_sets` | `Microsoft.Cdn/managedRuleSets` | generic | +| cdn | `azure_cdn_profiles` | `Microsoft.Cdn/profiles` | generic | +| cdn | `azure_cdn_rule_sets` | `Microsoft.Cdn/ruleSets` | generic | +| cdn | `azure_cdn_security_policies` | `Microsoft.Cdn/securityPolicies` | generic | +| cognitiveservices | `azure_cognitiveservices_account_capability_hosts` | `Microsoft.CognitiveServices/accountCapabilityHosts` | generic | +| cognitiveservices | `azure_cognitiveservices_account_connections` | `Microsoft.CognitiveServices/accountConnections` | generic | +| cognitiveservices | `azure_cognitiveservices_account_defender_for_ai_settings` | `Microsoft.CognitiveServices/accountDefenderForAiSettings` | generic | +| cognitiveservices | `azure_cognitiveservices_account_deployments` | `Microsoft.CognitiveServices/accountDeployments` | generic | +| cognitiveservices | `azure_cognitiveservices_account_encryption_scopes` | `Microsoft.CognitiveServices/accountEncryptionScopes` | generic | +| cognitiveservices | `azure_cognitiveservices_account_models` | `Microsoft.CognitiveServices/accountModels` | generic | +| cognitiveservices | `azure_cognitiveservices_account_network_security_perimeter_configurations` | `Microsoft.CognitiveServices/accountNetworkSecurityPerimeterConfigurations` | generic | +| cognitiveservices | `azure_cognitiveservices_account_private_endpoint_connections` | `Microsoft.CognitiveServices/accountPrivateEndpointConnections` | generic | +| cognitiveservices | `azure_cognitiveservices_account_private_link_resources` | `Microsoft.CognitiveServices/accountPrivateLinkResources` | generic | +| cognitiveservices | `azure_cognitiveservices_account_project_capability_hosts` | `Microsoft.CognitiveServices/accountProjectCapabilityHosts` | generic | +| cognitiveservices | `azure_cognitiveservices_account_project_connections` | `Microsoft.CognitiveServices/accountProjectConnections` | generic | +| cognitiveservices | `azure_cognitiveservices_account_projects` | `Microsoft.CognitiveServices/accountProjects` | generic | +| cognitiveservices | `azure_cognitiveservices_account_rai_blocklist_items` | `Microsoft.CognitiveServices/accountRaiBlocklistItems` | generic | +| cognitiveservices | `azure_cognitiveservices_account_rai_blocklists` | `Microsoft.CognitiveServices/accountRaiBlocklists` | generic | +| cognitiveservices | `azure_cognitiveservices_account_rai_policies` | `Microsoft.CognitiveServices/accountRaiPolicies` | generic | +| cognitiveservices | `azure_cognitiveservices_account_skus` | `Microsoft.CognitiveServices/accountSkus` | generic | +| cognitiveservices | `azure_cognitiveservices_account_usages` | `Microsoft.CognitiveServices/accountUsages` | generic | +| cognitiveservices | `azure_cognitiveservices_accounts` | `Microsoft.CognitiveServices/accounts` | generic | +| cognitiveservices | `azure_cognitiveservices_commitment_plans` | `Microsoft.CognitiveServices/commitmentPlans` | generic | +| cognitiveservices | `azure_cognitiveservices_deleted_accounts` | `Microsoft.CognitiveServices/deletedAccounts` | generic | +| cognitiveservices | `azure_cognitiveservices_resource_skus` | `Microsoft.CognitiveServices/resourceSkus` | generic | +| compute | `azure_compute_availability_sets` | `Microsoft.Compute/availabilitySets` | generic | +| compute | `azure_compute_capacity_reservation_groups` | `Microsoft.Compute/capacityReservationGroups` | generic | +| compute | `azure_compute_capacity_reservations` | `Microsoft.Compute/capacityReservations` | generic | +| compute | `azure_compute_dedicated_host_groups` | `Microsoft.Compute/dedicatedHostGroups` | generic | +| compute | `azure_compute_dedicated_hosts` | `Microsoft.Compute/dedicatedHosts` | generic | +| compute | `azure_compute_disk_accesses` | `Microsoft.Compute/diskAccesses` | generic | +| compute | `azure_compute_disk_encryption_sets` | `Microsoft.Compute/diskEncryptionSets` | generic | +| compute | `azure_compute_disks` | `Microsoft.Compute/disks` | typed | +| compute | `azure_compute_galleries` | `Microsoft.Compute/galleries` | generic | +| compute | `azure_compute_gallery_image_versions` | `Microsoft.Compute/galleryImageVersions` | generic | +| compute | `azure_compute_gallery_images` | `Microsoft.Compute/galleryImages` | generic | +| compute | `azure_compute_images` | `Microsoft.Compute/images` | generic | +| compute | `azure_compute_proximity_placement_groups` | `Microsoft.Compute/proximityPlacementGroups` | generic | +| compute | `azure_compute_restore_point_collections` | `Microsoft.Compute/restorePointCollections` | generic | +| compute | `azure_compute_skus` | `Microsoft.Compute/skus` | generic | +| compute | `azure_compute_snapshots` | `Microsoft.Compute/snapshots` | typed | +| compute | `azure_compute_ssh_public_keys` | `Microsoft.Compute/sshPublicKeys` | generic | +| compute | `azure_compute_virtual_machine_extensions` | `Microsoft.Compute/virtualMachineExtensions` | generic | +| compute | `azure_compute_virtual_machine_patch_assessments` | `Microsoft.Compute/virtualMachinePatchAssessments` | generic | +| compute | `azure_compute_virtual_machine_scale_set_network_interfaces` | `Microsoft.Compute/virtualMachineScaleSetNetworkInterfaces` | generic | +| compute | `azure_compute_virtual_machine_scale_set_vms` | `Microsoft.Compute/virtualMachineScaleSetVms` | generic | +| compute | `azure_compute_virtual_machine_scale_sets` | `Microsoft.Compute/virtualMachineScaleSets` | typed | +| compute | `azure_compute_virtual_machine_software_inventories` | `Microsoft.Compute/virtualMachineSoftwareInventories` | generic | +| compute | `azure_compute_virtual_machines` | `Microsoft.Compute/virtualMachines` | typed | +| confidentialledger | `azure_confidentialledger_ledgers` | `Microsoft.Confidentialledger/ledgers` | generic | +| confidentialledger | `azure_confidentialledger_operations` | `Microsoft.Confidentialledger/operations` | generic | +| confluent | `azure_confluent_marketplace_agreements` | `Microsoft.Confluent/marketplaceAgreements` | generic | +| connectedvmware | `azure_connectedvmware_clusters` | `Microsoft.ConnectedVMware/clusters` | generic | +| connectedvmware | `azure_connectedvmware_datastores` | `Microsoft.ConnectedVMware/datastores` | generic | +| connectedvmware | `azure_connectedvmware_hosts` | `Microsoft.ConnectedVMware/hosts` | generic | +| connectedvmware | `azure_connectedvmware_resource_pools` | `Microsoft.ConnectedVMware/resourcePools` | generic | +| connectedvmware | `azure_connectedvmware_v_centers` | `Microsoft.ConnectedVMware/vCenters` | generic | +| connectedvmware | `azure_connectedvmware_virtual_machine_templates` | `Microsoft.ConnectedVMware/virtualMachineTemplates` | generic | +| connectedvmware | `azure_connectedvmware_virtual_machines` | `Microsoft.ConnectedVMware/virtualMachines` | generic | +| connectedvmware | `azure_connectedvmware_virtual_networks` | `Microsoft.ConnectedVMware/virtualNetworks` | generic | +| consumption | `azure_consumption_billing_account_balances` | `Microsoft.Consumption/billingAccountBalances` | generic | +| consumption | `azure_consumption_billing_account_budgets` | `Microsoft.Consumption/billingAccountBudgets` | generic | +| consumption | `azure_consumption_billing_account_charges` | `Microsoft.Consumption/billingAccountCharges` | generic | +| consumption | `azure_consumption_billing_account_events` | `Microsoft.Consumption/billingAccountEvents` | generic | +| consumption | `azure_consumption_billing_account_legacy_usage_details` | `Microsoft.Consumption/billingAccountLegacyUsageDetails` | generic | +| consumption | `azure_consumption_billing_account_lots` | `Microsoft.Consumption/billingAccountLots` | generic | +| consumption | `azure_consumption_billing_account_marketplaces` | `Microsoft.Consumption/billingAccountMarketplaces` | generic | +| consumption | `azure_consumption_billing_account_modern_usage_details` | `Microsoft.Consumption/billingAccountModernUsageDetails` | generic | +| consumption | `azure_consumption_billing_account_profile_credits` | `Microsoft.Consumption/billingAccountProfileCredits` | generic | +| consumption | `azure_consumption_billing_account_reservation_recommendations` | `Microsoft.Consumption/billingAccountReservationRecommendations` | generic | +| consumption | `azure_consumption_billing_account_tags` | `Microsoft.Consumption/billingAccountTags` | generic | +| consumption | `azure_consumption_billing_profile_reservation_details` | `Microsoft.Consumption/billingProfileReservationDetails` | generic | +| consumption | `azure_consumption_billing_profile_reservation_recommendations` | `Microsoft.Consumption/billingProfileReservationRecommendations` | generic | +| consumption | `azure_consumption_billing_profile_reservation_summaries` | `Microsoft.Consumption/billingProfileReservationSummaries` | generic | +| consumption | `azure_consumption_billing_profile_reservation_transactions` | `Microsoft.Consumption/billingProfileReservationTransactions` | generic | +| consumption | `azure_consumption_subscription_budgets` | `Microsoft.Consumption/subscriptionBudgets` | generic | +| consumption | `azure_consumption_subscription_legacy_usage_details` | `Microsoft.Consumption/subscriptionLegacyUsageDetails` | generic | +| consumption | `azure_consumption_subscription_marketplaces` | `Microsoft.Consumption/subscriptionMarketplaces` | generic | +| consumption | `azure_consumption_subscription_price_sheets` | `Microsoft.Consumption/subscriptionPriceSheets` | generic | +| consumption | `azure_consumption_subscription_reservation_recommendations` | `Microsoft.Consumption/subscriptionReservationRecommendations` | generic | +| consumption | `azure_consumption_subscription_tags` | `Microsoft.Consumption/subscriptionTags` | generic | +| container | `azure_container_app_diagnostics_detectors` | `Microsoft.Container/appDiagnosticsDetectors` | generic | +| container | `azure_container_app_source_controls` | `Microsoft.Container/appSourceControls` | generic | +| containerapps | `azure_containerapps_connected_environment_certificates` | `Microsoft.Containerapps/connectedEnvironmentCertificates` | generic | +| containerapps | `azure_containerapps_connected_environment_dapr_components` | `Microsoft.Containerapps/connectedEnvironmentDaprComponents` | generic | +| containerapps | `azure_containerapps_connected_environment_storages` | `Microsoft.Containerapps/connectedEnvironmentStorages` | generic | +| containerapps | `azure_containerapps_connected_environments` | `Microsoft.Containerapps/connectedEnvironments` | generic | +| containerapps | `azure_containerapps_container_app_auth_configs` | `Microsoft.Containerapps/containerAppAuthConfigs` | generic | +| containerapps | `azure_containerapps_container_app_detector_revisions` | `Microsoft.Containerapps/containerAppDetectorRevisions` | generic | +| containerapps | `azure_containerapps_container_app_revision_replicas` | `Microsoft.Containerapps/containerAppRevisionReplicas` | generic | +| containerapps | `azure_containerapps_container_app_revisions` | `Microsoft.Containerapps/containerAppRevisions` | generic | +| containerapps | `azure_containerapps_container_apps` | `Microsoft.Containerapps/containerApps` | generic | +| containerapps | `azure_containerapps_job_executions` | `Microsoft.Containerapps/jobExecutions` | generic | +| containerapps | `azure_containerapps_jobs` | `Microsoft.Containerapps/jobs` | generic | +| containerapps | `azure_containerapps_managed_certificates` | `Microsoft.Containerapps/managedCertificates` | generic | +| containerapps | `azure_containerapps_managed_environment_certificates` | `Microsoft.Containerapps/managedEnvironmentCertificates` | generic | +| containerapps | `azure_containerapps_managed_environment_detectors` | `Microsoft.Containerapps/managedEnvironmentDetectors` | generic | +| containerapps | `azure_containerapps_managed_environment_storages` | `Microsoft.Containerapps/managedEnvironmentStorages` | generic | +| containerapps | `azure_containerapps_managed_environment_usages` | `Microsoft.Containerapps/managedEnvironmentUsages` | generic | +| containerapps | `azure_containerapps_managed_environments` | `Microsoft.Containerapps/managedEnvironments` | generic | +| containerapps | `azure_containerapps_operations` | `Microsoft.Containerapps/operations` | generic | +| containerinstance | `azure_containerinstance_container_groups` | `Microsoft.ContainerInstance/containerGroups` | generic | +| containerregistry | `azure_containerregistry_registries` | `Microsoft.ContainerRegistry/registries` | typed | +| containerservice | `azure_containerservice_managed_cluster_agent_pools` | `Microsoft.ContainerService/managedClusterAgentPools` | generic | +| containerservice | `azure_containerservice_managed_cluster_upgrade_profiles` | `Microsoft.ContainerService/managedClusterUpgradeProfiles` | generic | +| containerservice | `azure_containerservice_managed_clusters` | `Microsoft.ContainerService/managedClusters` | typed | +| containerservice | `azure_containerservice_snapshots` | `Microsoft.ContainerService/snapshots` | generic | +| cosmos | `azure_cosmos_cassandra_clusters` | `Microsoft.Cosmos/cassandraClusters` | generic | +| cosmos | `azure_cosmos_database_accounts` | `Microsoft.Cosmos/databaseAccounts` | generic | +| cosmos | `azure_cosmos_locations` | `Microsoft.Cosmos/locations` | generic | +| cosmos | `azure_cosmos_mongo_db_databases` | `Microsoft.Cosmos/mongoDbDatabases` | generic | +| cosmos | `azure_cosmos_restorable_database_accounts` | `Microsoft.Cosmos/restorableDatabaseAccounts` | generic | +| cosmos | `azure_cosmos_sql_databases` | `Microsoft.DocumentDB/databaseAccounts/sqlDatabases` | typed | +| costmanagement | `azure_costmanagement_subscription_costs` | `Microsoft.CostManagement/subscriptionCosts` | generic | +| costmanagement | `azure_costmanagement_view_queries` | `Microsoft.CostManagement/viewQueries` | generic | +| costmanagement | `azure_costmanagement_views` | `Microsoft.CostManagement/views` | generic | +| customerinsights | `azure_customerinsights_hubs` | `Microsoft.CustomerInsights/hubs` | generic | +| dashboard | `azure_dashboard_grafana` | `Microsoft.Dashboard/grafana` | generic | +| databox | `azure_databox_jobs` | `Microsoft.Databox/jobs` | generic | +| databricks | `azure_databricks_access_connectors` | `Microsoft.Databricks/accessConnectors` | generic | +| databricks | `azure_databricks_operations` | `Microsoft.Databricks/operations` | generic | +| databricks | `azure_databricks_outbound_network_dependencies_endpoints` | `Microsoft.Databricks/outboundNetworkDependenciesEndpoints` | generic | +| databricks | `azure_databricks_private_endpoint_connections` | `Microsoft.Databricks/privateEndpointConnections` | generic | +| databricks | `azure_databricks_private_link_resources` | `Microsoft.Databricks/privateLinkResources` | generic | +| databricks | `azure_databricks_virtual_network_peerings` | `Microsoft.Databricks/virtualNetworkPeerings` | generic | +| databricks | `azure_databricks_workspaces` | `Microsoft.Databricks/workspaces` | generic | +| datacatalog | `azure_datacatalog_catalogs` | `Microsoft.Datacatalog/catalogs` | generic | +| datadog | `azure_datadog_marketplace_agreements` | `Microsoft.Datadog/marketplaceAgreements` | generic | +| datadog | `azure_datadog_monitors` | `Microsoft.Datadog/monitors` | generic | +| datafactory | `azure_datafactory_factories` | `Microsoft.DataFactory/factories` | typed | +| datalakeanalytics | `azure_datalakeanalytics_accounts` | `Microsoft.Datalakeanalytics/accounts` | generic | +| datalakestore | `azure_datalakestore_accounts` | `Microsoft.Datalakestore/accounts` | generic | +| datamigration | `azure_datamigration_services` | `Microsoft.Datamigration/services` | generic | +| datashare | `azure_datashare_accounts` | `Microsoft.Datashare/accounts` | generic | +| desktopvirtualization | `azure_desktopvirtualization_application_groups` | `Microsoft.DesktopVirtualization/applicationGroups` | generic | +| desktopvirtualization | `azure_desktopvirtualization_host_pools` | `Microsoft.DesktopVirtualization/hostPools` | generic | +| desktopvirtualization | `azure_desktopvirtualization_workspaces` | `Microsoft.DesktopVirtualization/workspaces` | generic | +| devhub | `azure_devhub_workflow` | `Microsoft.Devhub/workflow` | generic | +| devops | `azure_devops_pipeline_template_definitions` | `Microsoft.Devops/pipelineTemplateDefinitions` | generic | +| devtestlabs | `azure_devtestlabs_global_schedules` | `Microsoft.Devtestlabs/globalSchedules` | generic | +| devtestlabs | `azure_devtestlabs_lab_artifact_source_arm_templates` | `Microsoft.Devtestlabs/labArtifactSourceArmTemplates` | generic | +| devtestlabs | `azure_devtestlabs_lab_artifact_source_artifact` | `Microsoft.Devtestlabs/labArtifactSourceArtifact` | generic | +| devtestlabs | `azure_devtestlabs_lab_artifact_sources` | `Microsoft.Devtestlabs/labArtifactSources` | generic | +| devtestlabs | `azure_devtestlabs_lab_custom_images` | `Microsoft.Devtestlabs/labCustomImages` | generic | +| devtestlabs | `azure_devtestlabs_lab_environments` | `Microsoft.Devtestlabs/labEnvironments` | generic | +| devtestlabs | `azure_devtestlabs_lab_formulas` | `Microsoft.Devtestlabs/labFormulas` | generic | +| devtestlabs | `azure_devtestlabs_lab_galery_images` | `Microsoft.Devtestlabs/labGaleryImages` | generic | +| devtestlabs | `azure_devtestlabs_lab_notification_channels` | `Microsoft.Devtestlabs/labNotificationChannels` | generic | +| devtestlabs | `azure_devtestlabs_lab_schedules` | `Microsoft.Devtestlabs/labSchedules` | generic | +| devtestlabs | `azure_devtestlabs_lab_user_disks` | `Microsoft.Devtestlabs/labUserDisks` | generic | +| devtestlabs | `azure_devtestlabs_lab_user_secrets` | `Microsoft.Devtestlabs/labUserSecrets` | generic | +| devtestlabs | `azure_devtestlabs_lab_users` | `Microsoft.Devtestlabs/labUsers` | generic | +| devtestlabs | `azure_devtestlabs_lab_virtual_machine_schedules` | `Microsoft.Devtestlabs/labVirtualMachineSchedules` | generic | +| devtestlabs | `azure_devtestlabs_lab_virtual_machines` | `Microsoft.Devtestlabs/labVirtualMachines` | generic | +| devtestlabs | `azure_devtestlabs_lab_virtual_networks` | `Microsoft.Devtestlabs/labVirtualNetworks` | generic | +| devtestlabs | `azure_devtestlabs_labs` | `Microsoft.Devtestlabs/labs` | generic | +| dns | `azure_dns_record_sets` | `Microsoft.Dns/recordSets` | generic | +| dns | `azure_dns_zones` | `Microsoft.Dns/zones` | generic | +| dnsresolver | `azure_dnsresolver_dns_forwarding_rulesets` | `Microsoft.Network/dnsForwardingRulesets` | generic | +| dnsresolver | `azure_dnsresolver_dns_resolvers` | `Microsoft.Network/dnsResolvers` | generic | +| elastic | `azure_elastic_monitors` | `Microsoft.Elastic/monitors` | generic | +| engagementfabric | `azure_engagementfabric_accounts` | `Microsoft.Engagementfabric/accounts` | generic | +| eventgrid | `azure_eventgrid_topic_types` | `Microsoft.EventGrid/topicTypes` | generic | +| eventhub | `azure_eventhub_clusters` | `Microsoft.EventHub/clusters` | generic | +| eventhub | `azure_eventhub_namespace_network_rule_sets` | `Microsoft.EventHub/namespaceNetworkRuleSets` | generic | +| eventhub | `azure_eventhub_namespaces` | `Microsoft.EventHub/namespaces` | generic | +| frontdoor | `azure_frontdoor_front_doors` | `Microsoft.Network/frontDoors` | generic | +| frontdoor | `azure_frontdoor_managed_rule_sets` | `Microsoft.Network/managedRuleSets` | generic | +| frontdoor | `azure_frontdoor_network_experiment_profiles` | `Microsoft.Network/networkExperimentProfiles` | generic | +| hanaonazure | `azure_hanaonazure_sap_monitors` | `Microsoft.Hanaonazure/sapMonitors` | generic | +| hdinsight | `azure_hdinsight_clusters` | `Microsoft.HDInsight/clusters` | generic | +| healthbot | `azure_healthbot_bots` | `Microsoft.Healthbot/bots` | generic | +| healthcareapis | `azure_healthcareapis_services` | `Microsoft.HealthcareApis/services` | generic | +| hybridcompute | `azure_hybridcompute_private_link_scopes` | `Microsoft.HybridCompute/privateLinkScopes` | generic | +| hybriddatamanager | `azure_hybriddatamanager_data_managers` | `Microsoft.Hybriddatamanager/dataManagers` | generic | +| keyvault | `azure_keyvault_certificate_issuers` | `Microsoft.KeyVault/vaults/certificates/issuers` | generic | +| keyvault | `azure_keyvault_certificate_policies` | `Microsoft.KeyVault/vaults/certificates/policies` | generic | +| keyvault | `azure_keyvault_certificates` | `Microsoft.KeyVault/vaults/certificates` | generic | +| keyvault | `azure_keyvault_key_rotation_policies` | `Microsoft.KeyVault/vaults/keys/rotationPolicies` | generic | +| keyvault | `azure_keyvault_keys` | `Microsoft.KeyVault/vaults/keys` | generic | +| keyvault | `azure_keyvault_keyvaults` | `Microsoft.KeyVault/vaults` | typed | +| keyvault | `azure_keyvault_managed_hsms` | `Microsoft.KeyVault/managedHSMs` | generic | +| keyvault | `azure_keyvault_secrets` | `Microsoft.KeyVault/vaults/secrets` | generic | +| kusto | `azure_kusto_clusters` | `Microsoft.Kusto/clusters` | generic | +| labservices | `azure_labservices_lab_plan_images` | `Microsoft.LabServices/labPlanImages` | generic | +| labservices | `azure_labservices_lab_plans` | `Microsoft.LabServices/labPlans` | generic | +| labservices | `azure_labservices_lab_schedules` | `Microsoft.LabServices/labSchedules` | generic | +| labservices | `azure_labservices_lab_usages` | `Microsoft.LabServices/labUsages` | generic | +| labservices | `azure_labservices_lab_users` | `Microsoft.LabServices/labUsers` | generic | +| labservices | `azure_labservices_lab_virtual_machines` | `Microsoft.LabServices/labVirtualMachines` | generic | +| labservices | `azure_labservices_labs` | `Microsoft.LabServices/labs` | generic | +| labservices | `azure_labservices_operations` | `Microsoft.LabServices/operations` | generic | +| labservices | `azure_labservices_skus` | `Microsoft.LabServices/skus` | generic | +| logic | `azure_logic_integration_account_agreements` | `Microsoft.Logic/integrationAccountAgreements` | generic | +| logic | `azure_logic_integration_account_assemblies` | `Microsoft.Logic/integrationAccountAssemblies` | generic | +| logic | `azure_logic_integration_account_certificates` | `Microsoft.Logic/integrationAccountCertificates` | generic | +| logic | `azure_logic_integration_account_maps` | `Microsoft.Logic/integrationAccountMaps` | generic | +| logic | `azure_logic_integration_account_partners` | `Microsoft.Logic/integrationAccountPartners` | generic | +| logic | `azure_logic_integration_account_schemas` | `Microsoft.Logic/integrationAccountSchemas` | generic | +| logic | `azure_logic_integration_account_sessions` | `Microsoft.Logic/integrationAccountSessions` | generic | +| logic | `azure_logic_integration_accounts` | `Microsoft.Logic/integrationAccounts` | generic | +| logic | `azure_logic_workflow_run_action_request_histories` | `Microsoft.Logic/workflowRunActionRequestHistories` | generic | +| logic | `azure_logic_workflow_run_actions` | `Microsoft.Logic/workflowRunActions` | generic | +| logic | `azure_logic_workflow_runs` | `Microsoft.Logic/workflowRuns` | generic | +| logic | `azure_logic_workflow_trigger_histories` | `Microsoft.Logic/workflowTriggerHistories` | generic | +| logic | `azure_logic_workflow_triggers` | `Microsoft.Logic/workflowTriggers` | generic | +| logic | `azure_logic_workflow_versions` | `Microsoft.Logic/workflowVersions` | generic | +| logic | `azure_logic_workflows` | `Microsoft.Logic/workflows` | generic | +| machinelearning | `azure_machinelearning_batch_deployments` | `Microsoft.MachineLearningServices/batchDeployments` | generic | +| machinelearning | `azure_machinelearning_batch_endpoints` | `Microsoft.MachineLearningServices/batchEndpoints` | generic | +| machinelearning | `azure_machinelearning_component_containers` | `Microsoft.MachineLearningServices/componentContainers` | generic | +| machinelearning | `azure_machinelearning_component_versions` | `Microsoft.MachineLearningServices/componentVersions` | generic | +| machinelearning | `azure_machinelearning_computes` | `Microsoft.MachineLearningServices/computes` | generic | +| machinelearning | `azure_machinelearning_data_containers` | `Microsoft.MachineLearningServices/dataContainers` | generic | +| machinelearning | `azure_machinelearning_data_versions` | `Microsoft.MachineLearningServices/dataVersions` | generic | +| machinelearning | `azure_machinelearning_datastores` | `Microsoft.MachineLearningServices/datastores` | generic | +| machinelearning | `azure_machinelearning_environment_containers` | `Microsoft.MachineLearningServices/environmentContainers` | generic | +| machinelearning | `azure_machinelearning_environment_versions` | `Microsoft.MachineLearningServices/environmentVersions` | generic | +| machinelearning | `azure_machinelearning_features` | `Microsoft.MachineLearningServices/features` | generic | +| machinelearning | `azure_machinelearning_featureset_containers` | `Microsoft.MachineLearningServices/featuresetContainers` | generic | +| machinelearning | `azure_machinelearning_featureset_versions` | `Microsoft.MachineLearningServices/featuresetVersions` | generic | +| machinelearning | `azure_machinelearning_featurestore_entity_containers` | `Microsoft.MachineLearningServices/featurestoreEntityContainers` | generic | +| machinelearning | `azure_machinelearning_featurestore_entity_versions` | `Microsoft.MachineLearningServices/featurestoreEntityVersions` | generic | +| machinelearning | `azure_machinelearning_jobs` | `Microsoft.MachineLearningServices/jobs` | generic | +| machinelearning | `azure_machinelearning_managed_network_setting_rules` | `Microsoft.MachineLearningServices/managedNetworkSettingRules` | generic | +| machinelearning | `azure_machinelearning_marketplace_subscriptions` | `Microsoft.MachineLearningServices/marketplaceSubscriptions` | generic | +| machinelearning | `azure_machinelearning_model_containers` | `Microsoft.MachineLearningServices/modelContainers` | generic | +| machinelearning | `azure_machinelearning_model_versions` | `Microsoft.MachineLearningServices/modelVersions` | generic | +| machinelearning | `azure_machinelearning_online_deployments` | `Microsoft.MachineLearningServices/onlineDeployments` | generic | +| machinelearning | `azure_machinelearning_online_endpoints` | `Microsoft.MachineLearningServices/onlineEndpoints` | generic | +| machinelearning | `azure_machinelearning_operations` | `Microsoft.MachineLearningServices/operations` | generic | +| machinelearning | `azure_machinelearning_private_endpoint_connections` | `Microsoft.MachineLearningServices/privateEndpointConnections` | generic | +| machinelearning | `azure_machinelearning_private_link_resources` | `Microsoft.MachineLearningServices/privateLinkResources` | generic | +| machinelearning | `azure_machinelearning_quotas` | `Microsoft.MachineLearningServices/quotas` | generic | +| machinelearning | `azure_machinelearning_registries` | `Microsoft.MachineLearningServices/registries` | generic | +| machinelearning | `azure_machinelearning_registry_code_containers` | `Microsoft.MachineLearningServices/registryCodeContainers` | generic | +| machinelearning | `azure_machinelearning_registry_code_versions` | `Microsoft.MachineLearningServices/registryCodeVersions` | generic | +| machinelearning | `azure_machinelearning_registry_component_containers` | `Microsoft.MachineLearningServices/registryComponentContainers` | generic | +| machinelearning | `azure_machinelearning_registry_component_versions` | `Microsoft.MachineLearningServices/registryComponentVersions` | generic | +| machinelearning | `azure_machinelearning_registry_data_containers` | `Microsoft.MachineLearningServices/registryDataContainers` | generic | +| machinelearning | `azure_machinelearning_registry_data_versions` | `Microsoft.MachineLearningServices/registryDataVersions` | generic | +| machinelearning | `azure_machinelearning_registry_environment_containers` | `Microsoft.MachineLearningServices/registryEnvironmentContainers` | generic | +| machinelearning | `azure_machinelearning_registry_environment_versions` | `Microsoft.MachineLearningServices/registryEnvironmentVersions` | generic | +| machinelearning | `azure_machinelearning_registry_model_containers` | `Microsoft.MachineLearningServices/registryModelContainers` | generic | +| machinelearning | `azure_machinelearning_registry_model_versions` | `Microsoft.MachineLearningServices/registryModelVersions` | generic | +| machinelearning | `azure_machinelearning_schedules` | `Microsoft.MachineLearningServices/schedules` | generic | +| machinelearning | `azure_machinelearning_serverless_endpoints` | `Microsoft.MachineLearningServices/serverlessEndpoints` | generic | +| machinelearning | `azure_machinelearning_usages` | `Microsoft.MachineLearningServices/usages` | generic | +| machinelearning | `azure_machinelearning_virtual_machine_sizes` | `Microsoft.MachineLearningServices/virtualMachineSizes` | generic | +| machinelearning | `azure_machinelearning_workspace_connections` | `Microsoft.MachineLearningServices/workspaceConnections` | generic | +| machinelearning | `azure_machinelearning_workspace_features` | `Microsoft.MachineLearningServices/workspaceFeatures` | generic | +| machinelearning | `azure_machinelearning_workspaces` | `Microsoft.MachineLearningServices/workspaces` | generic | +| maintenance | `azure_maintenance_configurations` | `Microsoft.Maintenance/configurations` | generic | +| maintenance | `azure_maintenance_public_maintenance_configurations` | `Microsoft.Maintenance/publicMaintenanceConfigurations` | generic | +| managedapplications | `azure_managedapplications_applications` | `Microsoft.Managedapplications/applications` | generic | +| management | `azure_management_locks` | `Microsoft.Management/locks` | generic | +| managementgroups | `azure_managementgroups_entities` | `Microsoft.Managementgroups/entities` | generic | +| managementgroups | `azure_managementgroups_management_groups` | `Microsoft.Managementgroups/managementGroups` | generic | +| mariadb | `azure_mariadb_server_configurations` | `Microsoft.DBforMariaDB/serverConfigurations` | generic | +| mariadb | `azure_mariadb_servers` | `Microsoft.DBforMariaDB/servers` | generic | +| marketplace | `azure_marketplace_private_store` | `Microsoft.Marketplace/privateStore` | generic | +| mediaservices | `azure_mediaservices_account_filters` | `Microsoft.Mediaservices/accountFilters` | generic | +| mediaservices | `azure_mediaservices_asset_filters` | `Microsoft.Mediaservices/assetFilters` | generic | +| mediaservices | `azure_mediaservices_asset_tracks` | `Microsoft.Mediaservices/assetTracks` | generic | +| mediaservices | `azure_mediaservices_assets` | `Microsoft.Mediaservices/assets` | generic | +| mediaservices | `azure_mediaservices_content_key_policies` | `Microsoft.Mediaservices/contentKeyPolicies` | generic | +| mediaservices | `azure_mediaservices_jobs` | `Microsoft.Mediaservices/jobs` | generic | +| mediaservices | `azure_mediaservices_live_event_outputs` | `Microsoft.Mediaservices/liveEventOutputs` | generic | +| mediaservices | `azure_mediaservices_live_events` | `Microsoft.Mediaservices/liveEvents` | generic | +| mediaservices | `azure_mediaservices_media_services` | `Microsoft.Mediaservices/mediaServices` | generic | +| mediaservices | `azure_mediaservices_private_endpoint_connections` | `Microsoft.Mediaservices/privateEndpointConnections` | generic | +| mediaservices | `azure_mediaservices_private_link_resources` | `Microsoft.Mediaservices/privateLinkResources` | generic | +| mediaservices | `azure_mediaservices_streaming_endpoints` | `Microsoft.Mediaservices/streamingEndpoints` | generic | +| mediaservices | `azure_mediaservices_streaming_locators` | `Microsoft.Mediaservices/streamingLocators` | generic | +| mediaservices | `azure_mediaservices_streaming_policies` | `Microsoft.Mediaservices/streamingPolicies` | generic | +| mediaservices | `azure_mediaservices_transforms` | `Microsoft.Mediaservices/transforms` | generic | +| monitor | `azure_monitor_action_groups` | `Microsoft.Insights/actionGroups` | generic | +| monitor | `azure_monitor_activity_log_alerts` | `Microsoft.Monitor/activityLogAlerts` | generic | +| monitor | `azure_monitor_autoscale_settings` | `Microsoft.Insights/autoscaleSettings` | generic | +| monitor | `azure_monitor_data_collection_rule_associations` | `Microsoft.Monitor/dataCollectionRuleAssociations` | generic | +| monitor | `azure_monitor_data_collection_rules` | `Microsoft.Monitor/dataCollectionRules` | generic | +| monitor | `azure_monitor_diagnostic_settings` | `Microsoft.Insights/diagnosticSettings` | generic | +| monitor | `azure_monitor_log_profiles` | `Microsoft.Monitor/logProfiles` | generic | +| monitor | `azure_monitor_metric_alerts` | `Microsoft.Insights/metricAlerts` | generic | +| monitor | `azure_monitor_metrics` | `Microsoft.Monitor/metrics` | generic | +| monitor | `azure_monitor_private_link_scopes` | `Microsoft.Monitor/privateLinkScopes` | generic | +| monitor | `azure_monitor_resources` | `Microsoft.Monitor/resources` | generic | +| monitor | `azure_monitor_scheduled_query_rules` | `Microsoft.Monitor/scheduledQueryRules` | generic | +| monitor | `azure_monitor_subscription_diagnostic_settings` | `Microsoft.Monitor/subscriptionDiagnosticSettings` | generic | +| monitor | `azure_monitor_tenant_activity_log_alerts` | `Microsoft.Monitor/tenantActivityLogAlerts` | generic | +| monitor | `azure_monitor_tenant_activity_logs` | `Microsoft.Monitor/tenantActivityLogs` | generic | +| mysql | `azure_mysql_server_configurations` | `Microsoft.DBforMySQL/serverConfigurations` | generic | +| mysql | `azure_mysql_server_databases` | `Microsoft.DBforMySQL/serverDatabases` | generic | +| mysql | `azure_mysql_server_firewall_rules` | `Microsoft.DBforMySQL/serverFirewallRules` | generic | +| mysql | `azure_mysql_servers` | `Microsoft.DBforMySQL/servers` | typed | +| mysqlflexibleservers | `azure_mysqlflexibleservers_server_configurations` | `Microsoft.Mysqlflexibleservers/serverConfigurations` | generic | +| mysqlflexibleservers | `azure_mysqlflexibleservers_server_firewall_rules` | `Microsoft.Mysqlflexibleservers/serverFirewallRules` | generic | +| mysqlflexibleservers | `azure_mysqlflexibleservers_servers` | `Microsoft.DBforMySQL/flexibleServers` | typed | +| netappfiles | `azure_netappfiles_account_backup_policies` | `Microsoft.NetApp/accountBackupPolicies` | generic | +| netappfiles | `azure_netappfiles_account_backup_vault_backups` | `Microsoft.NetApp/accountBackupVaultBackups` | generic | +| netappfiles | `azure_netappfiles_account_backup_vaults` | `Microsoft.NetApp/accountBackupVaults` | generic | +| netappfiles | `azure_netappfiles_account_pool_volume_quota_rules` | `Microsoft.NetApp/accountPoolVolumeQuotaRules` | generic | +| netappfiles | `azure_netappfiles_account_pool_volume_snapshots` | `Microsoft.NetApp/accountPoolVolumeSnapshots` | generic | +| netappfiles | `azure_netappfiles_account_pool_volume_subvolumes` | `Microsoft.NetApp/accountPoolVolumeSubvolumes` | generic | +| netappfiles | `azure_netappfiles_account_pool_volumes` | `Microsoft.NetApp/accountPoolVolumes` | generic | +| netappfiles | `azure_netappfiles_account_pools` | `Microsoft.NetApp/accountPools` | generic | +| netappfiles | `azure_netappfiles_account_snapshot_policies` | `Microsoft.NetApp/accountSnapshotPolicies` | generic | +| netappfiles | `azure_netappfiles_account_snapshot_policy_associated_volumes` | `Microsoft.NetApp/accountSnapshotPolicyAssociatedVolumes` | generic | +| netappfiles | `azure_netappfiles_account_volume_groups` | `Microsoft.NetApp/accountVolumeGroups` | generic | +| netappfiles | `azure_netappfiles_accounts` | `Microsoft.NetApp/accounts` | generic | +| netappfiles | `azure_netappfiles_quota_limits` | `Microsoft.NetApp/quotaLimits` | generic | +| netappfiles | `azure_netappfiles_region_infos` | `Microsoft.NetApp/regionInfos` | generic | +| network | `azure_network_application_gateways` | `Microsoft.Network/applicationGateways` | typed | +| network | `azure_network_application_security_groups` | `Microsoft.Network/applicationSecurityGroups` | generic | +| network | `azure_network_azure_firewall_fqdn_tags` | `Microsoft.Network/azureFirewallFqdnTags` | generic | +| network | `azure_network_azure_firewalls` | `Microsoft.Network/azureFirewalls` | generic | +| network | `azure_network_bastion_hosts` | `Microsoft.Network/bastionHosts` | generic | +| network | `azure_network_bgp_service_communities` | `Microsoft.Network/bgpServiceCommunities` | generic | +| network | `azure_network_custom_ip_prefixes` | `Microsoft.Network/customIpPrefixes` | generic | +| network | `azure_network_ddos_protection_plans` | `Microsoft.Network/ddosProtectionPlans` | generic | +| network | `azure_network_dscp_configuration` | `Microsoft.Network/dscpConfiguration` | generic | +| network | `azure_network_express_route_circuit_authorizations` | `Microsoft.Network/expressRouteCircuitAuthorizations` | generic | +| network | `azure_network_express_route_circuit_peerings` | `Microsoft.Network/expressRouteCircuitPeerings` | generic | +| network | `azure_network_express_route_circuits` | `Microsoft.Network/expressRouteCircuits` | generic | +| network | `azure_network_express_route_gateways` | `Microsoft.Network/expressRouteGateways` | generic | +| network | `azure_network_express_route_ports` | `Microsoft.Network/expressRoutePorts` | generic | +| network | `azure_network_express_route_ports_locations` | `Microsoft.Network/expressRoutePortsLocations` | generic | +| network | `azure_network_express_route_service_providers` | `Microsoft.Network/expressRouteServiceProviders` | generic | +| network | `azure_network_firewall_policies` | `Microsoft.Network/firewallPolicies` | generic | +| network | `azure_network_interface_effective_route_tables` | `Microsoft.Network/interfaceEffectiveRouteTables` | generic | +| network | `azure_network_interface_ip_configurations` | `Microsoft.Network/interfaceIpConfigurations` | generic | +| network | `azure_network_interfaces` | `Microsoft.Network/interfaces` | generic | +| network | `azure_network_ip_allocations` | `Microsoft.Network/ipAllocations` | generic | +| network | `azure_network_ip_groups` | `Microsoft.Network/ipGroups` | generic | +| network | `azure_network_load_balancers` | `Microsoft.Network/loadBalancers` | typed | +| network | `azure_network_nat_gateways` | `Microsoft.Network/natGateways` | generic | +| network | `azure_network_peering_route_tables` | `Microsoft.Network/peeringRouteTables` | generic | +| network | `azure_network_private_endpoints` | `Microsoft.Network/privateEndpoints` | generic | +| network | `azure_network_private_link_services` | `Microsoft.Network/privateLinkServices` | generic | +| network | `azure_network_profiles` | `Microsoft.Network/profiles` | generic | +| network | `azure_network_public_ip_addresses` | `Microsoft.Network/publicIPAddresses` | generic | +| network | `azure_network_public_ip_prefixes` | `Microsoft.Network/publicIpPrefixes` | generic | +| network | `azure_network_route_filters` | `Microsoft.Network/routeFilters` | generic | +| network | `azure_network_route_tables` | `Microsoft.Network/routeTables` | generic | +| network | `azure_network_security_groups` | `Microsoft.Network/networkSecurityGroups` | typed | +| network | `azure_network_security_partner_providers` | `Microsoft.Network/securityPartnerProviders` | generic | +| network | `azure_network_service_endpoint_policies` | `Microsoft.Network/serviceEndpointPolicies` | generic | +| network | `azure_network_subscription_network_manager_connections` | `Microsoft.Network/subscriptionNetworkManagerConnections` | generic | +| network | `azure_network_virtual_appliances` | `Microsoft.Network/virtualAppliances` | generic | +| network | `azure_network_virtual_hubs` | `Microsoft.Network/virtualHubs` | generic | +| network | `azure_network_virtual_network_gateway_connections` | `Microsoft.Network/virtualNetworkGatewayConnections` | generic | +| network | `azure_network_virtual_network_gateways` | `Microsoft.Network/virtualNetworkGateways` | generic | +| network | `azure_network_virtual_network_subnets` | `Microsoft.Network/virtualNetworkSubnets` | generic | +| network | `azure_network_virtual_network_taps` | `Microsoft.Network/virtualNetworkTaps` | generic | +| network | `azure_network_virtual_networks` | `Microsoft.Network/virtualNetworks` | typed | +| network | `azure_network_virtual_routers` | `Microsoft.Network/virtualRouters` | generic | +| network | `azure_network_virtual_wans` | `Microsoft.Network/virtualWans` | generic | +| network | `azure_network_vpn_gateways` | `Microsoft.Network/vpnGateways` | generic | +| network | `azure_network_vpn_server_configurations` | `Microsoft.Network/vpnServerConfigurations` | generic | +| network | `azure_network_vpn_sites` | `Microsoft.Network/vpnSites` | generic | +| network | `azure_network_watcher_flow_logs` | `Microsoft.Network/watcherFlowLogs` | generic | +| network | `azure_network_watchers` | `Microsoft.Network/networkWatchers` | generic | +| network | `azure_network_web_application_firewall_policies` | `Microsoft.Network/webApplicationFirewallPolicies` | generic | +| networkfunction | `azure_networkfunction_azure_traffic_collectors_by_subscription` | `Microsoft.Networkfunction/azureTrafficCollectorsBySubscription` | generic | +| nginx | `azure_nginx_deployments` | `Microsoft.Nginx/deployments` | generic | +| notificationhubs | `azure_notificationhubs_namespaces` | `Microsoft.NotificationHubs/namespaces` | generic | +| operationalinsights | `azure_operationalinsights_clusters` | `Microsoft.OperationalInsights/clusters` | generic | +| operationalinsights | `azure_operationalinsights_workspaces` | `Microsoft.OperationalInsights/workspaces` | generic | +| peering | `azure_peering_service_countries` | `Microsoft.Peering/serviceCountries` | generic | +| peering | `azure_peering_service_locations` | `Microsoft.Peering/serviceLocations` | generic | +| peering | `azure_peering_service_providers` | `Microsoft.Peering/serviceProviders` | generic | +| policy | `azure_policy_assignments` | `Microsoft.Policy/assignments` | generic | +| policy | `azure_policy_definition_versions` | `Microsoft.Policy/definitionVersions` | generic | +| policy | `azure_policy_definitions` | `Microsoft.Policy/definitions` | generic | +| policy | `azure_policy_set_definition_versions` | `Microsoft.Policy/setDefinitionVersions` | generic | +| policy | `azure_policy_set_definitions` | `Microsoft.Policy/setDefinitions` | generic | +| policyinsights | `azure_policyinsights_attestations` | `Microsoft.PolicyInsights/attestations` | generic | +| policyinsights | `azure_policyinsights_policy_events` | `Microsoft.PolicyInsights/policyEvents` | generic | +| policyinsights | `azure_policyinsights_policy_states` | `Microsoft.PolicyInsights/policyStates` | generic | +| policyinsights | `azure_policyinsights_policy_tracked_resources` | `Microsoft.PolicyInsights/policyTrackedResources` | generic | +| portal | `azure_portal_list_tenant_configuration_violations` | `Microsoft.Portal/listTenantConfigurationViolations` | generic | +| portal | `azure_portal_tenant_configurations` | `Microsoft.Portal/tenantConfigurations` | generic | +| postgresql | `azure_postgresql_databases` | `Microsoft.DBforPostgreSQL/servers/databases` | typed | +| postgresql | `azure_postgresql_server_configurations` | `Microsoft.DBforPostgreSQL/serverConfigurations` | generic | +| postgresql | `azure_postgresql_server_firewall_rules` | `Microsoft.DBforPostgreSQL/serverFirewallRules` | generic | +| postgresql | `azure_postgresql_servers` | `Microsoft.DBforPostgreSQL/servers` | generic | +| postgresqlflexibleservers | `azure_postgresqlflexibleservers_server_configurations` | `Microsoft.Postgresqlflexibleservers/serverConfigurations` | generic | +| postgresqlflexibleservers | `azure_postgresqlflexibleservers_server_firewall_rules` | `Microsoft.Postgresqlflexibleservers/serverFirewallRules` | generic | +| postgresqlflexibleservers | `azure_postgresqlflexibleservers_servers` | `Microsoft.Postgresqlflexibleservers/servers` | generic | +| postgresqlhsc | `azure_postgresqlhsc_server_groups` | `Microsoft.Postgresqlhsc/serverGroups` | generic | +| powerbidedicated | `azure_powerbidedicated_capacities` | `Microsoft.PowerBIDedicated/capacities` | generic | +| privatedns | `azure_privatedns_private_zone_record_sets` | `Microsoft.Network/privateZoneRecordSets` | generic | +| privatedns | `azure_privatedns_private_zone_virtual_network_links` | `Microsoft.Network/privateZoneVirtualNetworkLinks` | generic | +| privatedns | `azure_privatedns_private_zones` | `Microsoft.Network/privateZones` | generic | +| providerhub | `azure_providerhub_provider_registrations` | `Microsoft.Providerhub/providerRegistrations` | generic | +| purview | `azure_purview_account_keys` | `Microsoft.Purview/accountKeys` | generic | +| purview | `azure_purview_account_private_endpoint_connections` | `Microsoft.Purview/accountPrivateEndpointConnections` | generic | +| purview | `azure_purview_account_private_link_resources` | `Microsoft.Purview/accountPrivateLinkResources` | generic | +| purview | `azure_purview_accounts` | `Microsoft.Purview/accounts` | generic | +| purview | `azure_purview_operations` | `Microsoft.Purview/operations` | generic | +| quota | `azure_quota_quotas` | `Microsoft.Quota/quotas` | generic | +| quota | `azure_quota_usages` | `Microsoft.Quota/usages` | generic | +| recoveryservices | `azure_recoveryservices_backup_engines` | `Microsoft.RecoveryServices/backupEngines` | generic | +| recoveryservices | `azure_recoveryservices_backup_jobs` | `Microsoft.RecoveryServices/backupJobs` | generic | +| recoveryservices | `azure_recoveryservices_backup_policies` | `Microsoft.RecoveryServices/backupPolicies` | generic | +| recoveryservices | `azure_recoveryservices_backup_protected_items` | `Microsoft.RecoveryServices/backupProtectedItems` | generic | +| recoveryservices | `azure_recoveryservices_backup_protection_containers` | `Microsoft.RecoveryServices/backupProtectionContainers` | generic | +| recoveryservices | `azure_recoveryservices_backup_protection_intents` | `Microsoft.RecoveryServices/backupProtectionIntents` | generic | +| recoveryservices | `azure_recoveryservices_backup_usage_summaries` | `Microsoft.RecoveryServices/backupUsageSummaries` | generic | +| recoveryservices | `azure_recoveryservices_deleted_protection_containers` | `Microsoft.RecoveryServices/deletedProtectionContainers` | generic | +| recoveryservices | `azure_recoveryservices_operations` | `Microsoft.RecoveryServices/operations` | generic | +| recoveryservices | `azure_recoveryservices_private_link_resources` | `Microsoft.RecoveryServices/privateLinkResources` | generic | +| recoveryservices | `azure_recoveryservices_replication_alert_settings` | `Microsoft.RecoveryServices/replicationAlertSettings` | generic | +| recoveryservices | `azure_recoveryservices_replication_events` | `Microsoft.RecoveryServices/replicationEvents` | generic | +| recoveryservices | `azure_recoveryservices_replication_fabrics` | `Microsoft.RecoveryServices/replicationFabrics` | generic | +| recoveryservices | `azure_recoveryservices_replication_jobs` | `Microsoft.RecoveryServices/replicationJobs` | generic | +| recoveryservices | `azure_recoveryservices_replication_logical_networks` | `Microsoft.RecoveryServices/replicationLogicalNetworks` | generic | +| recoveryservices | `azure_recoveryservices_replication_migration_items` | `Microsoft.RecoveryServices/replicationMigrationItems` | generic | +| recoveryservices | `azure_recoveryservices_replication_network_mappings` | `Microsoft.RecoveryServices/replicationNetworkMappings` | generic | +| recoveryservices | `azure_recoveryservices_replication_networks` | `Microsoft.RecoveryServices/replicationNetworks` | generic | +| recoveryservices | `azure_recoveryservices_replication_policies` | `Microsoft.RecoveryServices/replicationPolicies` | generic | +| recoveryservices | `azure_recoveryservices_replication_protectable_items` | `Microsoft.RecoveryServices/replicationProtectableItems` | generic | +| recoveryservices | `azure_recoveryservices_replication_protected_items` | `Microsoft.RecoveryServices/replicationProtectedItems` | generic | +| recoveryservices | `azure_recoveryservices_replication_protection_containers` | `Microsoft.RecoveryServices/replicationProtectionContainers` | generic | +| recoveryservices | `azure_recoveryservices_replication_protection_ctnr_mappings` | `Microsoft.RecoveryServices/replicationProtectionCtnrMappings` | generic | +| recoveryservices | `azure_recoveryservices_replication_protection_intents` | `Microsoft.RecoveryServices/replicationProtectionIntents` | generic | +| recoveryservices | `azure_recoveryservices_replication_recovery_plans` | `Microsoft.RecoveryServices/replicationRecoveryPlans` | generic | +| recoveryservices | `azure_recoveryservices_replication_recovery_service_providers` | `Microsoft.RecoveryServices/replicationRecoveryServiceProviders` | generic | +| recoveryservices | `azure_recoveryservices_replication_storage_classif_mappings` | `Microsoft.RecoveryServices/replicationStorageClassifMappings` | generic | +| recoveryservices | `azure_recoveryservices_replication_storage_classifications` | `Microsoft.RecoveryServices/replicationStorageClassifications` | generic | +| recoveryservices | `azure_recoveryservices_replication_vault_settings` | `Microsoft.RecoveryServices/replicationVaultSettings` | generic | +| recoveryservices | `azure_recoveryservices_replication_vcenters` | `Microsoft.RecoveryServices/replicationVcenters` | generic | +| recoveryservices | `azure_recoveryservices_vault_replication_usages` | `Microsoft.RecoveryServices/vaultReplicationUsages` | generic | +| recoveryservices | `azure_recoveryservices_vault_usages` | `Microsoft.RecoveryServices/vaultUsages` | generic | +| recoveryservices | `azure_recoveryservices_vaults` | `Microsoft.RecoveryServices/vaults` | generic | +| redhatopenshift | `azure_redhatopenshift_open_shift_clusters` | `Microsoft.RedHatOpenShift/openShiftClusters` | generic | +| redis | `azure_redis_caches` | `Microsoft.Cache/Redis` | typed | +| redis | `azure_redis_firewall_rules` | `Microsoft.Redis/firewallRules` | generic | +| redis | `azure_redis_patch_schedules` | `Microsoft.Redis/patchSchedules` | generic | +| relay | `azure_relay_namespaces` | `Microsoft.Relay/namespaces` | generic | +| reservations | `azure_reservations_reservation` | `Microsoft.Capacity/reservation` | generic | +| reservations | `azure_reservations_reservation_order` | `Microsoft.Capacity/reservationOrder` | generic | +| resourcehealth | `azure_resourcehealth_availability_statuses` | `Microsoft.ResourceHealth/availabilityStatuses` | generic | +| resourcehealth | `azure_resourcehealth_emerging_issues` | `Microsoft.ResourceHealth/emergingIssues` | generic | +| resourcehealth | `azure_resourcehealth_event_impacted_resources` | `Microsoft.ResourceHealth/eventImpactedResources` | generic | +| resourcehealth | `azure_resourcehealth_events` | `Microsoft.ResourceHealth/events` | generic | +| resourcehealth | `azure_resourcehealth_security_advisory_impacted_resources` | `Microsoft.ResourceHealth/securityAdvisoryImpactedResources` | generic | +| resources | `azure_resources_links` | `Microsoft.Resources/links` | generic | +| resources | `azure_resources_providers` | `Microsoft.Resources/providers` | generic | +| resources | `azure_resources_resource_groups` | `Microsoft.Resources/resourceGroups` | typed | +| resources | `azure_resources_resources` | `-` | generic | +| role | `azure_role_management_policy_assignments` | `Microsoft.Role/managementPolicyAssignments` | generic | +| saas | `azure_saas_resources` | `Microsoft.Saas/resources` | generic | +| search | `azure_search_services` | `Microsoft.Search/searchServices` | generic | +| security | `azure_security_adaptive_application_controls` | `Microsoft.Security/adaptiveApplicationControls` | generic | +| security | `azure_security_alerts` | `Microsoft.Security/alerts` | generic | +| security | `azure_security_alerts_suppression_rules` | `Microsoft.Security/alertsSuppressionRules` | generic | +| security | `azure_security_allowed_connections` | `Microsoft.Security/allowedConnections` | generic | +| security | `azure_security_applications` | `Microsoft.Security/applications` | generic | +| security | `azure_security_assessments` | `Microsoft.Security/assessments` | generic | +| security | `azure_security_assessments_metadata` | `Microsoft.Security/assessmentsMetadata` | generic | +| security | `azure_security_auto_provisioning_settings` | `Microsoft.Security/autoProvisioningSettings` | generic | +| security | `azure_security_automations` | `Microsoft.Security/automations` | generic | +| security | `azure_security_connectors` | `Microsoft.Security/connectors` | generic | +| security | `azure_security_contacts` | `Microsoft.Security/contacts` | generic | +| security | `azure_security_container_registry_vulnerability_details` | `Microsoft.Security/containerRegistryVulnerabilityDetails` | generic | +| security | `azure_security_discovered_security_solutions` | `Microsoft.Security/discoveredSecuritySolutions` | generic | +| security | `azure_security_external_security_solutions` | `Microsoft.Security/externalSecuritySolutions` | generic | +| security | `azure_security_governance_rule` | `Microsoft.Security/governanceRule` | generic | +| security | `azure_security_jit_network_access_policies` | `Microsoft.Security/jitNetworkAccessPolicies` | generic | +| security | `azure_security_locations` | `Microsoft.Security/locations` | generic | +| security | `azure_security_pricings` | `Microsoft.Security/pricings` | generic | +| security | `azure_security_regulatory_compliance_assessments` | `Microsoft.Security/regulatoryComplianceAssessments` | generic | +| security | `azure_security_regulatory_compliance_controls` | `Microsoft.Security/regulatoryComplianceControls` | generic | +| security | `azure_security_regulatory_compliance_standards` | `Microsoft.Security/regulatoryComplianceStandards` | generic | +| security | `azure_security_secure_score_control_definitions` | `Microsoft.Security/secureScoreControlDefinitions` | generic | +| security | `azure_security_secure_score_controls` | `Microsoft.Security/secureScoreControls` | generic | +| security | `azure_security_secure_scores` | `Microsoft.Security/secureScores` | generic | +| security | `azure_security_server_vulnerability_details` | `Microsoft.Security/serverVulnerabilityDetails` | generic | +| security | `azure_security_settings` | `Microsoft.Security/settings` | generic | +| security | `azure_security_solutions` | `Microsoft.Security/solutions` | generic | +| security | `azure_security_sql_server_vulnerability_details` | `Microsoft.Security/sqlServerVulnerabilityDetails` | generic | +| security | `azure_security_sub_assessment_azure_resource_details` | `Microsoft.Security/subAssessmentAzureResourceDetails` | generic | +| security | `azure_security_sub_assessment_on_premise_resource_details` | `Microsoft.Security/subAssessmentOnPremiseResourceDetails` | generic | +| security | `azure_security_sub_assessment_on_premise_sql_resource_details` | `Microsoft.Security/subAssessmentOnPremiseSqlResourceDetails` | generic | +| security | `azure_security_sub_assessments` | `Microsoft.Security/subAssessments` | generic | +| security | `azure_security_tasks` | `Microsoft.Security/tasks` | generic | +| security | `azure_security_topology` | `Microsoft.Security/topology` | generic | +| security | `azure_security_workspace_settings` | `Microsoft.Security/workspaceSettings` | generic | +| servicebus | `azure_servicebus_namespace_topic_authorization_rules` | `Microsoft.ServiceBus/namespaceTopicAuthorizationRules` | generic | +| servicebus | `azure_servicebus_namespace_topic_rule_access_keys` | `Microsoft.ServiceBus/namespaceTopicRuleAccessKeys` | generic | +| servicebus | `azure_servicebus_namespace_topics` | `Microsoft.ServiceBus/namespaceTopics` | generic | +| servicebus | `azure_servicebus_namespaces` | `Microsoft.ServiceBus/namespaces` | typed | +| servicefabric | `azure_servicefabric_cluster_application_services` | `Microsoft.ServiceFabric/clusterApplicationServices` | generic | +| servicefabric | `azure_servicefabric_cluster_application_types` | `Microsoft.ServiceFabric/clusterApplicationTypes` | generic | +| servicefabric | `azure_servicefabric_cluster_applications` | `Microsoft.ServiceFabric/clusterApplications` | generic | +| servicefabric | `azure_servicefabric_clusters` | `Microsoft.ServiceFabric/clusters` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_cluster_application_services` | `Microsoft.Servicefabricmanaged/clusterApplicationServices` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_cluster_application_types` | `Microsoft.Servicefabricmanaged/clusterApplicationTypes` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_cluster_applications` | `Microsoft.Servicefabricmanaged/clusterApplications` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_cluster_node_type_skus` | `Microsoft.Servicefabricmanaged/clusterNodeTypeSkus` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_cluster_node_types` | `Microsoft.Servicefabricmanaged/clusterNodeTypes` | generic | +| servicefabricmanaged | `azure_servicefabricmanaged_clusters` | `Microsoft.Servicefabricmanaged/clusters` | generic | +| sql | `azure_sql_instance_pools` | `Microsoft.Sql/instancePools` | generic | +| sql | `azure_sql_managed_instance_encryption_protectors` | `Microsoft.Sql/managedInstanceEncryptionProtectors` | generic | +| sql | `azure_sql_managed_instance_vulnerability_assessments` | `Microsoft.Sql/managedInstanceVulnerabilityAssessments` | generic | +| sql | `azure_sql_managed_instances` | `Microsoft.Sql/managedInstances` | generic | +| sql | `azure_sql_server_admins` | `Microsoft.Sql/serverAdmins` | generic | +| sql | `azure_sql_server_advanced_threat_protection_settings` | `Microsoft.Sql/serverAdvancedThreatProtectionSettings` | generic | +| sql | `azure_sql_server_blob_auditing_policies` | `Microsoft.Sql/serverBlobAuditingPolicies` | generic | +| sql | `azure_sql_server_database_blob_auditing_policies` | `Microsoft.Sql/serverDatabaseBlobAuditingPolicies` | generic | +| sql | `azure_sql_server_database_long_term_retention_policies` | `Microsoft.Sql/serverDatabaseLongTermRetentionPolicies` | generic | +| sql | `azure_sql_server_database_threat_protections` | `Microsoft.Sql/serverDatabaseThreatProtections` | generic | +| sql | `azure_sql_server_database_vulnerability_assessment_scans` | `Microsoft.Sql/serverDatabaseVulnerabilityAssessmentScans` | generic | +| sql | `azure_sql_server_database_vulnerability_assessments` | `Microsoft.Sql/serverDatabaseVulnerabilityAssessments` | generic | +| sql | `azure_sql_server_databases` | `Microsoft.Sql/serverDatabases` | generic | +| sql | `azure_sql_server_encryption_protectors` | `Microsoft.Sql/serverEncryptionProtectors` | generic | +| sql | `azure_sql_server_failover_groups` | `Microsoft.Sql/serverFailoverGroups` | generic | +| sql | `azure_sql_server_firewall_rules` | `Microsoft.Sql/serverFirewallRules` | generic | +| sql | `azure_sql_server_security_alert_policies` | `Microsoft.Sql/serverSecurityAlertPolicies` | generic | +| sql | `azure_sql_server_virtual_network_rules` | `Microsoft.Sql/serverVirtualNetworkRules` | generic | +| sql | `azure_sql_server_vulnerability_assessments` | `Microsoft.Sql/serverVulnerabilityAssessments` | generic | +| sql | `azure_sql_servers` | `Microsoft.Sql/servers` | generic | +| sql | `azure_sql_transparent_data_encryptions` | `Microsoft.Sql/transparentDataEncryptions` | generic | +| sql | `azure_sql_virtual_clusters` | `Microsoft.Sql/virtualClusters` | generic | +| sqlvirtualmachine | `azure_sqlvirtualmachine_groups` | `Microsoft.Sqlvirtualmachine/groups` | generic | +| sqlvirtualmachine | `azure_sqlvirtualmachine_sql_virtual_machines` | `Microsoft.Sqlvirtualmachine/sqlVirtualMachines` | generic | +| storage | `azure_storage_account_keys` | `Microsoft.Storage/accountKeys` | generic | +| storage | `azure_storage_accounts` | `Microsoft.Storage/storageAccounts` | typed | +| storage | `azure_storage_blob_services` | `Microsoft.Storage/storageAccounts/blobServices` | generic | +| storage | `azure_storage_containers` | `Microsoft.Storage/containers` | generic | +| storage | `azure_storage_encryption_scopes` | `Microsoft.Storage/encryptionScopes` | generic | +| storage | `azure_storage_file_shares` | `Microsoft.Storage/storageAccounts/fileServices/shares` | generic | +| storage | `azure_storage_management_policies` | `Microsoft.Storage/managementPolicies` | generic | +| storage | `azure_storage_queue_acl` | `Microsoft.Storage/queueAcl` | generic | +| storage | `azure_storage_queue_services` | `Microsoft.Storage/storageAccounts/queueServices` | generic | +| storage | `azure_storage_queues` | `Microsoft.Storage/storageAccounts/queueServices/queues` | generic | +| storage | `azure_storage_tables` | `Microsoft.Storage/storageAccounts/tableServices/tables` | generic | +| storagecache | `azure_storagecache_caches` | `Microsoft.Storagecache/caches` | generic | +| storagemover | `azure_storagemover_agents` | `Microsoft.Storagemover/agents` | generic | +| storagemover | `azure_storagemover_endpoints` | `Microsoft.Storagemover/endpoints` | generic | +| storagemover | `azure_storagemover_job_definitions` | `Microsoft.Storagemover/jobDefinitions` | generic | +| storagemover | `azure_storagemover_job_runs` | `Microsoft.Storagemover/jobRuns` | generic | +| storagemover | `azure_storagemover_operations` | `Microsoft.Storagemover/operations` | generic | +| storagemover | `azure_storagemover_projects` | `Microsoft.Storagemover/projects` | generic | +| storagemover | `azure_storagemover_storagemovers` | `Microsoft.Storagemover/storagemovers` | generic | +| storagesync | `azure_storagesync_service_private_endpoint_connections` | `Microsoft.StorageSync/servicePrivateEndpointConnections` | generic | +| storagesync | `azure_storagesync_service_private_link_resources` | `Microsoft.StorageSync/servicePrivateLinkResources` | generic | +| storagesync | `azure_storagesync_service_registered_servers` | `Microsoft.StorageSync/serviceRegisteredServers` | generic | +| storagesync | `azure_storagesync_service_sync_group_server_endpoints` | `Microsoft.StorageSync/serviceSyncGroupServerEndpoints` | generic | +| storagesync | `azure_storagesync_service_sync_groups` | `Microsoft.StorageSync/serviceSyncGroups` | generic | +| storagesync | `azure_storagesync_services` | `Microsoft.StorageSync/services` | generic | +| streamanalytics | `azure_streamanalytics_streaming_jobs` | `Microsoft.StreamAnalytics/streamingjobs` | generic | +| subscription | `azure_subscription_subscription_locations` | `Microsoft.Subscription/subscriptionLocations` | generic | +| subscription | `azure_subscription_subscriptions` | `Microsoft.Subscription/subscriptions` | typed | +| subscription | `azure_subscription_tenants` | `Microsoft.Subscription/tenants` | generic | +| support | `azure_support_services` | `Microsoft.Support/services` | generic | +| support | `azure_support_tickets` | `Microsoft.Support/tickets` | generic | +| synapse | `azure_synapse_ip_firewall_rules` | `Microsoft.Synapse/ipFirewallRules` | generic | +| synapse | `azure_synapse_keys` | `Microsoft.Synapse/keys` | generic | +| synapse | `azure_synapse_private_link_hubs` | `Microsoft.Synapse/privateLinkHubs` | generic | +| synapse | `azure_synapse_restorable_dropped_sql_pools` | `Microsoft.Synapse/restorableDroppedSqlPools` | generic | +| synapse | `azure_synapse_sql_pool_blob_auditing_policies` | `Microsoft.Synapse/sqlPoolBlobAuditingPolicies` | generic | +| synapse | `azure_synapse_sql_pool_current_sensitivity_labels` | `Microsoft.Synapse/sqlPoolCurrentSensitivityLabels` | generic | +| synapse | `azure_synapse_sql_pool_geo_backup_policies` | `Microsoft.Synapse/sqlPoolGeoBackupPolicies` | generic | +| synapse | `azure_synapse_sql_pool_operations` | `Microsoft.Synapse/sqlPoolOperations` | generic | +| synapse | `azure_synapse_sql_pool_recommended_sensitivity_labels` | `Microsoft.Synapse/sqlPoolRecommendedSensitivityLabels` | generic | +| synapse | `azure_synapse_sql_pool_replication_links` | `Microsoft.Synapse/sqlPoolReplicationLinks` | generic | +| synapse | `azure_synapse_sql_pool_restore_points` | `Microsoft.Synapse/sqlPoolRestorePoints` | generic | +| synapse | `azure_synapse_sql_pool_schema_table_columns` | `Microsoft.Synapse/sqlPoolSchemaTableColumns` | generic | +| synapse | `azure_synapse_sql_pool_schema_tables` | `Microsoft.Synapse/sqlPoolSchemaTables` | generic | +| synapse | `azure_synapse_sql_pool_schemas` | `Microsoft.Synapse/sqlPoolSchemas` | generic | +| synapse | `azure_synapse_sql_pool_security_alert_policies` | `Microsoft.Synapse/sqlPoolSecurityAlertPolicies` | generic | +| synapse | `azure_synapse_sql_pool_transparent_data_encryptions` | `Microsoft.Synapse/sqlPoolTransparentDataEncryptions` | generic | +| synapse | `azure_synapse_sql_pool_usages` | `Microsoft.Synapse/sqlPoolUsages` | generic | +| synapse | `azure_synapse_sql_pools` | `Microsoft.Synapse/workspaces/sqlPools` | generic | +| synapse | `azure_synapse_workspaces` | `Microsoft.Synapse/workspaces` | generic | +| trafficmanager | `azure_trafficmanager_profiles` | `Microsoft.Trafficmanager/profiles` | generic | +| windowsiot | `azure_windowsiot_services` | `Microsoft.WindowsIoT/services` | generic | +| workloads | `azure_workloads_monitors` | `Microsoft.Workloads/monitors` | generic | + diff --git a/docs/authoring/indexed-resources/azure.md b/docs/authoring/indexed-resources/azure.md new file mode 100644 index 000000000..46cd39bb2 --- /dev/null +++ b/docs/authoring/indexed-resources/azure.md @@ -0,0 +1,221 @@ +# Azure indexer + +The native Azure indexer (`azureapi`) discovers Azure resources via the +official Microsoft `azure-mgmt-*` Python SDKs and ships full coverage of +the Azure resource catalog. This page is the authoritative reference for +**what gets indexed** and **what data your generation rules will see**. + +For the engineering deep dive (dispatch, selective discovery, how to add +a new typed collector), see +[Azure indexer internals](../../architecture/azure-indexer-internals.md). + +> [!NOTE] +> A second `cloudquery` backend exists for legacy reasons but is being +> phased out. New workspaces should set `azureIndexerBackend: azureapi`. + +## How to enable + +In your `workspaceInfo.yaml`: + +```yaml +azureIndexerBackend: azureapi +cloudConfig: + azure: + tenantId: ${AZ_TENANT_ID} + clientId: ${AZ_CLIENT_ID} + clientSecret: ${AZ_CLIENT_SECRET} + subscriptionId: + defaultLOD: detailed + # Or 'none' + per-RG overrides for selective mode. +``` + +Any of the three credential fields may be omitted to fall back to +`DefaultAzureCredential` (managed identity, az CLI cache, env vars). If +*some* are populated and *some* are blank, the indexer fails fast with a +clear error rather than silently falling back. + +## What can be indexed + +Every Azure resource type that the CloudQuery Azure plugin tabulates is +indexable by the native indexer too — **619 resource types total**, full +parity with the legacy backend. The full sortable catalog lives at +[`azure-resource-catalog.md`](./azure-resource-catalog.md); the rest of +this page covers the model. + +Indexable types fall into two tiers: + +### Typed (rich-payload) tier + +Hand-written `azure-mgmt-*` collectors that return the full SDK model +for each resource. Use these when your generation rules need to match +against `properties.*` (status, configuration, network rules, +diagnostics, etc.). + +| CloudQuery table name | ARM type | Aliases | +| --- | --- | --- | +| `azure_resources_resource_groups` | `Microsoft.Resources/resourceGroups` | `resource_group` | +| `azure_subscription_subscriptions` | `Microsoft.Subscription/subscriptions` | - | +| `azure_compute_virtual_machines` | `Microsoft.Compute/virtualMachines` | `virtual_machine` | +| `azure_compute_disks` | `Microsoft.Compute/disks` | - | +| `azure_compute_snapshots` | `Microsoft.Compute/snapshots` | - | +| `azure_compute_virtual_machine_scale_sets` | `Microsoft.Compute/virtualMachineScaleSets` | - | +| `azure_storage_accounts` | `Microsoft.Storage/storageAccounts` | - | +| `azure_keyvault_keyvaults` | `Microsoft.KeyVault/vaults` | `azure_keyvault_vaults`, `azure_keyvault_keyvault` | +| `azure_network_virtual_networks` | `Microsoft.Network/virtualNetworks` | - | +| `azure_network_security_groups` | `Microsoft.Network/networkSecurityGroups` | - | +| `azure_network_load_balancers` | `Microsoft.Network/loadBalancers` | - | +| `azure_network_application_gateways` | `Microsoft.Network/applicationGateways` | - | +| `azure_containerservice_managed_clusters` | `Microsoft.ContainerService/managedClusters` | - | +| `azure_containerregistry_registries` | `Microsoft.ContainerRegistry/registries` | - | +| `azure_appservice_plans` | `Microsoft.Web/serverFarms` | - | +| `azure_appservice_web_apps` | `Microsoft.Web/sites` | - | +| `azure_mysql_servers` | `Microsoft.DBforMySQL/servers` | - | +| `azure_mysqlflexibleservers_servers` | `Microsoft.DBforMySQL/flexibleServers` | - | +| `azure_postgresql_databases` | `Microsoft.DBforPostgreSQL/servers/databases` | walks parent servers | +| `azure_redis_caches` | `Microsoft.Cache/Redis` | - | +| `azure_servicebus_namespaces` | `Microsoft.ServiceBus/namespaces` | - | +| `azure_datafactory_factories` | `Microsoft.DataFactory/factories` | - | +| `azure_apimanagement_service` | `Microsoft.ApiManagement/service` | - | +| `azure_cosmos_sql_databases` | `Microsoft.DocumentDB/databaseAccounts/sqlDatabases` | walks parent accounts | +| `azure_azurearcdata_sql_server_instances` | `Microsoft.AzureArcData/sqlServerInstances` | - | + +### Generic (basic-envelope) tier + +Every other registry entry — covered by a single catch-all that calls +`ResourceManagementClient.resources.list()` once per subscription (or +once per resource group in selective mode) and routes each +`GenericResource` back to its registry entry by ARM type. No extra +configuration is required; gen rules referencing any registered type +"just work". + +The full list — 594 types today, sorted by service — is in +[`azure-resource-catalog.md`](./azure-resource-catalog.md). + +## What the data looks like + +After collection, every resource is normalized into a flat dict so that +downstream code (parsers, generation rules, the resource store) doesn't +care which tier produced it. + +**Common fields (always present)**: + +```yaml +id: /subscriptions//resourceGroups//providers// +name: +resource_type: azure__ # canonical RWL name +subscription_id: +resource_group: # extracted from id +location: +tags: {} # always a dict +sku: { ... } # when the resource exposes one +identity: { ... } # when the resource has a managed identity +kind: # storage account kind, web app kind, etc. +managed_by: # when set +``` + +**Typed-tier-only fields**: + +```yaml +properties: # full SDK payload, verbatim + +plan: { ... } # rare; some marketplace resources +``` + +The exact contract is pinned by tests in +`src/indexers/test_azureapi_normalizers.py`. + +> [!IMPORTANT] +> Generic-tier resources do **not** carry `properties`. The Azure ARM +> Resources API doesn't expand them in list calls. If a generation rule +> needs to match on `properties.*` for a non-typed type, request a typed +> collector be added — see +> [adding a new typed collector](../../architecture/azure-indexer-internals.md#adding-a-new-typed-collector). + +### Per-type highlights (typed tier) + +A handful of typed types have idiosyncratic shapes worth knowing about: + +* `azure_storage_accounts.properties.primaryEndpoints` - the public URLs + for blob/queue/table/file. Useful for runbook templates. +* `azure_keyvault_keyvaults.properties.vaultUri` - the data-plane URL. +* `azure_compute_virtual_machines.properties.storageProfile.osDisk.managedDisk.id` - + ARM ID of the OS disk. You can join against `azure_compute_disks` to + write rules that pull both VM and disk into a single SLX. +* `azure_postgresql_databases.id` - includes the parent server segment + (`/.../servers//databases/`); the parent name + is exposed as `properties.parentName` for convenience. +* `azure_subscription_subscriptions` - emitted exactly once per + configured subscription, regardless of LOD. The `name` field is the + subscription's display name; `id` is `/subscriptions/`. + +## Selective discovery + +When the workspace declares `defaultLOD: none` plus a per-RG whitelist, +the indexer skips the subscription-wide list endpoints (typed *and* +generic) and calls the per-RG variants instead. This keeps API budget +and latency bounded on large subscriptions. + +```yaml +cloudConfig: + azure: + subscriptionId: my-sub + defaultLOD: none + subscriptions: + - subscriptionId: my-sub + defaultLOD: none + resourceGroupLevelOfDetails: + rg-prod: detailed + rg-staging: basic + rg-legacy: none # explicit deny +``` + +In that example the indexer enumerates only `rg-prod` and `rg-staging` +across every supported type (typed and generic). `rg-legacy` is +excluded entirely; everything else in the subscription is invisible. + +Two types ignore selective scoping by design: + +* `azure_resources_resource_groups` — we always need the full RG list + to compute scope in the first place. +* `azure_subscription_subscriptions` — one subscription resource per + configured subscription, period. + +## When the typed and generic passes overlap + +The dispatcher runs the typed pass first, tracks every ARM ID it +emitted, and then runs the generic pass with two filters: + +1. ARM types owned by the typed pass are skipped wholesale (a generic + row would just be a sparser copy of one we already wrote). +2. ARM types not referenced by any loaded generation rule are dropped + (the indexer is gen-rule-driven; we don't balloon the resource + store with unused rows). + +Net effect: every type the contrib CodeBundles reference is indexed +exactly once, with the richest payload available. + +## Adding a new typed collector + +When you need a richer payload for a resource type that's currently in +the generic tier, promote it to typed: + +1. Implement `_collect__all` and `_collect__in_rg` in + `src/indexers/azureapi_resource_types.py`. +2. Register the pair in `_TYPED_COLLECTORS`. +3. Add the table name to `typed_collectors:` in + `scripts/azure/azure_resource_type_overrides.yaml`. +4. Re-run `scripts/azure/sync_azure_resource_type_registry.py`. +5. Re-run `scripts/azure/dump_azure_resource_catalog.py` to refresh + this page's sibling catalog. +6. Update unit tests in + `src/indexers/test_azure_resource_type_registry.py::test_typed_collectors_present`. + +The full mechanical recipe (with examples) is in +[Azure indexer internals](../../architecture/azure-indexer-internals.md#adding-a-new-typed-collector). + +## See also + +* [Azure resource catalog](./azure-resource-catalog.md) — sortable table of all 619 indexable types. +* [Azure cloud-discovery user guide](../../user-guide/cloud-discovery/azure.md) — credentials, `workspaceInfo.yaml` snippets, troubleshooting auth. +* [Generation rules: examples](../generation-rules/examples/) — worked examples that match Azure resources. +* [Azure indexer internals](../../architecture/azure-indexer-internals.md) — engineering deep dive. diff --git a/docs/authoring/indexed-resources/gcp.md b/docs/authoring/indexed-resources/gcp.md new file mode 100644 index 000000000..2a81bcb35 --- /dev/null +++ b/docs/authoring/indexed-resources/gcp.md @@ -0,0 +1,155 @@ +# GCP indexer + +RunWhen Local can discover GCP resources two ways, selected by +`gcpIndexerBackend` in `workspaceInfo.yaml`: + +* **`cloudquery`** (default): invokes the + [CloudQuery GCP plugin](https://hub.cloudquery.io/plugins/source/cloudquery/gcp) + against the project(s) you've configured and reads the resulting SQLite + intermediate. +* **`gcpapi`**: the native indexer — uses first-party `google-cloud-*` SDK + collectors as its functional baseline, plus an optional Google Cloud Asset + Inventory accelerator, with no CloudQuery binary or `gcloud` + subprocesses. It discovers only the resource types your generation rules + reference, per project, and respects per-project `projectLevelOfDetails` + (projects with LOD `none` are skipped). See + [GCP indexer internals](../../architecture/gcp-indexer-internals.md) for the + design. + +```yaml +# workspaceInfo.yaml +gcpIndexerBackend: gcpapi +``` + +Either way the CodeBundle-facing contract is the same: generation rules +reference the **CloudQuery table name** as `resource_type` (e.g. +`gcp_compute_instances`), and field shapes follow the CloudQuery GCP plugin +output. The native backend normalizes Cloud Asset Inventory payloads into that +same shape, so rules don't change when you flip the backend. Per-table schemas +live in the +[plugin's table reference](https://hub.cloudquery.io/plugins/source/cloudquery/gcp/latest/tables). + +For credential setup (service-account JSON, ADC, project scoping) and +`workspaceInfo.yaml` snippets, see the user guide's +[GCP cloud-discovery page](../../user-guide/cloud-discovery/gcp.md). + +## Discovery tiers: the typed baseline and the optional CAI accelerator + +The native `gcpapi` backend has two discovery tiers: + +1. The **typed fallback collectors** — the supported **functional baseline**. + A set of hand-written collectors built on first-party `google-cloud-*` SDKs + for the high-value types. They need only the relevant per-service viewer + roles and run whether or not Cloud Asset Inventory is available. +2. An **optional Cloud Asset Inventory (CAI) accelerator** — a catch-all generic + pass that *broadens* coverage to the long tail of resource types that have no + typed collector. CAI is **not required**; its absence is normal and never + fails discovery. + +The registry (`src/indexers/gcp_resource_type_registry.yaml`) currently +tracks **404** CloudQuery tables. **403** of them carry a CAI asset-type mapping +and are therefore discoverable by the generic pass (the one exception, +`gcp_billing_billing_accounts`, has no CAI mapping). Of those 403, **12** also +have a typed collector and **390** are served **only** by the CAI generic pass. + +### Baseline — typed fallback collectors (no CAI required) + +The supported baseline for native GCP discovery is the typed fallback tier. The +12 typed types below are discovered via their own SDK clients (each is +automatically excluded from the CAI pass so it is written exactly once), plus +the synthesized **`gcp_projects`** anchor (always emitted, no API call). They +require only the matching **per-service viewer roles** +(compute / storage / container / pubsub / iam) — there is no dependency on Cloud +Asset Inventory. Because discovery is selective, only the subset your loaded +generation rules actually reference is collected: + +| CloudQuery table | SDK client | Required role | +|------------------|-----------|---------------| +| `gcp_projects` (anchor) | — (synthesized) | — | +| `gcp_compute_instances` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_disks` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_snapshots` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_networks` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_subnetworks` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_firewalls` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_compute_addresses` | `google-cloud-compute` | `roles/compute.viewer` | +| `gcp_storage_buckets` | `google-cloud-storage` | `roles/storage.objectViewer` | +| `gcp_container_clusters` (GKE) | `google-cloud-container` | `roles/container.viewer` | +| `gcp_pubsub_topics` | `google-cloud-pubsub` | `roles/pubsub.viewer` | +| `gcp_pubsub_subscriptions` | `google-cloud-pubsub` | `roles/pubsub.viewer` | +| `gcp_iam_service_accounts` | `google-cloud-iam` | `roles/iam.serviceAccountViewer` | + +### Optional accelerator — Cloud Asset Inventory (broadens coverage only) + +CAI is an **optional** accelerator: enabling it broadens coverage to the **390 +CAI-only** types that have no typed collector, but it is not needed for the +baseline. If you want that extra breadth, enable the API and grant a CAI viewer +role: + +```bash +# OPTIONAL — only to broaden coverage beyond the typed baseline. +gcloud services enable cloudasset.googleapis.com --project +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:" \ + --role="roles/cloudasset.viewer" +``` + +Without CAI, the 390 CAI-only types are simply not discovered; the typed +baseline is unaffected. + +`gcp_sql_instances` and `gcp_run_services` are **deferred** — they do not yet +have a typed collector (Cloud SQL Admin / Cloud Run lack an idiomatic +`google-cloud-*` client suited to the fallback pattern), so they remain +CAI-served. They, along with the other ~388 long-tail types, are only discovered +when the optional CAI accelerator is enabled. + +### Permission matrix + +| Tier | Roles / API | Status | +|------|-------------|--------| +| Typed fallback (baseline) | `roles/compute.viewer`, `roles/storage.objectViewer`, `roles/container.viewer`, `roles/pubsub.viewer`, `roles/iam.serviceAccountViewer` | **Required baseline** — these gate the always-on typed types | +| Cloud Asset Inventory | `roles/cloudasset.viewer` + `cloudasset.googleapis.com` enabled | **Optional accelerator** — broadens coverage to the other 390 types; not required | +| Convenience superset | `roles/viewer` | Optional — covers the typed tier in one grant for sandbox/test projects | + +### When Cloud Asset Inventory is unavailable (normal, non-fatal) + +If the optional CAI call is rejected with `403 ... cloudasset.assets.listResource` +(or the API is not enabled), the indexer logs an **informational** +`GCP_CAI_PERMISSION_DENIED` note (at INFO — not an error, no banner, no warning) +and **continues normally**: the typed baseline discovers as usual and the +CAI-only types are simply skipped. The `gcpapi` CI smoke test +(`.test/gcp/gcp-and-k8s`, `task ci-test-gcpapi-baseline`) **passes** with CAI +unavailable — it is driven entirely by the typed fallback collectors and asserts +their function (project anchor, storage buckets / GKE clusters, and the default +`gcp_compute_networks` / `gcp_compute_firewalls`). + +## Common matchable types + +Generation rules in the contrib CodeBundles most often target: + +* `gcp_compute_instances` +* `gcp_compute_disks`, `gcp_compute_snapshots` +* `gcp_storage_buckets` +* `gcp_container_clusters` (GKE) +* `gcp_sql_instances` +* `gcp_pubsub_topics`, `gcp_pubsub_subscriptions` +* `gcp_iam_service_accounts` + +Use the CloudQuery table name as `resource_type` in your generation +rule. Field shapes follow the CloudQuery GCP plugin output verbatim. + +Note that `gcp_sql_instances` is currently a **CAI-only** type (see +[Optional accelerator — Cloud Asset Inventory](#optional-accelerator--cloud-asset-inventory-broadens-coverage-only)), +so it is only discovered when the optional Cloud Asset Inventory accelerator is +enabled. The +[typed fallback collectors](#baseline--typed-fallback-collectors-no-cai-required) +table lists the baseline types that are discovered with or without CAI. The full mapping +of CloudQuery table → CAI asset type lives in the generated registry +(`src/indexers/gcp_resource_type_registry.yaml`). + +## Roadmap + +The native `gcpapi` indexer is the path toward removing the CloudQuery +dependency entirely (alongside `azureapi` and a future native AWS indexer). +`cloudquery` remains the default backend until the native path has been +validated across the contrib CodeBundles. diff --git a/docs/authoring/indexed-resources/kubernetes.md b/docs/authoring/indexed-resources/kubernetes.md new file mode 100644 index 000000000..91256119f --- /dev/null +++ b/docs/authoring/indexed-resources/kubernetes.md @@ -0,0 +1,82 @@ +# Kubernetes indexer + +The Kubernetes indexer talks to a cluster via the official Kubernetes +Python client, using whatever kubeconfig / in-cluster service account the +container has access to. + +For credential setup and per-cluster `workspaceInfo.yaml` snippets, see +the user guide's [Kubernetes cloud-discovery +page](../../user-guide/cloud-discovery/kubernetes.md). For the full +namespace-scoping mechanics, see the +[Kubernetes-LOD architecture docs](../../architecture/kubernetes-lod/README.md). + +## Discovered resource types + +The indexer discovers the standard Kubernetes object kinds that the +contrib CodeBundles target: + +* `cluster` - one per workspaceInfo cluster entry. +* `namespace` - via `CoreV1Api.list_namespace()`. +* `deployment`, `statefulset`, `daemonset` (apps/v1). +* `pod`, `service`, `configmap`, `secret` (core/v1) - + visibility depends on the per-namespace LOD. +* `cronjob`, `job` (batch/v1). +* `ingress` (networking.k8s.io/v1). +* `gatewayclass`, `gateway`, `httproute` (gateway.networking.k8s.io/v1) + when the CRDs are installed. +* Any custom resource explicitly listed under + `cloudConfig.kubernetes.customResourceTypes`. + +Each object lands in the resource store with its full +`apiVersion`/`kind`/`metadata`/`spec`/`status` payload, plus +`subscription_id` set to the cluster name (the resource store treats the +cluster like a "subscription" so a single workspace can span many). + +## Level of detail per namespace + +`workspaceInfo.yaml` lets you set discovery scope per namespace. The +classic invocation looks like: + +```yaml +cloudConfig: + kubernetes: + contexts: + - name: prod-west + clusterName: prod-west + defaultLOD: none + namespaceLevelOfDetails: + payments: detailed + orders: basic + kube-system: none +``` + +Resources whose effective LOD resolves to `NONE` are dropped at indexing +time, exactly like the Azure indexer drops out-of-scope resource groups. +The decision tree (workspace default, per-context default, namespace +override, custom-resource override) is documented in detail in +[`architecture/kubernetes-lod/`](../../architecture/kubernetes-lod/README.md). + +## What generation rules can match + +Generation rules can match against any of the discovered types by +`resource_type`: + +```yaml +match: + resource_type: deployment + predicates: + - jsonpath: $.metadata.namespace + equals: payments + - jsonpath: $.spec.replicas + greater_than: 1 +``` + +`predicates` operate on the original Kubernetes object payload (see +[generation rules](../generation-rules/README.md) for the full predicate +grammar). + +## See also + +* [User guide: Kubernetes cloud discovery](../../user-guide/cloud-discovery/kubernetes.md) +* [Architecture: Kubernetes-LOD internals](../../architecture/kubernetes-lod/README.md) +* [AKS namespace-LOD addendum](../../architecture/kubernetes-lod/aks-namespace-lods.md) diff --git a/docs/cloud-discovery-configuration/google-cloud-platform.md b/docs/cloud-discovery-configuration/google-cloud-platform.md deleted file mode 100644 index eef8c1402..000000000 --- a/docs/cloud-discovery-configuration/google-cloud-platform.md +++ /dev/null @@ -1,145 +0,0 @@ -# Amazon Web Services - -{% hint style="info" %} -AWS discovery is supported from 0.5.7 onwards. -{% endhint %} - -## AWS Credentials - -AWS discovery leverages [cloudquery](https://github.com/cloudquery/cloudquery) with the AWS source plugin to build up an inventory of cloud resources that should be matched with troubleshooting commands. - -### Authentication Methods - -Multiple authentication methods are available, evaluated in the following priority order: - -1. **Explicit Access Keys** - Credentials specified directly in `workspaceInfo.yaml` -2. **Kubernetes Secret** - Credentials stored in a Kubernetes secret (via `awsSecretName`) -3. **Workload Identity (IRSA)** - EKS IAM Roles for Service Accounts -4. **Assume Role** - IAM role assumption using default credential chain -5. **Default Credential Chain** - boto3 default credential provider (environment variables, AWS CLI profile, instance profile, etc.) - -### AWS CloudQuery Version Details - -* Currently supported source plugin: AWS [v23.6.0](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v23.6.0/docs) -* Available resources: See [this link](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v23.6.0/tables) - -## AWS WorkspaceInfo Configuration - -### Method 1: Explicit Access Keys - -Provide AWS credentials directly in `workspaceInfo.yaml`: - -```yaml -cloudConfig: - aws: - region: us-east-1 - awsAccessKeyId: AKIA... - awsSecretAccessKey: ... - awsSessionToken: ... # Optional, for temporary credentials -``` - -### Method 2: Kubernetes Secret - -Store credentials in a Kubernetes secret and reference it: - -```yaml -cloudConfig: - aws: - region: us-east-1 - awsSecretName: aws-credentials -``` - -Create the Kubernetes secret with the following keys: -```bash -kubectl create secret generic aws-credentials \ - --from-literal=awsAccessKeyId=AKIA... \ - --from-literal=awsSecretAccessKey=... \ - --from-literal=region=us-east-1 -``` - -### Method 3: Workload Identity (EKS IRSA) - -For EKS clusters with IAM Roles for Service Accounts configured: - -```yaml -cloudConfig: - aws: - region: us-east-1 - useWorkloadIdentity: true -``` - -This automatically uses the `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` environment variables set by EKS. - -### Method 4: Assume Role - -Assume an IAM role using the default credential chain: - -```yaml -cloudConfig: - aws: - region: us-east-1 - assumeRoleArn: arn:aws:iam::123456789012:role/RunWhenDiscoveryRole - assumeRoleExternalId: optional-external-id # Optional - assumeRoleSessionName: runwhen-local # Optional, defaults to 'runwhen-local-session' - assumeRoleDurationSeconds: 3600 # Optional, defaults to 3600 -``` - -### Method 5: Default Credential Chain - -If no explicit configuration is provided, the default boto3 credential chain is used: - -```yaml -cloudConfig: - aws: - region: us-east-1 -``` - -This will use credentials from (in order): -- Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) -- Shared credentials file (`~/.aws/credentials`) -- AWS config file (`~/.aws/config`) -- Instance profile (for EC2/ECS) - -### Combined: Assume Role with Other Methods - -Assume role can be combined with other authentication methods: - -```yaml -# Explicit keys + assume role -cloudConfig: - aws: - awsAccessKeyId: AKIA... - awsSecretAccessKey: ... - assumeRoleArn: arn:aws:iam::TARGET_ACCOUNT:role/DiscoveryRole - -# Kubernetes secret + assume role -cloudConfig: - aws: - awsSecretName: aws-credentials - assumeRoleArn: arn:aws:iam::TARGET_ACCOUNT:role/DiscoveryRole -``` - -## Configuration Reference - -| Field Name | Description | -| ------------------------- | ----------------------------------------------------- | -| region | AWS region (default: us-east-1) | -| awsAccessKeyId | Explicit access key ID | -| awsSecretAccessKey | Explicit secret access key | -| awsSessionToken | Session token for temporary credentials | -| awsSecretName | Name of Kubernetes secret containing credentials | -| useWorkloadIdentity | Enable EKS IRSA authentication (true/false) | -| assumeRoleArn | ARN of IAM role to assume | -| assumeRoleExternalId | External ID for role assumption | -| assumeRoleSessionName | Session name for role assumption | -| assumeRoleDurationSeconds | Duration for assumed role session (default: 3600) | - -## Level of Detail - -To configure the specific Level of Detail (LoD) that is collected with a discovered AWS resource, AWS tags can be applied to the resource. Supported tag keys: - -- `lod` -- `levelofdetail` -- `level-of-detail` - -Example tag: `lod=detailed` or `lod=basic` diff --git a/docs/community/discord-slack-chat.md b/docs/community/discord-slack-chat.md deleted file mode 100644 index 6e1f95bb7..000000000 --- a/docs/community/discord-slack-chat.md +++ /dev/null @@ -1,11 +0,0 @@ -# Slack - -Want to get in touch with us? Join our [Slack](https://runwhen.slack.com/join/shared\_invite/zt-1l7t3tdzl-IzB8gXDsWtHkT8C5nufm2A#/shared-invite/email) instance! - -Chat is best used for real-time, or near real-time discussions around: - -* Troubleshooting topics (the tool itself, or Kubernetes systems, cloud systems, and so on) -* Discussing the function of a troubleshooting script or command -* Ideas around new troubleshooting commands -* Learning how to write your own troubleshooting commands - diff --git a/docs/community/github-discussions.md b/docs/community/github-discussions.md deleted file mode 100644 index 992b4def5..000000000 --- a/docs/community/github-discussions.md +++ /dev/null @@ -1,10 +0,0 @@ -# GitHub Discussions - -GitHub discussions are: - -* Available [here](https://github.com/orgs/runwhen-contrib/discussions) -* automatically created when new troubleshooting tasks are added to any of the public runwhen troubleshooting libraries (referred to as code collections) - in the category of Troubleshooting Commands -* can be started by any GitHub user for any topic -* are a great way to publicly have an asynchronous discussion which other users may benefit from - -

GitHub Discussions

diff --git a/docs/configuration/cheat-sheet-features/README.md b/docs/configuration/cheat-sheet-features/README.md deleted file mode 100644 index ed205afcc..000000000 --- a/docs/configuration/cheat-sheet-features/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Cheat Sheet Features - diff --git a/docs/configuration/cheat-sheet-features/terminal-configuration.md b/docs/configuration/cheat-sheet-features/terminal-configuration.md deleted file mode 100644 index 640768c60..000000000 --- a/docs/configuration/cheat-sheet-features/terminal-configuration.md +++ /dev/null @@ -1,40 +0,0 @@ -# Terminal Configuration - -RunWhen Local provides a browser accessible terminal. This terminal connects directly to the RunWhen Local container, with the goal of providing faster access to running cheat sheet commands & scripts as the `runwhen` terminal user. - -

Terminal Example

- -### Disabling the Terminal - -The terminal is enabled by default; to disable the terminal, set the environment variable `RW_LOCAL_TERMINAL_DISABLED` to `True` - - - -By example: - -{% tabs %} -{% tab title="Docker" %} -``` -docker run --name RunWhenLocal -p 8081:8081 -e RW_LOCAL_TERMINAL_DISABLED=true -v $workdir/shared:/shared -d ghcr.io/runwhen-contrib/runwhen-local:latest -``` -{% endtab %} - -{% tab title="Kubernetes" %} -``` -... - spec: - containers: - - name: runwhen-local - image: ghcr.io/runwhen-contrib/runwhen-local:latest - imagePullPolicy: Always - ports: - - containerPort: 8081 - - containerPort: 8000 - - containerPort: 7687 - env: - - name: RW_LOCAL_TERMINAL_DISABLED - value: "True" -... -``` -{% endtab %} -{% endtabs %} diff --git a/docs/github-issues-request-or-share-commands-report-bugs-request-features.md b/docs/github-issues-request-or-share-commands-report-bugs-request-features.md deleted file mode 100644 index cd1de348c..000000000 --- a/docs/github-issues-request-or-share-commands-report-bugs-request-features.md +++ /dev/null @@ -1,17 +0,0 @@ -# GitHub Issues - Request or Share Commands, Report Bugs, Request Features - -GitHub issues are: - -* Available [here](https://github.com/runwhen-contrib/runwhen-local/issues/new/choose) -* Used for: - * Requesting a command that doesn't exist - * Sharing a command that you would like to see in the cheat sheet - * Reporting bugs or requesting features - -Three primary templates are used for each of the topics above, and should make the effort of requesting a command or sharing a command easy. - -

GitHub Issue Templates

- -

Requesting a Command

- -

Sharing an Awesome Command

diff --git a/docs/plans/aws-authentication-enhancement-plan.md b/docs/plans/aws-authentication-enhancement-plan.md deleted file mode 100644 index 3bca88323..000000000 --- a/docs/plans/aws-authentication-enhancement-plan.md +++ /dev/null @@ -1,1131 +0,0 @@ -# AWS Authentication Enhancement Plan - -## Overview - -This document outlines the plan to enhance AWS authentication support in RunWhen Local, following the patterns established for Azure and GCP. Currently, AWS authentication only supports explicit access key/secret key pairs. This enhancement will add support for: - -- **Kubernetes Secret-based credentials** (like Azure's `spSecretName` and GCP's `saSecretName`) -- **Default AWS credential chain** (IAM roles, instance profiles, environment variables) -- **AWS Workload Identity (EKS IRSA)** - IAM Roles for Service Accounts -- **Assume Role support** for cross-account access - -## Current State Analysis - -### Azure Authentication (Reference Pattern) - -Azure supports three authentication methods in priority order: - -1. **Explicit Service Principal** - Direct credentials in `workspaceInfo.yaml` - - Fields: `tenantId`, `clientId`, `clientSecret`, `subscriptionId` - - Auth type: `azure_explicit` - -2. **Kubernetes Secret Service Principal** - Credentials from K8s secret - - Field: `spSecretName` - - Auth type: `azure_service_principal_secret` - -3. **Managed Identity/DefaultAzureCredential** - Automatic credential chain - - No explicit credentials needed - - Auth type: `azure_managed_identity` or `azure_identity` - -Key implementation files: -- `src/azure_utils.py` - Core authentication logic -- `src/enrichers/azure.py` - Credential caching and enricher integration -- `src/templates/azure-auth.yaml` - Template for auth configuration in generated workspace -- `src/indexers/cloudquery.py` - CloudQuery integration - -### GCP Authentication (Reference Pattern) - -GCP supports four authentication methods: - -1. **Explicit Service Account** - Key in `workspaceInfo.yaml` - - Fields: `projectId`, `serviceAccountKey` - - Auth type: `gcp_service_account` - -2. **Kubernetes Secret Service Account** - Key from K8s secret - - Field: `saSecretName` - - Auth type: `gcp_service_account_secret` - -3. **Application Credentials File** - Local file reference - - Field: `applicationCredentialsFile` - - Auth type: `gcp_service_account_file` - -4. **Application Default Credentials (ADC)** - Automatic credential chain - - Auth type: `gcp_adc` - -### Current AWS Authentication (Limited) - -Currently, AWS only supports: -- `awsAccessKeyId` and `awsSecretAccessKey` in `workspaceInfo.yaml` -- Optional `awsSessionToken` for temporary credentials -- No fallback to AWS credential chain -- No Kubernetes secret support -- `boto3` is imported but unused - -## Proposed AWS Authentication Methods - -### Priority Order (Matching Azure/GCP Pattern) - -1. **Explicit Access Keys** (`aws_explicit`) - - Fields: `awsAccessKeyId`, `awsSecretAccessKey`, `awsSessionToken` (optional) - - Existing functionality - keep as-is - -2. **Kubernetes Secret** (`aws_secret`) - - Field: `awsSecretName` - - Secret keys: `awsAccessKeyId`, `awsSecretAccessKey`, `awsSessionToken` (optional) - -3. **Assume Role** (`aws_assume_role`) - - Field: `assumeRoleArn` (with optional `assumeRoleExternalId`, `assumeRoleSessionName`) - - Can be combined with other auth methods as the base credential - -4. **AWS Workload Identity / IRSA** (`aws_workload_identity` or `aws_irsa`) - - Field: `useWorkloadIdentity: true` or auto-detected in EKS - - Uses web identity token from projected service account token - -5. **Default Credential Chain** (`aws_default_chain`) - - No explicit credentials - fallback behavior - - Supports: environment variables, shared credentials file, instance profile, etc. - -## Implementation Plan - -### Phase 1: Create `aws_utils.py` Module - -Create a new module following the pattern of `azure_utils.py` and `gcp_utils.py`. - -**File: `src/aws_utils.py`** - -```python -""" -AWS authentication utilities for RunWhen Local. - -Supports multiple authentication methods: -1. Explicit access keys from workspaceInfo.yaml -2. Access keys from Kubernetes secret -3. Assume Role (with or without base credentials) -4. EKS Workload Identity (IRSA) -5. Default AWS credential chain -""" - -import base64 -import json -import logging -import os -import sys -from typing import Any, Optional, Dict, Tuple - -import boto3 -from botocore.credentials import RefreshableCredentials -from botocore.session import get_session - -from k8s_utils import get_secret -from utils import mask_string - -logger = logging.getLogger(__name__) - -# Cache for AWS credentials -_aws_credentials = { - "access_key_id": None, - "secret_access_key": None, - "session_token": None, - "region": None, - "auth_type": None, - "auth_secret": None, - "session": None, -} - -def set_aws_credentials( - access_key_id: str = None, - secret_access_key: str = None, - session_token: str = None, - region: str = None, - auth_type: str = None, - auth_secret: str = None, - session: boto3.Session = None -): - """Set AWS credentials for reuse across modules.""" - global _aws_credentials - if access_key_id: - _aws_credentials["access_key_id"] = access_key_id - if secret_access_key: - _aws_credentials["secret_access_key"] = secret_access_key - if session_token: - _aws_credentials["session_token"] = session_token - if region: - _aws_credentials["region"] = region - if auth_type: - _aws_credentials["auth_type"] = auth_type - if auth_secret: - _aws_credentials["auth_secret"] = auth_secret - if session: - _aws_credentials["session"] = session - - logger.info(f"Set AWS credentials with auth type: {auth_type}") - - -def get_aws_credential(workspace_info: dict) -> Tuple[boto3.Session, str, str, str, str, str, str]: - """ - Get AWS credentials using workspace configuration. - - Returns: - Tuple of (session, region, access_key_id, secret_access_key, session_token, auth_type, auth_secret) - """ - auth_type = None - auth_secret = None - aws_config = workspace_info.get('cloudConfig', {}).get('aws', {}) - - region = aws_config.get('region') or aws_config.get('defaultRegion') or 'us-east-1' - access_key_id = aws_config.get('awsAccessKeyId') - secret_access_key = aws_config.get('awsSecretAccessKey') - session_token = aws_config.get('awsSessionToken') - aws_secret_name = aws_config.get('awsSecretName') - assume_role_arn = aws_config.get('assumeRoleArn') - use_workload_identity = aws_config.get('useWorkloadIdentity', False) - - # Method 1: Explicit access keys in workspaceInfo.yaml - if access_key_id and secret_access_key: - logger.info("Using explicit AWS access keys from workspaceInfo.yaml") - auth_type = "aws_explicit" - session = create_boto_session(access_key_id, secret_access_key, session_token, region) - - # Handle assume role if specified - if assume_role_arn: - session, access_key_id, secret_access_key, session_token = assume_role( - session, assume_role_arn, aws_config, region - ) - auth_type = "aws_explicit_assume_role" - - return session, region, access_key_id, secret_access_key, session_token, auth_type, auth_secret - - # Method 2: Credentials from Kubernetes secret - if aws_secret_name: - logger.info(f"Using AWS credentials from Kubernetes secret: {mask_string(aws_secret_name)}") - try: - secret_data = get_secret(aws_secret_name) - access_key_id = base64.b64decode(secret_data.get('awsAccessKeyId')).decode('utf-8') - secret_access_key = base64.b64decode(secret_data.get('awsSecretAccessKey')).decode('utf-8') - session_token = None - if secret_data.get('awsSessionToken'): - session_token = base64.b64decode(secret_data.get('awsSessionToken')).decode('utf-8') - if secret_data.get('region'): - region = base64.b64decode(secret_data.get('region')).decode('utf-8') - - auth_type = "aws_secret" - auth_secret = aws_secret_name - session = create_boto_session(access_key_id, secret_access_key, session_token, region) - - # Handle assume role if specified - if assume_role_arn: - session, access_key_id, secret_access_key, session_token = assume_role( - session, assume_role_arn, aws_config, region - ) - auth_type = "aws_secret_assume_role" - - return session, region, access_key_id, secret_access_key, session_token, auth_type, auth_secret - - except Exception as e: - logger.error(f"Failed to retrieve AWS credentials from Kubernetes secret '{aws_secret_name}': {e}") - sys.exit(1) - - # Method 3: Assume Role with Web Identity (EKS IRSA / Workload Identity) - if use_workload_identity or os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE'): - logger.info("Using AWS Workload Identity (IRSA) for authentication") - auth_type = "aws_workload_identity" - - try: - session = boto3.Session(region_name=region) - - # Handle assume role if specified (in addition to IRSA) - if assume_role_arn: - session, access_key_id, secret_access_key, session_token = assume_role( - session, assume_role_arn, aws_config, region - ) - auth_type = "aws_workload_identity_assume_role" - - return session, region, None, None, None, auth_type, auth_secret - - except Exception as e: - logger.error(f"Failed to use AWS Workload Identity: {e}") - sys.exit(1) - - # Method 4: Assume Role only (relies on default chain for base credentials) - if assume_role_arn: - logger.info(f"Using AWS Assume Role with default credential chain: {mask_string(assume_role_arn)}") - auth_type = "aws_assume_role" - - try: - base_session = boto3.Session(region_name=region) - session, access_key_id, secret_access_key, session_token = assume_role( - base_session, assume_role_arn, aws_config, region - ) - return session, region, access_key_id, secret_access_key, session_token, auth_type, auth_secret - - except Exception as e: - logger.error(f"Failed to assume role: {e}") - sys.exit(1) - - # Method 5: Default AWS credential chain - logger.info("Using default AWS credential chain for authentication") - auth_type = "aws_default_chain" - - try: - session = boto3.Session(region_name=region) - # Verify credentials are available - sts = session.client('sts') - identity = sts.get_caller_identity() - logger.info(f"Authenticated as: {mask_string(identity['Arn'])}") - - return session, region, None, None, None, auth_type, auth_secret - - except Exception as e: - logger.error(f"Failed to authenticate using default AWS credential chain: {e}") - sys.exit(1) - - -def create_boto_session( - access_key_id: str, - secret_access_key: str, - session_token: str = None, - region: str = None -) -> boto3.Session: - """Create a boto3 session with explicit credentials.""" - return boto3.Session( - aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key, - aws_session_token=session_token, - region_name=region - ) - - -def assume_role( - session: boto3.Session, - role_arn: str, - aws_config: dict, - region: str -) -> Tuple[boto3.Session, str, str, str]: - """ - Assume an IAM role and return a new session with the assumed credentials. - """ - external_id = aws_config.get('assumeRoleExternalId') - session_name = aws_config.get('assumeRoleSessionName', 'runwhen-local-session') - duration_seconds = aws_config.get('assumeRoleDurationSeconds', 3600) - - logger.info(f"Assuming role: {mask_string(role_arn)}") - - sts = session.client('sts', region_name=region) - - assume_role_params = { - 'RoleArn': role_arn, - 'RoleSessionName': session_name, - 'DurationSeconds': duration_seconds - } - - if external_id: - assume_role_params['ExternalId'] = external_id - - response = sts.assume_role(**assume_role_params) - - credentials = response['Credentials'] - access_key_id = credentials['AccessKeyId'] - secret_access_key = credentials['SecretAccessKey'] - session_token = credentials['SessionToken'] - - new_session = boto3.Session( - aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key, - aws_session_token=session_token, - region_name=region - ) - - logger.info(f"Successfully assumed role: {mask_string(role_arn)}") - - return new_session, access_key_id, secret_access_key, session_token - - -def get_account_id(session: boto3.Session) -> str: - """Get the AWS account ID for the current session.""" - try: - sts = session.client('sts') - identity = sts.get_caller_identity() - return identity['Account'] - except Exception as e: - logger.error(f"Failed to get AWS account ID: {e}") - return None - - -def get_account_alias(session: boto3.Session) -> Optional[str]: - """Get the AWS account alias for the current session.""" - try: - iam = session.client('iam') - aliases = iam.list_account_aliases() - if aliases['AccountAliases']: - return aliases['AccountAliases'][0] - return None - except Exception as e: - logger.warning(f"Failed to get AWS account alias: {e}") - return None - - -def enumerate_regions(session: boto3.Session, service: str = 'ec2') -> list: - """Enumerate all available AWS regions for a service.""" - try: - ec2 = session.client('ec2', region_name='us-east-1') - regions = ec2.describe_regions() - return [r['RegionName'] for r in regions['Regions']] - except Exception as e: - logger.warning(f"Failed to enumerate AWS regions: {e}") - return ['us-east-1'] # Fallback to us-east-1 - - -def validate_aws_credentials(session: boto3.Session) -> bool: - """Validate AWS credentials by calling STS GetCallerIdentity.""" - try: - sts = session.client('sts') - sts.get_caller_identity() - return True - except Exception as e: - logger.error(f"AWS credential validation failed: {e}") - return False -``` - -### Phase 2: Update CloudQuery Indexer - -**File: `src/indexers/cloudquery.py`** - -Update the AWS credential handling section: - -```python -# Add import at top -from aws_utils import get_aws_credential, set_aws_credentials, get_account_id - -# Replace the existing AWS credential handling (around line 841) -elif platform_name == "aws": - session, region, akid, sak, stkn, auth_type, auth_secret = get_aws_credential(workspace_info) - - # Set environment variables for CloudQuery - if akid and sak: - cq_process_environment_vars["AWS_ACCESS_KEY_ID"] = akid - cq_process_environment_vars["AWS_SECRET_ACCESS_KEY"] = sak - if stkn: - cq_process_environment_vars["AWS_SESSION_TOKEN"] = stkn - - if region: - cq_process_environment_vars["AWS_DEFAULT_REGION"] = region - - # Store auth type for template rendering - platform_cfg["_auth_type"] = auth_type - platform_cfg["_auth_secret"] = auth_secret - - # Update enrichers.aws module with credentials - try: - from enrichers.aws import set_aws_credentials as set_enricher_aws_credentials - set_enricher_aws_credentials(session=session, auth_type=auth_type) - except ImportError: - pass - -# Update get_auth_type function to include AWS -def get_auth_type(platform_name, platform_config_data: dict[str,Any]): - auth_secret = None - auth_type = None - - if platform_name == "azure": - # ... existing Azure logic ... - - elif platform_name == "aws": - if platform_config_data.get("awsAccessKeyId"): - auth_type = "aws_explicit" - elif platform_config_data.get("awsSecretName"): - auth_secret = platform_config_data.get("awsSecretName") - auth_type = "aws_secret" - elif platform_config_data.get("useWorkloadIdentity") or os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE'): - auth_type = "aws_workload_identity" - elif platform_config_data.get("assumeRoleArn"): - auth_type = "aws_assume_role" - else: - auth_type = "aws_default_chain" - - # Check for assume role modifier - if platform_config_data.get("assumeRoleArn") and auth_type != "aws_assume_role": - auth_type = auth_type + "_assume_role" - - return auth_type, auth_secret -``` - -### Phase 3: Update AWS Enricher - -**File: `src/enrichers/aws.py`** - -Add credential caching support: - -```python -# Add at the top of the file, after imports -_aws_credentials = { - "session": None, - "auth_type": None, -} - -def set_aws_credentials(session: 'boto3.Session' = None, auth_type: str = None): - """Set AWS credentials for reuse in enricher operations.""" - global _aws_credentials - if session: - _aws_credentials["session"] = session - if auth_type: - _aws_credentials["auth_type"] = auth_type - logger.info(f"Set AWS enricher credentials with auth type: {auth_type}") - -def get_aws_session(): - """Get the cached AWS session or create a new one.""" - global _aws_credentials - if _aws_credentials["session"]: - return _aws_credentials["session"] - return boto3.Session() -``` - -### Phase 4: Create AWS Auth Template - -**File: `src/templates/aws-auth.yaml`** - -```yaml -{% if cluster is defined and cluster.cluster_type | default('') == "eks" and cluster.auth_type | default('') == "aws_explicit" %} - - name: aws_credentials - workspaceKey: aws:access_key@cli - - name: aws_access_key_id - workspaceKey: k8s:file@secret/{{ custom.aws_credentials_secret_name | default('undefined') }}:awsAccessKeyId - - name: aws_secret_access_key - workspaceKey: k8s:file@secret/{{ custom.aws_credentials_secret_name | default('undefined') }}:awsSecretAccessKey -{% elif cluster is defined and cluster.cluster_type | default('') == "eks" and cluster.auth_type | default('') == "aws_secret" %} - - name: aws_credentials - workspaceKey: aws:access_key@cli - - name: aws_access_key_id - workspaceKey: k8s:file@secret/{{ cluster.auth_secret | default('undefined') }}:awsAccessKeyId - - name: aws_secret_access_key - workspaceKey: k8s:file@secret/{{ cluster.auth_secret | default('undefined') }}:awsSecretAccessKey -{% elif match_resource is defined and match_resource.auth_type | default('') == "aws_explicit" %} - - name: aws_credentials - workspaceKey: aws:access_key@cli - - name: aws_access_key_id - workspaceKey: k8s:file@secret/{{ custom.aws_credentials_secret_name | default('undefined') }}:awsAccessKeyId - - name: aws_secret_access_key - workspaceKey: k8s:file@secret/{{ custom.aws_credentials_secret_name | default('undefined') }}:awsSecretAccessKey -{% elif match_resource is defined and match_resource.auth_type | default('') == "aws_secret" %} - - name: aws_credentials - workspaceKey: aws:access_key@cli - - name: aws_access_key_id - workspaceKey: k8s:file@secret/{{ match_resource.auth_secret | default('undefined') }}:awsAccessKeyId - - name: aws_secret_access_key - workspaceKey: k8s:file@secret/{{ match_resource.auth_secret | default('undefined') }}:awsSecretAccessKey -{% elif match_resource is defined and match_resource.auth_type | default('') == "aws_workload_identity" %} - - name: aws_credentials - workspaceKey: aws:irsa@cli -{% elif match_resource is defined and match_resource.auth_type | default('') == "aws_assume_role" %} - - name: aws_credentials - workspaceKey: aws:assume_role@cli - - name: aws_role_arn - value: "{{ match_resource.assume_role_arn | default('undefined') }}" -{% elif match_resource is defined and match_resource.auth_type | default('') == "aws_default_chain" %} - - name: aws_credentials - workspaceKey: aws:default_chain@cli -{% elif custom is defined and custom.aws_credentials_key is defined %} - - name: aws_credentials - workspaceKey: "{{ custom.aws_credentials_key }}" -{% else %} - - name: aws_credentials - workspaceKey: AUTH DETAILS NOT FOUND -{% endif %} -``` - -### Phase 5: Update Kubernetes Auth Template for EKS - -**File: `src/templates/kubernetes-auth.yaml`** (add EKS section) - -Add support for EKS clusters with various auth types, similar to AKS handling. - -### Phase 6: Configuration Schema Updates - -**workspaceInfo.yaml schema additions:** - -```yaml -cloudConfig: - aws: - # Existing fields - awsAccessKeyId: "AKIA..." # Explicit access key - awsSecretAccessKey: "..." # Explicit secret key - awsSessionToken: "..." # Optional session token - - # New fields - region: "us-east-1" # Default region - defaultRegion: "us-east-1" # Alias for region - - # Kubernetes secret auth - awsSecretName: "aws-credentials" # K8s secret containing credentials - - # Workload Identity (IRSA) - useWorkloadIdentity: true # Enable IRSA authentication - - # Assume Role - assumeRoleArn: "arn:aws:iam::123456789012:role/MyRole" - assumeRoleExternalId: "external-id" # Optional external ID - assumeRoleSessionName: "my-session" # Optional session name - assumeRoleDurationSeconds: 3600 # Optional duration (default 3600) - - # Multi-account support (future) - accounts: - - accountId: "123456789012" - assumeRoleArn: "arn:aws:iam::123456789012:role/MyRole" - regions: - - "us-east-1" - - "us-west-2" -``` - ---- - -## Test Infrastructure Plan - -### Directory Structure - -``` -.test/aws/ -├── basic/ # Existing - update for multiple auth methods -│ ├── workspaceInfo.yaml -│ ├── Taskfile.yaml -│ └── README.md -├── secret-auth/ # NEW: Kubernetes secret authentication -│ ├── workspaceInfo.yaml -│ ├── Taskfile.yaml -│ └── README.md -├── assume-role/ # NEW: Assume role testing -│ ├── terraform/ -│ │ ├── main.tf -│ │ ├── provider.tf -│ │ ├── variables.tf -│ │ ├── outputs.tf -│ │ └── backend.tf -│ ├── workspaceInfo.yaml -│ ├── Taskfile.yaml -│ └── README.md -├── workload-identity-eks/ # NEW: EKS IRSA testing -│ ├── terraform/ -│ │ ├── main.tf # EKS cluster + IRSA config -│ │ ├── provider.tf -│ │ ├── iam.tf # IAM roles for IRSA -│ │ ├── variables.tf -│ │ ├── outputs.tf -│ │ └── backend.tf -│ ├── workspaceInfo.yaml -│ ├── Taskfile.yaml -│ └── README.md -├── multi-account/ # NEW: Multi-account cross-account access -│ ├── terraform/ -│ │ ├── main.tf -│ │ ├── cross-account-role.tf -│ │ ├── provider.tf -│ │ ├── variables.tf -│ │ └── outputs.tf -│ ├── workspaceInfo.yaml -│ ├── Taskfile.yaml -│ └── README.md -└── ecr-registry-sync/ # Existing -``` - -### Test Case: Kubernetes Secret Authentication - -**File: `.test/aws/secret-auth/terraform/main.tf`** - -```hcl -# Create IAM user with limited permissions for testing -resource "aws_iam_user" "test_user" { - name = "runwhen-test-user-${var.resource_suffix}" - tags = local.common_tags -} - -resource "aws_iam_access_key" "test_user" { - user = aws_iam_user.test_user.name -} - -resource "aws_iam_user_policy" "test_user_policy" { - name = "runwhen-test-policy" - user = aws_iam_user.test_user.name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "ec2:Describe*", - "s3:List*", - "s3:GetBucket*", - "iam:GetUser", - "iam:ListUsers", - "sts:GetCallerIdentity" - ] - Resource = "*" - } - ] - }) -} - -# Output for creating K8s secret -output "aws_access_key_id" { - value = aws_iam_access_key.test_user.id - sensitive = true -} - -output "aws_secret_access_key" { - value = aws_iam_access_key.test_user.secret - sensitive = true -} -``` - -**File: `.test/aws/secret-auth/workspaceInfo.yaml`** - -```yaml -workspaceName: "aws-secret-auth-test" -workspaceOwnerEmail: test@runwhen.com -defaultLocation: location-01 -defaultLOD: detailed -cloudConfig: - aws: - region: "us-east-1" - awsSecretName: "aws-credentials" -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/aws-c7n-codecollection" - branch: "main" - codeBundles: ["aws-c7n-s3-health"] -``` - -**File: `.test/aws/secret-auth/Taskfile.yaml`** - -```yaml -version: '3' - -vars: - TEST_NAME: aws-secret-auth - NAMESPACE: runwhen-local - -tasks: - default: - cmds: - - task: setup-k8s-secret - - task: run-rwl-discovery - - task: verify-results - - build-terraform-infra: - dir: terraform - cmds: - - terraform init - - terraform apply -auto-approve - - setup-k8s-secret: - desc: Create K8s secret from Terraform outputs - cmds: - - | - cd terraform - ACCESS_KEY=$(terraform output -raw aws_access_key_id) - SECRET_KEY=$(terraform output -raw aws_secret_access_key) - kubectl create secret generic aws-credentials \ - --namespace={{.NAMESPACE}} \ - --from-literal=awsAccessKeyId=$ACCESS_KEY \ - --from-literal=awsSecretAccessKey=$SECRET_KEY \ - --dry-run=client -o yaml | kubectl apply -f - - - run-rwl-discovery: - desc: Run RunWhen Local discovery - cmds: - - | - docker run --rm \ - -v $(pwd):/shared \ - -v ~/.kube/config:/root/.kube/config \ - runwhen-local:test \ - python run.py --config /shared/workspaceInfo.yaml - - verify-results: - desc: Verify discovery results - cmds: - - | - # Check that AWS resources were discovered - if [ -f output/resources.yaml ]; then - grep -q "platform: aws" output/resources.yaml && echo "SUCCESS: AWS resources found" - else - echo "FAILED: No resources output" - exit 1 - fi - - cleanup: - desc: Clean up test resources - cmds: - - kubectl delete secret aws-credentials --namespace={{.NAMESPACE}} --ignore-not-found - - task: cleanup-terraform-infra - - cleanup-terraform-infra: - dir: terraform - cmds: - - terraform destroy -auto-approve -``` - -### Test Case: Assume Role - -**File: `.test/aws/assume-role/terraform/main.tf`** - -```hcl -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.region -} - -data "aws_caller_identity" "current" {} - -locals { - common_tags = { - Project = "runwhen-local" - Environment = "test" - Purpose = "assume-role-testing" - Lifecycle = "deleteme" - } -} - -# IAM Role that can be assumed -resource "aws_iam_role" "assume_role_test" { - name = "runwhen-assume-role-test-${var.resource_suffix}" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Principal = { - AWS = data.aws_caller_identity.current.arn - } - Action = "sts:AssumeRole" - Condition = { - StringEquals = { - "sts:ExternalId" = var.external_id - } - } - } - ] - }) - - tags = local.common_tags -} - -resource "aws_iam_role_policy" "assume_role_test_policy" { - name = "runwhen-test-policy" - role = aws_iam_role.assume_role_test.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "ec2:Describe*", - "s3:List*", - "s3:GetBucket*", - "sts:GetCallerIdentity" - ] - Resource = "*" - } - ] - }) -} - -variable "region" { - default = "us-east-1" -} - -variable "resource_suffix" { - default = "test" -} - -variable "external_id" { - default = "runwhen-test-external-id" -} - -output "assume_role_arn" { - value = aws_iam_role.assume_role_test.arn -} - -output "external_id" { - value = var.external_id -} -``` - -**File: `.test/aws/assume-role/workspaceInfo.yaml`** - -```yaml -workspaceName: "aws-assume-role-test" -workspaceOwnerEmail: test@runwhen.com -defaultLocation: location-01 -defaultLOD: detailed -cloudConfig: - aws: - region: "us-east-1" - assumeRoleArn: "${ASSUME_ROLE_ARN}" - assumeRoleExternalId: "${EXTERNAL_ID}" - assumeRoleSessionName: "runwhen-test-session" -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/aws-c7n-codecollection" - branch: "main" - codeBundles: ["aws-c7n-s3-health"] -``` - -### Test Case: EKS Workload Identity (IRSA) - -**File: `.test/aws/workload-identity-eks/terraform/main.tf`** - -```hcl -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.region -} - -data "aws_caller_identity" "current" {} - -locals { - common_tags = { - Project = "runwhen-local" - Environment = "test" - Purpose = "workload-identity-testing" - Lifecycle = "deleteme" - } - cluster_name = "runwhen-irsa-test-${var.resource_suffix}" -} - -# VPC for EKS -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = "${local.cluster_name}-vpc" - cidr = "10.0.0.0/16" - - azs = ["${var.region}a", "${var.region}b"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24"] - public_subnets = ["10.0.101.0/24", "10.0.102.0/24"] - - enable_nat_gateway = true - single_nat_gateway = true - - tags = local.common_tags -} - -# EKS Cluster -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.0" - - cluster_name = local.cluster_name - cluster_version = "1.28" - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - cluster_endpoint_public_access = true - - eks_managed_node_groups = { - default = { - min_size = 1 - max_size = 2 - desired_size = 1 - - instance_types = ["t3.medium"] - } - } - - tags = local.common_tags -} - -# OIDC Provider for IRSA -data "tls_certificate" "eks" { - url = module.eks.cluster_oidc_issuer_url -} - -# IAM Role for Service Account (IRSA) -resource "aws_iam_role" "runwhen_irsa" { - name = "runwhen-irsa-${var.resource_suffix}" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Principal = { - Federated = module.eks.oidc_provider_arn - } - Action = "sts:AssumeRoleWithWebIdentity" - Condition = { - StringEquals = { - "${module.eks.oidc_provider}:aud" = "sts.amazonaws.com" - "${module.eks.oidc_provider}:sub" = "system:serviceaccount:runwhen-local:runwhen-local" - } - } - } - ] - }) - - tags = local.common_tags -} - -resource "aws_iam_role_policy" "runwhen_irsa_policy" { - name = "runwhen-discovery-policy" - role = aws_iam_role.runwhen_irsa.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "ec2:Describe*", - "s3:List*", - "s3:GetBucket*", - "iam:GetUser", - "iam:ListUsers", - "sts:GetCallerIdentity" - ] - Resource = "*" - } - ] - }) -} - -variable "region" { - default = "us-east-1" -} - -variable "resource_suffix" { - default = "test" -} - -output "cluster_name" { - value = module.eks.cluster_name -} - -output "cluster_endpoint" { - value = module.eks.cluster_endpoint -} - -output "irsa_role_arn" { - value = aws_iam_role.runwhen_irsa.arn -} - -output "oidc_provider_arn" { - value = module.eks.oidc_provider_arn -} -``` - -**File: `.test/aws/workload-identity-eks/workspaceInfo.yaml`** - -```yaml -workspaceName: "aws-workload-identity-test" -workspaceOwnerEmail: test@runwhen.com -defaultLocation: location-01 -defaultLOD: detailed -cloudConfig: - aws: - region: "us-east-1" - useWorkloadIdentity: true -codeCollections: - - repoURL: "https://github.com/runwhen-contrib/aws-c7n-codecollection" - branch: "main" - codeBundles: ["aws-c7n-s3-health"] -``` - ---- - -## Implementation Phases - -### Phase 1: Core Infrastructure (Week 1-2) -- [ ] Create `src/aws_utils.py` module -- [ ] Add unit tests for `aws_utils.py` -- [ ] Update `src/indexers/cloudquery.py` for enhanced AWS auth -- [ ] Update `get_auth_type()` function for AWS - -### Phase 2: Enricher Integration (Week 2) -- [ ] Update `src/enrichers/aws.py` with credential caching -- [ ] Add `set_aws_credentials()` function -- [ ] Test enricher with various auth types - -### Phase 3: Template Support (Week 2-3) -- [ ] Create `src/templates/aws-auth.yaml` -- [ ] Update `src/templates/kubernetes-auth.yaml` for EKS -- [ ] Test template rendering with all auth types - -### Phase 4: Test Infrastructure (Week 3-4) -- [ ] Create `.test/aws/secret-auth/` test case -- [ ] Create `.test/aws/assume-role/` test case -- [ ] Create `.test/aws/workload-identity-eks/` test case -- [ ] Create `.test/aws/multi-account/` test case -- [ ] Update existing `.test/aws/basic/` for comprehensive testing - -### Phase 5: Documentation (Week 4) -- [ ] Update `docs/cloud-discovery-configuration/` for AWS auth methods -- [ ] Add examples in `src/examples/` for AWS configurations -- [ ] Update main README with AWS auth options - ---- - -## Testing Strategy - -### Unit Tests -- Test each authentication method in isolation -- Test credential resolution priority -- Test error handling for invalid credentials -- Test assume role functionality - -### Integration Tests -- Test CloudQuery indexer with each auth type -- Test enricher with real AWS resources -- Test template rendering for workspace generation - -### End-to-End Tests -- Full discovery workflow with Kubernetes secret auth -- Full discovery workflow with assume role -- Full discovery workflow with IRSA on EKS -- Multi-account discovery test - ---- - -## Risk Mitigation - -1. **Backwards Compatibility**: Existing `awsAccessKeyId`/`awsSecretAccessKey` configurations continue to work unchanged. - -2. **Credential Security**: - - Never log credentials - - Use `mask_string()` utility for all credential-related logging - - Session tokens have limited lifetime - -3. **Error Handling**: - - Clear error messages for auth failures - - Graceful fallback behavior - - Don't expose sensitive info in error messages - -4. **Testing Coverage**: - - Each auth method has dedicated test infrastructure - - CI/CD tests for regression prevention - ---- - -## Success Criteria - -1. All five AWS authentication methods work correctly -2. Existing configurations continue to work (backwards compatible) -3. Auth type correctly propagates to templates -4. Test infrastructure covers all scenarios -5. Documentation is complete and accurate -6. CloudQuery successfully indexes AWS resources with all auth types diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index ffd54cbba..000000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,5 +0,0 @@ -# Roadmap - -The RunWhen Local roadmap is managed through GitHub Projects [here](https://github.com/orgs/runwhen-contrib/projects/2). - -

RunWhen Local Roadmap

diff --git a/docs/user-guide/README.md b/docs/user-guide/README.md new file mode 100644 index 000000000..f77979619 --- /dev/null +++ b/docs/user-guide/README.md @@ -0,0 +1,61 @@ +# User Guide + +How to deploy and operate RunWhen Local against your own cloud and Kubernetes +estate. Start with [getting-started](./getting-started.md) if you've never run +it before. + +## Install + +* [Local Docker / Podman](./installation/local-docker.md) +* [Kubernetes (standalone)](./installation/kubernetes-standalone.md) +* [Kubernetes self-hosted runner (connected to the platform)](./installation/kubernetes-self-hosted/README.md) + +## Configure + +The single source of truth is `workspaceInfo.yaml`. Start with the reference +and dip into the specific guides as needed. + +* [`workspaceInfo.yaml` reference](./configuration/workspace-info.md) +* [Discovery level of detail](./configuration/level-of-detail.md) +* [Group / map customizations](./configuration/group-and-map-customizations.md) +* [CodeCollection configuration](./configuration/codecollection.md) +* [File-watching configuration](./configuration/file-watching.md) +* [Helm configuration](./configuration/helm.md) +* [Proxy & outbound connections](./configuration/proxy-and-outbound.md) +* [Private container registry](./configuration/private-registry.md) + +Examples: + +* [`workspaceinfo-overrides-example.yaml`](./configuration/workspaceinfo-overrides-example.yaml) +* [`workspaceinfo-multi-context-example.yaml`](./configuration/workspaceinfo-multi-context-example.yaml) + +## Connect cloud / Kubernetes platforms + +RunWhen Local discovers resources from each platform you point it at. +Per-platform setup (credentials, scope, supported resource types): + +* [Microsoft Azure](./cloud-discovery/azure.md) +* [Amazon Web Services](./cloud-discovery/aws.md) - + [IAM key reference](./cloud-discovery/aws-iam-keys.md) +* [Google Cloud Platform](./cloud-discovery/gcp.md) - + extras: [extras-1](./cloud-discovery/gcp-extras-1.md), + [extras-2](./cloud-discovery/gcp-extras-2.md) +* [Kubernetes](./cloud-discovery/kubernetes.md) + +## Use the generated workspace + +* [Feature overview](./features/README.md) +* [Workspace builder](./features/workspace-builder.md) +* [Upload to RunWhen Platform](./features/upload-to-runwhen-platform.md) + +## Operate + +* [Privacy and security](./privacy-and-security.md) +* [Release notes](./release-notes.md) + +## Troubleshoot + +* [Stuck? Read this](./troubleshooting/stuck.md) +* [`configProvided` overrides](./troubleshooting/config-overrides.md) - + [troubleshooting them](./troubleshooting/config-overrides-troubleshooting.md) +* [CloudQuery debug logging](./troubleshooting/cloudquery-debug-logging.md) diff --git a/docs/cloud-discovery-configuration/aws-workspace-key-reference.md b/docs/user-guide/cloud-discovery/aws-iam-keys.md similarity index 100% rename from docs/cloud-discovery-configuration/aws-workspace-key-reference.md rename to docs/user-guide/cloud-discovery/aws-iam-keys.md diff --git a/docs/cloud-discovery-configuration/amazon-web-services.md b/docs/user-guide/cloud-discovery/aws.md similarity index 100% rename from docs/cloud-discovery-configuration/amazon-web-services.md rename to docs/user-guide/cloud-discovery/aws.md diff --git a/docs/cloud-discovery-configuration/microsoft-azure.md b/docs/user-guide/cloud-discovery/azure.md similarity index 75% rename from docs/cloud-discovery-configuration/microsoft-azure.md rename to docs/user-guide/cloud-discovery/azure.md index d82e264fa..37cd1b888 100644 --- a/docs/cloud-discovery-configuration/microsoft-azure.md +++ b/docs/user-guide/cloud-discovery/azure.md @@ -31,7 +31,41 @@ cloudConfig: ## Azure Cloud Resource Discovery -Azure discovery leverages [cloudquery](https://github.com/cloudquery/cloudquery) with the Azure source plugin to build up an inventory of cloud resources that should be matched with troubleshooting commands. +Azure discovery builds up an inventory of cloud resources that gets matched with troubleshooting commands. RunWhen Local supports two backends for Azure resource discovery: + +* **`cloudquery`** (default) — Uses [cloudquery](https://github.com/cloudquery/cloudquery) with the Azure source plugin. Long-standing path; still the default while the new backend bakes in. +* **`azureapi`** — Uses the native Azure management SDK (`azure-mgmt-*`) directly. Removes the CloudQuery process / binary requirement, integrates better with airgapped images, and is the path forward for Azure (AWS / GCP migrations to follow). + +Both backends produce the same registry shape, so generation rules and SLX templates do not need to change when switching between them. + +### Selecting the backend + +Set `azureIndexerBackend` in `workspaceInfo.yaml`: + +```yaml +azureIndexerBackend: azureapi # or "cloudquery" (default) + +cloudConfig: + azure: + subscriptionId: "[subscription-id]" + # ... rest of the existing azure config ... +``` + +When `azureIndexerBackend: azureapi` is set, the CloudQuery indexer skips Azure (it still runs for AWS / GCP if those are configured) and the native `azureapi` indexer takes over. When the value is omitted or set to `cloudquery`, behavior is unchanged from previous releases. + +### Supported resource types (azureapi backend) + +Out of the box the native backend can discover: + +* `resource_group` (always) +* `virtual_machine` +* `azure_storage_accounts` +* `azure_network_virtual_networks` +* `azure_network_security_groups` +* `azure_keyvault_vaults` +* `azure_containerservice_managed_clusters` + +Additional types can be added by registering a collector in `src/indexers/azureapi_resource_types.py`. If a generation rule references an Azure resource type with no registered collector under the `azureapi` backend, the build emits a warning and continues; under the legacy `cloudquery` backend the behavior is unchanged. ## Azure Credentials diff --git a/docs/cloud-discovery-configuration/google-cloud-platform-1.md b/docs/user-guide/cloud-discovery/gcp-extras-1.md similarity index 100% rename from docs/cloud-discovery-configuration/google-cloud-platform-1.md rename to docs/user-guide/cloud-discovery/gcp-extras-1.md diff --git a/docs/cloud-discovery-configuration/google-cloud-platform-expanded.md b/docs/user-guide/cloud-discovery/gcp-extras-2.md similarity index 100% rename from docs/cloud-discovery-configuration/google-cloud-platform-expanded.md rename to docs/user-guide/cloud-discovery/gcp-extras-2.md diff --git a/docs/user-guide/cloud-discovery/gcp.md b/docs/user-guide/cloud-discovery/gcp.md new file mode 100644 index 000000000..190063173 --- /dev/null +++ b/docs/user-guide/cloud-discovery/gcp.md @@ -0,0 +1,170 @@ +# Google Cloud Platform + +{% hint style="warning" %} +This page previously contained AWS content by mistake. It has been rewritten to +document GCP cloud discovery. For AWS, see the AWS cloud-discovery page. +{% endhint %} + +RunWhen Local discovers GCP resources and matches them with troubleshooting +commands. Discovery is configured under `cloudConfig.gcp` in +`workspaceInfo.yaml`, plus a couple of top-level toggles. + +## Choosing a discovery backend + +Two backends are available, selected by the top-level `gcpIndexerBackend` key +(or the `WB_GCP_INDEXER_BACKEND` environment variable): + +```yaml +# workspaceInfo.yaml +gcpIndexerBackend: gcpapi # or: cloudquery (default) +``` + +* **`cloudquery`** (default): runs the + [CloudQuery GCP plugin](https://hub.cloudquery.io/plugins/source/cloudquery/gcp) + against the configured project(s). +* **`gcpapi`**: the native indexer — first-party `google-cloud-*` SDK collectors + as its functional baseline, plus an optional Google Cloud Asset Inventory (CAI) + accelerator, with no CloudQuery binary or `gcloud` subprocesses. + +Both backends expose the **same** generation-rule contract (the CloudQuery +table name is the `resource_type`), so rules don't change when you flip the +backend. The `gcpapi` baseline needs only per-service viewer roles; Cloud Asset +Inventory is an optional add-on — see +[Cloud Asset Inventory (optional accelerator)](#cloud-asset-inventory-optional-accelerator) +below. + +You can optionally select where discovered resources are persisted with the +top-level `resourceStoreBackend` (e.g. `sqlite` or `memory`): + +```yaml +gcpIndexerBackend: gcpapi +resourceStoreBackend: sqlite +``` + +## GCP credentials + +Credentials are resolved in the following priority order (see +`src/indexers/gcp_common.py`): + +1. **Kubernetes secret** — `saSecretName` points at a secret whose + `serviceAccountKey` (and optionally `projectId`) keys are base64-encoded. +2. **Inline service-account key** — `serviceAccountKey` (raw JSON or base64). +3. **Application credentials file** — `applicationCredentialsFile`, a path to a + service-account JSON key mounted into the container (the common local / + sandbox setup). +4. **Application Default Credentials (ADC)** — if none of the above are set, + the `google.auth` default chain is used (e.g. Workload Identity, metadata + server, or `GOOGLE_APPLICATION_CREDENTIALS`). + +### Method 1: Service-account JSON file (`applicationCredentialsFile`) + +Mount the service-account key into the container and point at it: + +```yaml +cloudConfig: + gcp: + applicationCredentialsFile: /shared/gcp.secret + projects: + - my-gcp-project +``` + +### Method 2: Inline service-account key + +```yaml +cloudConfig: + gcp: + serviceAccountKey: | + { "type": "service_account", "project_id": "my-gcp-project", ... } + projects: + - my-gcp-project +``` + +### Method 3: Kubernetes secret + +```yaml +cloudConfig: + gcp: + saSecretName: gcp-credentials +``` + +Create the secret with a base64-encoded service-account key: + +```bash +kubectl create secret generic gcp-credentials \ + --from-file=serviceAccountKey=./gcp-sa-key.json +``` + +### Method 4: Application Default Credentials + +If no explicit credentials are configured, ADC is used. The project is then +taken from `projects` / `projectId` in config, or from the +`GOOGLE_CLOUD_PROJECT` / `GCP_PROJECT` environment variables. + +```yaml +cloudConfig: + gcp: + projects: + - my-gcp-project +``` + +## Project scoping and Level of Detail + +GCP discovery is scoped **per project**. List the projects to discover under +`projects`, and set how much is collected per project with +`projectLevelOfDetails` (`detailed`, `basic`, or `none`). A project whose +effective LOD is `none` is skipped entirely. + +```yaml +defaultLOD: detailed +cloudConfig: + gcp: + applicationCredentialsFile: /shared/gcp.secret + projects: + - my-gcp-project + - my-other-project + projectLevelOfDetails: + my-gcp-project: detailed + my-other-project: none +``` + +The native `gcpapi` backend is also **generation-rule-driven**: within an +in-scope project it only collects the resource types your loaded generation +rules reference (plus the mandatory `gcp_projects` anchor). + +## Configuration reference + +| Field | Scope | Description | +| ----- | ----- | ----------- | +| `gcpIndexerBackend` | top-level | `cloudquery` (default) or `gcpapi`. Env: `WB_GCP_INDEXER_BACKEND` | +| `resourceStoreBackend` | top-level | Where discovered resources are persisted, e.g. `sqlite`, `memory` | +| `applicationCredentialsFile` | `cloudConfig.gcp` | Path to a mounted service-account JSON key | +| `serviceAccountKey` | `cloudConfig.gcp` | Inline service-account key (raw JSON or base64) | +| `saSecretName` | `cloudConfig.gcp` | Name of a Kubernetes secret holding `serviceAccountKey` / `projectId` | +| `projects` | `cloudConfig.gcp` | List of project IDs to discover | +| `projectId` | `cloudConfig.gcp` | Single project ID (alternative to `projects`) | +| `projectLevelOfDetails` | `cloudConfig.gcp` | Per-project LOD map: `detailed` / `basic` / `none` | +| `includeTags` / `excludeTags` | `cloudConfig.gcp` | Optional label-based include/exclude filters | + +## Cloud Asset Inventory (optional accelerator) + +When using the native `gcpapi` backend, the supported **functional baseline** is +the per-service typed `google-cloud-*` SDK collectors (plus the synthesized +`gcp_projects` anchor): the **12** high-value types are discovered using only +the relevant per-service viewer roles +(compute / storage / container / pubsub / iam), **with or without** Cloud Asset +Inventory. + +**Cloud Asset Inventory (CAI)** is an **optional** accelerator that *broadens* +coverage. The registry tracks **404** resource tables; **403** are discoverable +via CAI, of which **390** are served **only** by CAI. Enabling CAI adds those +390 long-tail types; it is **not required** for the baseline. + +If the service account lacks `roles/cloudasset.viewer` (or the +`cloudasset.googleapis.com` API is disabled), discovery runs **normally** on the +typed baseline: the indexer logs an **informational** `GCP_CAI_PERMISSION_DENIED` +note (not an error) and simply skips the 390 CAI-only types. No action is needed. + +For the permission matrix, the exact list of typed types, the deferred CAI-only +types (`gcp_sql_instances`, `gcp_run_services`), and the optional `gcloud` +enable/grant commands, see +[GCP indexer → discovery tiers](../../authoring/indexed-resources/gcp.md#discovery-tiers-the-typed-baseline-and-the-optional-cai-accelerator). diff --git a/docs/cloud-discovery-configuration/kubernetes-configuration.md b/docs/user-guide/cloud-discovery/kubernetes.md similarity index 100% rename from docs/cloud-discovery-configuration/kubernetes-configuration.md rename to docs/user-guide/cloud-discovery/kubernetes.md diff --git a/docs/configuration/codecollection-configuration.md b/docs/user-guide/configuration/codecollection.md similarity index 100% rename from docs/configuration/codecollection-configuration.md rename to docs/user-guide/configuration/codecollection.md diff --git a/docs/configuration/file-watching-configuration.md b/docs/user-guide/configuration/file-watching.md similarity index 100% rename from docs/configuration/file-watching-configuration.md rename to docs/user-guide/configuration/file-watching.md diff --git a/docs/configuration/user_reference.md b/docs/user-guide/configuration/group-and-map-customizations.md similarity index 98% rename from docs/configuration/user_reference.md rename to docs/user-guide/configuration/group-and-map-customizations.md index 3a089c4ea..b9e9d78db 100644 --- a/docs/configuration/user_reference.md +++ b/docs/user-guide/configuration/group-and-map-customizations.md @@ -2,7 +2,7 @@ When resources are discovered, they are automatically grouped for easier organization: -* In RunWhen Local, these groups help with organization and navigation of the troubleshooting tech docs that are rendered in mkdocs +* In RunWhen Local, these groups help organize discovered resources and generated configuration output * In the RunWhen Platform, these groups help with moving across the [map](https://docs.runwhen.com/public/runwhen-platform/feature-overview/maps) and provide additional context for the [Digital Assistants](https://docs.runwhen.com/public/runwhen-platform/feature-overview/digital-assistants) to help provide automated troubleshooting In general, the default grouping often follow a simple pattern, such as: diff --git a/docs/configuration/helm-configuration.md b/docs/user-guide/configuration/helm.md similarity index 100% rename from docs/configuration/helm-configuration.md rename to docs/user-guide/configuration/helm.md diff --git a/docs/configuration/level-of-detail.md b/docs/user-guide/configuration/level-of-detail.md similarity index 100% rename from docs/configuration/level-of-detail.md rename to docs/user-guide/configuration/level-of-detail.md diff --git a/docs/configuration/using-a-private-container-registry.md b/docs/user-guide/configuration/private-registry.md similarity index 100% rename from docs/configuration/using-a-private-container-registry.md rename to docs/user-guide/configuration/private-registry.md diff --git a/docs/configuration/proxy-configuration-and-outbound-connections.md b/docs/user-guide/configuration/proxy-and-outbound.md similarity index 96% rename from docs/configuration/proxy-configuration-and-outbound-connections.md rename to docs/user-guide/configuration/proxy-and-outbound.md index 09a8e15fb..e6a236dc5 100644 --- a/docs/configuration/proxy-configuration-and-outbound-connections.md +++ b/docs/user-guide/configuration/proxy-and-outbound.md @@ -17,7 +17,7 @@ As of v0.5.3 users can now support RunWhen Local running behind an HTTP proxy. A {% tab title="Docker Command" %} ``` -docker run --name RunWhenLocal --network host -p 8081:8081 -e NO_PROXY="localhost" -e HTTP_PROXY="http://proxy_host:proxy_port" -e HTTPS_PROXY="http://proxy_host:proxy_port" -v $workdir/shared:/shared -d runwhen-local:test +docker run --name RunWhenLocal --network host -p 8000:8000 -e NO_PROXY="localhost" -e HTTP_PROXY="http://proxy_host:proxy_port" -e HTTPS_PROXY="http://proxy_host:proxy_port" -v $workdir/shared:/shared -d runwhen-local:test ``` {% endtab %} diff --git a/docs/configuration/workspaceinfo-customization.md b/docs/user-guide/configuration/workspace-info.md similarity index 100% rename from docs/configuration/workspaceinfo-customization.md rename to docs/user-guide/configuration/workspace-info.md diff --git a/docs/examples/workspaceInfo-multi-context-example.yaml b/docs/user-guide/configuration/workspaceinfo-multi-context-example.yaml similarity index 100% rename from docs/examples/workspaceInfo-multi-context-example.yaml rename to docs/user-guide/configuration/workspaceinfo-multi-context-example.yaml diff --git a/docs/workspaceInfo-overrides-example.yaml b/docs/user-guide/configuration/workspaceinfo-overrides-example.yaml similarity index 100% rename from docs/workspaceInfo-overrides-example.yaml rename to docs/user-guide/configuration/workspaceinfo-overrides-example.yaml diff --git a/docs/user-guide/features/README.md b/docs/user-guide/features/README.md index 40fed9f63..1867cc604 100644 --- a/docs/user-guide/features/README.md +++ b/docs/user-guide/features/README.md @@ -2,7 +2,8 @@ Here are some of the features of RunWhen Local: -* **Tailored troubleshooting commands:** RunWhen Local provides a curated collection of troubleshooting commands that are specific to your Kubernetes & Cloud environment. -* **Searchable web interface:** The cheat sheet is easily searchable, so you can quickly find the commands you need. -* **Copy-paste friendly:** The commands are formatted for easy copying and pasting into your terminal. -* **Open source:** RunWhen Local is open source, so you can contribute to its development. You can also add your own troubleshooting commands to public CodeCollections, or to your own private CodeCollections for sharing very specific commands with your own team. +* **Tailored troubleshooting commands:** RunWhen Local provides a curated collection of troubleshooting commands that are specific to your Kubernetes and cloud environment. +* **Workspace Builder pipeline:** Discovery, enrichment, and rendering run through a composable pipeline exposed via a REST API and `./run.sh`. +* **[Agentic access via MCP](./mcp-server.md):** A built-in Model Context Protocol server lets Claude Code, Cursor, Claude Desktop, and other agents search and read your generated Skills. +* **Copy-paste friendly output:** Generated SLX configuration and command lists are written to `/shared/output` for use in your terminal or for upload to the RunWhen Platform. +* **Open source:** RunWhen Local is open source, so you can contribute to its development. You can also add your own troubleshooting commands to public CodeCollections, or to your own private CodeCollections for sharing very specific commands with your own team. diff --git a/docs/user-guide/features/mcp-server.md b/docs/user-guide/features/mcp-server.md new file mode 100644 index 000000000..f95781d1a --- /dev/null +++ b/docs/user-guide/features/mcp-server.md @@ -0,0 +1,226 @@ +# MCP server: agentic access to your discovered Skills + +RunWhen Local includes a built-in **Model Context Protocol (MCP)** server +so AI agents — Claude Code, Cursor, Claude Desktop, and any other +MCP-compatible client — can search, browse, and read the agentic Skills +that the workspace-builder has generated for *your* environment. + +This is the lightweight, OSS-tier counterpart to the production-grade +governance and execution that the [RunWhen Platform](https://www.runwhen.com) +provides. v1 is **read-only**: it lets agents discover Skills and ground +their reasoning in the resources you actually have. Execution against +those Skills lives on the RunWhen Platform today, and a sandboxed local +micro-runtime is on the roadmap. + +## What you get + +The MCP server exposes **twelve read-only tools** and **four canned +prompts** to any agent that connects. + +### Tools + +#### Workspace introspection + +| Tool | What it does | +| --- | --- | +| `get_workspace_summary` | Top-line stats: counts of resources by platform / type, number of generated Skills. Good first call for an agent. | +| `get_workspace_health` | Whether the last discovery run succeeded, when it ran, how long it took, warnings / parsing errors. Use this to answer "did discovery work?". | +| `list_codebundles` | Which CodeCollections are loaded, where each was cloned from. Lets the agent explain Skill provenance. | + +#### Skill discovery and reading + +| Tool | What it does | +| --- | --- | +| `search_skills` | Rank-search Skills by a curated query. Returns ranked matches with a short snippet around the first match. | +| `recommend_skills` | Like `search_skills` but takes free-text context (an error trace, a user message, a log line) and ranks **every** Skill against it. Use when a query string feels too narrow. | +| `list_skills` | Browse all generated Skills with pagination, optionally filtered by platform / resource type. | +| `get_skill` | Return the full bundle for one Skill: SLX yaml, SLI yaml, runbook yaml, and the SKILL.md describing what it does. | +| `preview_skill_invocation` | Return what *would* run for a Skill — runbook content + an illustrative `runwhen-cli` invocation — without executing anything. The handoff tool while the sandboxed micro-runtime is on the roadmap. | + +#### Resource graph + +| Tool | What it does | +| --- | --- | +| `search_resources` | Search the indexed resource graph (Kubernetes / Azure / AWS / GCP). | +| `get_resource` | Drill into one resource by `(platform, resource_type, qualified_name)` with the full attribute payload. | +| `get_resource_neighbors` | Walk one hop in the resource graph: forward refs (this Deployment → its Pods / Service / ReplicaSet) and reverse refs (the Namespace's members). | +| `get_skills_for_resource` | The agent's natural "I'm looking at this resource, what runbooks apply?" entry point. Returns Skill bundles bound to the given resource. | + +The server reads from the same SQLite resource store that powers the +workspace explorer at `/explorer/`, so what an agent sees over MCP is +always consistent with what a human sees in the UI. + +### Prompts + +MCP prompts are pre-built starter templates that show up in your +client's slash-menu (e.g. `/triage-namespace` in Cursor or Claude +Code). Each one orchestrates calls to the tools above and gives the +agent a curated investigation flow rather than "figure it out from +scratch": + +| Prompt | Arguments | What it does | +| --- | --- | --- | +| `kickoff_investigation` | — | Get oriented in a freshly-indexed workspace. Calls summary + health + list-codebundles + a sample of Skills, then proposes 2–3 investigation directions. The recommended first prompt after wiring the server up. | +| `triage_kubernetes_namespace` | `namespace` | Triage a Kubernetes namespace: enumerate its resources, find Skills bound to each, and produce a checklist with Skill links. | +| `diagnose_failing_deployment` | `namespace`, `deployment` | Drill into one failing Deployment: pull attributes, walk the resource graph, find matching Skills, propose the single most-likely-relevant runbook to read. | +| `audit_azure_keyvaults` | — | Audit indexed Azure Key Vaults for rotation / expiry / access-policy concerns. Returns a per-vault checklist linked to Skills. | + +## Endpoint + +The MCP server is mounted into the existing FastAPI process at: + +``` +http://:8000/mcp/ +``` + +It speaks the [Streamable HTTP](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#streamable-http) +transport from the MCP spec. + +> The trailing slash matters - clients that POST to `/mcp` (no slash) +> will be 307-redirected to `/mcp/`. Most MCP clients follow this +> automatically, but if you see "Method Not Allowed" or empty +> responses, double-check the URL. + +## Connecting an MCP client + +### Claude Code + +Add a `claude_code` MCP server entry pointing at your runwhen-local +instance: + +```json +{ + "mcpServers": { + "runwhen-local": { + "type": "http", + "url": "http://localhost:8000/mcp/" + } + } +} +``` + +If you're running runwhen-local in Docker, the URL is whatever you've +mapped port `8000` to on the host. + +### Cursor + +In your Cursor settings, add a remote MCP server: + +```json +{ + "mcp.servers": { + "runwhen-local": { + "url": "http://localhost:8000/mcp/" + } + } +} +``` + +### Claude Desktop + +Claude Desktop currently prefers stdio transports for local servers. +The cleanest way to bridge HTTP into stdio is the +[`mcp-remote`](https://www.npmjs.com/package/mcp-remote) helper: + +```json +{ + "mcpServers": { + "runwhen-local": { + "command": "npx", + "args": ["-y", "mcp-remote", "http://localhost:8000/mcp/"] + } + } +} +``` + +### Any HTTP-capable client + +Send a standard MCP `initialize` request to `/mcp/`: + +```bash +curl -sL -X POST http://localhost:8000/mcp/ \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + -H "MCP-Protocol-Version: 2025-06-18" \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize", + "params":{"protocolVersion":"2025-06-18", + "capabilities":{}, + "clientInfo":{"name":"my-client","version":"0"}}}' +``` + +The response includes the negotiated protocol version, our server +capabilities, and a `mcp-session-id` header to use on subsequent +requests. + +## Worked example: an agent investigating a problem + +A typical interaction loop looks like this: + +1. **Ground**: agent calls `get_workspace_summary` and + `get_workspace_health` to learn what the user has indexed (e.g. + "12 AKS clusters, 47 Azure Key Vaults, 230 Kubernetes Deployments, + discovery last succeeded 4 minutes ago"). +2. **Search**: user asks "are any of my key vaults about to expire?" + The agent calls + `search_skills(query="key vault expiry rotation")` and gets back + the `azure-keyvault-rotation` Skill ranked first, with a snippet + from its SKILL.md. +3. **Read**: agent calls `get_skill(slx_name="azure-keyvault-rotation")` + to read the full runbook + SKILL.md so it can describe the right + command set. +4. **Drill**: if the user wants to know which specific vaults are + affected, the agent calls + `search_resources(platform="azure", resource_type="azure_keyvault_vaults")` + and presents the list. +5. **Bind**: for each interesting vault, agent calls + `get_skills_for_resource(...)` to find runbooks bound to that + specific vault, and `get_resource_neighbors(...)` to walk to + related resources (private endpoints, network ACL rules). +6. **Preview**: when the user picks a Skill to run, agent calls + `preview_skill_invocation(slx_name=...)` to show the exact command + without executing it. + +Or — much simpler — the user just types `/audit-azure-keyvaults` in +their MCP client. The pre-built prompt drives the same loop with no +hand-prompting. + +## Configuration + +| Environment variable | Default | Effect | +| --- | --- | --- | +| `RW_MCP_DISABLED` | `false` | Set to `true` / `1` to mount the FastAPI app **without** the MCP route. The rest of the server (REST API, explorer UI, health) keeps running. | +| `RW_RESOURCE_STORE_PATH` | `/shared/output/resources.sqlite` | Path to the SQLite store the MCP tools read from. Same setting the explorer uses. | + +The MCP server reads the same SQLite database the workspace-builder +writes during a discovery run. If you call any MCP tool *before* +discovery has run, the server returns a friendly empty response with +a `discovery_complete: false` flag so the agent can prompt the user +to run discovery. + +## Security model (v1) + +* **Read-only**: every tool is a SELECT against the SQLite store. No + cloud calls are issued by MCP; the workspace-builder owns all + outbound traffic. +* **No auth on the route by default.** The server inherits the + same trust model as the rest of `:8000` - it expects to be exposed + on `localhost` or behind a reverse proxy you control. **Do not + publish `/mcp/` to the public internet without a proxy that adds + authentication.** +* **No execution.** Skills are returned as data (YAML + Markdown). + Running a Skill against your environment is the + [RunWhen Platform's](https://www.runwhen.com) job today. + +## Roadmap + +| Phase | What | +| --- | --- | +| **v1.1 (today)** | 12 read-only tools and 4 canned prompts. Search, browse, drill-in, resource ↔ Skill joins, free-text recommendations, invocation previews. | +| **Next** | Optional bearer-token auth on `/mcp/`, semantic / embedding-based ranking opt-in, more cloud-specific prompts (AWS RDS audit, GCP IAM, GKE namespace triage). | +| **Later** | A sandboxed micro-runtime that lets agents *execute* one Skill at a time against the user's environment. `preview_skill_invocation` is the placeholder until then. | + +## See also + +* [Workspace explorer](../../user-guide/features/workspace-builder.md) - the human-facing counterpart that reads the same resource store. +* [Resource store query API](../../architecture/resource-store-query-api.md) - the lower-level REST API powering both the explorer and the MCP tools. +* [Concepts: CodeBundle / Skill / SLX / Runbook](../../authoring/concepts.md) - the vocabulary the MCP tools speak in. diff --git a/docs/user-guide/features/upload-to-runwhen-platform.md b/docs/user-guide/features/upload-to-runwhen-platform.md index 9b73f8ce7..a28b19ad7 100644 --- a/docs/user-guide/features/upload-to-runwhen-platform.md +++ b/docs/user-guide/features/upload-to-runwhen-platform.md @@ -30,13 +30,9 @@ RunWhen Local will look for the file called `uploadInfo.yaml` in the `/shared` d With this file in place, re-run the discovery process with the `--upload` flag: ``` -docker exec -w /workspace-builder -e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true" -- RunWhenLocal ./run.sh --upload +docker exec -w /workspace-builder -- RunWhenLocal ./run.sh --upload ``` -{% hint style="info" %} -To speed up this step, we also skip the cheat sheet rendering in this step by passing `-e WB_DEBUG_SUPPRESS_CHEAT_SHEET="true"` in the docker exec command -{% endhint %} - #### Additional Upload Options for Merge Conflicts Additional upload options are available to handle certain cases where the same SLX already exists on the platform and RunWhen Local has generated new configuration under the same name: diff --git a/docs/user-guide/features/user_guide-feature_overview.md b/docs/user-guide/features/user_guide-feature_overview.md index 8adfe51a0..7f2901703 100644 --- a/docs/user-guide/features/user_guide-feature_overview.md +++ b/docs/user-guide/features/user_guide-feature_overview.md @@ -1,84 +1,41 @@ -# Troubleshooting Cheat Sheet +# Discovery Output -The Troubleshooting Cheat Sheet first performs a discovery process and builds up an inventory of your Kubernetes and cloud resources, matching them with troubleshooting commands that are from open-source "CodeCollection" libraries (such as [this](https://github.com/runwhen-contrib/rw-cli-codecollection)). See [broken-reference](broken-reference/ "mention")for more details on how this works. +RunWhen Local performs discovery on your Kubernetes and cloud resources, matches them with troubleshooting commands from open-source CodeCollection libraries (such as [this](https://github.com/runwhen-contrib/rw-cli-codecollection)), and writes the results to `/shared/output`. See [workspace-builder.md](workspace-builder.md) for how the pipeline works. -{% hint style="info" %} -See our [RunWhen Authors](https://docs.runwhen.com/public/runwhen-authors/getting-started-with-codecollection-development) for more about open source troubleshooting CodeCollections. -{% endhint %} +## Running discovery -## Troubleshooting Home Page +From a running container: -When discovery is complete, the RunWhen Local home page provides a main search bar to help you find useful troubleshooting commands, along with some details about the discovery process, such as: +``` +docker exec -w /workspace-builder RunWhenLocal ./run.sh +``` -* The number of commands generated -* The number of clusters scanned -* How many community authors wrote the troubleshooting commands -* Who ran the discovery process +When discovery completes, generated configuration and resource data are available under `$workdir/shared/output` on the host (or `/shared/output` inside the container). +## REST API +The workspace-builder service exposes a FastAPI server on port **8000**: -

RunWhen Local Cheat Sheet Home Page

+* `GET /info/` — version, available indexers/enrichers/renderers, and settings catalog +* `POST /run/` — run the discovery pipeline (used internally by `./run.sh`) +* `GET /health/` — service health and last-run status -#### Home Page Quick Links +Example: -Within the home page, you will also notice some quick links that will allow you to easily navigate through your tailored troubleshooting commands, or interact with the open source community. +``` +curl http://localhost:8000/health/ +``` -
+Browse indexed resources in the Workspace Explorer at [http://localhost:8000/explorer/](http://localhost:8000/explorer/) after running discovery with `resourceStoreBackend: sqlite`. The explorer has three tabs: **SLX Bundles** (each card groups the rendered SLX, SLI, runbook, and overlaid `Skill.md` for one SLX directory), **Discovered Resources** (the indexer graph), and **All Artifacts**. For HTTP and SQL examples, see [Resource store query API](../architecture/resource-store-query-api.md). -## Searching for Content +### Terminology -Search bars are everywhere! The home page sports a large search bar to help find the right troubleshooting command. There is also a search bar in the top right corner of every page - you can input items such as the name of your application, namespace, or a particular type of resource you are trying to troubleshoot. +In the Skills Registry vocabulary used by the explorer: -

Home Page Search Bar

+- A **CodeBundle** is a directory under `codebundles/` in a CodeCollection git repo. It defines a **Skill** (or "Skill Template") that an agent can invoke. +- A **Skill** is documented by an optional Skill markdown file at the CodeBundle root (case-insensitive — commonly `SKILL.md` or `Skill.md`). +- An **SLX** is an *instance* of a Skill, bound to a specific resource. RunWhen Local renders an SLX directory (`output/workspaces//slxs//`) containing `slx.yaml`, `sli.yaml`, `runbook.yaml`, and a verbatim copy of the source CodeBundle's Skill markdown file (when present), preserving the upstream filename casing. -

In-Page Search Bar

+## Upload to RunWhen Platform -## Command Details - -Each page is generated based on a particular resource that was discovered and combines troubleshooting commands from the open source community that best match the resource. - -

Example List of Troiubleshooting Commands for the Argo Namespace

- -### Links/Resources - -At the top of this page, we see a few additional resources to learn more about the commands, such as: - -* The GitHub user that authored the commands -* How many troubleshooting commands the page includes -* The last time the troubleshooting command source code was updated -* The links to the (open source) troubleshooting source code -* The link to view the \*private\* configuration, which tailors the commands to make them copy/paste ready but is never shared outside of your container -* A link to GitHub Discussions to engage with the community about the commands found on this page - -

Example Page Hearder with Communtiy Resources

- -### Commands - -The list of commands in a copy & paste format, ready to be used in your terminal: - -
- -### Explanations - -If you want to understand what each command does, click the **What does it do? tab** to display documentation about the command. All documentation is statically generated and stored with the public source code of the command and does not have access to, or context of, your resource names, namespace names, and so on. These explanations are generic and do not share any of your resource details with RunWhen. - -### Running Commands from the Terminal - -{% hint style="danger" %} -The built in terminal provides **unauthenticated access** to the container, with access to tools like `kubectl`. This feature should be disabled ([terminal-configuration.md](../../configuration/cheat-sheet-features/terminal-configuration.md "mention")) if running in a shared cluster - or the service account permissions thoroughly reviewed. -{% endhint %} - -A simple terminal has been added to the RunWhen Local application in order to provide a quicker way to run commands from within the same window. The terminal has command line tools like `kubectl` installed, and is configured to use the same KUBECONFIG that was used to discover cluster resources. - -

Running Commands in the Terminal

- -## Feedback - -This tool gets better with feedback and contributions from a growing community. If you are passionte about sharing your best troubleshooting commands or finding other commands that suit your needs, please consider sharing your experience in any of the following avenues: - -* [Fill out a very short feedback form](https://docs.google.com/forms/d/e/1FAIpQLScuso8SQMdj9d-0VnxxBMcvdZrcZ2M389EbwE355flnkQOUFQ/viewform) -* [Open up a GitHub issue](https://github.com/runwhen-contrib/runwhen-local/issues/new/choose) - * Ask for help - * Contribute a command - * Provide any other ideas or issues -* [Join our GitHub Discussions](https://github.com/orgs/runwhen-contrib/discussions) +To push discovery results to the RunWhen Platform instead of only writing local output files, see [upload-to-runwhen-platform.md](upload-to-runwhen-platform.md). diff --git a/docs/user-guide/features/workspace-builder.md b/docs/user-guide/features/workspace-builder.md index f52611450..53a1e66dd 100644 --- a/docs/user-guide/features/workspace-builder.md +++ b/docs/user-guide/features/workspace-builder.md @@ -5,5 +5,5 @@ Workspace Builder is the engine within RunWhen Local that: * Performs discovery on Kubernetes & Cloud (AWS, Azure, GCP) resources * Matches resources with relevant open source [CodeBundles](https://docs.runwhen.com/public/runwhen-platform/terms-and-concepts#codebundle) (health, troubleshooting, and automation tasks and scripts) * Generates the RunWhen configuration necessary to: - * Display tailored troubleshooting commands in the [user\_guide-feature\_overview.md](user\_guide-feature\_overview.md "mention") + * Display tailored troubleshooting commands in generated output (see [Discovery Output](user\_guide-feature\_overview.md "mention")) * Create a[ RunWhen Platform Workspace](https://docs.runwhen.com/public/runwhen-platform/feature-overview/workspaces), where Engineering Assistants can suggest, run, and investigate the output of these [open source CodeBundles](https://registry.runwhen.com) diff --git a/docs/user-guide/getting-started.md b/docs/user-guide/getting-started.md index 1f011abf9..188ace26d 100644 --- a/docs/user-guide/getting-started.md +++ b/docs/user-guide/getting-started.md @@ -27,7 +27,7 @@ RunWhen Local is distributed as a container image and can be deployed in the fol 1. **In Kubernetes via Helm (Connected)** * This method integrates a self-hosted runner, which executes health checks, troubleshooting, and automation tasks on behalf of the RunWhen Platform SaaS. 2. **Standalone (Disconnected Mode)** - * RunWhen Local can also be deployed to provide resource discovery and the Troubleshooting Cheat Sheet, but without any connection or interaction with the RunWhen Platform service. + * RunWhen Local can also be deployed to provide resource discovery and local output files, but without any connection or interaction with the RunWhen Platform service. #### Which deployment method is right for you? @@ -36,7 +36,7 @@ RunWhen Local is distributed as a container image and can be deployed in the fol -* **I just want a cheat sheet of automatically generated troubleshooting commands from a community of smart engineers** +* **I just want locally generated troubleshooting configuration from a community of smart engineers** * Want to run it in Kubernetes? The [kubernetes\_standalone.md](../installation/kubernetes\_standalone.md "mention") is for you. * Prefer to run it on your laptop? The [getting\_started-running\_locally.md](../installation/getting\_started-running\_locally.md "mention") is your best choice. diff --git a/docs/installation/kubernetes_self_hosted_runner/README.md b/docs/user-guide/installation/kubernetes-self-hosted/README.md similarity index 94% rename from docs/installation/kubernetes_self_hosted_runner/README.md rename to docs/user-guide/installation/kubernetes-self-hosted/README.md index a6ab785dd..0ff093b73 100644 --- a/docs/installation/kubernetes_self_hosted_runner/README.md +++ b/docs/user-guide/installation/kubernetes-self-hosted/README.md @@ -121,21 +121,6 @@ kubectl create secret generic runner-registration-token --from-literal=token="[T * From the workspace creation wizard, select **Generate Upload Configuration** (alternatively, if the wizard is gone, this can be performed from Configuration -> Workspace -> Admin Tools) {% tabs %} -{% tab title="From the Web UI" %} -* Port-forward the RunWhen Local UI (or leverage an ingress object) to access the Upload Configuration Screen - -``` -kubectl port-forward deployment/runwhen-local 8081:8081 -n $namespace -``` - -* Navigate to [http://127.0.0.1:8081/platform-upload/](http://127.0.0.1:8081/platform-upload/) -* Select **Upload Configuration** - -
- - -{% endtab %} - {% tab title="From the CLI" %} * Create a secret in the namespace with the contents of uploadInfo.yaml (ensure to update the file name accordingly) diff --git a/docs/installation/kubernetes_self_hosted_runner/runner-network-requirements.md b/docs/user-guide/installation/kubernetes-self-hosted/runner-network-requirements.md similarity index 100% rename from docs/installation/kubernetes_self_hosted_runner/runner-network-requirements.md rename to docs/user-guide/installation/kubernetes-self-hosted/runner-network-requirements.md diff --git a/docs/installation/kubernetes_standalone.md b/docs/user-guide/installation/kubernetes-standalone.md similarity index 68% rename from docs/installation/kubernetes_standalone.md rename to docs/user-guide/installation/kubernetes-standalone.md index ab9fd14c4..4c518a6f6 100644 --- a/docs/installation/kubernetes_standalone.md +++ b/docs/user-guide/installation/kubernetes-standalone.md @@ -14,10 +14,10 @@ If you have any issues with this process, feel free to reach out on [Slack](http ## Overview -Some teams might benefit from running a single instance of RunWhen Local directly from a Kubernetes cluster, sharing copy & paste-able troubleshooting commands with an entire team. +Some teams might benefit from running a single instance of RunWhen Local directly from a Kubernetes cluster, sharing discovery output and generated troubleshooting configuration with an entire team. {% hint style="info" %} -The commands generated in the Troubleshooting Cheat Sheet include the specific kubeconfig context. In order for this tool to be of the greatest use to all users, each user should have their kubeconfig context set to the identical name as the one that is used to generate the cheat sheet. +Generated commands include the specific kubeconfig context from discovery. For the greatest use to all users, each user should have their kubeconfig context set to the identical name as the one used during discovery. {% endhint %} As we also host this in Kubernetes for the purposes of an online demo, this document will share the manifests that we have used in [our own demo environment](https://runwhen-local.sandbox.runwhen.com). @@ -64,7 +64,7 @@ For additional resources on creating a long-lived service account and Kubeconfig Deploying RunWhen Local to a Kubernetes cluster can be achieved with the following manifests: * Deployment: - * Supports an environment variable titled `AUTORUN_WORKSPACE_BUILDER_INTERVAL` to control how often the Troubleshooting Cheat Sheet content is refreshed + * Supports an environment variable titled `AUTORUN_WORKSPACE_BUILDER_INTERVAL` to control how often discovery runs * Includes automatic file watching for configuration changes - see [File Watching Configuration](../configuration/file-watching-configuration.md) for details * Defines the following volumes to mount into the container: * configmap-volume: mounts the workspaceInto.yaml file into the container @@ -74,9 +74,9 @@ Deploying RunWhen Local to a Kubernetes cluster can be achieved with the followi * Ingress * The ingress object supports access from outside of the cluster to the RunWhen container. An example ingress manifest is not provided, as this will vary from cluster to cluster. * ConfigMap - * Stores the `workspaceInfo.yaml` file, which is the main configuration file that is used to customize how RunWhen Local builds it's Troubleshooting Cheat Sheet. See [Broken link](broken-reference "mention")for more details on how to modify this file. + * Stores the `workspaceInfo.yaml` file, which is the main configuration file used to customize how RunWhen Local runs discovery. See [workspaceinfo-customization.md](../configuration/workspaceinfo-customization.md) for details. * Secret - * A kubeconfig secret that contains all contexts that should be included in the Troubleshooting Cheat Sheet. This is typically a user or service account that has view-only access to the resources you wish to be included in the Troubleshooting Cheat Sheet. + * A kubeconfig secret that contains all contexts that should be included in discovery. This is typically a user or service account that has view-only access to the resources you wish to be discovered. Example deployment manifests (as used in the online demo environment) are in the [runwhen-local GitHub repo](https://github.com/runwhen-contrib/runwhen-local/tree/main/deploy/kubernetes). There is an all-in-one.yaml manifest that provides the fastest path to deployment. @@ -142,7 +142,7 @@ helm install runwhen-local runwhen-contrib/runwhen-local -n $namespace \ --set ingress.className="ingress-nginx" \ --set ingress.hosts[0].host=${hostname} \ --set ingress.hosts[0].paths[0].backend.service.name="runwhen-local" \ - --set ingress.hosts[0].paths[0].backend.service.port.number=8081 \ + --set ingress.hosts[0].paths[0].backend.service.port.number=8000 \ --set ingress.hosts[0].paths[0].path="/" \ --set ingress.hosts[0].paths[0].pathType="Prefix" \ --set ingress.tls[0].hosts[0]=${hostname} \ @@ -174,55 +174,15 @@ If you choose to deploy an ingress object (or loadbalancer type service) with an Without an ingress object or loadbalancer IP address, you can port-forward the instance to your local machine: ``` -kubectl port-forward svc/runwhen-local 8081:8081 -n $namespace +kubectl port-forward svc/runwhen-local 8000:8000 -n $namespace ``` -With the service available on your local machine, you can access the interface by opening a browser to [http://localhost:8081](http://localhost:8081) +With the service available on your local machine, you can check health and run discovery: -
- -### Optional: Add a CLI Shortcut - -If you would like a shortcut from the CLI to open your cheatsheet, the following may help: +``` +curl http://localhost:8000/health/ +kubectl exec -n $namespace deploy/runwhen-local -- ./run.sh +``` -{% tabs %} -{% tab title="Linux/MacOS" %} -* **Edit the Bash Profile**: - * For **Linux**, you'll typically edit the `.bashrc` file. For **macOS**, you'll edit the `.bash_profile` or `.zshrc` if you're using zsh. - * Use a text editor like nano or vim. For example, type `nano ~/.bashrc` (Linux) or `nano ~/.bash_profile` (macOS) and press Enter. -* **Add the Alias**: - * At the end of the file, add the following line: - - ```bash - alias runwhen-local='open http://[INGRESS_URL or http://127.0.0.1:8081] &>/dev/null &' - ``` - * For macOS, `open` is the command to open the URL in your default browser. For Linux, you might need to use `xdg-open` instead of `open`. -* **Save and Exit**: - * For nano, press `CTRL + X`, then `Y` to confirm, and `Enter` to exit. - * For vim, press `Esc`, type `:wq`, and press `Enter`. -* **Activate the Alias**: - * To make the alias available, you need to reload the profile. Type `source ~/.bashrc` (Linux) or `source ~/.bash_profile` (macOS) and press Enter. -* **Test the Alias**: - * Simply type `runwhen-local` in your terminal and press Enter. It should open your default browser to the specified website. -{% endtab %} +Generated output is written to the container's `/shared/output` volume. -{% tab title="Microsoft PowerShell" %} -* **Check if a Profile Already Exists**: - * In PowerShell, type `Test-Path $PROFILE` and press Enter. If it returns `True`, then you already have a profile. -* **Create or Edit the Profile**: - * If you don't have a profile, create one by typing `New-Item -path $PROFILE -type file -force`. - * Open the profile in a text editor, such as Notepad, by typing `notepad $PROFILE`. -* **Add the Function and Alias to Your Profile**: - * Add the following lines to the profile script: - - ```powershell - function Open-RunWhenLocal { Start-Process http://[INGRESS_URL or http://127.0.0.1:8081] } - Set-Alias -Name runwhen-local Open-RunWhenLocal - ``` - * Save and close the file. -* **Reload Your Profile** (or restart PowerShell): - * Type `. $PROFILE` to reload your profile in the current session. -* **Test the Alias Again**: - * Type `runwhen-local` and press Enter. It should open your default browser to the specified website. -{% endtab %} -{% endtabs %} diff --git a/docs/installation/getting_started-running_locally.md b/docs/user-guide/installation/local-docker.md similarity index 74% rename from docs/installation/getting_started-running_locally.md rename to docs/user-guide/installation/local-docker.md index 2ee779c8e..4d2b3a33f 100644 --- a/docs/installation/getting_started-running_locally.md +++ b/docs/user-guide/installation/local-docker.md @@ -1,7 +1,7 @@ --- description: >- These steps get you started with downloading the RunWhen Local container image - and preparing it to produce a troubleshooting cheat sheet that is tailored for + and preparing it to run resource discovery tailored for your environment. --- @@ -143,7 +143,7 @@ EOF {% endtabs %} {% hint style="info" %} -Everything in the workspaceInfo.yaml file that has a \[placeholder] beside it is not required for RunWhen Local to perform discovery or render the Troubleshooting Cheat Sheet. These values are required, however, when [uploading](../user-guide/features/upload-to-runwhen-platform.md) configurations to the RunWhen Platform (and are generated automatically when this activity is performed). +Everything in the workspaceInfo.yaml file that has a \[placeholder] beside it is not required for RunWhen Local to perform discovery. These values are required, however, when [uploading](../user-guide/features/upload-to-runwhen-platform.md) configurations to the RunWhen Platform (and are generated automatically when this activity is performed). {% endhint %} ### Generating your Kubeconfig @@ -197,9 +197,9 @@ cp ${newFile} $workdir/shared/kubeconfig ### -### Generating your Cheat Sheet +### Running discovery -With the working directory in place, there are two more steps to generate the your troubleshooting cheat sheet: +With the working directory in place, there are two more steps to run discovery: * Run the container image @@ -211,7 +211,7 @@ Run this command within the same terminal that was used to prepare $workdir. {% tab title="Docker" %} {% code overflow="wrap" %} ``` -docker run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared -d ghcr.io/runwhen-contrib/runwhen-local:latest +docker run --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared -d ghcr.io/runwhen-contrib/runwhen-local:latest ``` {% endcode %} {% endtab %} @@ -220,7 +220,7 @@ docker run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared -d ghcr.i {% code overflow="wrap" %} ``` # Run the container image -podman run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest +podman run --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest ``` {% endcode %} {% endtab %} @@ -229,7 +229,7 @@ podman run --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared --userns {% code overflow="wrap" %} ``` # Run the container image -podman run --platform=linux/arm64 --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest +podman run --platform=linux/arm64 --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest ``` {% endcode %} {% endtab %} @@ -242,13 +242,13 @@ With SELinux enabled, adding "--security-opt label=disable" can get things going {% code overflow="wrap" %} ``` # Run the container image -podman run --platform=linux/arm64 --name RunWhenLocal -p 8081:8081 -v $workdir/shared:/shared --security-opt label=disable --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest +podman run --platform=linux/arm64 --name RunWhenLocal -p 8000:8000 -v $workdir/shared:/shared --security-opt label=disable --userns=keep-id:uid=999,gid=999 ghcr.io/runwhen-contrib/runwhen-local:latest ``` {% endcode %} {% endtab %} {% endtabs %} -* Execute the script to perform discovery and build documentation: +* Execute the script to perform discovery: {% tabs %} {% tab title="Docker" %} @@ -268,60 +268,15 @@ podman exec -w /workspace-builder -- RunWhenLocal ./run.sh Depending on the amount of resources to in your cluster(s), the discovery process may take a few minutes to complete. {% endhint %} -### Viewing the Troubleshooting Cheat Sheet +### Viewing discovery output -When the process has completed, you can navigate to [http://localhost:8081](http://localhost:8081) to view the troubleshooting commands generated for your environment. +When the process has completed, generated files are available under `$workdir/shared/output`. You can also verify the service is healthy: -
- -### - -### Optional: Add a CLI Shortcut - -If you would like a shortcut from the CLI to open your cheatsheet, the following may help: - -{% tabs %} -{% tab title="Linux/MacOS" %} -* **Open Terminal**: This can usually be found in your applications or by searching. -* **Edit the Bash Profile**: - * For **Linux**, you'll typically edit the `.bashrc` file. For **macOS**, you'll edit the `.bash_profile` or `.zshrc` if you're using zsh. - * Use a text editor like nano or vim. For example, type `nano ~/.bashrc` (Linux) or `nano ~/.bash_profile` (macOS) and press Enter. -* **Add the Alias**: - * At the end of the file, add the following line: - - ```bash - alias runwhen-local='open http://127.0.0.1:8081 &>/dev/null &' - ``` - * For macOS, `open` is the command to open the URL in your default browser. For Linux, you might need to use `xdg-open` instead of `open`. -* **Save and Exit**: - * For nano, press `CTRL + X`, then `Y` to confirm, and `Enter` to exit. - * For vim, press `Esc`, type `:wq`, and press `Enter`. -* **Activate the Alias**: - * To make the alias available, you need to reload the profile. Type `source ~/.bashrc` (Linux) or `source ~/.bash_profile` (macOS) and press Enter. -* **Test the Alias**: - * Simply type `runwhen-local` in your terminal and press Enter. It should open your default browser to the specified website. -{% endtab %} +``` +curl http://localhost:8000/health/ +``` -{% tab title="Microsoft PowerShell" %} -* **Check if a Profile Already Exists**: - * In PowerShell, type `Test-Path $PROFILE` and press Enter. If it returns `True`, then you already have a profile. -* **Create or Edit the Profile**: - * If you don't have a profile, create one by typing `New-Item -path $PROFILE -type file -force`. - * Open the profile in a text editor, such as Notepad, by typing `notepad $PROFILE`. -* **Add the Function and Alias to Your Profile**: - * Add the following lines to the profile script: - - ```powershell - function Open-RunWhenLocal { Start-Process "http://127.0.0.1:8081" } - Set-Alias -Name runwhen-local Open-RunWhenLocal - ``` - * Save and close the file. -* **Reload Your Profile** (or restart PowerShell): - * Type `. $PROFILE` to reload your profile in the current session. -* **Test the Alias Again**: - * Type `runwhen-local` and press Enter. It should open your default browser to the specified website. -{% endtab %} -{% endtabs %} +See [Discovery Output](../user-guide/features/user_guide-feature_overview.md) for details on the REST API. ### Cleanup diff --git a/docs/User_Guide-Privacy_and_Security.md b/docs/user-guide/privacy-and-security.md similarity index 100% rename from docs/User_Guide-Privacy_and_Security.md rename to docs/user-guide/privacy-and-security.md diff --git a/docs/User_Guide-Release_Notes.md b/docs/user-guide/release-notes.md similarity index 100% rename from docs/User_Guide-Release_Notes.md rename to docs/user-guide/release-notes.md diff --git a/docs/cloudquery-debug-logging.md b/docs/user-guide/troubleshooting/cloudquery-debug-logging.md similarity index 100% rename from docs/cloudquery-debug-logging.md rename to docs/user-guide/troubleshooting/cloudquery-debug-logging.md diff --git a/docs/configProvided-overrides-troubleshooting.md b/docs/user-guide/troubleshooting/config-overrides-troubleshooting.md similarity index 100% rename from docs/configProvided-overrides-troubleshooting.md rename to docs/user-guide/troubleshooting/config-overrides-troubleshooting.md diff --git a/docs/configProvided-overrides.md b/docs/user-guide/troubleshooting/config-overrides.md similarity index 100% rename from docs/configProvided-overrides.md rename to docs/user-guide/troubleshooting/config-overrides.md diff --git a/docs/User_Guide-Stuck_Read_This.md b/docs/user-guide/troubleshooting/stuck.md similarity index 90% rename from docs/User_Guide-Stuck_Read_This.md rename to docs/user-guide/troubleshooting/stuck.md index 9b40d7284..6582b868c 100644 --- a/docs/User_Guide-Stuck_Read_This.md +++ b/docs/user-guide/troubleshooting/stuck.md @@ -35,20 +35,11 @@ In order to get more insights into the root cause of the issue, you can attach t ``` $ docker attach RunWhenLocal Directory /shared/output has been created. -Starting up neo4j -Waiting a bit before starting prodgraph REST server -WARNING - Config value 'build': Unrecognised configuration name: build -WARNING - Config value 'dev_addr': The use of the IP address '0.0.0.0' suggests a production environment or the use of a proxy to connect to the MkDocs server. However, the MkDocs' server is intended for local development purposes only. Please use a third party production-ready server instead. -INFO - Building documentation... -INFO - Cleaning site directory -INFO - Documentation built in 1.48 seconds -INFO - [13:28:09] Watching paths for changes: 'cmd-assist-docs/docs', 'cmd-assist-docs/mkdocs.yml' -INFO - [13:28:09] Serving on http://0.0.0.0:8081/ Operations to perform: Apply all migrations: admin, auth, contenttypes, sessions Running migrations: ... -Starting prodgraph REST server +Starting workspace builder REST server ``` * In many cases, you may just see the following generic error, which is often related to [#authentication-issues](User\_Guide-Stuck\_Read\_This.md#authentication-issues "mention") diff --git a/scripts/aws/aws_cloudquery_tables.txt b/scripts/aws/aws_cloudquery_tables.txt new file mode 100644 index 000000000..950dcccdc --- /dev/null +++ b/scripts/aws/aws_cloudquery_tables.txt @@ -0,0 +1,1122 @@ +# CloudQuery AWS plugin table list (parity source) +# Source: https://hub.cloudquery.io/plugins/source/cloudquery/aws/latest/tables +# Plugin version: v33.26.0 | tables: 1119 | captured: 2026-05-29 +aws_accessanalyzer_analyzer_archive_rules +aws_accessanalyzer_analyzer_findings +aws_accessanalyzer_analyzer_findings_v2 +aws_accessanalyzer_analyzers +aws_account_alternate_contacts +aws_account_contacts +aws_acm_certificates +aws_acmpca_certificate_authorities +aws_amp_rule_groups_namespaces +aws_amp_workspaces +aws_amplify_apps +aws_apigateway_accounts +aws_apigateway_api_keys +aws_apigateway_client_certificates +aws_apigateway_domain_name_base_path_mappings +aws_apigateway_domain_names +aws_apigateway_rest_api_authorizers +aws_apigateway_rest_api_deployments +aws_apigateway_rest_api_documentation_parts +aws_apigateway_rest_api_documentation_versions +aws_apigateway_rest_api_gateway_responses +aws_apigateway_rest_api_models +aws_apigateway_rest_api_request_validators +aws_apigateway_rest_api_resource_method_integrations +aws_apigateway_rest_api_resource_methods +aws_apigateway_rest_api_resources +aws_apigateway_rest_api_stages +aws_apigateway_rest_apis +aws_apigateway_usage_plan_keys +aws_apigateway_usage_plans +aws_apigateway_vpc_links +aws_apigatewayv2_api_authorizers +aws_apigatewayv2_api_deployments +aws_apigatewayv2_api_integration_responses +aws_apigatewayv2_api_integrations +aws_apigatewayv2_api_models +aws_apigatewayv2_api_route_responses +aws_apigatewayv2_api_routes +aws_apigatewayv2_api_stages +aws_apigatewayv2_apis +aws_apigatewayv2_domain_name_rest_api_mappings +aws_apigatewayv2_domain_names +aws_apigatewayv2_vpc_links +aws_appconfig_applications +aws_appconfig_configuration_profiles +aws_appconfig_deployment_strategies +aws_appconfig_environments +aws_appconfig_hosted_configuration_versions +aws_appflow_flows +aws_applicationautoscaling_policies +aws_applicationautoscaling_scalable_targets +aws_applicationautoscaling_scaling_activities +aws_applicationautoscaling_scheduled_actions +aws_appmesh_meshes +aws_appmesh_virtual_gateways +aws_appmesh_virtual_nodes +aws_appmesh_virtual_routers +aws_appmesh_virtual_services +aws_apprunner_auto_scaling_configurations +aws_apprunner_connections +aws_apprunner_custom_domains +aws_apprunner_observability_configurations +aws_apprunner_operations +aws_apprunner_services +aws_apprunner_vpc_connectors +aws_apprunner_vpc_ingress_connections +aws_appstream_app_blocks +aws_appstream_application_fleet_associations +aws_appstream_applications +aws_appstream_directory_configs +aws_appstream_fleets +aws_appstream_image_builders +aws_appstream_images +aws_appstream_stack_entitlements +aws_appstream_stack_user_associations +aws_appstream_stacks +aws_appstream_usage_report_subscriptions +aws_appstream_users +aws_appsync_graphql_apis +aws_athena_data_catalog_database_tables +aws_athena_data_catalog_databases +aws_athena_data_catalogs +aws_athena_work_group_named_queries +aws_athena_work_group_prepared_statements +aws_athena_work_group_query_executions +aws_athena_work_groups +aws_auditmanager_assessments +aws_autoscaling_group_lifecycle_hooks +aws_autoscaling_group_scaling_policies +aws_autoscaling_groups +aws_autoscaling_launch_configurations +aws_autoscaling_plan_resources +aws_autoscaling_plans +aws_autoscaling_scheduled_actions +aws_autoscaling_warm_pools +aws_availability_zones +aws_backup_frameworks +aws_backup_global_settings +aws_backup_jobs +aws_backup_plan_selections +aws_backup_plans +aws_backup_protected_resources +aws_backup_region_settings +aws_backup_report_plans +aws_backup_restore_testing_plans +aws_backup_restore_testing_selections +aws_backup_tiering_configurations +aws_backup_vault_recovery_points +aws_backup_vaults +aws_backupgateway_gateways +aws_batch_compute_environments +aws_batch_job_definitions +aws_batch_job_queues +aws_batch_jobs +aws_bedrock_agent_versions +aws_bedrock_agents +aws_bedrock_custom_models +aws_bedrock_evaluation_jobs +aws_bedrock_foundation_models +aws_bedrock_guardrails +aws_bedrock_inference_profiles +aws_bedrock_model_copy_jobs +aws_bedrock_model_customization_jobs +aws_bedrock_provisioned_model_throughputs +aws_budgets_actions +aws_budgets_budgets +aws_cloudformation_stack_instance_resource_drifts +aws_cloudformation_stack_instance_summaries +aws_cloudformation_stack_resources +aws_cloudformation_stack_set_operation_results +aws_cloudformation_stack_set_operations +aws_cloudformation_stack_sets +aws_cloudformation_stack_templates +aws_cloudformation_stacks +aws_cloudformation_template_summaries +aws_cloudfront_cache_policies +aws_cloudfront_distributions +aws_cloudfront_functions +aws_cloudfront_key_value_stores +aws_cloudfront_origin_access_identities +aws_cloudfront_origin_request_policies +aws_cloudfront_response_headers_policies +aws_cloudhsmv2_backups +aws_cloudhsmv2_clusters +aws_cloudtrail_channels +aws_cloudtrail_events +aws_cloudtrail_imports +aws_cloudtrail_trail_event_selectors +aws_cloudtrail_trails +aws_cloudwatch_alarms +aws_cloudwatch_metric_data +aws_cloudwatch_metric_statistics +aws_cloudwatch_metric_streams +aws_cloudwatch_metrics +aws_cloudwatchlogs_deliveries +aws_cloudwatchlogs_delivery_destinations +aws_cloudwatchlogs_delivery_sources +aws_cloudwatchlogs_log_group_data_protection_policies +aws_cloudwatchlogs_log_group_subscription_filters +aws_cloudwatchlogs_log_groups +aws_cloudwatchlogs_log_streams +aws_cloudwatchlogs_metric_filters +aws_cloudwatchlogs_resource_policies +aws_codeartifact_domains +aws_codeartifact_repositories +aws_codebuild_builds +aws_codebuild_projects +aws_codebuild_source_credentials +aws_codecommit_repositories +aws_codedeploy_applications +aws_codedeploy_deployment_configs +aws_codedeploy_deployment_groups +aws_codedeploy_deployments +aws_codegurureviewer_repository_associations +aws_codepipeline_pipelines +aws_codepipeline_webhooks +aws_codestar_connections_managed +aws_cognito_identity_pools +aws_cognito_user_pool_identity_providers +aws_cognito_user_pools +aws_comprehend_document_classification_jobs +aws_comprehend_document_classifiers +aws_comprehend_dominant_language_detection_jobs +aws_comprehend_endpoints +aws_comprehend_entities_detection_jobs +aws_comprehend_entity_recognizers +aws_comprehend_events_detection_jobs +aws_comprehend_flywheel_datasets +aws_comprehend_flywheel_iteration_histories +aws_comprehend_flywheels +aws_comprehend_keyphrases_detection_jobs +aws_comprehend_pii_entities_etection_jobs +aws_comprehend_sentiment_detection_jobs +aws_comprehend_targeted_sentiment_detection_jobs +aws_comprehend_topics_detection_jobs +aws_computeoptimizer_autoscaling_group_recommendations +aws_computeoptimizer_ebs_volume_recommendations +aws_computeoptimizer_ec2_instance_recommendations +aws_computeoptimizer_ecs_service_recommendations +aws_computeoptimizer_enrollment_statuses +aws_computeoptimizer_lambda_function_recommendations +aws_computeoptimizer_rds_database_recommendations +aws_computeoptimizerautomation_accounts +aws_config_config_rule_compliance_details +aws_config_config_rule_compliances +aws_config_config_rules +aws_config_configuration_aggregators +aws_config_configuration_recorders +aws_config_conformance_pack_rule_compliances +aws_config_conformance_packs +aws_config_delivery_channel_statuses +aws_config_delivery_channels +aws_config_remediation_configurations +aws_config_retention_configurations +aws_connect_agent_queues +aws_connect_agent_statuses +aws_connect_approved_origins +aws_connect_authentication_profiles +aws_connect_contact_evaluations +aws_connect_contact_flow_modules +aws_connect_contact_references +aws_connect_contacts +aws_connect_default_vocabularies +aws_connect_evaluation_form_versions +aws_connect_evaluation_forms +aws_connect_flow_associations +aws_connect_hours_of_operations +aws_connect_instance_storage_configs +aws_connect_instances +aws_connect_integration_associations +aws_connect_lambda_functions +aws_connect_lex_bots +aws_connect_lex_bots_v1 +aws_connect_lex_bots_v2 +aws_connect_lex_v1_bots +aws_connect_lex_v2_bots +aws_connect_phone_numbers +aws_connect_prompts +aws_connect_queue_quick_connects +aws_connect_queues +aws_connect_quick_connects +aws_connect_routing_profile_queues +aws_connect_routing_profiles +aws_connect_rules +aws_connect_security_keys +aws_connect_security_profiles +aws_connect_task_templates +aws_connect_traffic_distribution_group_users +aws_connect_traffic_distribution_groups +aws_connect_use_cases +aws_connect_user_hierarchy_groups +aws_connect_user_proficiencies +aws_connect_users +aws_connect_view_versions +aws_connect_views +aws_costexplorer_cost_30d +aws_costexplorer_cost_custom +aws_costexplorer_cost_forecast_30d +aws_costexplorer_reservation_coverages +aws_costexplorer_reservation_utilizations +aws_costoptimizationhub_recommendations +aws_datapipeline_pipelines +aws_datasync_agents +aws_datasync_azureblob_locations +aws_datasync_efs_locations +aws_datasync_fsxlustre_locations +aws_datasync_fsxontap_locations +aws_datasync_fsxopenzfs_locations +aws_datasync_fsxwindows_locations +aws_datasync_hdfs_locations +aws_datasync_locations +aws_datasync_nfs_locations +aws_datasync_objectstorage_locations +aws_datasync_s3_locations +aws_datasync_smb_locations +aws_dax_clusters +aws_detective_graph_members +aws_detective_graphs +aws_devopsguru_anomalies +aws_devopsguru_events +aws_devopsguru_insights +aws_devopsguru_log_groups +aws_devopsguru_monitored_resources +aws_devopsguru_notification_channels +aws_devopsguru_recommendations +aws_directconnect_connections +aws_directconnect_gateway_associations +aws_directconnect_gateway_attachments +aws_directconnect_gateways +aws_directconnect_lags +aws_directconnect_locations +aws_directconnect_virtual_gateways +aws_directconnect_virtual_interfaces +aws_directoryservice_directories +aws_dlm_lifecycle_policies +aws_dms_certificates +aws_dms_event_subscriptions +aws_dms_replication_instances +aws_dms_replication_subnet_groups +aws_dms_replication_tasks +aws_docdb_certificates +aws_docdb_cluster_parameter_groups +aws_docdb_cluster_parameters +aws_docdb_cluster_snapshots +aws_docdb_clusters +aws_docdb_engine_versions +aws_docdb_event_categories +aws_docdb_event_subscriptions +aws_docdb_events +aws_docdb_global_clusters +aws_docdb_instances +aws_docdb_orderable_db_instance_options +aws_docdb_pending_maintenance_actions +aws_docdb_subnet_groups +aws_dsql_cluster_policies +aws_dsql_clusters +aws_dynamodb_backups +aws_dynamodb_exports +aws_dynamodb_global_tables +aws_dynamodb_table_continuous_backups +aws_dynamodb_table_replica_auto_scalings +aws_dynamodb_table_resource_policies +aws_dynamodb_table_stream_resource_policies +aws_dynamodb_tables +aws_dynamodbstreams_streams +aws_ebs_default_kms_key_ids +aws_ebs_encryption_by_defaults +aws_ec2_account_attributes +aws_ec2_byoip_cidrs +aws_ec2_capacity_reservation_topologies +aws_ec2_capacity_reservations +aws_ec2_customer_gateways +aws_ec2_dhcp_options +aws_ec2_ebs_snapshot_attributes +aws_ec2_ebs_snapshots +aws_ec2_ebs_volume_statuses +aws_ec2_ebs_volumes +aws_ec2_egress_only_internet_gateways +aws_ec2_eips +aws_ec2_flow_logs +aws_ec2_hosts +aws_ec2_image_block_public_access_states +aws_ec2_image_last_launched_times +aws_ec2_image_launch_permissions +aws_ec2_image_references +aws_ec2_images +aws_ec2_instance_connect_endpoints +aws_ec2_instance_credit_specifications +aws_ec2_instance_disable_api_stop +aws_ec2_instance_disable_api_termination +aws_ec2_instance_statuses +aws_ec2_instance_topologies +aws_ec2_instance_types +aws_ec2_instance_user_data +aws_ec2_instances +aws_ec2_internet_gateways +aws_ec2_ipam_address_history +aws_ec2_ipam_byoasns +aws_ec2_ipam_discovered_accounts +aws_ec2_ipam_discovered_public_addresses +aws_ec2_ipam_discovered_resource_cidrs +aws_ec2_ipam_pool_allocations +aws_ec2_ipam_pool_cidrs +aws_ec2_ipam_pools +aws_ec2_ipam_resource_cidrs +aws_ec2_ipam_resource_discoveries +aws_ec2_ipam_resource_discovery_associations +aws_ec2_ipam_scopes +aws_ec2_ipams +aws_ec2_key_pairs +aws_ec2_launch_template_versions +aws_ec2_launch_templates +aws_ec2_managed_prefix_list_entries +aws_ec2_managed_prefix_lists +aws_ec2_nat_gateways +aws_ec2_network_acls +aws_ec2_network_interfaces +aws_ec2_prefix_lists +aws_ec2_regional_configs +aws_ec2_replace_root_volume_tasks +aws_ec2_reserved_instances +aws_ec2_route_tables +aws_ec2_security_group_rules +aws_ec2_security_groups +aws_ec2_serial_console_access_statuses +aws_ec2_snapshot_block_public_access_states +aws_ec2_spot_fleet_instances +aws_ec2_spot_fleet_requests +aws_ec2_spot_instance_requests +aws_ec2_subnets +aws_ec2_traffic_mirror_filters +aws_ec2_traffic_mirror_sessions +aws_ec2_traffic_mirror_targets +aws_ec2_transit_gateway_attachments +aws_ec2_transit_gateway_connect_peers +aws_ec2_transit_gateway_multicast_domains +aws_ec2_transit_gateway_peering_attachments +aws_ec2_transit_gateway_route_tables +aws_ec2_transit_gateway_routes +aws_ec2_transit_gateway_vpc_attachments +aws_ec2_transit_gateways +aws_ec2_vpc_endpoint_connections +aws_ec2_vpc_endpoint_service_configurations +aws_ec2_vpc_endpoint_service_permissions +aws_ec2_vpc_endpoint_services +aws_ec2_vpc_endpoints +aws_ec2_vpc_peering_connections +aws_ec2_vpcs +aws_ec2_vpn_connections +aws_ec2_vpn_gateways +aws_ecr_pull_through_cache_rules +aws_ecr_registries +aws_ecr_registry_policies +aws_ecr_repositories +aws_ecr_repository_image_scan_findings +aws_ecr_repository_images +aws_ecr_repository_lifecycle_policies +aws_ecr_repository_policies +aws_ecrpublic_repositories +aws_ecrpublic_repository_images +aws_ecs_cluster_container_instances +aws_ecs_cluster_services +aws_ecs_cluster_task_sets +aws_ecs_cluster_tasks +aws_ecs_clusters +aws_ecs_task_definitions +aws_efs_access_points +aws_efs_filesystems +aws_eks_access_policies +aws_eks_cluster_access_entries +aws_eks_cluster_addons +aws_eks_cluster_associated_access_policies +aws_eks_cluster_node_groups +aws_eks_cluster_oidc_identity_provider_configs +aws_eks_cluster_versions +aws_eks_clusters +aws_eks_fargate_profiles +aws_elasticache_clusters +aws_elasticache_engine_versions +aws_elasticache_events +aws_elasticache_global_replication_groups +aws_elasticache_parameter_groups +aws_elasticache_replication_groups +aws_elasticache_reserved_cache_nodes +aws_elasticache_reserved_cache_nodes_offerings +aws_elasticache_serverless_cache_snapshots +aws_elasticache_serverless_caches +aws_elasticache_service_updates +aws_elasticache_snapshots +aws_elasticache_subnet_groups +aws_elasticache_update_actions +aws_elasticache_user_groups +aws_elasticache_users +aws_elasticbeanstalk_application_versions +aws_elasticbeanstalk_applications +aws_elasticbeanstalk_configuration_options +aws_elasticbeanstalk_configuration_settings +aws_elasticbeanstalk_environments +aws_elasticbeanstalk_platform_versions +aws_elasticsearch_domains +aws_elasticsearch_packages +aws_elasticsearch_reserved_instances +aws_elasticsearch_versions +aws_elasticsearch_vpc_endpoints +aws_elbv1_load_balancer_policies +aws_elbv1_load_balancers +aws_elbv2_listener_certificates +aws_elbv2_listener_rules +aws_elbv2_listeners +aws_elbv2_load_balancer_attributes +aws_elbv2_load_balancer_capacity_reservations +aws_elbv2_load_balancer_web_acls +aws_elbv2_load_balancers +aws_elbv2_target_group_attributes +aws_elbv2_target_group_target_health_descriptions +aws_elbv2_target_groups +aws_emr_block_public_access_configs +aws_emr_cluster_instance_fleets +aws_emr_cluster_instance_groups +aws_emr_cluster_instances +aws_emr_clusters +aws_emr_notebook_executions +aws_emr_release_labels +aws_emr_security_configurations +aws_emr_steps +aws_emr_studio_session_mappings +aws_emr_studios +aws_emr_supported_instance_types +aws_eventbridge_api_destinations +aws_eventbridge_archives +aws_eventbridge_connections +aws_eventbridge_endpoints +aws_eventbridge_event_bus_rules +aws_eventbridge_event_bus_targets +aws_eventbridge_event_buses +aws_eventbridge_event_sources +aws_eventbridge_replays +aws_firehose_delivery_streams +aws_fis_actions +aws_fis_experiment_resolved_targets +aws_fis_experiment_templates +aws_fis_experiments +aws_fis_target_account_configurations +aws_fis_target_resource_types +aws_frauddetector_batch_imports +aws_frauddetector_batch_predictions +aws_frauddetector_detectors +aws_frauddetector_entity_types +aws_frauddetector_event_types +aws_frauddetector_external_models +aws_frauddetector_labels +aws_frauddetector_model_versions +aws_frauddetector_models +aws_frauddetector_outcomes +aws_frauddetector_rules +aws_frauddetector_variables +aws_freetier_usages +aws_fsx_backups +aws_fsx_data_repository_associations +aws_fsx_data_repository_tasks +aws_fsx_file_caches +aws_fsx_file_systems +aws_fsx_snapshots +aws_fsx_storage_virtual_machines +aws_fsx_volumes +aws_glacier_data_retrieval_policies +aws_glacier_vault_access_policies +aws_glacier_vault_lock_policies +aws_glacier_vault_notifications +aws_glacier_vaults +aws_globalaccelerator_accelerators +aws_globalaccelerator_custom_routing_accelerators +aws_globalaccelerator_endpoint_groups +aws_globalaccelerator_listeners +aws_glue_catalogs +aws_glue_classifiers +aws_glue_connections +aws_glue_crawlers +aws_glue_database_table_indexes +aws_glue_database_table_storage_optimizers +aws_glue_database_tables +aws_glue_databases +aws_glue_datacatalog_encryption_settings +aws_glue_dev_endpoints +aws_glue_job_runs +aws_glue_jobs +aws_glue_ml_transform_task_runs +aws_glue_ml_transforms +aws_glue_registries +aws_glue_registry_schema_versions +aws_glue_registry_schemas +aws_glue_security_configurations +aws_glue_triggers +aws_glue_workflows +aws_grafana_permissions +aws_grafana_versions +aws_grafana_workspace_service_account_tokens +aws_grafana_workspace_service_accounts +aws_grafana_workspaces +aws_guardduty_detector_coverages +aws_guardduty_detector_filters +aws_guardduty_detector_findings +aws_guardduty_detector_intel_sets +aws_guardduty_detector_ip_sets +aws_guardduty_detector_members +aws_guardduty_detector_publishing_destinations +aws_guardduty_detectors +aws_health_affected_entities +aws_health_event_details +aws_health_events +aws_health_org_event_details +aws_health_organization_affected_entities +aws_health_organization_events +aws_healthlake_fhir_datastores +aws_iam_account_authorization_details +aws_iam_accounts +aws_iam_credential_reports +aws_iam_group_attached_policies +aws_iam_group_last_accessed_details +aws_iam_group_policies +aws_iam_groups +aws_iam_instance_profiles +aws_iam_mfa_devices +aws_iam_openid_connect_identity_providers +aws_iam_outbound_web_identity_federations +aws_iam_password_policies +aws_iam_policies +aws_iam_policy_default_versions +aws_iam_policy_last_accessed_details +aws_iam_policy_versions +aws_iam_role_attached_policies +aws_iam_role_last_accessed_details +aws_iam_role_policies +aws_iam_roles +aws_iam_saml_identity_providers +aws_iam_server_certificates +aws_iam_signing_certificates +aws_iam_ssh_public_keys +aws_iam_user_access_keys +aws_iam_user_attached_policies +aws_iam_user_groups +aws_iam_user_last_accessed_details +aws_iam_user_policies +aws_iam_users +aws_iam_virtual_mfa_devices +aws_identitystore_group_memberships +aws_identitystore_groups +aws_identitystore_users +aws_imagebuilder_distribution_configurations +aws_imagebuilder_images +aws_imagebuilder_workflows +aws_inspector2_cis_scan_result_details +aws_inspector2_cis_scans +aws_inspector2_cis_target_resource_aggregations +aws_inspector2_covered_resources +aws_inspector2_findings +aws_inspector_findings +aws_invoicing_invoice_units +aws_iot_billing_groups +aws_iot_ca_certificates +aws_iot_certificates +aws_iot_jobs +aws_iot_policies +aws_iot_security_profiles +aws_iot_streams +aws_iot_thing_groups +aws_iot_thing_types +aws_iot_things +aws_iot_topic_rules +aws_kafka_cluster_operations +aws_kafka_cluster_policies +aws_kafka_clusters +aws_kafka_configurations +aws_kafka_nodes +aws_kendra_access_control_configurations +aws_kendra_data_source_sync_jobs +aws_kendra_data_sources +aws_kendra_experience_entities +aws_kendra_experience_entity_personas +aws_kendra_experiences +aws_kendra_faqs +aws_kendra_featured_results_sets +aws_kendra_indices +aws_kendra_query_suggestions_block_lists +aws_kendra_query_suggestions_configs +aws_kendra_thesauri +aws_keyspaces_keyspaces +aws_keyspaces_tables +aws_kinesis_stream_consumers +aws_kinesis_stream_shards +aws_kinesis_streams +aws_kinesisanalytics_application_operations +aws_kinesisanalytics_application_snapshots +aws_kinesisanalytics_application_versions +aws_kinesisanalytics_applications +aws_kinesisvideo_streams +aws_kms_aliases +aws_kms_key_grants +aws_kms_key_policies +aws_kms_key_rotation_statuses +aws_kms_key_rotations +aws_kms_keys +aws_lakeformation_data_cells_filters +aws_lakeformation_opt_ins +aws_lakeformation_permissions +aws_lakeformation_resource_tags +aws_lakeformation_resources +aws_lakeformation_tags +aws_lakeformation_transactions +aws_lambda_function_aliases +aws_lambda_function_concurrency_configs +aws_lambda_function_event_invoke_configs +aws_lambda_function_event_source_mappings +aws_lambda_function_url_configs +aws_lambda_function_versions +aws_lambda_functions +aws_lambda_layer_version_policies +aws_lambda_layer_versions +aws_lambda_layers +aws_lambda_runtimes +aws_lex_bot_aliases +aws_lex_bot_channel_associations +aws_lex_bot_version_utterances_views +aws_lex_bot_versions +aws_lex_bots +aws_lex_builtin_intents +aws_lex_builtin_slot_types +aws_lex_intent_versions +aws_lex_intents +aws_lex_migrations +aws_lex_slot_type_versions +aws_lex_slot_types +aws_lexv2_bot_aliases +aws_lexv2_bots +aws_lightsail_alarms +aws_lightsail_bucket_access_keys +aws_lightsail_buckets +aws_lightsail_certificates +aws_lightsail_container_service_deployments +aws_lightsail_container_service_images +aws_lightsail_container_services +aws_lightsail_database_events +aws_lightsail_database_log_events +aws_lightsail_database_parameters +aws_lightsail_database_snapshots +aws_lightsail_databases +aws_lightsail_disk_snapshots +aws_lightsail_disks +aws_lightsail_distributions +aws_lightsail_instance_port_states +aws_lightsail_instance_snapshots +aws_lightsail_instances +aws_lightsail_load_balancer_tls_certificates +aws_lightsail_load_balancers +aws_lightsail_static_ips +aws_location_keys +aws_location_maps +aws_macie2_allow_lists +aws_macie2_automated_discovery_accounts +aws_macie2_classification_jobs +aws_macie2_classification_scopes +aws_macie2_custom_data_identifiers +aws_macie2_findings +aws_macie2_invitations +aws_macie2_managed_data_identifiers +aws_macie2_members +aws_macie2_sensitivity_inspection_templates +aws_macie2_usage_totals +aws_memorydb_reserved_nodes +aws_mpa_teams +aws_mq_broker_configuration_revisions +aws_mq_broker_configurations +aws_mq_broker_users +aws_mq_brokers +aws_mwaa_environments +aws_neptune_cluster_parameter_group_parameters +aws_neptune_cluster_parameter_groups +aws_neptune_cluster_snapshots +aws_neptune_clusters +aws_neptune_db_parameter_group_db_parameters +aws_neptune_db_parameter_groups +aws_neptune_event_subscriptions +aws_neptune_global_clusters +aws_neptune_instances +aws_neptune_subnet_groups +aws_networkfirewall_firewall_policies +aws_networkfirewall_firewalls +aws_networkfirewall_rule_groups +aws_networkfirewall_tls_inspection_configurations +aws_networkmanager_attachments +aws_networkmanager_core_network_policy_versions +aws_networkmanager_core_networks +aws_networkmanager_global_networks +aws_networkmanager_links +aws_networkmanager_sites +aws_networkmanager_transit_gateway_registrations +aws_odb_autonomous_virtual_machines +aws_odb_cloud_autonomous_vm_clusters +aws_odb_cloud_exadata_infrastructures +aws_odb_cloud_vm_clusters +aws_odb_db_nodes +aws_odb_db_servers +aws_odb_db_system_shapes +aws_odb_gi_versions +aws_odb_networks +aws_odb_peering_connections +aws_odb_system_versions +aws_opensearch_domain_auto_tunes +aws_opensearch_domain_configs +aws_opensearch_domain_data_sources +aws_opensearch_domain_health +aws_opensearch_domain_maintenances +aws_opensearch_domain_nodes +aws_opensearch_domain_packages +aws_opensearch_domain_scheduled_actions +aws_opensearch_domains +aws_opensearch_inbound_connections +aws_opensearch_outbound_connections +aws_opensearch_reserved_instances +aws_opensearch_versions +aws_opensearch_vpc_endpoints +aws_organization_resource_policies +aws_organizations +aws_organizations_account_parents +aws_organizations_accounts +aws_organizations_delegated_administrators +aws_organizations_delegated_services +aws_organizations_organizational_unit_parents +aws_organizations_organizational_units +aws_organizations_policies +aws_organizations_policy_targets +aws_organizations_roots +aws_pinpoint_apps +aws_pinpoint_campaign_versions +aws_pinpoint_campaigns +aws_pinpoint_export_jobs +aws_pinpoint_import_jobs +aws_pinpoint_recommender_configurations +aws_pinpoint_segments +aws_pinpoint_template_versions +aws_pinpoint_templates +aws_polly_lexicons +aws_polly_speech_synthesis_tasks +aws_polly_voices +aws_quicksight_analyses +aws_quicksight_dashboards +aws_quicksight_data_sets +aws_quicksight_data_sources +aws_quicksight_folders +aws_quicksight_group_members +aws_quicksight_groups +aws_quicksight_ingestions +aws_quicksight_templates +aws_quicksight_users +aws_ram_principals +aws_ram_resource_share_associations +aws_ram_resource_share_invitations +aws_ram_resource_share_permissions +aws_ram_resource_shares +aws_ram_resource_types +aws_ram_resources +aws_rds_certificates +aws_rds_cluster_backtracks +aws_rds_cluster_parameter_group_parameters +aws_rds_cluster_parameter_groups +aws_rds_cluster_parameters +aws_rds_cluster_snapshots +aws_rds_clusters +aws_rds_db_parameter_group_db_parameters +aws_rds_db_parameter_groups +aws_rds_db_proxies +aws_rds_db_proxy_endpoints +aws_rds_db_proxy_target_groups +aws_rds_db_proxy_targets +aws_rds_db_security_groups +aws_rds_db_snapshots +aws_rds_engine_versions +aws_rds_event_subscriptions +aws_rds_events +aws_rds_global_clusters +aws_rds_instance_resource_metrics +aws_rds_instances +aws_rds_major_engine_versions +aws_rds_option_groups +aws_rds_pending_maintenance_actions +aws_rds_reserved_instances +aws_rds_subnet_groups +aws_redshift_cluster_parameter_groups +aws_redshift_cluster_parameters +aws_redshift_clusters +aws_redshift_data_shares +aws_redshift_endpoint_accesses +aws_redshift_endpoint_authorizations +aws_redshift_event_subscriptions +aws_redshift_events +aws_redshift_reserved_nodes +aws_redshift_snapshots +aws_redshift_subnet_groups +aws_regions +aws_rekognition_collection_faces +aws_rekognition_collections +aws_rekognition_media_analysis_jobs +aws_rekognition_project_versions +aws_rekognition_projects +aws_rekognition_stream_processors +aws_resiliencehub_alarm_recommendations +aws_resiliencehub_app_assessments +aws_resiliencehub_app_component_compliances +aws_resiliencehub_app_version_resource_mappings +aws_resiliencehub_app_version_resources +aws_resiliencehub_app_versions +aws_resiliencehub_apps +aws_resiliencehub_component_recommendations +aws_resiliencehub_recommendation_templates +aws_resiliencehub_resiliency_policies +aws_resiliencehub_sop_recommendations +aws_resiliencehub_suggested_resiliency_policies +aws_resiliencehub_test_recommendations +aws_resourcegroups_resource_groups +aws_route53_delegation_sets +aws_route53_domains +aws_route53_health_checks +aws_route53_hosted_zone_dnssecs +aws_route53_hosted_zone_query_logging_configs +aws_route53_hosted_zone_resource_record_sets +aws_route53_hosted_zone_traffic_policy_instances +aws_route53_hosted_zones +aws_route53_operations +aws_route53_profiles +aws_route53_traffic_policies +aws_route53_traffic_policy_versions +aws_route53recoverycontrolconfig_clusters +aws_route53recoverycontrolconfig_control_panels +aws_route53recoverycontrolconfig_routing_controls +aws_route53recoverycontrolconfig_safety_rules +aws_route53recoveryreadiness_cells +aws_route53recoveryreadiness_readiness_checks +aws_route53recoveryreadiness_recovery_groups +aws_route53recoveryreadiness_resource_sets +aws_route53resolver_firewall_configs +aws_route53resolver_firewall_domain_lists +aws_route53resolver_firewall_rule_group_associations +aws_route53resolver_firewall_rule_groups +aws_route53resolver_resolver_endpoints +aws_route53resolver_resolver_query_log_config_associations +aws_route53resolver_resolver_query_log_configs +aws_route53resolver_resolver_rule_associations +aws_route53resolver_resolver_rules +aws_s3_access_grant_instances +aws_s3_access_grants +aws_s3_access_points +aws_s3_accounts +aws_s3_bucket_cors_rules +aws_s3_bucket_encryption_rules +aws_s3_bucket_grants +aws_s3_bucket_lifecycles +aws_s3_bucket_loggings +aws_s3_bucket_notification_configurations +aws_s3_bucket_object_grants +aws_s3_bucket_object_heads +aws_s3_bucket_object_lock_configurations +aws_s3_bucket_objects +aws_s3_bucket_ownership_controls +aws_s3_bucket_policies +aws_s3_bucket_public_access_blocks +aws_s3_bucket_replications +aws_s3_bucket_versionings +aws_s3_bucket_websites +aws_s3_buckets +aws_s3_directory_buckets +aws_s3_multi_region_access_points +aws_s3_storage_lens_configurations +aws_s3_storage_lens_groups +aws_s3tables_bucket_policies +aws_s3tables_buckets +aws_s3tables_namespaces +aws_s3vectors_bucket_policies +aws_s3vectors_buckets +aws_s3vectors_indexes +aws_sagemaker_apps +aws_sagemaker_domains +aws_sagemaker_endpoint_configurations +aws_sagemaker_endpoints +aws_sagemaker_hyperparameter_tuning_jobs +aws_sagemaker_image_versions +aws_sagemaker_images +aws_sagemaker_mlflow_apps +aws_sagemaker_mlflow_tracking_servers +aws_sagemaker_models +aws_sagemaker_notebook_instance_lifecycle_configs +aws_sagemaker_notebook_instances +aws_sagemaker_processing_jobs +aws_sagemaker_spaces +aws_sagemaker_studio_lifecycle_configs +aws_sagemaker_training_jobs +aws_sagemaker_transform_jobs +aws_sagemaker_user_profiles +aws_savingsplans_plans +aws_scheduler_schedule_groups +aws_scheduler_schedules +aws_secretsmanager_secret_versions +aws_secretsmanager_secrets +aws_securityhub_enabled_standards +aws_securityhub_findings +aws_securityhub_hubs +aws_servicecatalog_launch_paths +aws_servicecatalog_portfolios +aws_servicecatalog_products +aws_servicecatalog_provisioned_products +aws_servicecatalog_provisioning_artifacts +aws_servicecatalog_provisioning_parameters +aws_servicediscovery_instances +aws_servicediscovery_namespaces +aws_servicediscovery_services +aws_servicequotas_awsdefaultservicequotas +aws_servicequotas_quota_utilizations +aws_servicequotas_quotas +aws_servicequotas_services +aws_ses_active_receipt_rule_sets +aws_ses_configuration_set_event_destinations +aws_ses_configuration_sets +aws_ses_contact_lists +aws_ses_custom_verification_email_templates +aws_ses_identities +aws_ses_suppressed_destinations +aws_ses_templates +aws_shield_attacks +aws_shield_protection_groups +aws_shield_protections +aws_shield_subscriptions +aws_signer_signing_profiles +aws_snowball_addresses +aws_snowball_cluster_jobs +aws_snowball_clusters +aws_snowball_compatible_images +aws_snowball_jobs +aws_snowball_long_term_pricing +aws_snowball_pickup_locations +aws_sns_subscriptions +aws_sns_topic_data_protection_policies +aws_sns_topics +aws_sqs_queues +aws_ssm_associations +aws_ssm_command_invocations +aws_ssm_compliance_summary_items +aws_ssm_document_contents +aws_ssm_document_versions +aws_ssm_documents +aws_ssm_instance_compliance_items +aws_ssm_instance_patch_states +aws_ssm_instance_patches +aws_ssm_instances +aws_ssm_inventories +aws_ssm_inventory_entries +aws_ssm_inventory_schemas +aws_ssm_maintenance_window_executions +aws_ssm_maintenance_window_schedules +aws_ssm_maintenance_window_targets +aws_ssm_maintenance_window_tasks +aws_ssm_maintenance_windows +aws_ssm_parameters +aws_ssm_patch_baselines +aws_ssm_sessions +aws_ssmincidents_incident_findings +aws_ssmincidents_incident_related_items +aws_ssmincidents_incident_timeline_events +aws_ssmincidents_incidents +aws_ssmincidents_response_plans +aws_ssoadmin_instances +aws_ssoadmin_permission_set_account_assignments +aws_ssoadmin_permission_set_customer_managed_policies +aws_ssoadmin_permission_set_inline_policies +aws_ssoadmin_permission_set_managed_policies +aws_ssoadmin_permission_set_permissions_boundaries +aws_ssoadmin_permission_sets +aws_ssoadmin_trusted_token_issuers +aws_stepfunctions_activities +aws_stepfunctions_executions +aws_stepfunctions_map_run_executions +aws_stepfunctions_map_runs +aws_stepfunctions_state_machines +aws_storagegateway_automatic_tape_creation_policies +aws_storagegateway_cache_reports +aws_storagegateway_file_shares +aws_storagegateway_file_system_associations +aws_storagegateway_gateways +aws_storagegateway_local_disks +aws_storagegateway_tape_pools +aws_storagegateway_tapes +aws_storagegateway_volume_recovery_points +aws_storagegateway_volumes +aws_support_case_communications +aws_support_cases +aws_support_services +aws_support_severity_levels +aws_support_trusted_advisor_check_results +aws_support_trusted_advisor_check_summaries +aws_support_trusted_advisor_checks +aws_swf_activity_types +aws_swf_closed_workflow_executions +aws_swf_domains +aws_swf_open_workflow_executions +aws_swf_workflow_types +aws_timestream_databases +aws_timestream_tables +aws_transcribe_call_analytics_categories +aws_transcribe_call_analytics_jobs +aws_transcribe_language_models +aws_transcribe_medical_scribe_jobs +aws_transcribe_medical_transcription_jobs +aws_transcribe_medical_vocabularies +aws_transcribe_transcription_jobs +aws_transcribe_vocabularies +aws_transcribe_vocabulary_filters +aws_transfer_agreements +aws_transfer_certificates +aws_transfer_connectors +aws_transfer_profiles +aws_transfer_servers +aws_transfer_users +aws_transfer_workflows +aws_trustedadvisor_organization_recommendation_accounts +aws_trustedadvisor_organization_recommendation_resources +aws_trustedadvisor_organization_recommendations +aws_trustedadvisor_recommendation_resources +aws_trustedadvisor_recommendations +aws_vpc_lattice_resource_configurations +aws_vpc_lattice_resource_gateways +aws_vpc_lattice_service_networks +aws_vpc_lattice_services +aws_waf_ipsets +aws_waf_rule_groups +aws_waf_rules +aws_waf_subscribed_rule_groups +aws_waf_web_acls +aws_wafregional_rate_based_rules +aws_wafregional_rule_groups +aws_wafregional_rules +aws_wafregional_web_acls +aws_wafv2_ipsets +aws_wafv2_managed_rule_groups +aws_wafv2_regex_pattern_sets +aws_wafv2_rule_groups +aws_wafv2_web_acls +aws_wellarchitected_lens_review_improvements +aws_wellarchitected_lens_reviews +aws_wellarchitected_lenses +aws_wellarchitected_share_invitations +aws_wellarchitected_workload_milestones +aws_wellarchitected_workload_shares +aws_wellarchitected_workloads +aws_workspaces_connection_alias_permissions +aws_workspaces_connection_aliases +aws_workspaces_directories +aws_workspaces_workspaces +aws_xray_encryption_configs +aws_xray_groups +aws_xray_resource_policies +aws_xray_sampling_rules diff --git a/scripts/aws/aws_resource_type_overrides.yaml b/scripts/aws/aws_resource_type_overrides.yaml new file mode 100644 index 000000000..78366da67 --- /dev/null +++ b/scripts/aws/aws_resource_type_overrides.yaml @@ -0,0 +1,354 @@ +metadata: + description: >- + Manual overrides applied by sync_aws_resource_type_registry.py. Edit by hand, + then re-run the sync script to regenerate + src/indexers/aws_resource_type_registry.yaml. + + The registry maps every CloudQuery AWS plugin table to its AWS Cloud Control + API resource type -- the CloudFormation resource type name + (``AWS::::``, e.g. ``AWS::EC2::Instance``) -- plus metadata + used by the native ``awsapi`` indexer. The CloudFormation (CFN) type is the + join key for generic discovery via the Cloud Control API + (``cloudcontrol`` boto3 client, ``list_resources`` / ``get_resource``), + exactly like ARM types are for Azure and Cloud Asset Inventory asset types + are for GCP. + +# --------------------------------------------------------------------------- +# service_cfn_names +# Maps the CloudQuery service token (the ```` in +# ``aws__``) to the CloudFormation *service* segment used in +# the CFN resource type name. Many AWS services are irregular acronyms +# (EC2, S3, RDS) or PascalCase brand names (CloudFront, DynamoDB) that the +# plain capitalisation heuristic gets wrong, so pin them here. When a token +# is absent the heuristic falls back to ``token.capitalize()``. +# --------------------------------------------------------------------------- +service_cfn_names: + # --- acronym services (all-caps) --- + ec2: EC2 + s3: S3 + rds: RDS + iam: IAM + eks: EKS + ecs: ECS + ecr: ECR + efs: EFS + emr: EMR + sns: SNS + sqs: SQS + ses: SES + kms: KMS + mq: AmazonMQ + dms: DMS + dax: DAX + fsx: FSx + waf: WAF + wafv2: WAFv2 + wafregional: WAFRegional + mwaa: MWAA + dsql: DSQL + amp: APS + # --- PascalCase brand names --- + cloudfront: CloudFront + cloudwatch: CloudWatch + cloudwatchlogs: Logs + cloudtrail: CloudTrail + cloudformation: CloudFormation + cloudhsmv2: CloudHSM + dynamodb: DynamoDB + dynamodbstreams: DynamoDB + lambda: Lambda + apigateway: ApiGateway + apigatewayv2: ApiGatewayV2 + autoscaling: AutoScaling + applicationautoscaling: ApplicationAutoScaling + route53: Route53 + route53resolver: Route53Resolver + route53recoverycontrolconfig: Route53RecoveryControl + route53recoveryreadiness: Route53RecoveryReadiness + secretsmanager: SecretsManager + ssm: SSM + ssmincidents: SSMIncidents + ssoadmin: SSO + kinesis: Kinesis + kinesisanalytics: KinesisAnalyticsV2 + kinesisvideo: KinesisVideo + firehose: KinesisFirehose + glue: Glue + athena: Athena + sagemaker: SageMaker + redshift: Redshift + elasticache: ElastiCache + elasticbeanstalk: ElasticBeanstalk + elasticsearch: Elasticsearch + opensearch: OpenSearchService + guardduty: GuardDuty + macie2: Macie + codebuild: CodeBuild + codepipeline: CodePipeline + codecommit: CodeCommit + codedeploy: CodeDeploy + codeartifact: CodeArtifact + codegurureviewer: CodeGuruReviewer + codestar: CodeStarConnections + organizations: Organizations + servicediscovery: ServiceDiscovery + servicecatalog: ServiceCatalog + servicequotas: ServiceQuotas + batch: Batch + backup: Backup + backupgateway: BackupGateway + docdb: DocDB + neptune: Neptune + networkfirewall: NetworkFirewall + networkmanager: NetworkManager + globalaccelerator: GlobalAccelerator + transfer: Transfer + datasync: DataSync + datapipeline: DataPipeline + directconnect: DirectConnect + directoryservice: DirectoryService + eventbridge: Events + scheduler: Scheduler + pinpoint: Pinpoint + cognito: Cognito + appsync: AppSync + amplify: Amplify + appconfig: AppConfig + appflow: AppFlow + appmesh: AppMesh + apprunner: AppRunner + appstream: AppStream + timestream: Timestream + memorydb: MemoryDB + lakeformation: LakeFormation + lex: Lex + lexv2: LexV2 + imagebuilder: ImageBuilder + inspector2: InspectorV2 + signer: Signer + shield: Shield + detective: Detective + frauddetector: FraudDetector + comprehend: Comprehend + transcribe: Transcribe + rekognition: Rekognition + polly: Polly + bedrock: Bedrock + quicksight: QuickSight + connect: Connect + acmpca: ACMPCA + acm: CertificateManager + accessanalyzer: AccessAnalyzer + auditmanager: AuditManager + budgets: Budgets + config: Config + cognitoidentity: Cognito + devopsguru: DevOpsGuru + dlm: DLM + ebs: EC2 + glacier: Glacier + healthlake: HealthLake + identitystore: IdentityStore + iot: IoT + kafka: MSK + keyspaces: Cassandra + lightsail: Lightsail + location: Location + resiliencehub: ResilienceHub + resourcegroups: ResourceGroups + savingsplans: SavingsPlans + securityhub: SecurityHub + snowball: Snowball + storagegateway: StorageGateway + swf: SWF + vpc_lattice: VpcLattice + wellarchitected: WellArchitected + workspaces: WorkSpaces + xray: XRay + elbv1: ElasticLoadBalancing + elbv2: ElasticLoadBalancingV2 + +# --------------------------------------------------------------------------- +# cfn_type_overrides +# Pin the full CloudFormation resource type for a table when the heuristic +# gets the service or entity wrong, OR when the table has no Cloud Control +# resource type (set to null so generic discovery skips it -- a typed +# collector handles it instead, or it is simply not discoverable). These are +# the high-value, frequently-referenced resources first. +# --------------------------------------------------------------------------- +cfn_type_overrides: + # --- compute / EC2 (most-referenced surface; entity tokens are irregular) --- + aws_ec2_instances: AWS::EC2::Instance + aws_ec2_ebs_volumes: AWS::EC2::Volume + aws_ec2_ebs_snapshots: AWS::EC2::Snapshot + aws_ec2_vpcs: AWS::EC2::VPC + aws_ec2_subnets: AWS::EC2::Subnet + aws_ec2_security_groups: AWS::EC2::SecurityGroup + aws_ec2_network_interfaces: AWS::EC2::NetworkInterface + aws_ec2_internet_gateways: AWS::EC2::InternetGateway + aws_ec2_nat_gateways: AWS::EC2::NatGateway + aws_ec2_route_tables: AWS::EC2::RouteTable + aws_ec2_network_acls: AWS::EC2::NetworkAcl + aws_ec2_eips: AWS::EC2::EIP + aws_ec2_vpc_peering_connections: AWS::EC2::VPCPeeringConnection + aws_ec2_vpn_connections: AWS::EC2::VPNConnection + aws_ec2_vpn_gateways: AWS::EC2::VPNGateway + aws_ec2_customer_gateways: AWS::EC2::CustomerGateway + aws_ec2_transit_gateways: AWS::EC2::TransitGateway + aws_ec2_flow_logs: AWS::EC2::FlowLog + aws_ec2_images: AWS::EC2::Image + aws_ec2_key_pairs: AWS::EC2::KeyPair + aws_ec2_launch_templates: AWS::EC2::LaunchTemplate + aws_ec2_dhcp_options: AWS::EC2::DHCPOptions + aws_ec2_vpc_endpoints: AWS::EC2::VPCEndpoint + + # --- containers / serverless --- + aws_ecs_clusters: AWS::ECS::Cluster + aws_ecs_cluster_services: AWS::ECS::Service + aws_ecs_task_definitions: AWS::ECS::TaskDefinition + aws_eks_clusters: AWS::EKS::Cluster + aws_eks_cluster_node_groups: AWS::EKS::Nodegroup + aws_eks_fargate_profiles: AWS::EKS::FargateProfile + aws_lambda_functions: AWS::Lambda::Function + aws_lambda_layers: AWS::Lambda::LayerVersion + aws_batch_compute_environments: AWS::Batch::ComputeEnvironment + aws_batch_job_queues: AWS::Batch::JobQueue + aws_batch_job_definitions: AWS::Batch::JobDefinition + aws_apprunner_services: AWS::AppRunner::Service + + # --- load balancing / networking --- + aws_elbv2_load_balancers: AWS::ElasticLoadBalancingV2::LoadBalancer + aws_elbv2_target_groups: AWS::ElasticLoadBalancingV2::TargetGroup + aws_elbv2_listeners: AWS::ElasticLoadBalancingV2::Listener + aws_elbv1_load_balancers: AWS::ElasticLoadBalancing::LoadBalancer + aws_cloudfront_distributions: AWS::CloudFront::Distribution + aws_globalaccelerator_accelerators: AWS::GlobalAccelerator::Accelerator + aws_route53_hosted_zones: AWS::Route53::HostedZone + aws_apigateway_rest_apis: AWS::ApiGateway::RestApi + aws_apigatewayv2_apis: AWS::ApiGatewayV2::Api + + # --- storage / data --- + aws_s3_buckets: AWS::S3::Bucket + aws_s3_access_points: AWS::S3::AccessPoint + aws_efs_filesystems: AWS::EFS::FileSystem + aws_fsx_file_systems: AWS::FSx::FileSystem + aws_dynamodb_tables: AWS::DynamoDB::Table + aws_dynamodb_global_tables: AWS::DynamoDB::GlobalTable + aws_glacier_vaults: AWS::Glacier::Vault + aws_backup_vaults: AWS::Backup::BackupVault + aws_backup_plans: AWS::Backup::BackupPlan + + # --- databases --- + aws_rds_instances: AWS::RDS::DBInstance + aws_rds_clusters: AWS::RDS::DBCluster + aws_rds_db_proxies: AWS::RDS::DBProxy + aws_rds_subnet_groups: AWS::RDS::DBSubnetGroup + aws_rds_global_clusters: AWS::RDS::GlobalCluster + aws_docdb_clusters: AWS::DocDB::DBCluster + aws_docdb_instances: AWS::DocDB::DBInstance + aws_neptune_clusters: AWS::Neptune::DBCluster + aws_neptune_instances: AWS::Neptune::DBInstance + aws_elasticache_clusters: AWS::ElastiCache::CacheCluster + aws_elasticache_replication_groups: AWS::ElastiCache::ReplicationGroup + aws_redshift_clusters: AWS::Redshift::Cluster + aws_memorydb_reserved_nodes: AWS::MemoryDB::Cluster + + # --- messaging / streaming --- + aws_sns_topics: AWS::SNS::Topic + aws_sns_subscriptions: AWS::SNS::Subscription + aws_sqs_queues: AWS::SQS::Queue + aws_kinesis_streams: AWS::Kinesis::Stream + aws_firehose_delivery_streams: AWS::KinesisFirehose::DeliveryStream + aws_kafka_clusters: AWS::MSK::Cluster + aws_mq_brokers: AWS::AmazonMQ::Broker + aws_eventbridge_event_buses: AWS::Events::EventBus + + # --- security / identity --- + aws_kms_keys: AWS::KMS::Key + aws_kms_aliases: AWS::KMS::Alias + aws_iam_roles: AWS::IAM::Role + aws_iam_users: AWS::IAM::User + aws_iam_groups: AWS::IAM::Group + aws_iam_policies: AWS::IAM::ManagedPolicy + aws_iam_instance_profiles: AWS::IAM::InstanceProfile + aws_secretsmanager_secrets: AWS::SecretsManager::Secret + aws_acm_certificates: AWS::CertificateManager::Certificate + aws_wafv2_web_acls: AWS::WAFv2::WebACL + aws_guardduty_detectors: AWS::GuardDuty::Detector + aws_cognito_user_pools: AWS::Cognito::UserPool + + # --- observability / ops --- + aws_cloudwatch_alarms: AWS::CloudWatch::Alarm + aws_cloudwatchlogs_log_groups: AWS::Logs::LogGroup + aws_cloudformation_stacks: AWS::CloudFormation::Stack + aws_ssm_parameters: AWS::SSM::Parameter + aws_ssm_documents: AWS::SSM::Document + aws_sagemaker_endpoints: AWS::SageMaker::Endpoint + aws_sagemaker_notebook_instances: AWS::SageMaker::NotebookInstance + aws_stepfunctions_state_machines: AWS::StepFunctions::StateMachine + aws_autoscaling_groups: AWS::AutoScaling::AutoScalingGroup + aws_autoscaling_launch_configurations: AWS::AutoScaling::LaunchConfiguration + + # --- the account anchor: synthesized from config, not via Cloud Control --- + # ``aws_iam_accounts`` is the AWS analogue of ``gcp_projects`` / + # ``azure_resources_resource_groups``: the mandatory anchor every other AWS + # resource is scoped under (account_id / account_name). It is synthesized + # directly from the resolved credentials, so it has no Cloud Control type + # (null) and is excluded from the generic pass. + aws_iam_accounts: null + + # --- tables with no Cloud Control resource type (skip in generic pass) --- + # Reports, metrics, recommendations, cost rollups, and pseudo-tables that + # Cloud Control does not expose as discrete resources. A typed collector + # would be required if any of these are ever referenced by a gen rule. + aws_regions: null + aws_availability_zones: null + aws_iam_credential_reports: null + aws_iam_account_authorization_details: null + aws_cloudwatch_metric_data: null + aws_cloudwatch_metric_statistics: null + aws_cloudwatch_metrics: null + aws_costexplorer_cost_30d: null + aws_costexplorer_cost_custom: null + aws_costexplorer_cost_forecast_30d: null + aws_costexplorer_reservation_coverages: null + aws_costexplorer_reservation_utilizations: null + aws_costoptimizationhub_recommendations: null + aws_ec2_account_attributes: null + aws_ec2_instance_statuses: null + aws_ec2_instance_types: null + +# --------------------------------------------------------------------------- +# aliases +# Extra RWL resource_type_name aliases that should resolve to a table, on +# top of the canonical CloudQuery table name. Mirrors the legacy preset +# registry names (e.g. ``account``, ``ec2_instance``) so existing generation +# rules keep working. +# --------------------------------------------------------------------------- +aliases: + aws_iam_accounts: + - account + - aws_account + aws_ec2_instances: + - ec2_instance + +# --------------------------------------------------------------------------- +# typed_collectors +# Tables for which awsapi_resource_types.py ships a dedicated boto3 service +# collector (richer payload than the generic Cloud Control pass), plus +# ``aws_iam_accounts`` which is synthesized directly from config as the +# anchor. Every other table is covered by the Cloud Control generic pass. +# --------------------------------------------------------------------------- +typed_collectors: +- aws_iam_accounts +- aws_ec2_instances +- aws_s3_buckets + +# --------------------------------------------------------------------------- +# mandatory +# Tables always collected regardless of generation-rule scope. +# ``aws_iam_accounts`` is the anchor: every other AWS resource is scoped +# under its account. +# --------------------------------------------------------------------------- +mandatory: +- aws_iam_accounts diff --git a/scripts/aws/sync_aws_resource_type_registry.py b/scripts/aws/sync_aws_resource_type_registry.py new file mode 100644 index 000000000..43a6def2f --- /dev/null +++ b/scripts/aws/sync_aws_resource_type_registry.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +""" +Sync ``src/indexers/aws_resource_type_registry.yaml``. + +The registry maps every CloudQuery AWS plugin table to its AWS Cloud Control +API resource type -- the CloudFormation resource type name +(``AWS::::``, e.g. ``AWS::EC2::Instance``) -- plus metadata +used by the native ``awsapi`` indexer. The CloudFormation (CFN) type is the +join key for generic discovery, exactly like ARM types are for the Azure +indexer and Cloud Asset Inventory asset types are for the GCP indexer. + +The data is materialised from three inputs: + +1. **A list of CloudQuery AWS table names.** Source of truth for which tables + exist. Provided either by re-reading the previous registry snapshot + (default), reading a local file (e.g. ``scripts/aws/aws_cloudquery_tables.txt``), + or fetching the live list from CloudQuery's public hub. + +2. **A heuristic** that converts ``aws__`` into + ``AWS::::``. The service segment uses + ``service_cfn_names`` (a hand-curated dictionary that handles services + whose CFN segment differs from a simple capitalisation of the table token, + e.g. ``ec2`` -> ``EC2``, ``cloudfront`` -> ``CloudFront``); the entity + segment is singularised and PascalCased. + +3. **A manual overrides YAML** at + ``scripts/aws/aws_resource_type_overrides.yaml`` that pins the CFN resource + type, aliases, ``typed_collector`` flag, and ``mandatory`` flag for any + table whose heuristic value is wrong or where extra metadata is needed. + Set a ``cfn_type_override`` to ``null`` for tables that have no Cloud + Control resource type (reports, metrics, cost rollups, the synthesized + account anchor...) so generic discovery skips them. + +Hand-edit the overrides YAML; never hand-edit the registry YAML. After editing +the overrides, re-run this script to regenerate the registry. + +Usage: + + # Round-trip the current registry (most common - picks up new overrides + # without touching the table list): + python scripts/aws/sync_aws_resource_type_registry.py + + # Use a fresh table list from a file: + python scripts/aws/sync_aws_resource_type_registry.py \\ + --from-file scripts/aws/aws_cloudquery_tables.txt + + # Fetch the latest list from CloudQuery's public hub: + python scripts/aws/sync_aws_resource_type_registry.py --from-cloudquery + + # Print summary of changes without writing: + python scripts/aws/sync_aws_resource_type_registry.py --dry-run +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import re +import sys +from pathlib import Path +from typing import Iterable, Optional + +try: + import yaml +except ImportError: # pragma: no cover - tooling environment must have yaml + print("PyYAML is required: pip install pyyaml", file=sys.stderr) + sys.exit(2) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +REGISTRY_PATH = REPO_ROOT / "src" / "indexers" / "aws_resource_type_registry.yaml" +OVERRIDES_PATH = REPO_ROOT / "scripts" / "aws" / "aws_resource_type_overrides.yaml" +DEFAULT_TABLES_FILE = REPO_ROOT / "scripts" / "aws" / "aws_cloudquery_tables.txt" +CLOUDQUERY_TABLES_URL = ( + "https://www.cloudquery.io/hub/plugins/source/cloudquery/aws/latest/tables" +) + + +# --------------------------------------------------------------------------- +# Heuristic +# --------------------------------------------------------------------------- + +_AWS_TABLE_PREFIX = "aws_" + +# Irregular plural -> singular forms that the rule-based singulariser gets +# wrong. Keep this small; pin anything fancier via cfn_type_overrides. +_IRREGULAR_SINGULARS = { + "addresses": "address", + "indices": "index", + "indexes": "index", + "policies": "policy", + "proxies": "proxy", + "registries": "registry", + "repositories": "repository", + "gateways": "gateway", + "schemas": "schema", + "metadata": "metadata", + "settings": "setting", + "series": "series", + "aliases": "alias", + "analyses": "analysis", + "thesauri": "thesaurus", + "faqs": "faq", +} + + +def _singularize(word: str) -> str: + """Best-effort English singularisation of a lowercase noun.""" + if not word: + return word + if word in _IRREGULAR_SINGULARS: + return _IRREGULAR_SINGULARS[word] + if word.endswith("ies") and len(word) > 3: + return word[:-3] + "y" + # buses, statuses, addresses, boxes, watches, dishes -> drop "es" + if re.search(r"(s|x|z|ch|sh)es$", word): + return word[:-2] + if word.endswith("ses") and len(word) > 3: + return word[:-2] # databases -> database + if word.endswith("s") and not word.endswith("ss"): + return word[:-1] + return word + + +def _snake_to_pascal(snake: str) -> str: + """Convert ``foo_bar_baz`` to ``FooBarBaz``. Empty input -> empty output.""" + parts = [p for p in snake.split("_") if p] + return "".join(p[:1].upper() + p[1:] for p in parts) + + +def _default_service_segment(token: str) -> str: + """Fallback CFN service segment for an unmapped service token. + + Most AWS CloudFormation service segments are PascalCase brand names, so a + simple capitalisation of the (single-word) token is the best default. The + acronym services (EC2, S3, RDS, ...) are pinned via ``service_cfn_names``. + """ + if not token: + return token + return token[:1].upper() + token[1:] + + +def infer_cfn_type( + cloudquery_table: str, + service_cfn_names: dict[str, str], +) -> Optional[str]: + """Best-effort heuristic mapping CQ table -> ``AWS::::``. + + Returns None if the input doesn't follow the ``aws__`` + shape. The entity segment is singularised (only the trailing token is + singularised; e.g. ``node_groups`` -> ``NodeGroup``) and PascalCased. + """ + if not cloudquery_table.startswith(_AWS_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_AWS_TABLE_PREFIX):] + if "_" not in rest: + # e.g. "aws_regions" - no entity segment; treat the token as both. + token = rest + service = service_cfn_names.get(token, _default_service_segment(token)) + entity = _snake_to_pascal(_singularize(token)) + return f"AWS::{service}::{entity}" if entity else None + + service_token, entity_snake = rest.split("_", 1) + service = service_cfn_names.get(service_token, _default_service_segment(service_token)) + + # Singularise only the final word of the entity, then PascalCase the whole. + entity_parts = entity_snake.split("_") + entity_parts[-1] = _singularize(entity_parts[-1]) + entity = _snake_to_pascal("_".join(entity_parts)) + if not entity: + return None + return f"AWS::{service}::{entity}" + + +def _category_for(cloudquery_table: str) -> Optional[str]: + if not cloudquery_table.startswith(_AWS_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_AWS_TABLE_PREFIX):] + return rest.split("_", 1)[0] if rest else None + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + +def load_overrides(path: Path = OVERRIDES_PATH) -> dict: + if not path.exists(): + raise FileNotFoundError( + f"Overrides YAML not found at {path}. " + f"This file holds the hand-curated metadata; do not delete it." + ) + payload = yaml.safe_load(path.read_text()) or {} + # cfn_type_overrides may legitimately carry null values (tables with no + # Cloud Control equivalent), so preserve the key set rather than filtering + # falsy values. + return { + "service_cfn_names": payload.get("service_cfn_names") or {}, + "cfn_type_overrides": payload.get("cfn_type_overrides") or {}, + "aliases": payload.get("aliases") or {}, + "typed_collectors": set(payload.get("typed_collectors") or []), + "mandatory": set(payload.get("mandatory") or []), + } + + +def tables_from_registry(path: Path = REGISTRY_PATH) -> list[str]: + if not path.exists(): + return [] + payload = yaml.safe_load(path.read_text()) or {} + types = payload.get("types") or {} + return sorted(types.keys()) + + +def tables_from_file(path: Path) -> list[str]: + text = path.read_text() + if path.suffix.lower() in {".yaml", ".yml"}: + payload = yaml.safe_load(text) or {} + if isinstance(payload, dict) and "tables" in payload: + return sorted(str(t) for t in payload["tables"]) + if isinstance(payload, list): + return sorted(str(t) for t in payload) + raise ValueError( + f"YAML file {path} must be a list of names or a mapping with a 'tables' key" + ) + # Treat anything else as one-table-name-per-line (comments with '#' ignored). + tables: set[str] = set() + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + tables.add(line) + return sorted(tables) + + +def tables_from_cloudquery_hub(url: str = CLOUDQUERY_TABLES_URL) -> list[str]: + """Fetch and parse the CloudQuery hub tables page. Best-effort: this is a + public HTML page and the markup may shift; if parsing fails the caller + should fall back to --from-file with an explicit table list. + """ + try: + from urllib.request import Request, urlopen + except ImportError: # pragma: no cover + raise RuntimeError("urllib is unavailable in this Python build") + + req = Request(url, headers={"User-Agent": "rwl-sync-script/1.0"}) + with urlopen(req, timeout=45) as resp: + html = resp.read().decode("utf-8", errors="replace") + + tables = sorted(set(re.findall(r"\baws_[a-z][a-z0-9_]+\b", html))) + if not tables: + raise RuntimeError( + f"No aws_* table names found at {url}. The page layout may have " + f"changed; pass --from-file with an explicit table list instead." + ) + return tables + + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +def build_registry( + table_names: Iterable[str], + overrides: dict, + snapshot_date: Optional[str] = None, +) -> dict: + cfn_overrides: dict = overrides["cfn_type_overrides"] + aliases: dict = overrides["aliases"] + typed_collectors: set[str] = overrides["typed_collectors"] + mandatory: set[str] = overrides["mandatory"] + service_cfn_names: dict = overrides["service_cfn_names"] + + types: dict[str, dict] = {} + typed_collector_count = 0 + cfn_types_assigned = 0 + + for name in sorted(set(table_names)): + if name in cfn_overrides: + cfn_type = cfn_overrides[name] + cfn_type_source = "override" + else: + cfn_type = infer_cfn_type(name, service_cfn_names) + cfn_type_source = "heuristic" if cfn_type else None + + is_typed = name in typed_collectors + is_mandatory = name in mandatory + + if is_typed: + typed_collector_count += 1 + if cfn_type: + cfn_types_assigned += 1 + + types[name] = { + "cfn_type": cfn_type, + "cfn_type_source": cfn_type_source, + "category": _category_for(name), + "aliases": list(aliases.get(name, [])), + "typed_collector": is_typed, + "mandatory": is_mandatory, + } + + metadata = { + "source": CLOUDQUERY_TABLES_URL, + "snapshot_date": snapshot_date or _dt.date.today().isoformat(), + "total_tables": len(types), + "typed_collectors": typed_collector_count, + "cfn_types_assigned": cfn_types_assigned, + "generator": "scripts/aws/sync_aws_resource_type_registry.py", + "notes": ( + "Generated file. To change the CloudFormation resource type for a " + "table, edit scripts/aws/aws_resource_type_overrides.yaml and " + "re-run the sync script. Hand-edits to this file will be " + "overwritten." + ), + } + + return {"metadata": metadata, "types": types} + + +def diff_summary(old: dict, new: dict) -> str: + old_types = (old or {}).get("types") or {} + new_types = new.get("types") or {} + added = sorted(set(new_types) - set(old_types)) + removed = sorted(set(old_types) - set(new_types)) + changed = [] + for name in sorted(set(new_types) & set(old_types)): + if new_types[name] != old_types[name]: + changed.append(name) + + lines = [ + f" total tables: {len(new_types)} (was {len(old_types)})", + f" added : {len(added)}", + f" removed : {len(removed)}", + f" changed : {len(changed)}", + ] + for label, items in (("added", added), ("removed", removed), ("changed", changed)): + if not items: + continue + sample = items[:10] + more = f" ... (+{len(items) - len(sample)} more)" if len(items) > len(sample) else "" + lines.append(f" {label}: {sample}{more}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + src = parser.add_mutually_exclusive_group() + src.add_argument( + "--from-registry", + action="store_true", + help="Read the table list from the existing registry YAML (default).", + ) + src.add_argument( + "--from-file", + type=Path, + help="Read the table list from a local file (one name per line, or YAML).", + ) + src.add_argument( + "--from-cloudquery", + action="store_true", + help=f"Fetch the table list from {CLOUDQUERY_TABLES_URL}", + ) + parser.add_argument( + "--overrides", + type=Path, + default=OVERRIDES_PATH, + help="Path to the overrides YAML.", + ) + parser.add_argument( + "--out", + type=Path, + default=REGISTRY_PATH, + help="Path to write the registry YAML to.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a diff summary and exit without writing.", + ) + parser.add_argument( + "--snapshot-date", + help="Override the metadata.snapshot_date value (defaults to today).", + ) + args = parser.parse_args(argv) + + if args.from_file: + tables = tables_from_file(args.from_file) + source_label = f"file:{args.from_file}" + elif args.from_cloudquery: + tables = tables_from_cloudquery_hub() + source_label = "cloudquery hub" + else: + # Default: round-trip the existing registry, but bootstrap from the + # checked-in table list the first time the registry doesn't exist yet. + tables = tables_from_registry(args.out) + if tables: + source_label = "existing registry" + elif DEFAULT_TABLES_FILE.exists(): + tables = tables_from_file(DEFAULT_TABLES_FILE) + source_label = f"file:{DEFAULT_TABLES_FILE}" + else: + source_label = "existing registry" + + if not tables: + print( + "No tables found in the requested source. " + "Aborting; at least one input must yield a non-empty list.", + file=sys.stderr, + ) + return 2 + + overrides = load_overrides(args.overrides) + new_registry = build_registry(tables, overrides, snapshot_date=args.snapshot_date) + + old_registry: dict = {} + if args.out.exists(): + old_registry = yaml.safe_load(args.out.read_text()) or {} + + print(f"Source: {source_label} tables={len(tables)}") + print(diff_summary(old_registry, new_registry)) + + if args.dry_run: + print("\nDry run; not writing.") + return 0 + + args.out.write_text( + yaml.safe_dump(new_registry, sort_keys=False, default_flow_style=False, width=200) + ) + print(f"\nWrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/azure/__pycache__/sync_azure_resource_type_registry.cpython-312.pyc b/scripts/azure/__pycache__/sync_azure_resource_type_registry.cpython-312.pyc new file mode 100644 index 000000000..48c59a554 Binary files /dev/null and b/scripts/azure/__pycache__/sync_azure_resource_type_registry.cpython-312.pyc differ diff --git a/scripts/azure/azure_resource_type_overrides.yaml b/scripts/azure/azure_resource_type_overrides.yaml new file mode 100644 index 000000000..654f3410d --- /dev/null +++ b/scripts/azure/azure_resource_type_overrides.yaml @@ -0,0 +1,238 @@ +metadata: + description: Manual overrides applied by sync_azure_resource_type_registry.py. Edit by hand, then re-run the sync script to regenerate src/indexers/azure_resource_type_registry.yaml. +service_namespace_casings: + analysisservices: AnalysisServices + apimanagement: ApiManagement + appcomplianceautomation: AppComplianceAutomation + automation: Automation + azurearcdata: AzureArcData + batch: Batch + botservice: Botservice + cdn: Cdn + cognitiveservices: CognitiveServices + compute: Compute + confidentialledger: Confidentialledger + confluent: Confluent + connectedvmware: ConnectedVMware + consumption: Consumption + container: Container + containerapps: Containerapps + containerservice: ContainerService + cosmos: Cosmos + costmanagement: CostManagement + customerinsights: CustomerInsights + dashboard: Dashboard + databox: Databox + databricks: Databricks + datacatalog: Datacatalog + datadog: Datadog + datalakeanalytics: Datalakeanalytics + datalakestore: Datalakestore + datamigration: Datamigration + datashare: Datashare + desktopvirtualization: DesktopVirtualization + devhub: Devhub + devops: Devops + devtestlabs: Devtestlabs + dns: Dns + elastic: Elastic + engagementfabric: Engagementfabric + eventgrid: EventGrid + eventhub: EventHub + hanaonazure: Hanaonazure + hdinsight: HDInsight + healthbot: Healthbot + healthcareapis: HealthcareApis + hybridcompute: HybridCompute + hybriddatamanager: Hybriddatamanager + kusto: Kusto + labservices: LabServices + logic: Logic + maintenance: Maintenance + managedapplications: Managedapplications + management: Management + managementgroups: Managementgroups + marketplace: Marketplace + mediaservices: Mediaservices + monitor: Monitor + mysqlflexibleservers: Mysqlflexibleservers + network: Network + networkfunction: Networkfunction + nginx: Nginx + operationalinsights: OperationalInsights + peering: Peering + policy: Policy + policyinsights: PolicyInsights + portal: Portal + postgresqlflexibleservers: Postgresqlflexibleservers + postgresqlhsc: Postgresqlhsc + powerbidedicated: PowerBIDedicated + providerhub: Providerhub + purview: Purview + quota: Quota + recoveryservices: RecoveryServices + redhatopenshift: RedHatOpenShift + redis: Redis + relay: Relay + resourcehealth: ResourceHealth + role: Role + saas: Saas + security: Security + servicebus: ServiceBus + servicefabric: ServiceFabric + servicefabricmanaged: Servicefabricmanaged + sql: Sql + sqlvirtualmachine: Sqlvirtualmachine + storage: Storage + storagecache: Storagecache + storagemover: Storagemover + storagesync: StorageSync + subscription: Subscription + support: Support + synapse: Synapse + trafficmanager: Trafficmanager + windowsiot: WindowsIoT + workloads: Workloads +service_namespace_remaps: + appservice: Web + dnsresolver: Network + frontdoor: Network + machinelearning: MachineLearningServices + mariadb: DBforMariaDB + mysql: DBforMySQL + netappfiles: NetApp + postgresql: DBforPostgreSQL + privatedns: Network + reservations: Capacity +arm_type_overrides: + azure_advisor_recommendation_metadata: Microsoft.Advisor/metadata + azure_advisor_recommendations: Microsoft.Advisor/recommendations + azure_advisor_suppressions: Microsoft.Advisor/suppressions + azure_appconfiguration_configuration_stores: Microsoft.AppConfiguration/configurationStores + azure_applicationinsights_components: Microsoft.Insights/components + azure_applicationinsights_web_tests: Microsoft.Insights/webtests + azure_appservice_certificate_orders: Microsoft.CertificateRegistration/certificateOrders + azure_appservice_certificates: Microsoft.Web/certificates + azure_appservice_deleted_web_apps: Microsoft.Web/deletedSites + azure_appservice_domains: Microsoft.DomainRegistration/domains + azure_appservice_environments: Microsoft.Web/hostingEnvironments + azure_appservice_plans: Microsoft.Web/serverFarms + azure_appservice_recommendations: Microsoft.Web/recommendations + azure_appservice_resource_health_metadata: Microsoft.Web/sites/resourceHealthMetadata + azure_appservice_static_sites: Microsoft.Web/staticSites + azure_appservice_top_level_domains: Microsoft.DomainRegistration/topLevelDomains + azure_appservice_web_apps: Microsoft.Web/sites + azure_authorization_classic_administrators: Microsoft.Authorization/classicAdministrators + azure_authorization_provider_operations_metadata: Microsoft.Authorization/providerOperations + azure_authorization_role_assignments: Microsoft.Authorization/roleAssignments + azure_authorization_role_definitions: Microsoft.Authorization/roleDefinitions + azure_billing_accounts: Microsoft.Billing/billingAccounts + azure_billing_enrollment_accounts: Microsoft.Billing/enrollmentAccounts + azure_billing_periods: Microsoft.Billing/billingPeriods + azure_cdn_endpoints: Microsoft.Cdn/profiles/endpoints + azure_cdn_profiles: Microsoft.Cdn/profiles + azure_cognitiveservices_accounts: Microsoft.CognitiveServices/accounts + azure_compute_disk_encryption_sets: Microsoft.Compute/diskEncryptionSets + azure_compute_disks: Microsoft.Compute/disks + azure_compute_images: Microsoft.Compute/images + azure_compute_snapshots: Microsoft.Compute/snapshots + azure_compute_virtual_machine_scale_sets: Microsoft.Compute/virtualMachineScaleSets + azure_compute_virtual_machines: Microsoft.Compute/virtualMachines + azure_containerinstance_container_groups: Microsoft.ContainerInstance/containerGroups + azure_containerregistry_registries: Microsoft.ContainerRegistry/registries + azure_containerservice_managed_clusters: Microsoft.ContainerService/managedClusters + azure_cosmos_sql_databases: Microsoft.DocumentDB/databaseAccounts/sqlDatabases + azure_datafactory_factories: Microsoft.DataFactory/factories + azure_eventhub_namespaces: Microsoft.EventHub/namespaces + azure_keyvault_certificate_issuers: Microsoft.KeyVault/vaults/certificates/issuers + azure_keyvault_certificate_policies: Microsoft.KeyVault/vaults/certificates/policies + azure_keyvault_certificates: Microsoft.KeyVault/vaults/certificates + azure_keyvault_key_rotation_policies: Microsoft.KeyVault/vaults/keys/rotationPolicies + azure_keyvault_keys: Microsoft.KeyVault/vaults/keys + azure_keyvault_keyvaults: Microsoft.KeyVault/vaults + azure_keyvault_managed_hsms: Microsoft.KeyVault/managedHSMs + azure_keyvault_secrets: Microsoft.KeyVault/vaults/secrets + azure_logic_workflows: Microsoft.Logic/workflows + azure_machinelearning_workspaces: Microsoft.MachineLearningServices/workspaces + azure_mariadb_servers: Microsoft.DBforMariaDB/servers + azure_monitor_action_groups: Microsoft.Insights/actionGroups + azure_monitor_autoscale_settings: Microsoft.Insights/autoscaleSettings + azure_monitor_diagnostic_settings: Microsoft.Insights/diagnosticSettings + azure_monitor_metric_alerts: Microsoft.Insights/metricAlerts + azure_mysql_servers: Microsoft.DBforMySQL/servers + azure_mysqlflexibleservers_servers: Microsoft.DBforMySQL/flexibleServers + azure_network_application_gateways: Microsoft.Network/applicationGateways + azure_network_bastion_hosts: Microsoft.Network/bastionHosts + azure_network_ddos_protection_plans: Microsoft.Network/ddosProtectionPlans + azure_network_express_route_circuits: Microsoft.Network/expressRouteCircuits + azure_network_load_balancers: Microsoft.Network/loadBalancers + azure_network_nat_gateways: Microsoft.Network/natGateways + azure_network_private_endpoints: Microsoft.Network/privateEndpoints + azure_network_public_ip_addresses: Microsoft.Network/publicIPAddresses + azure_network_route_tables: Microsoft.Network/routeTables + azure_network_security_groups: Microsoft.Network/networkSecurityGroups + azure_network_virtual_hubs: Microsoft.Network/virtualHubs + azure_network_virtual_network_gateways: Microsoft.Network/virtualNetworkGateways + azure_network_virtual_networks: Microsoft.Network/virtualNetworks + azure_network_virtual_wans: Microsoft.Network/virtualWans + azure_network_vpn_gateways: Microsoft.Network/vpnGateways + azure_network_watchers: Microsoft.Network/networkWatchers + azure_notificationhubs_namespaces: Microsoft.NotificationHubs/namespaces + azure_operationalinsights_workspaces: Microsoft.OperationalInsights/workspaces + azure_postgresql_databases: Microsoft.DBforPostgreSQL/servers/databases + azure_postgresql_servers: Microsoft.DBforPostgreSQL/servers + azure_recoveryservices_vaults: Microsoft.RecoveryServices/vaults + azure_redis_caches: Microsoft.Cache/Redis + azure_resources_links: Microsoft.Resources/links + azure_resources_providers: Microsoft.Resources/providers + azure_resources_resource_groups: Microsoft.Resources/resourceGroups + azure_resources_resources: null + azure_search_services: Microsoft.Search/searchServices + azure_servicebus_namespaces: Microsoft.ServiceBus/namespaces + azure_sql_managed_instances: Microsoft.Sql/managedInstances + azure_sql_servers: Microsoft.Sql/servers + azure_storage_accounts: Microsoft.Storage/storageAccounts + azure_storage_blob_services: Microsoft.Storage/storageAccounts/blobServices + azure_storage_file_shares: Microsoft.Storage/storageAccounts/fileServices/shares + azure_storage_queue_services: Microsoft.Storage/storageAccounts/queueServices + azure_storage_queues: Microsoft.Storage/storageAccounts/queueServices/queues + azure_storage_tables: Microsoft.Storage/storageAccounts/tableServices/tables + azure_streamanalytics_streaming_jobs: Microsoft.StreamAnalytics/streamingjobs + azure_synapse_sql_pools: Microsoft.Synapse/workspaces/sqlPools + azure_synapse_workspaces: Microsoft.Synapse/workspaces +aliases: + azure_compute_virtual_machines: + - virtual_machine + azure_keyvault_keyvaults: + - azure_keyvault_vaults + - azure_keyvault_keyvault + azure_resources_resource_groups: + - resource_group +typed_collectors: +- azure_apimanagement_service +- azure_appservice_plans +- azure_appservice_web_apps +- azure_azurearcdata_sql_server_instances +- azure_compute_disks +- azure_compute_snapshots +- azure_compute_virtual_machine_scale_sets +- azure_compute_virtual_machines +- azure_containerregistry_registries +- azure_containerservice_managed_clusters +- azure_cosmos_sql_databases +- azure_datafactory_factories +- azure_keyvault_keyvaults +- azure_mysql_servers +- azure_mysqlflexibleservers_servers +- azure_network_application_gateways +- azure_network_load_balancers +- azure_network_security_groups +- azure_network_virtual_networks +- azure_postgresql_databases +- azure_redis_caches +- azure_resources_resource_groups +- azure_servicebus_namespaces +- azure_storage_accounts +- azure_subscription_subscriptions +mandatory: +- azure_resources_resource_groups diff --git a/scripts/azure/dump_azure_resource_catalog.py b/scripts/azure/dump_azure_resource_catalog.py new file mode 100755 index 000000000..b7a7ebc07 --- /dev/null +++ b/scripts/azure/dump_azure_resource_catalog.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Regenerate ``docs/authoring/indexed-resources/azure-resource-catalog.md``. + +The catalog is the user-facing companion to ``azure.md``: a single sortable +table of every Azure resource type the ``azureapi`` indexer knows about, +grouped by service. It's derived directly from +``src/indexers/azure_resource_type_registry.yaml`` (which itself is generated +from the CloudQuery Azure plugin's table list and the manual overrides in +``scripts/azure/azure_resource_type_overrides.yaml``). + +Run this script after editing ``azure_resource_type_overrides.yaml`` and +re-running ``sync_azure_resource_type_registry.py``. Hand-edits to the +generated catalog get overwritten; touch the registry / overrides instead. + +Usage:: + + python scripts/azure/dump_azure_resource_catalog.py +""" + +from __future__ import annotations + +import datetime as _dt +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SRC = _REPO_ROOT / "src" +sys.path.insert(0, str(_SRC)) + +from indexers.azure_resource_type_registry import load_registry # noqa: E402 + +_OUTPUT = ( + _REPO_ROOT + / "docs" + / "authoring" + / "indexed-resources" + / "azure-resource-catalog.md" +) + + +def main() -> None: + registry = load_registry() + rows = sorted( + registry, + key=lambda e: ((e.category or "~"), e.cloudquery_table_name), + ) + typed_count = sum(1 for e in rows if e.typed_collector) + generated_at = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%d") + + lines: list[str] = [] + lines.append("# Azure resource catalog") + lines.append("") + lines.append( + "Every Azure resource type the native `azureapi` indexer can " + "discover. This page is the companion catalog for " + "[`azure.md`](./azure.md); see that page for how to enable the " + "indexer, what data each row carries, and the typed/generic " + "distinction." + ) + lines.append("") + lines.append( + f"_{len(rows)} resource types - {typed_count} typed (rich-payload), " + f"{len(rows) - typed_count} generic (basic envelope). " + f"Generated {generated_at} from " + "`src/indexers/azure_resource_type_registry.yaml`._" + ) + lines.append("") + lines.append( + "_Regenerate with `python scripts/azure/dump_azure_resource_catalog.py` " + "after touching the registry or overrides; do not hand-edit this file._" + ) + lines.append("") + lines.append( + "* `typed` - hand-written `azure-mgmt-*` collector returns the " + "full SDK payload (rich `properties`)." + ) + lines.append( + "* `generic` - covered by the ARM-resources catch-all " + "(`ResourceManagementClient.resources.list[_by_resource_group]`); " + "row carries the basic envelope (`id`, `name`, `type`, `location`, " + "`tags`, `sku`, `kind`, `identity`, `managed_by`) but **no** " + "`properties` (an ARM API limitation, not a workspace-builder one)." + ) + lines.append("") + lines.append("| Service | CloudQuery table name | ARM type | Tier |") + lines.append("| --- | --- | --- | --- |") + for e in rows: + cat = e.category or "-" + cq = e.cloudquery_table_name + arm = e.arm_type or "-" + tier = "typed" if e.typed_collector else "generic" + lines.append(f"| {cat} | `{cq}` | `{arm}` | {tier} |") + lines.append("") + + _OUTPUT.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Wrote {_OUTPUT.relative_to(_REPO_ROOT)} ({len(rows)} entries)") + + +if __name__ == "__main__": + main() diff --git a/scripts/azure/sync_azure_resource_type_registry.py b/scripts/azure/sync_azure_resource_type_registry.py new file mode 100644 index 000000000..86f249dff --- /dev/null +++ b/scripts/azure/sync_azure_resource_type_registry.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +""" +Sync ``src/indexers/azure_resource_type_registry.yaml``. + +The registry maps every CloudQuery Azure plugin table to its ARM resource +type plus metadata used by the native ``azureapi`` indexer. The data is +materialised from three inputs: + +1. **A list of CloudQuery Azure table names.** Source of truth for which + tables exist. Provided either by re-reading the previous registry + snapshot (default), reading a local file, or fetching the live list + from CloudQuery's public hub. + +2. **A heuristic** that converts ``azure__`` into + ``Microsoft./``. The service segment uses + the casing in ``service_namespace_casings`` (a hand-curated dictionary + that handles multi-word service names like ``apimanagement`` -> + ``ApiManagement``); the entity segment uses snake-case-to-camelCase. + +3. **A manual overrides YAML** at + ``scripts/azure/azure_resource_type_overrides.yaml`` that pins the + ARM type, aliases, ``typed_collector`` flag, and ``mandatory`` flag + for any table whose heuristic value is wrong or where extra metadata + is needed. + +Hand-edit the overrides YAML; never hand-edit the registry YAML. After +editing the overrides, re-run this script to regenerate the registry. + +Usage: + + # Round-trip the current registry (most common - picks up new + # overrides without touching the table list): + python scripts/azure/sync_azure_resource_type_registry.py + + # Use a fresh table list from a file: + python scripts/azure/sync_azure_resource_type_registry.py \\ + --from-file path/to/azure_tables.txt + + # Fetch the latest list from CloudQuery's public hub: + python scripts/azure/sync_azure_resource_type_registry.py --from-cloudquery + + # Print summary of changes without writing: + python scripts/azure/sync_azure_resource_type_registry.py --dry-run +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import re +import sys +from pathlib import Path +from typing import Iterable, Optional + +try: + import yaml +except ImportError: # pragma: no cover - tooling environment must have yaml + print("PyYAML is required: pip install pyyaml", file=sys.stderr) + sys.exit(2) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +REGISTRY_PATH = REPO_ROOT / "src" / "indexers" / "azure_resource_type_registry.yaml" +OVERRIDES_PATH = REPO_ROOT / "scripts" / "azure" / "azure_resource_type_overrides.yaml" +CLOUDQUERY_TABLES_URL = ( + "https://www.cloudquery.io/hub/plugins/source/cloudquery/azure/latest/tables" +) + + +# --------------------------------------------------------------------------- +# Heuristic +# --------------------------------------------------------------------------- + +_AZURE_TABLE_PREFIX = "azure_" + + +def _snake_to_camel(snake: str) -> str: + """Convert ``foo_bar_baz`` to ``fooBarBaz``. Empty input -> empty output.""" + if not snake: + return "" + parts = [p for p in snake.split("_") if p] + if not parts: + return "" + first, *rest = parts + return first + "".join(p[:1].upper() + p[1:] for p in rest) + + +def infer_arm_type( + cloudquery_table: str, + service_namespace_casings: dict[str, str], + service_namespace_remaps: Optional[dict[str, str]] = None, +) -> Optional[str]: + """Best-effort heuristic mapping CQ table -> Microsoft.X/Y. Returns None + if the input doesn't follow the ``azure__`` shape. + + Resolution order for the service namespace: + 1. ``service_namespace_remaps`` - full namespace substitution + (e.g. ``frontdoor`` -> ``Network``). + 2. ``service_namespace_casings`` - same service, custom casing + (e.g. ``apimanagement`` -> ``ApiManagement``). + 3. Title-case fallback (``compute`` -> ``Compute``). + """ + remaps = service_namespace_remaps or {} + if not cloudquery_table.startswith(_AZURE_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_AZURE_TABLE_PREFIX):] + if "_" not in rest: + return None + + service_token, entity_snake = rest.split("_", 1) + if service_token in remaps: + namespace = remaps[service_token] + elif service_token in service_namespace_casings: + namespace = service_namespace_casings[service_token] + else: + namespace = service_token[:1].upper() + service_token[1:] + entity = _snake_to_camel(entity_snake) + if not entity: + return None + return f"Microsoft.{namespace}/{entity}" + + +def _category_for(cloudquery_table: str) -> Optional[str]: + if not cloudquery_table.startswith(_AZURE_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_AZURE_TABLE_PREFIX):] + return rest.split("_", 1)[0] if rest else None + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + +def load_overrides(path: Path = OVERRIDES_PATH) -> dict: + if not path.exists(): + raise FileNotFoundError( + f"Overrides YAML not found at {path}. " + f"This file holds the hand-curated metadata; do not delete it." + ) + payload = yaml.safe_load(path.read_text()) or {} + return { + "service_namespace_casings": payload.get("service_namespace_casings") or {}, + "service_namespace_remaps": payload.get("service_namespace_remaps") or {}, + "arm_type_overrides": payload.get("arm_type_overrides") or {}, + "aliases": payload.get("aliases") or {}, + "typed_collectors": set(payload.get("typed_collectors") or []), + "mandatory": set(payload.get("mandatory") or []), + } + + +def tables_from_registry(path: Path = REGISTRY_PATH) -> list[str]: + if not path.exists(): + return [] + payload = yaml.safe_load(path.read_text()) or {} + types = payload.get("types") or {} + return sorted(types.keys()) + + +def tables_from_file(path: Path) -> list[str]: + text = path.read_text() + if path.suffix.lower() in {".yaml", ".yml"}: + payload = yaml.safe_load(text) or {} + if isinstance(payload, dict) and "tables" in payload: + return sorted(str(t) for t in payload["tables"]) + if isinstance(payload, list): + return sorted(str(t) for t in payload) + raise ValueError( + f"YAML file {path} must be a list of names or a mapping with a 'tables' key" + ) + # Treat anything else as one-table-name-per-line. + tables: set[str] = set() + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + tables.add(line) + return sorted(tables) + + +def tables_from_cloudquery_hub(url: str = CLOUDQUERY_TABLES_URL) -> list[str]: + """Fetch and parse the CloudQuery hub tables page. Best-effort: this is + a public HTML page and the markup may shift; if parsing fails the script + falls back to whatever tables are encoded in the page text via the + ``azure_`` prefix. + """ + try: + from urllib.request import Request, urlopen + except ImportError: # pragma: no cover + raise RuntimeError("urllib is unavailable in this Python build") + + req = Request(url, headers={"User-Agent": "rwl-sync-script/1.0"}) + with urlopen(req, timeout=30) as resp: + html = resp.read().decode("utf-8", errors="replace") + + tables = sorted(set(re.findall(r"\bazure_[a-z][a-z0-9_]+\b", html))) + if not tables: + raise RuntimeError( + f"No azure_* table names found at {url}. The page layout may have " + f"changed; pass --from-file with an explicit table list instead." + ) + return tables + + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +def build_registry( + table_names: Iterable[str], + overrides: dict, + snapshot_date: Optional[str] = None, +) -> dict: + arm_overrides: dict = overrides["arm_type_overrides"] + aliases: dict = overrides["aliases"] + typed_collectors: set[str] = overrides["typed_collectors"] + mandatory: set[str] = overrides["mandatory"] + service_casings: dict = overrides["service_namespace_casings"] + service_remaps: dict = overrides["service_namespace_remaps"] + + types: dict[str, dict] = {} + typed_collector_count = 0 + arm_types_assigned = 0 + + for name in sorted(set(table_names)): + if name in arm_overrides: + arm_type = arm_overrides[name] + arm_type_source = "override" + else: + arm_type = infer_arm_type(name, service_casings, service_remaps) + arm_type_source = "heuristic" if arm_type else None + + is_typed = name in typed_collectors + is_mandatory = name in mandatory + + if is_typed: + typed_collector_count += 1 + if arm_type: + arm_types_assigned += 1 + + types[name] = { + "arm_type": arm_type, + "arm_type_source": arm_type_source, + "category": _category_for(name), + "aliases": list(aliases.get(name, [])), + "typed_collector": is_typed, + "mandatory": is_mandatory, + } + + metadata = { + "source": CLOUDQUERY_TABLES_URL, + "snapshot_date": snapshot_date or _dt.date.today().isoformat(), + "total_tables": len(types), + "typed_collectors": typed_collector_count, + "arm_types_assigned": arm_types_assigned, + "generator": "scripts/azure/sync_azure_resource_type_registry.py", + "notes": ( + "Generated file. To change ARM type for a table, edit " + "scripts/azure/azure_resource_type_overrides.yaml and re-run " + "the sync script. Hand-edits to this file will be overwritten." + ), + } + + return {"metadata": metadata, "types": types} + + +def diff_summary(old: dict, new: dict) -> str: + old_types = (old or {}).get("types") or {} + new_types = new.get("types") or {} + added = sorted(set(new_types) - set(old_types)) + removed = sorted(set(old_types) - set(new_types)) + changed = [] + for name in sorted(set(new_types) & set(old_types)): + if new_types[name] != old_types[name]: + changed.append(name) + + lines = [ + f" total tables: {len(new_types)} (was {len(old_types)})", + f" added : {len(added)}", + f" removed : {len(removed)}", + f" changed : {len(changed)}", + ] + for label, items in (("added", added), ("removed", removed), ("changed", changed)): + if not items: + continue + sample = items[:10] + more = f" ... (+{len(items) - len(sample)} more)" if len(items) > len(sample) else "" + lines.append(f" {label}: {sample}{more}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + src = parser.add_mutually_exclusive_group() + src.add_argument( + "--from-registry", + action="store_true", + help="Read the table list from the existing registry YAML (default).", + ) + src.add_argument( + "--from-file", + type=Path, + help="Read the table list from a local file (one name per line, or YAML).", + ) + src.add_argument( + "--from-cloudquery", + action="store_true", + help=f"Fetch the table list from {CLOUDQUERY_TABLES_URL}", + ) + parser.add_argument( + "--overrides", + type=Path, + default=OVERRIDES_PATH, + help="Path to the overrides YAML.", + ) + parser.add_argument( + "--out", + type=Path, + default=REGISTRY_PATH, + help="Path to write the registry YAML to.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a diff summary and exit without writing.", + ) + parser.add_argument( + "--snapshot-date", + help="Override the metadata.snapshot_date value (defaults to today).", + ) + args = parser.parse_args(argv) + + if args.from_file: + tables = tables_from_file(args.from_file) + source_label = f"file:{args.from_file}" + elif args.from_cloudquery: + tables = tables_from_cloudquery_hub() + source_label = "cloudquery hub" + else: + tables = tables_from_registry(args.out) + source_label = "existing registry" + + if not tables: + print( + "No tables found in the requested source. " + "Aborting; at least one input must yield a non-empty list.", + file=sys.stderr, + ) + return 2 + + overrides = load_overrides(args.overrides) + new_registry = build_registry(tables, overrides, snapshot_date=args.snapshot_date) + + old_registry: dict = {} + if args.out.exists(): + old_registry = yaml.safe_load(args.out.read_text()) or {} + + print(f"Source: {source_label} tables={len(tables)}") + print(diff_summary(old_registry, new_registry)) + + if args.dry_run: + print("\nDry run; not writing.") + return 0 + + args.out.write_text( + yaml.safe_dump(new_registry, sort_keys=False, default_flow_style=False, width=200) + ) + print(f"\nWrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gcp/gcp_cloudquery_tables.txt b/scripts/gcp/gcp_cloudquery_tables.txt new file mode 100644 index 000000000..bcfad1d9f --- /dev/null +++ b/scripts/gcp/gcp_cloudquery_tables.txt @@ -0,0 +1,407 @@ +# CloudQuery GCP plugin table list (parity source) +# Source: https://www.cloudquery.io/hub/plugins/source/cloudquery/gcp/latest/tables +# Plugin version: v22.1.0 | tables: 404 | captured: 2026-05-29 +gcp_accessapproval_folder_approval_requests +gcp_accessapproval_folder_service_accounts +gcp_accessapproval_folder_settings +gcp_accessapproval_organization_approval_requests +gcp_accessapproval_organization_service_accounts +gcp_accessapproval_organization_settings +gcp_accessapproval_project_approval_requests +gcp_accessapproval_project_service_accounts +gcp_accessapproval_project_settings +gcp_aiplatform_batch_prediction_jobs +gcp_aiplatform_custom_jobs +gcp_aiplatform_dataset_locations +gcp_aiplatform_datasets +gcp_aiplatform_endpoint_locations +gcp_aiplatform_endpoints +gcp_aiplatform_featurestore_locations +gcp_aiplatform_featurestores +gcp_aiplatform_hyperparameter_tuning_jobs +gcp_aiplatform_index_endpoints +gcp_aiplatform_index_locations +gcp_aiplatform_indexendpoint_locations +gcp_aiplatform_indexes +gcp_aiplatform_job_locations +gcp_aiplatform_metadata_locations +gcp_aiplatform_metadata_stores +gcp_aiplatform_model_deployment_monitoring_jobs +gcp_aiplatform_model_locations +gcp_aiplatform_models +gcp_aiplatform_operations +gcp_aiplatform_pipeline_jobs +gcp_aiplatform_pipeline_locations +gcp_aiplatform_specialist_pools +gcp_aiplatform_specialistpool_locations +gcp_aiplatform_studies +gcp_aiplatform_tensorboard_locations +gcp_aiplatform_tensorboards +gcp_aiplatform_training_pipelines +gcp_aiplatform_vizier_locations +gcp_alloydb_clusters +gcp_alloydb_instances +gcp_apigateway_apis +gcp_apigateway_gateways +gcp_apikeys_keys +gcp_appengine_apps +gcp_appengine_authorized_certificates +gcp_appengine_authorized_domains +gcp_appengine_domain_mappings +gcp_appengine_firewall_ingress_rules +gcp_appengine_instances +gcp_appengine_services +gcp_appengine_versions +gcp_applicationintegration_authconfigs +gcp_applicationintegration_certificates +gcp_applicationintegration_integration_execution_suspensions +gcp_applicationintegration_integration_executions +gcp_applicationintegration_integration_versions +gcp_applicationintegration_integrations +gcp_applicationintegration_sfdc_channels +gcp_applicationintegration_sfdc_instances +gcp_artifactregistry_docker_images +gcp_artifactregistry_files +gcp_artifactregistry_locations +gcp_artifactregistry_packages +gcp_artifactregistry_repositories +gcp_artifactregistry_tags +gcp_artifactregistry_versions +gcp_baremetalsolution_instances +gcp_baremetalsolution_networks +gcp_baremetalsolution_nfs_shares +gcp_baremetalsolution_volume_luns +gcp_baremetalsolution_volumes +gcp_batch_jobs +gcp_batch_task_groups +gcp_batch_tasks +gcp_beyondcorp_app_connections +gcp_beyondcorp_app_connectors +gcp_beyondcorp_app_gateways +gcp_bigquery_datasets +gcp_bigquery_tables +gcp_bigquerydatatransfer_configs +gcp_bigquerydatatransfer_datasources +gcp_bigquerydatatransfer_locations +gcp_bigquerydatatransfer_logs +gcp_bigquerydatatransfer_runs +gcp_bigtableadmin_app_profiles +gcp_bigtableadmin_backups +gcp_bigtableadmin_clusters +gcp_bigtableadmin_instances +gcp_bigtableadmin_tables +gcp_billing_billing_account_subaccounts +gcp_billing_billing_accounts +gcp_billing_budgets +gcp_billing_projects +gcp_billing_service_skus +gcp_billing_services +gcp_binaryauthorization_assertors +gcp_certificatemanager_certificate_issuance_configs +gcp_certificatemanager_certificate_map_entries +gcp_certificatemanager_certificate_maps +gcp_certificatemanager_certificates +gcp_certificatemanager_dns_authorizations +gcp_cloudassetinventory_assets +gcp_cloudassetinventory_assets_access_policies +gcp_cloudassetinventory_assets_history +gcp_cloudassetinventory_assets_iam_policies +gcp_cloudassetinventory_assets_org_policies +gcp_cloudassetinventory_assets_os_inventories +gcp_cloudassetinventory_assets_relationships +gcp_cloudassetinventory_assets_resources +gcp_cloudassetinventory_effective_iam_policies +gcp_cloudassetinventory_feeds +gcp_cloudassetinventory_org_assets +gcp_cloudassetinventory_org_assets_access_policies +gcp_cloudassetinventory_org_assets_iam_policies +gcp_cloudassetinventory_org_assets_org_policies +gcp_cloudassetinventory_org_assets_os_inventories +gcp_cloudassetinventory_org_assets_relationships +gcp_cloudassetinventory_org_assets_resources +gcp_cloudassetinventory_savedqueries +gcp_cloudbuild_builds +gcp_cloudbuild_connection_repositories +gcp_cloudbuild_connections +gcp_cloudbuild_triggers +gcp_cloudbuild_worker_pools +gcp_clouddeploy_delivery_pipelines +gcp_clouddeploy_job_runs +gcp_clouddeploy_releases +gcp_clouddeploy_rollouts +gcp_clouddeploy_targets +gcp_clouderrorreporting_error_events +gcp_clouderrorreporting_error_group_stats +gcp_cloudresourcemanager_organizations +gcp_cloudscheduler_jobs +gcp_cloudscheduler_locations +gcp_cloudsupport_cases +gcp_cloudtasks_locations +gcp_cloudtasks_queues +gcp_cloudtasks_tasks +gcp_cloudtrace_traces +gcp_composer_environments +gcp_composer_image_versions +gcp_composer_operations +gcp_compute_addresses +gcp_compute_autoscalers +gcp_compute_backend_buckets +gcp_compute_backend_services +gcp_compute_disk_types +gcp_compute_disks +gcp_compute_external_vpn_gateways +gcp_compute_firewalls +gcp_compute_forwarding_rules +gcp_compute_global_addresses +gcp_compute_global_forwarding_rules +gcp_compute_health_checks +gcp_compute_image_policies +gcp_compute_images +gcp_compute_instance_group_instances +gcp_compute_instance_group_managers +gcp_compute_instance_group_regional_instances +gcp_compute_instance_groups +gcp_compute_instance_tag_bindings +gcp_compute_instance_templates +gcp_compute_instances +gcp_compute_interconnect_attachments +gcp_compute_interconnect_locations +gcp_compute_interconnect_remote_locations +gcp_compute_interconnects +gcp_compute_machine_types +gcp_compute_network_endpoint_groups +gcp_compute_network_firewall_policies +gcp_compute_networks +gcp_compute_osconfig_inventories +gcp_compute_osconfig_os_patch_deployments +gcp_compute_osconfig_os_patch_jobs +gcp_compute_osconfig_os_patch_jobs_instance_details +gcp_compute_osconfig_os_policy_assignment_reports +gcp_compute_osconfig_os_policy_assignments +gcp_compute_osconfig_os_vulnerability_reports +gcp_compute_packet_mirrorings +gcp_compute_projects +gcp_compute_region_instance_templates +gcp_compute_region_network_firewall_policies +gcp_compute_router_nat_mapping_infos +gcp_compute_routers +gcp_compute_routes +gcp_compute_security_policies +gcp_compute_snapshots +gcp_compute_ssl_certificates +gcp_compute_ssl_policies +gcp_compute_subnetworks +gcp_compute_target_grpc_proxies +gcp_compute_target_http_proxies +gcp_compute_target_https_proxies +gcp_compute_target_instances +gcp_compute_target_pools +gcp_compute_target_ssl_proxies +gcp_compute_target_tcp_proxies +gcp_compute_target_vpn_gateways +gcp_compute_url_maps +gcp_compute_vpn_gateways +gcp_compute_vpn_tunnels +gcp_compute_zones +gcp_container_clusters +gcp_container_node_pools +gcp_containeranalysis_occurrences +gcp_databasemigration_locations +gcp_databasemigration_migration_jobs +gcp_databasemigration_operations +gcp_dataflow_job_messages +gcp_dataflow_job_metrics +gcp_dataflow_jobs +gcp_dataflow_snapshots +gcp_datafusion_available_versions +gcp_datafusion_instance_operations +gcp_datafusion_instances +gcp_dataproc_autoscaling_policies +gcp_dataproc_cluster_nodegroups +gcp_dataproc_clusters +gcp_dataproc_jobs +gcp_dataproc_regions +gcp_deploymentmanager_deployments +gcp_deploymentmanager_manifests +gcp_deploymentmanager_operations +gcp_deploymentmanager_resources +gcp_deploymentmanager_types +gcp_dns_managed_zones +gcp_dns_policies +gcp_dns_resource_record_sets +gcp_domains_registrations +gcp_essentialcontacts_folder_contacts +gcp_essentialcontacts_organization_contacts +gcp_essentialcontacts_project_contacts +gcp_eventarc_channels +gcp_eventarc_providers +gcp_eventarc_triggers +gcp_filestore_backups +gcp_filestore_instances +gcp_filestore_locations +gcp_filestore_operations +gcp_filestore_snapshots +gcp_firebase_hosting_site_channels +gcp_firebase_hosting_site_custom_domains +gcp_firebase_hosting_site_releases +gcp_firebase_hosting_site_versions +gcp_firebase_hosting_sites +gcp_firebase_rules_releases +gcp_firebase_rules_rulesets +gcp_firebaseappcheck_app_attest_configs +gcp_firebaseappcheck_device_check_configs +gcp_firebaseappcheck_play_integrity_configs +gcp_firebaseappcheck_recaptcha_configs +gcp_firebaseappcheck_recaptcha_enterprise_configs +gcp_firebaseappcheck_safety_net_configs +gcp_firestore_databases +gcp_functions_function_policies +gcp_functions_functions +gcp_functionsv2_function_policies +gcp_functionsv2_functions +gcp_iam_deny_policies +gcp_iam_organizational_roles +gcp_iam_predefined_roles +gcp_iam_roles +gcp_iam_service_account_keys +gcp_iam_service_account_policies +gcp_iam_service_accounts +gcp_iam_workload_identity_pool_providers +gcp_iam_workload_identity_pools +gcp_identitytoolkit_accounts +gcp_identitytoolkit_default_supported_idps +gcp_identitytoolkit_inbound_saml_configs +gcp_identitytoolkit_oauth_idp_configs +gcp_identitytoolkit_project_configs +gcp_identitytoolkit_project_public_configs +gcp_identitytoolkit_recaptcha_params +gcp_identitytoolkit_tenants +gcp_kms_crypto_key_versions +gcp_kms_crypto_keys +gcp_kms_ekm_connections +gcp_kms_import_jobs +gcp_kms_keyrings +gcp_kms_locations +gcp_livestream_channels +gcp_livestream_inputs +gcp_logging_audit_logs +gcp_logging_metrics +gcp_logging_sinks +gcp_looker_instance_operations +gcp_looker_instances +gcp_memorystore_instances +gcp_memorystore_locations +gcp_memorystore_operations +gcp_monitoring_alert_policies +gcp_networkconnectivity_internal_ranges +gcp_networkconnectivity_locations +gcp_networkintelligencecenter_connectivity_tests +gcp_networkintelligencecenter_locations +gcp_networkintelligencecenter_operations +gcp_networksecurity_address_groups +gcp_networksecurity_firewall_endpoints +gcp_networksecurity_security_profile_groups +gcp_networksecurity_security_profiles +gcp_networksecurity_tls_inspection_policies +gcp_networkservices_endpoint_policies +gcp_networkservices_gateways +gcp_networkservices_grpc_routes +gcp_networkservices_http_routes +gcp_networkservices_meshes +gcp_networkservices_service_bindings +gcp_networkservices_tcp_routes +gcp_networkservices_tls_routes +gcp_organization_folders_policies +gcp_organization_policies +gcp_organization_projects_policies +gcp_policyanalyzer_activities +gcp_privateca_authorities +gcp_privateca_certificates +gcp_privateca_pools +gcp_projects +gcp_pubsub_schema_revisions +gcp_pubsub_schemas +gcp_pubsub_snapshots +gcp_pubsub_subscriptions +gcp_pubsub_topics +gcp_recommendations_folders +gcp_recommendations_folders_insights +gcp_recommendations_folders_locations +gcp_recommendations_organizations +gcp_recommendations_organizations_insights +gcp_recommendations_organizations_locations +gcp_recommendations_projects +gcp_recommendations_projects_insights +gcp_recommendations_projects_locations +gcp_redis_instances +gcp_resourcemanager_folder_policies +gcp_resourcemanager_folders +gcp_resourcemanager_organization_policies +gcp_resourcemanager_organization_projects +gcp_resourcemanager_organization_tag_keys +gcp_resourcemanager_organization_tag_values +gcp_resourcemanager_project_policies +gcp_resourcemanager_project_tag_bindings +gcp_resourcemanager_project_tag_keys +gcp_resourcemanager_project_tag_values +gcp_resourcemanager_projects +gcp_resourcemanager_projects_search +gcp_resourcemanager_subfolders +gcp_run_executions +gcp_run_job_policies +gcp_run_jobs +gcp_run_locations +gcp_run_revisions +gcp_run_service_policies +gcp_run_services +gcp_run_tasks +gcp_run_worker_pool_policies +gcp_run_worker_pools +gcp_secretmanager_secrets +gcp_securitycenter_folder_event_threat_detection +gcp_securitycenter_folder_findings +gcp_securitycenter_org_event_threat_detection_settings +gcp_securitycenter_organization_findings +gcp_securitycenter_project_event_threat_detection +gcp_securitycenter_project_findings +gcp_servicehealth_events +gcp_servicehealth_locations +gcp_serviceusage_service_project_quota_metrics +gcp_serviceusage_services +gcp_sourcerepo_config +gcp_sourcerepo_repos +gcp_spanner_databases +gcp_spanner_instances +gcp_sql_backups +gcp_sql_databases +gcp_sql_instances +gcp_sql_ssl_certs +gcp_sql_users +gcp_storage_bucket_objects +gcp_storage_bucket_policies +gcp_storage_bucket_tag_bindings +gcp_storage_buckets +gcp_storage_hmac_keys +gcp_storagetransfer_agent_pools +gcp_storagetransfer_transfer_jobs +gcp_storagetransfer_transfer_operations +gcp_translate_glossaries +gcp_videotranscoder_job_templates +gcp_videotranscoder_jobs +gcp_vision_product_reference_images +gcp_vision_products +gcp_vmmigration_groups +gcp_vmmigration_source_datacenter_connectors +gcp_vmmigration_source_migrating_vm_clone_jobs +gcp_vmmigration_source_migrating_vm_cutover_jobs +gcp_vmmigration_source_migrating_vms +gcp_vmmigration_source_utilization_reports +gcp_vmmigration_sources +gcp_vmmigration_target_projects +gcp_vpcaccess_connectors +gcp_vpcaccess_locations +gcp_websecurityscanner_scan_config_scan_run_crawled_urls +gcp_websecurityscanner_scan_config_scan_run_findings +gcp_websecurityscanner_scan_config_scan_runs +gcp_websecurityscanner_scan_configs +gcp_workflows_workflows diff --git a/scripts/gcp/gcp_resource_type_overrides.yaml b/scripts/gcp/gcp_resource_type_overrides.yaml new file mode 100644 index 000000000..20037cf0b --- /dev/null +++ b/scripts/gcp/gcp_resource_type_overrides.yaml @@ -0,0 +1,182 @@ +metadata: + description: >- + Manual overrides applied by sync_gcp_resource_type_registry.py. Edit by hand, + then re-run the sync script to regenerate + src/indexers/gcp_resource_type_registry.yaml. + + The registry maps every CloudQuery GCP plugin table to its Cloud Asset + Inventory (CAI) asset type (``.googleapis.com/``) plus + metadata used by the native ``gcpapi`` indexer. CAI asset types are the join + key for generic discovery, exactly like ARM types are for Azure. + +# --------------------------------------------------------------------------- +# service_api_hosts +# Maps the CloudQuery service token (the ```` in +# ``gcp__``) to the Google API host that CAI uses in the +# asset type. When a token is absent here the heuristic falls back to +# ``.googleapis.com``. Only list tokens whose host differs from the +# token, or where we want to be explicit. +# --------------------------------------------------------------------------- +service_api_hosts: + cloudassetinventory: cloudasset + resourcemanager: cloudresourcemanager + projects: cloudresourcemanager + organization: cloudresourcemanager + sql: sqladmin + kms: cloudkms + functions: cloudfunctions + functionsv2: cloudfunctions + filestore: file + cloudtasks: cloudtasks + cloudscheduler: cloudscheduler + cloudbuild: cloudbuild + clouddeploy: clouddeploy + cloudtrace: cloudtrace + cloudsupport: cloudsupport + clouderrorreporting: clouderrorreporting + databasemigration: datamigration + bigquerydatatransfer: bigquerydatatransfer + identitytoolkit: identitytoolkit + privateca: privateca + websecurityscanner: websecurityscanner + networkintelligencecenter: networkmanagement + sourcerepo: sourcerepo + videotranscoder: transcoder + +# --------------------------------------------------------------------------- +# cai_type_overrides +# Pin the full CAI asset type for a table when the heuristic gets the host +# or entity wrong, or when the table has no CAI equivalent (set to null so +# generic discovery skips it and a typed collector handles it instead). +# These are the high-value, frequently-referenced resources first. +# --------------------------------------------------------------------------- +cai_type_overrides: + # --- compute (most-referenced surface) --- + gcp_compute_instances: compute.googleapis.com/Instance + gcp_compute_disks: compute.googleapis.com/Disk + gcp_compute_networks: compute.googleapis.com/Network + gcp_compute_subnetworks: compute.googleapis.com/Subnetwork + gcp_compute_addresses: compute.googleapis.com/Address + gcp_compute_global_addresses: compute.googleapis.com/GlobalAddress + gcp_compute_firewalls: compute.googleapis.com/Firewall + gcp_compute_forwarding_rules: compute.googleapis.com/ForwardingRule + gcp_compute_global_forwarding_rules: compute.googleapis.com/GlobalForwardingRule + gcp_compute_routers: compute.googleapis.com/Router + gcp_compute_routes: compute.googleapis.com/Route + gcp_compute_vpn_gateways: compute.googleapis.com/VpnGateway + gcp_compute_vpn_tunnels: compute.googleapis.com/VpnTunnel + gcp_compute_target_pools: compute.googleapis.com/TargetPool + gcp_compute_backend_services: compute.googleapis.com/BackendService + gcp_compute_url_maps: compute.googleapis.com/UrlMap + gcp_compute_images: compute.googleapis.com/Image + gcp_compute_snapshots: compute.googleapis.com/Snapshot + gcp_compute_instance_groups: compute.googleapis.com/InstanceGroup + gcp_compute_instance_templates: compute.googleapis.com/InstanceTemplate + gcp_compute_autoscalers: compute.googleapis.com/Autoscaler + gcp_compute_ssl_certificates: compute.googleapis.com/SslCertificate + + # --- containers / serverless --- + gcp_container_clusters: container.googleapis.com/Cluster + gcp_container_node_pools: container.googleapis.com/NodePool + gcp_run_services: run.googleapis.com/Service + gcp_run_jobs: run.googleapis.com/Job + gcp_appengine_apps: appengine.googleapis.com/Application + gcp_appengine_services: appengine.googleapis.com/Service + gcp_appengine_versions: appengine.googleapis.com/Version + gcp_functions_functions: cloudfunctions.googleapis.com/CloudFunction + gcp_functionsv2_functions: cloudfunctions.googleapis.com/Function + + # --- storage / data --- + gcp_storage_buckets: storage.googleapis.com/Bucket + gcp_sql_instances: sqladmin.googleapis.com/Instance + gcp_bigquery_datasets: bigquery.googleapis.com/Dataset + gcp_bigquery_tables: bigquery.googleapis.com/Table + gcp_spanner_instances: spanner.googleapis.com/Instance + gcp_spanner_databases: spanner.googleapis.com/Database + gcp_redis_instances: redis.googleapis.com/Instance + gcp_filestore_instances: file.googleapis.com/Instance + gcp_filestore_backups: file.googleapis.com/Backup + gcp_bigtableadmin_instances: bigtableadmin.googleapis.com/Instance + gcp_bigtableadmin_tables: bigtableadmin.googleapis.com/Table + gcp_bigtableadmin_clusters: bigtableadmin.googleapis.com/Cluster + gcp_bigtableadmin_app_profiles: bigtableadmin.googleapis.com/AppProfile + gcp_alloydb_clusters: alloydb.googleapis.com/Cluster + gcp_alloydb_instances: alloydb.googleapis.com/Instance + + # --- messaging --- + gcp_pubsub_topics: pubsub.googleapis.com/Topic + gcp_pubsub_subscriptions: pubsub.googleapis.com/Subscription + gcp_pubsub_snapshots: pubsub.googleapis.com/Snapshot + + # --- security / identity --- + gcp_kms_crypto_keys: cloudkms.googleapis.com/CryptoKey + gcp_kms_crypto_key_versions: cloudkms.googleapis.com/CryptoKeyVersion + gcp_kms_keyrings: cloudkms.googleapis.com/KeyRing + gcp_kms_import_jobs: cloudkms.googleapis.com/ImportJob + gcp_iam_roles: iam.googleapis.com/Role + gcp_iam_service_accounts: iam.googleapis.com/ServiceAccount + gcp_secretmanager_secrets: secretmanager.googleapis.com/Secret + gcp_dns_managed_zones: dns.googleapis.com/ManagedZone + gcp_dns_policies: dns.googleapis.com/Policy + gcp_artifactregistry_repositories: artifactregistry.googleapis.com/Repository + + # --- resource manager (project is the anchor type) --- + gcp_projects: cloudresourcemanager.googleapis.com/Project + gcp_resourcemanager_projects: cloudresourcemanager.googleapis.com/Project + gcp_resourcemanager_folders: cloudresourcemanager.googleapis.com/Folder + + # --- tables with no CAI asset-type equivalent --- + # These are IAM policy bindings, billing rollups, metadata, etc. CAI does not + # expose them as discrete assets, so generic discovery must skip them + # (cai_asset_type: null). A typed collector handles them if ever needed. + gcp_billing_billing_accounts: null + +# --------------------------------------------------------------------------- +# aliases +# Extra RWL resource_type_name aliases that should resolve to a table, on +# top of the canonical CloudQuery table name. Mirrors the legacy preset +# registry names (e.g. ``project``, ``compute_instance``) so existing +# generation rules keep working. +# --------------------------------------------------------------------------- +aliases: + gcp_projects: + - project + gcp_compute_instances: + - compute_instance + +# --------------------------------------------------------------------------- +# typed_collectors +# Tables for which gcpapi_resource_types.py ships a dedicated SDK collector +# (richer payload than the generic Cloud Asset Inventory pass). +# --------------------------------------------------------------------------- +# typed_collectors +# Tables for which gcpapi_resource_types.py ships a dedicated google-cloud-* +# SDK collector (richer payload than the generic Cloud Asset Inventory pass), +# plus ``gcp_projects`` which is synthesized directly from config as the +# anchor. Every other table is covered by the Cloud Asset Inventory pass. +typed_collectors: +- gcp_projects +- gcp_compute_instances +- gcp_storage_buckets +- gcp_container_clusters +# Tier 1 high-value compute fallbacks (google-cloud-compute, already a dep) so +# these survive without Cloud Asset Inventory. +- gcp_compute_disks +- gcp_compute_snapshots +- gcp_compute_networks +- gcp_compute_subnetworks +- gcp_compute_firewalls +- gcp_compute_addresses +# Tier 2 idiomatic single-call service clients (google-cloud-pubsub, +# google-cloud-iam) for the remaining common matchable types. +- gcp_pubsub_topics +- gcp_pubsub_subscriptions +- gcp_iam_service_accounts + +# --------------------------------------------------------------------------- +# mandatory +# Tables always collected regardless of generation-rule scope. ``gcp_projects`` +# is the anchor: every other GCP resource links to its parent project. +# --------------------------------------------------------------------------- +mandatory: +- gcp_projects diff --git a/scripts/gcp/sync_gcp_resource_type_registry.py b/scripts/gcp/sync_gcp_resource_type_registry.py new file mode 100644 index 000000000..af87f1fe0 --- /dev/null +++ b/scripts/gcp/sync_gcp_resource_type_registry.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +""" +Sync ``src/indexers/gcp_resource_type_registry.yaml``. + +The registry maps every CloudQuery GCP plugin table to its Cloud Asset +Inventory (CAI) asset type plus metadata used by the native ``gcpapi`` +indexer. CAI asset types (``.googleapis.com/``) are the join +key for generic discovery, exactly like ARM types are for the Azure indexer. + +The data is materialised from three inputs: + +1. **A list of CloudQuery GCP table names.** Source of truth for which tables + exist. Provided either by re-reading the previous registry snapshot + (default), reading a local file (e.g. ``scripts/gcp/gcp_cloudquery_tables.txt``), + or fetching the live list from CloudQuery's public hub. + +2. **A heuristic** that converts ``gcp__`` into + ``.googleapis.com/``. The host segment uses + ``service_api_hosts`` (a hand-curated dictionary that handles services whose + API host differs from the table token, e.g. ``sql`` -> ``sqladmin``); the + entity segment is singularised and PascalCased. + +3. **A manual overrides YAML** at + ``scripts/gcp/gcp_resource_type_overrides.yaml`` that pins the CAI asset + type, aliases, ``typed_collector`` flag, and ``mandatory`` flag for any + table whose heuristic value is wrong or where extra metadata is needed. + Set a ``cai_type_override`` to ``null`` for tables that have no CAI asset + type (IAM bindings, recommendations, billing...) so generic discovery skips + them. + +Hand-edit the overrides YAML; never hand-edit the registry YAML. After editing +the overrides, re-run this script to regenerate the registry. + +Usage: + + # Round-trip the current registry (most common - picks up new overrides + # without touching the table list): + python scripts/gcp/sync_gcp_resource_type_registry.py + + # Use a fresh table list from a file: + python scripts/gcp/sync_gcp_resource_type_registry.py \\ + --from-file scripts/gcp/gcp_cloudquery_tables.txt + + # Fetch the latest list from CloudQuery's public hub: + python scripts/gcp/sync_gcp_resource_type_registry.py --from-cloudquery + + # Print summary of changes without writing: + python scripts/gcp/sync_gcp_resource_type_registry.py --dry-run +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import re +import sys +from pathlib import Path +from typing import Iterable, Optional + +try: + import yaml +except ImportError: # pragma: no cover - tooling environment must have yaml + print("PyYAML is required: pip install pyyaml", file=sys.stderr) + sys.exit(2) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +REGISTRY_PATH = REPO_ROOT / "src" / "indexers" / "gcp_resource_type_registry.yaml" +OVERRIDES_PATH = REPO_ROOT / "scripts" / "gcp" / "gcp_resource_type_overrides.yaml" +DEFAULT_TABLES_FILE = REPO_ROOT / "scripts" / "gcp" / "gcp_cloudquery_tables.txt" +CLOUDQUERY_TABLES_URL = ( + "https://www.cloudquery.io/hub/plugins/source/cloudquery/gcp/latest/tables" +) + + +# --------------------------------------------------------------------------- +# Heuristic +# --------------------------------------------------------------------------- + +_GCP_TABLE_PREFIX = "gcp_" + +# Irregular plural -> singular forms that the rule-based singulariser gets +# wrong. Keep this small; pin anything fancier via cai_type_overrides. +_IRREGULAR_SINGULARS = { + "addresses": "address", + "indices": "index", + "indexes": "index", + "policies": "policy", + "proxies": "proxy", + "registries": "registry", + "repositories": "repository", + "gateways": "gateway", + "schemas": "schema", + "metadata": "metadata", + "settings": "setting", + "series": "series", +} + + +def _singularize(word: str) -> str: + """Best-effort English singularisation of a lowercase noun.""" + if not word: + return word + if word in _IRREGULAR_SINGULARS: + return _IRREGULAR_SINGULARS[word] + if word.endswith("ies") and len(word) > 3: + return word[:-3] + "y" + # buses, statuses, addresses, boxes, watches, dishes -> drop "es" + if re.search(r"(s|x|z|ch|sh)es$", word): + return word[:-2] + if word.endswith("ses") and len(word) > 3: + return word[:-2] # databases -> database + if word.endswith("s") and not word.endswith("ss"): + return word[:-1] + return word + + +def _snake_to_pascal(snake: str) -> str: + """Convert ``foo_bar_baz`` to ``FooBarBaz``. Empty input -> empty output.""" + parts = [p for p in snake.split("_") if p] + return "".join(p[:1].upper() + p[1:] for p in parts) + + +def infer_cai_type( + cloudquery_table: str, + service_api_hosts: dict[str, str], +) -> Optional[str]: + """Best-effort heuristic mapping CQ table -> ``.googleapis.com/``. + + Returns None if the input doesn't follow the ``gcp__`` + shape. The entity segment is singularised (only the trailing token is + singularised; e.g. ``node_pools`` -> ``NodePool``) and PascalCased. + """ + if not cloudquery_table.startswith(_GCP_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_GCP_TABLE_PREFIX):] + if "_" not in rest: + # e.g. "gcp_projects" - no entity segment; treat token as both. + token = rest + host = service_api_hosts.get(token, token) + entity = _snake_to_pascal(_singularize(token)) + return f"{host}.googleapis.com/{entity}" if entity else None + + service_token, entity_snake = rest.split("_", 1) + host = service_api_hosts.get(service_token, service_token) + + # Singularise only the final word of the entity, then PascalCase the whole. + entity_parts = entity_snake.split("_") + entity_parts[-1] = _singularize(entity_parts[-1]) + entity = _snake_to_pascal("_".join(entity_parts)) + if not entity: + return None + return f"{host}.googleapis.com/{entity}" + + +def _category_for(cloudquery_table: str) -> Optional[str]: + if not cloudquery_table.startswith(_GCP_TABLE_PREFIX): + return None + rest = cloudquery_table[len(_GCP_TABLE_PREFIX):] + return rest.split("_", 1)[0] if rest else None + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + +def load_overrides(path: Path = OVERRIDES_PATH) -> dict: + if not path.exists(): + raise FileNotFoundError( + f"Overrides YAML not found at {path}. " + f"This file holds the hand-curated metadata; do not delete it." + ) + payload = yaml.safe_load(path.read_text()) or {} + # cai_type_overrides may legitimately carry null values (tables with no CAI + # equivalent), so preserve the key set rather than filtering falsy values. + return { + "service_api_hosts": payload.get("service_api_hosts") or {}, + "cai_type_overrides": payload.get("cai_type_overrides") or {}, + "aliases": payload.get("aliases") or {}, + "typed_collectors": set(payload.get("typed_collectors") or []), + "mandatory": set(payload.get("mandatory") or []), + } + + +def tables_from_registry(path: Path = REGISTRY_PATH) -> list[str]: + if not path.exists(): + return [] + payload = yaml.safe_load(path.read_text()) or {} + types = payload.get("types") or {} + return sorted(types.keys()) + + +def tables_from_file(path: Path) -> list[str]: + text = path.read_text() + if path.suffix.lower() in {".yaml", ".yml"}: + payload = yaml.safe_load(text) or {} + if isinstance(payload, dict) and "tables" in payload: + return sorted(str(t) for t in payload["tables"]) + if isinstance(payload, list): + return sorted(str(t) for t in payload) + raise ValueError( + f"YAML file {path} must be a list of names or a mapping with a 'tables' key" + ) + # Treat anything else as one-table-name-per-line (comments with '#' ignored). + tables: set[str] = set() + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + tables.add(line) + return sorted(tables) + + +def tables_from_cloudquery_hub(url: str = CLOUDQUERY_TABLES_URL) -> list[str]: + """Fetch and parse the CloudQuery hub tables page. Best-effort: this is a + public HTML page and the markup may shift; if parsing fails the caller + should fall back to --from-file with an explicit table list. + """ + try: + from urllib.request import Request, urlopen + except ImportError: # pragma: no cover + raise RuntimeError("urllib is unavailable in this Python build") + + req = Request(url, headers={"User-Agent": "rwl-sync-script/1.0"}) + with urlopen(req, timeout=45) as resp: + html = resp.read().decode("utf-8", errors="replace") + + tables = sorted(set(re.findall(r"\bgcp_[a-z][a-z0-9_]+\b", html))) + if not tables: + raise RuntimeError( + f"No gcp_* table names found at {url}. The page layout may have " + f"changed; pass --from-file with an explicit table list instead." + ) + return tables + + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +def build_registry( + table_names: Iterable[str], + overrides: dict, + snapshot_date: Optional[str] = None, +) -> dict: + cai_overrides: dict = overrides["cai_type_overrides"] + aliases: dict = overrides["aliases"] + typed_collectors: set[str] = overrides["typed_collectors"] + mandatory: set[str] = overrides["mandatory"] + service_hosts: dict = overrides["service_api_hosts"] + + types: dict[str, dict] = {} + typed_collector_count = 0 + cai_types_assigned = 0 + + for name in sorted(set(table_names)): + if name in cai_overrides: + cai_type = cai_overrides[name] + cai_type_source = "override" + else: + cai_type = infer_cai_type(name, service_hosts) + cai_type_source = "heuristic" if cai_type else None + + is_typed = name in typed_collectors + is_mandatory = name in mandatory + + if is_typed: + typed_collector_count += 1 + if cai_type: + cai_types_assigned += 1 + + types[name] = { + "cai_asset_type": cai_type, + "cai_asset_type_source": cai_type_source, + "category": _category_for(name), + "aliases": list(aliases.get(name, [])), + "typed_collector": is_typed, + "mandatory": is_mandatory, + } + + metadata = { + "source": CLOUDQUERY_TABLES_URL, + "snapshot_date": snapshot_date or _dt.date.today().isoformat(), + "total_tables": len(types), + "typed_collectors": typed_collector_count, + "cai_types_assigned": cai_types_assigned, + "generator": "scripts/gcp/sync_gcp_resource_type_registry.py", + "notes": ( + "Generated file. To change the CAI asset type for a table, edit " + "scripts/gcp/gcp_resource_type_overrides.yaml and re-run the sync " + "script. Hand-edits to this file will be overwritten." + ), + } + + return {"metadata": metadata, "types": types} + + +def diff_summary(old: dict, new: dict) -> str: + old_types = (old or {}).get("types") or {} + new_types = new.get("types") or {} + added = sorted(set(new_types) - set(old_types)) + removed = sorted(set(old_types) - set(new_types)) + changed = [] + for name in sorted(set(new_types) & set(old_types)): + if new_types[name] != old_types[name]: + changed.append(name) + + lines = [ + f" total tables: {len(new_types)} (was {len(old_types)})", + f" added : {len(added)}", + f" removed : {len(removed)}", + f" changed : {len(changed)}", + ] + for label, items in (("added", added), ("removed", removed), ("changed", changed)): + if not items: + continue + sample = items[:10] + more = f" ... (+{len(items) - len(sample)} more)" if len(items) > len(sample) else "" + lines.append(f" {label}: {sample}{more}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + src = parser.add_mutually_exclusive_group() + src.add_argument( + "--from-registry", + action="store_true", + help="Read the table list from the existing registry YAML (default).", + ) + src.add_argument( + "--from-file", + type=Path, + help="Read the table list from a local file (one name per line, or YAML).", + ) + src.add_argument( + "--from-cloudquery", + action="store_true", + help=f"Fetch the table list from {CLOUDQUERY_TABLES_URL}", + ) + parser.add_argument( + "--overrides", + type=Path, + default=OVERRIDES_PATH, + help="Path to the overrides YAML.", + ) + parser.add_argument( + "--out", + type=Path, + default=REGISTRY_PATH, + help="Path to write the registry YAML to.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a diff summary and exit without writing.", + ) + parser.add_argument( + "--snapshot-date", + help="Override the metadata.snapshot_date value (defaults to today).", + ) + args = parser.parse_args(argv) + + if args.from_file: + tables = tables_from_file(args.from_file) + source_label = f"file:{args.from_file}" + elif args.from_cloudquery: + tables = tables_from_cloudquery_hub() + source_label = "cloudquery hub" + else: + # Default: round-trip the existing registry, but bootstrap from the + # checked-in table list the first time the registry doesn't exist yet. + tables = tables_from_registry(args.out) + if tables: + source_label = "existing registry" + elif DEFAULT_TABLES_FILE.exists(): + tables = tables_from_file(DEFAULT_TABLES_FILE) + source_label = f"file:{DEFAULT_TABLES_FILE}" + else: + source_label = "existing registry" + + if not tables: + print( + "No tables found in the requested source. " + "Aborting; at least one input must yield a non-empty list.", + file=sys.stderr, + ) + return 2 + + overrides = load_overrides(args.overrides) + new_registry = build_registry(tables, overrides, snapshot_date=args.snapshot_date) + + old_registry: dict = {} + if args.out.exists(): + old_registry = yaml.safe_load(args.out.read_text()) or {} + + print(f"Source: {source_label} tables={len(tables)}") + print(diff_summary(old_registry, new_registry)) + + if args.dry_run: + print("\nDry run; not writing.") + return 0 + + args.out.write_text( + yaml.safe_dump(new_registry, sort_keys=False, default_flow_style=False, width=200) + ) + print(f"\nWrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/.gitignore b/src/.gitignore index 545a282bc..7c4df941c 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -6,7 +6,6 @@ BackupGeneratedWorkspaces generation-rules-test templates-test dummyoutput -cheat-sheet-sample *.pyc .DS_Store rw-cli-codecollection-cache \ No newline at end of file diff --git a/src/Dockerfile b/src/Dockerfile index eba5312b4..48ae55fee 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -55,9 +55,12 @@ RUN curl -sSL https://install.python-poetry.org | python3 - --version $POETRY_VE # Copy project requirement files here to ensure they will be cached. WORKDIR $PYSETUP_PATH -COPY pyproject.toml ./ +COPY pyproject.toml poetry.lock ./ -# Install Python Dependencies +# Install Python Dependencies. The lockfile pins every transitive dep so the +# image build is reproducible and is not exposed to upstream resolution flakes +# (e.g. a freshly-published transitive that doesn't yet have wheels for the +# Python ABI we target). RUN poetry run pip install --upgrade pip setuptools wheel RUN poetry install --only main @@ -88,15 +91,6 @@ RUN apt-get update \ WORKDIR $RUNWHEN_HOME -RUN pip install --no-cache-dir \ - mkdocs \ - mkdocs-material \ - mkdocs-macros-plugin \ - pymdown-extensions \ - ruamel.yaml \ - gitpython \ - pyyaml - ENV NODE_PATH="/var/lib/neo4j/node_modules" ENV CLOUDQUERY_PLUGINS_PREINSTALLED=true @@ -203,5 +197,5 @@ RUN set -eux; \ chown -R runwhen:runwhen "$CODE_COLLECTION_CACHE_ROOT" USER runwhen -EXPOSE 8081 +EXPOSE 8000 ENTRYPOINT ["/bin/sh", "-c", "./entrypoint.sh"] diff --git a/src/cheat-sheet-docs/docs/about.md b/src/cheat-sheet-docs/docs/about.md deleted file mode 100644 index 8fd7d894b..000000000 --- a/src/cheat-sheet-docs/docs/about.md +++ /dev/null @@ -1,108 +0,0 @@ - -
-
- About Icon -

About This Tool

-
-
-
-

What does it do?

-
-

- This tool runs on your local machine, or in your own cluster, and: -

-
-
-
- Icon -
Scans your Kubernetes-based cluster
-
-
- Icon -
Combines your environment with community-powered troubleshooting commands
-
-
- Icon -
Generates troubleshooting commands that are ready to Copy and Paste
-
-
- - -

- -

Command Line Utilities Used

-

These commands are intended to be run from the comfort of your laptop, but will require the appropriate binaries. These may include:

-
-
-
- kubectl Icon - kubectl -
-
- oc Icon - oc -
-
- jq Icon - jq -
-
- jp Icon - jp -
-
- jp Icon - cURL -
-
-
- -

-

Connect, Share Feedback, Suggest Improvements

-

This open source tool gets better by providing your feedback. Join in the discussion, open issues, contribute suggested commands, or connect through slack:

- - -

-

Additional Links

-

Looking for docs, videos, or other resources:

- - - diff --git a/src/cheat-sheet-docs/docs/assets/background_blue.png b/src/cheat-sheet-docs/docs/assets/background_blue.png deleted file mode 100644 index 47e89ff36..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/background_blue.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/background_blue_dots.png b/src/cheat-sheet-docs/docs/assets/background_blue_dots.png deleted file mode 100644 index 5e9c16ff0..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/background_blue_dots.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/flow.png b/src/cheat-sheet-docs/docs/assets/flow.png deleted file mode 100644 index 34d855c84..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/flow.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/icon.png b/src/cheat-sheet-docs/docs/assets/icon.png deleted file mode 100644 index e76212908..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/icon.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/logo_large.png b/src/cheat-sheet-docs/docs/assets/logo_large.png deleted file mode 100644 index d6b7e0356..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/logo_large.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/logo_small.png b/src/cheat-sheet-docs/docs/assets/logo_small.png deleted file mode 100644 index b0c2e4f25..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/logo_small.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/runwhen_icon_orange.png b/src/cheat-sheet-docs/docs/assets/runwhen_icon_orange.png deleted file mode 100644 index 67dd49d3b..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/runwhen_icon_orange.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/runwhen_map_icon_blue.png b/src/cheat-sheet-docs/docs/assets/runwhen_map_icon_blue.png deleted file mode 100644 index 346feab14..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/runwhen_map_icon_blue.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/secret.gif b/src/cheat-sheet-docs/docs/assets/secret.gif deleted file mode 100644 index aa6f78f45..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/secret.gif and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/white_runwhen_logo_transparent_bg.png b/src/cheat-sheet-docs/docs/assets/white_runwhen_logo_transparent_bg.png deleted file mode 100644 index a183298f6..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/white_runwhen_logo_transparent_bg.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/assets/workspaceUpload.png b/src/cheat-sheet-docs/docs/assets/workspaceUpload.png deleted file mode 100644 index 4f19e7384..000000000 Binary files a/src/cheat-sheet-docs/docs/assets/workspaceUpload.png and /dev/null differ diff --git a/src/cheat-sheet-docs/docs/css/custom.css b/src/cheat-sheet-docs/docs/css/custom.css deleted file mode 100644 index 5f5f4c65e..000000000 --- a/src/cheat-sheet-docs/docs/css/custom.css +++ /dev/null @@ -1,1215 +0,0 @@ -:root { - --md-primary-color: #2f80ed; - --md-accent-color: #faa629; - --tooltip-background: #333; - --tooltip-text-color: #ccc; - --icon-filter: brightness(0) saturate(100%) invert(39%) sepia(84%) saturate(907%) hue-rotate(188deg) brightness(95%) contrast(96%); -} - -[data-md-color-scheme="slate"] { - --md-hue: 230; - --tooltip-background: #ccc; - --tooltip-text-color: #333; - --icon-filter: invert(97%) sepia(0%) saturate(0%) hue-rotate(187deg) brightness(86%) contrast(90%); -} - -a { - color: var(--md-primary-color); -} - -a:hover { - color: var(--md-accent-color); -} - -pre code, -.md:not(.use-csslab) pre code { - white-space: pre-wrap; - background-color: #282d33; - color: #f8f8f2; -} - -blockquote { - background-color: #f0f0f0; - border-left-color: var(--md-accent-color); -} - -[data-md-color-scheme="slate"] blockquote { - background-color: #282d33; - color: #ccc; -} - -[data-md-color-scheme="slate"] .tag-card{ - background-color: #333; /* Darker background color */ - color: var(--md-accent-color); /* Orange text color */ - box-shadow: 0 2px 4px rgba(255, 255, 255, 0.1); /* Lighter box shadow */ -} - -[data-md-color-scheme="slate"] .tag-card a { - color: var(--md-accent-color); /* Orange color for links */ -} - -[data-md-color-scheme="slate"] .card{ - background-color: #333; /* Darker background color */ - color: var(--md-accent-color); /* Orange text color */ - box-shadow: 0 2px 4px rgba(255, 255, 255, 0.1); /* Lighter box shadow */ -} - -[data-md-color-scheme="slate"] .card a { - color: var(--md-accent-color); /* Orange color for links */ -} - -[data-md-color-scheme="slate"] .tag-card .tag-card-icon { - filter: invert(100%) sepia(0%) saturate(0%) hue-rotate(187deg) brightness(100%) contrast(100%); /* Ensure icons are visible */ -} - -[data-md-color-scheme="slate"] .card .card-icon { - filter: invert(100%) sepia(0%) saturate(0%) hue-rotate(187deg) brightness(100%) contrast(100%); /* Ensure icons are visible */ -} - -[data-md-color-scheme="slate"] .demo-card { - background-color: #333; /* Darker background color */ - color: #faa629; /* Orange text color */ - box-shadow: 0 2px 4px rgba(255, 255, 255, 0.1); /* Lighter box shadow */ -} - -[data-md-color-scheme="slate"] .demo-card a { - color: #faa629; /* Orange color for links */ -} - -[data-md-color-scheme="slate"] .demo-card .demo-card-icon { - filter: brightness(1.5) contrast(1.2) sepia(1) hue-rotate(30deg) saturate(2); /* Adjusted filter for slight hint of accent color */ -} - -[data-md-color-scheme="slate"] .bio-icon { - filter: invert(100%) brightness(2) !important; /* Ensure black icons are visible by inverting them and increasing brightness */ -} - -table th { - background-color: #f0f0f0; - color: #000; -} - -table { - border-color: var(--md-accent-color); -} - -.md-header { - background-color: var(--md-primary-color); - color: #ffffff; -} - -.md-button { - background-color: #ffffff; - color: var(--md-primary-color) !important; -} - -.md-header__button img { - filter: brightness(0) saturate(100%) invert(1); -} - -.md-button:hover { - background-color: var(--md-accent-color); -} - -.popup { - position: fixed; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - max-width: 60%; - background-color: white; - border: 1px solid #ccc; - border-radius: 5px; - box-shadow: 0 0 10px rgba(0, 0, 0, 0.3); - z-index: 9999; - padding: 20px; - overflow: hidden; - font-family: "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 18px; - white-space: pre-wrap; - color: #3f3f3f; -} - - - -.popup h1, -.popup h2 { - font-size: 20px; - color: #858484; -} - -.close { - position: absolute; - top: 10px; - right: 10px; - cursor: pointer; - font-size: 20px; - font-weight: bold; - color: #aaa; -} - -.close:hover { - color: #3f3f3f; -} - -ul ul { - list-style-type: circle; -} - -ul ul ul { - list-style-type: square; -} - -.invisible-table { - border: none; - border-collapse: collapse; - width: 100%; -} - -.invisible-table td { - border: none; - padding: 0; -} - -.icon-cell { - width: 120px; - text-align: left; -} - -.icon-cell img { - width: 80px; - height: 80px; - border-radius: 12px; - border: 1px solid #e7e7e7; - padding: 9px; -} - -.bullet-cell { - padding-left: 10px; -} - -.bullet-cell ul { - list-style-type: disc; - margin: 0; - padding-left: 20px; -} - -.bullet-cell li { - margin-bottom: 5px; -} - -h1 { - color: #000; -} - -h2, -h3, -h4, -h5, -h6, -p, -li { - color: #858484; -} - - .card-grid { - display: grid; - /* grid-template-columns: repeat(1, 1fr); */ - grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); - grid-gap: 10px; -} - - -.card { - display: flex; - background-color: #fff; - border-radius: 4px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - padding: 5px; - min-height: 75px; -} - -.cc-card-grid { - display: grid; - grid-template-columns: 1fr; /* One column that fills the row */ - gap: 20px; - justify-content: center; - padding: 20px; -} - -.cc-card { - display: flex; - flex-direction: column; /* Stack the children vertically */ - align-items: stretch; /* Stretch children to fill the width */ - background-color: #fff; - border-radius: 4px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - padding: 20px; /* Adjusted for consistent spacing */ - min-height: 300px; -} -.card-icon { - flex-shrink: 0; - width: 50px; - height: 50px; - margin-right: 10px; - border-radius: 12px; - padding: 9px; - filter: brightness(0) saturate(100%) invert(39%) sepia(84%) saturate(907%) hue-rotate(188deg) brightness(95%) contrast(96%); - vertical-align: middle; - object-fit: contain; /* Contain the image within the element, preserving its aspect ratio */ - -} - -.tag-card-grid { - display: grid; - grid-template-columns: repeat(4, 1fr); - /* grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); */ - grid-gap: 10px; -} - -.tag-card { - display: flex; - flex-direction: column; /* Stack children vertically */ - align-items: center; /* Center-align the flex items horizontally */ - justify-content: center; /* Center-align the flex items vertically, if you want */ - background-color: #fff; - border-radius: 4px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - padding: 5px; - min-height: 75px; - text-align: center; /* Ensure the text is centered under the icon */ -} - -.tag-card-icon { - flex-shrink: 0; - width: 100px; - height: 100px; - margin-bottom: 10px; /* Add some space between the icon and the text */ - border-radius: 12px; - padding: 9px; -} - -.main-page-search-icon { - flex-shrink: 0; - width: 50px; - height: 50px; - margin-right: 10px; - border-radius: 0px; - padding: 15px 1px 0 1px; /* Adjust the top padding value as needed */ - filter: brightness(0) saturate(100%) invert(39%) sepia(84%) saturate(907%) hue-rotate(188deg) brightness(95%) contrast(96%); - /* vertical-align: baseline; or baseline */ - -} - -.card-title { - font-size: 8px; - font-weight: bold; - margin-bottom: 2px; -} - -.card-description { - font-size: 14px; - color: #666; -} - -[data-md-color-scheme="slate"] .card { - background-color: #282d33; - color: #ccc; -} - -[data-md-color-scheme="slate"] .card-title, -[data-md-color-scheme="slate"] .card-description { - color: #ccc; -} - -.custom-section { - background-image: url('../assets/background_blue_dots.png'); - background-size: cover; - background-repeat: no-repeat; - background-position: top center; - position: relative; - display: flex; - flex-direction: column; - justify-content: flex-end; - align-items: center; - min-height: 30vh; /* Changed from height to min-height */ - background-color: var(--md-primary-color); - color: #ffffff; - font-size: 2vw; /* Example of using viewport width for sizing */ -} - -/* Media query for larger screens */ -@media (min-width: 1024px) { - .custom-section { - font-size: 1.5rem; /* Fixed size for larger screens */ - } -} - -/* Media query for smaller screens */ -@media (max-width: 768px) { - .custom-section { - font-size: 1rem; /* Smaller font size for mobile devices */ - } -} - - -.custom-section .md-search { - position: relative; /* Change position to relative */ - width: 100%; - background-color: #ccc; - align-items: center; - border-radius: 8px; - padding: 8px; - -} - -.custom-section .md-search__inner { - width: 80%; - background-color: #ccc; - -} - -[data-md-color-scheme="slate"] .custom-section .md-search { - background-color: #ccc; - color: #666; - -} - -.custom-section .md-search__input { - width: 80%; - color: #666; - font-size: 24px; - -} - -.custom-section .md-search__form { - position: relative; - width: 100%; - background-color: #ccc; - color: #666; - -} - -.custom-section .md-search__output { - position: absolute; - top: 100%; - left: 0; - width: 100%; - max-height: 400px; - overflow-y: clip; - z-index: 999; -} - -.custom-section .md-search__output .md-search-result { - padding: 2px; - border-radius: 1px; - margin-bottom: 1x; - border: 2px solid #bebebe; /* Adjust the border color as desired */ -} - -.custom-section .md-search__suggest { - margin-top: 10px; - width: 100%; -} - -.custom-section .md-search__icon:hover { - color: var(--md-primary-color); - width: 100%; -} - -.custom-section .md-search__options { - display: flex; - align-items: center; - margin-left: 10px; - width: 100%; -} - -.custom-section .md-search__scrollwrap { - display: flex; - flex-direction: column; - overflow-y: auto; -} - - -[data-md-toggle=search]:checked~.md-container .md-search__output { - box-shadow: 0 6px 10px 0 rgba(0, 0, 0, 0.14), 0 1px 18px 0 rgba(0, 0, 0, 0.12), 0 3px 5px -1px rgba(0, 0, 0, 0.4); - opacity: 1; - overflow: visible; - width: auto; -} - -[data-md-toggle=search]:checked~.md-container .md-search__overlay { - width: 100%; - height: 100%; - transition: width 0s, height 0s, opacity 0.25s; - opacity: 1; -} - -[data-md-toggle=search]:checked~.md-container .md-search__scrollwrap { - max-height: 75vh; - width: 100%; -} - -[data-md-color-scheme="slate"] .code-block { - background-color: #666; -} - -.code-block { - font-size: 18px; -} - -.copy-button-icon { - margin-right: 6px; - font-size: 18px; -} - -.copied-message { - position: fixed; - bottom: 20px; - right: 20px; - padding: 10px 20px; - background-color: var(--md-primary-color); - color: #fff; - font-size: 14px; - border-radius: 4px; - opacity: 0; - transition: opacity 0.3s; - pointer-events: none; - z-index: 9999; -} - -.copied-message.show { - opacity: 1; -} - -.home-column-container { - display: flex; -} - -.home-column { - flex: 1; - padding: 0 10px; -} - -.home-title { - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 28px; - font-weight: bold; - /* text-transform: uppercase; */ -} - -.home-subtitle { - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 16px; - text-align: center; - color: #ffffff -} - -.home-title .primary-color { - color: var(--md-primary-color); -} - -.home-title .accent-color { - color: var(--md-accent-color); -} - -.fade-announcement { - display: flex; - justify-content: center; - align-items: center; - height: 60%; - color: #666; -} - -.fade-announcement-text { - animation: fade 6s ease-in-out infinite; - opacity: 0; - color: #ffffff; - text-align: center; -} - -.fade-announcement a { - color: white; -} - -.fade-announcement a:hover { - color: white; -} - -@keyframes fade { - 0%, 100% { - opacity: 0.33; - } - 50% { - opacity: 1; - } -} - -.md-banner { - background-color: var(--md-primary-color); -} - - -.tabbed-content a{ - margin-right: 8px; - text-decoration: none; - color: #777; - filter: invert(49%) sepia(17%) saturate(0%) hue-rotate(246deg) brightness(91%) contrast(81%); -} - -hr.custom-hr { - margin-top: 0px; /* Adjust the top margin as needed */ - margin-bottom: 5px; /* Adjust the bottom margin as needed */ - border-color: rgba(119, 119, 119, 0.2); /* Adjust the opacity as needed */ - -} - -.command-header-grid { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - font-size: 12px; /* Adjust the value as needed */ - color: rgba(119, 119, 119, 0.8); /* Adjust the opacity as needed */ - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - padding: 0px 0; /* Adjust the values as needed */ - -} - -.command-header-grid .grid-item { - display: flex; - align-items: center; - margin: 0 10px; /* Adjust the horizontal margin as needed */ - -} - -.command-header-grid .grid-item a { - text-decoration: none; - color: inherit; -} - -.command-header-grid .card-icon { - margin-right: 0px; - filter: invert(47%) sepia(0%) saturate(372%) hue-rotate(168deg) brightness(99%) contrast(88%); - -} - -.command-tools-grid { - display: grid; - grid-template-columns: repeat(5, minmax(0, 1fr)); - gap: 8px; - font-size: 14px; - color: #777; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - padding: 0px; -} - -.command-tools-grid .grid-item { - display: flex; - align-items: left; -} - -.command-tools-grid .grid-item a { - text-decoration: none; - color: inherit; -} - -.command-tools-grid .grid-item a:hover { - color: var(--md-accent-color); -} - -.command-tools-grid .card-icon { - margin-right: 8px; - filter: invert(47%) sepia(0%) saturate(372%) hue-rotate(168deg) brightness(99%) contrast(88%); -} - -.author-block { - display: flex; - align-items: center; -} - -.author-avatar { - width: 30px; - height: 30px; - border-radius: 50%; - object-fit: cover; - margin-right: 10px; - filter: grayscale(100%); -} - -.author-info { - display: flex; - flex-direction: column; -} - -.author-name { - font-size: 14px; - font-weight: bold; - margin: 0; -} - -.author-bio { - font-size: 14px; - margin: 0; - color: #888888; - display: flex; - align-items: center; - gap: 5px; /* Adjust the gap value to control the spacing between the icon and text */ -} - -.bio-icon { - width: 16px; - height: 16px; -} - -.author-block a { - text-decoration: none; - color: inherit; -} - -/* Footer */ -.md-footer { - background-color: var(--md-primary-color); - color: #fff; - padding: 20px; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - display: flex; - align-items: center; - justify-content: space-around; -} - -.md-footer__section { - display: flex; - align-items: center; - gap: 5px; -} - -.md-footer__section i { - font-size: 20px; -} - -.md-footer__section a { - text-decoration: none; - color: #fff; -} - -.md-footer__section a:hover { - text-decoration: underline; -} - -.md-footer-meta__inner { - display: flex; - align-items: center; - justify-content: center; - gap: px; - flex-wrap: wrap; -} - - -.md-footer-meta__inner > * { - margin: 20px; /* Adjust the margin as needed */ -} - - -.about-description ul { - list-style-type: none; - padding-left: 0; -} - -.about-description li { - display: flex; - align-items: center; - margin-bottom: 10px; -} - -.about-description li:before { - content: "•"; - margin-right: 10px; -} - -.about-content { - display: flex; - flex-direction: column; - align-items: center; - padding: 20px; - text-align: center; -} - -.about-header { - margin-bottom: 20px; -} - -.about-header img { - width: 80px; - height: 80px; -} - -.about-links { - display: flex; - justify-content: center; - flex-wrap: wrap; - gap: 20px; - margin-top: 40px; -} - -.about-link { - display: flex; - align-items: center; - padding: 10px; - border-radius: 8px; - background-color: #f5f5f5; - text-decoration: none; -} - -.about-link img { - width: 20px; - height: 20px; - margin-right: 10px; -} - -.about-link-text { - font-weight: bold; -} - -.about-link:hover { - background-color: #e0e0e0; -} - -.about-utilities { - text-align: center; - margin-top: 40px; -} - -.about-utilities .utility-cards-container { - display: flex; - justify-content: center; - flex-wrap: wrap; - gap: 20px; -} - -.about-utilities .utility-card { - display: flex; - align-items: center; - padding: 10px; - border-radius: 8px; - background-color: #f5f5f5; - text-decoration: none; -} - -.about-utilities .utility-card img { - width: 20px; - height: 20px; - margin-right: 10px; -} - -.about-utilities .utility-card-text { - font-weight: bold; -} - -.admonition.note { - border-left: 4px solid #cccccc; - /* border-color: #777 !important; */ -} - -.admonition-block { - background-color: #f5f5f5; - border-left: 4px solid #cccccc; - padding: 10px; - margin-top: 20px; -} - -.admonition-block p { - margin-bottom: 0; -} - - -.about-column { - display: flex; - justify-content: left; /* Align cards to the left */ - gap: 20px; - margin-top: 40px; -} - -.command-title { - color: var(--md-primary-color); - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 16px; - font-weight: bold; - padding: 0px; -} - -.command-details { - color: #777; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 14px; - font-weight: bold; - padding: 0px; -} - -.command-explanation { - color: #777; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 14px; - padding: 0px; -} -.md-typeset .admonition.runwhen, -.md-typeset details.runwhen { -} -.md-typeset .runwhen > .admonition-title, -.md-typeset .runwhen > summary { - color: #777; -} -.md-typeset .runwhen > .admonition-title::before, -.md-typeset .runwhen > summary::before { -} -.md-typeset details.runwhen > h4 { - visibility: hidden; -} - -#terminalContainer { - display: none; - border: 1px solid #fff; - position: fixed; - background-color: #fff; - z-index: 1000; - bottom: 0; - left: 0; - width: 100%; - height: 30%; /* or whatever default height you want */ - z-index: 1000; - overflow: hidden; -} - -#toggleTerminal img { - filter: invert(1); -} - - -#terminalControls { - display: flex; - justify-content: flex-end; - gap: 10px; - background-color: #3f3f3f; /* Optional background for better visibility */ - padding: 5px 10px; - border-bottom: 1px solid #ccc; -} - -#terminalControls button { - margin: 0 5px; - font-family: "Roboto", "Helvetica", "Arial", sans-serif; - color: #ffffff; -} - -#terminal { - padding: 0; - margin: 0; - overflow: auto; - width: 100%; - height: 95%; -} - - -/* Input Style */ -input[type="text"] { - border: 1px solid #ccc; /* Light gray border */ - padding: 8px 12px; /* Comfortable padding for text */ - border-radius: 4px; /* Rounded corners */ - font-size: 16px; /* Font size similar to other text elements */ - color: #3f3f3f; /* Dark color for text */ - transition: border-color 0.3s ease; /* Smooth transition for focus effect */ -} - -/* Focus Effect */ -input[type="text"]:focus { - border-color: var(--md-primary-color); /* Change border color to primary color on focus */ - outline: none; /* Remove the default focus outline */ -} - -/* Light Theme */ -[data-md-color-scheme="light"] .theme-input { - background-color: #fff; /* Set the background color for the light theme */ - color: #3f3f3f; /* Set the text color for the light theme */ - border: 1px solid #ccc; -} - -/* Dark Theme */ -[data-md-color-scheme="slate"] .theme-input { - background-color: #282d33; /* Set the background color for the dark theme */ - color: #ccc; /* Set the text color for the dark theme */ - border: 1px solid #ccc; -} - -.info-icon { - position: relative; - display: inline-block; - cursor: pointer; - /* background-image: url('https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/info.svg'); - background-size: 18px; */ - width: 18px; - height: 18px; - -} - -.info-icon img { - filter: var(--icon-filter); -} - -/* Style the tooltip (hidden by default) */ -.info-icon .tooltip { - display: none; - position: absolute; - top: 100%; - left: 0; - width: 200px; - background-color: var(--tooltip-background) !important; - color: var(--tooltip-text-color) !important; - border-radius: 4px; - padding: 8px; - font-size: 14px; - z-index: 1; - /* Add a box-shadow for better visibility */ - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); -} - -/* Style the tooltip arrow */ -.info-icon .tooltip::after { - content: ""; - position: absolute; - top: -10px; - left: 50%; - transform: translateX(-50%); - border-width: 0 8px 8px 8px; - border-style: solid; - /* Use CSS variables for arrow color */ - border-color: transparent transparent var(--tooltip-background) transparent; -} - -/* Show the tooltip on hover */ -.info-icon:hover .tooltip { - display: block; -} - - -.button-container { - display: flex; - justify-content: space-around; /* Adjust as needed for your layout */ - padding: 20px; -} - -.front-page-button { - background-color: #ffffff; - color: var(--md-primary-color); - border: none; - padding: 10px 20px; - cursor: pointer; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-weight: bold; - font-size: 16px; - border-radius: 4px; - transition: background-color 0.3s; - margin: 0 20px; /* Adds horizontal margin */ -} - -.header-button { - background-color: #3f3f3f; - color: #f8f8f2; - border: none; - padding: 10px 20px; - cursor: pointer; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-size: 16px; - border-radius: 4px; - transition: background-color 0.3s; - margin: 0 20px; -} - -.header-button:hover { - color: #ffffff !important; -} - -.main-content .button-container .terraform-button:hover { - background-color: var(--md-accent-color); - color: #ffffff !important; - -} - - -/* .card-stats { - display: flex; - justify-content: space-between; - - flex-direction: column; - padding: 6px; - width: 100%; - -} */ - -/* .stat-item { - display: flex; - justify-content: space-between; - padding: 1px 0; - font-size: 12px; -} */ -/* .stat-item { - display: flex; - justify-content: space-between; - align-items: center; - width: 100%; - font-size: 14px; -} -.stat-item span { - font-weight: bold; - -} */ - -.card-stats { - display: flex; - flex-direction: column; - gap: 10px; /* Add space between stat items */ - padding: 6px; - width: 100%; -} - -.stat-item { - display: flex; - justify-content: space-between; - align-items: center; - width: 100%; - font-size: 14px; -} - -.stat-item span { - font-weight: bold; -} - -.card-footer { - display: flex; - margin-top: auto; - padding: 10px; - align-items: center; - background-color: #f0f0f0; - justify-content: center; - -} - -.card-header { - display: flex; - justify-content: left; - align-items: center; - width: 100%; -} -.header-content { - flex-grow: 1; /* Allows the header content to take up remaining space */ - padding: 0 10px; /* Adds some padding around the text */ -} - - -.home-card { - display: flex; - background-color: #fff; - border-radius: 4px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - padding: 5px; - min-height: 75px; -} - -.codebundle-button { - display: inline-flex; - align-items: center; - justify-content: center; - gap: 8px; /* Space between icon and text */ - padding: 5px 16px; - font-size: 12px; - font-weight: 600; - color: #24292e !important; - background-color: #fafbfc; - border: 1px solid rgba(27,31,35,.15); - border-radius: 6px; - cursor: pointer; -} - -.codebundle-button-icon { - width: 16px; - height: 16px; - /* filter: var(--icon-filter); */ -} -.demo-card { - max-width: 100%; - margin: auto; - padding: 20px; - box-shadow: 0 2px 4px rgba(0,0,0,0.2); - border-radius: 4px; - background-color: #fff; -} - -.action-container { - display: flex; - align-items: center; - padding: 16px; - width: 95%; /* Set the width of the divs */ - -} - -.action-container label, -.action-container input { - margin-right: 8px; - justify-content: right; -} - -.content-page-button { - margin-left: auto; /* This will push the button to the right */ - background-color: var(--md-primary-color); - color: #ffffff; - border: none; - padding: 10px 20px; - cursor: pointer; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-weight: bold; - font-size: 16px; - border-radius: 4px; - transition: background-color 0.3s; -} - - -.md-typeset .action-container a.content-page-button, -.md-typeset .action-container a.action-button { - display: inline-block; - background-color: var(--md-primary-color); - color: #ffffff !important; /* Ensuring color overrides */ - border: none; - padding: 10px 20px; - cursor: pointer; - font-family: "Railway", "Roboto", "Helvetica", "Arial", sans-serif; - font-weight: bold; - font-size: 16px; - border-radius: 4px; - text-align: center; - text-decoration: none; - transition: background-color 0.3s; - margin-left: auto; -} - -.md-typeset .action-container a.content-page-button:hover, -.md-typeset .action-container a.action-button:hover, -.md-typeset .action-container a.content-page-button:focus, -.md-typeset .action-container a.action-button:focus { - color: #ffffff !important; -} - -.md-typeset .action-container a.content-page-button:visited, -.md-typeset .action-container a.action-button:visited { - color: #ffffff !important; -} - -.action-container .right-align { - text-align: right; -} - -.action-container .button-container { - display: flex; - justify-content: flex-end; -} - -.action-container:not(:last-child) { - border-bottom: 1px solid #ccc; /* Adjust the color and thickness as needed */ - padding-bottom: 10px; /* Provides some spacing between the content and the border */ - margin-bottom: 10px; /* Provides some spacing between this div and the next */ -} \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/index.md b/src/cheat-sheet-docs/docs/index.md deleted file mode 100644 index 7c9b3a614..000000000 --- a/src/cheat-sheet-docs/docs/index.md +++ /dev/null @@ -1,64 +0,0 @@ -# Community Powered Troubleshooting Cheat sheet - -Welcome to your troubleshooting cheat sheet: - - ➡️ a local tool, run by you, on your system - ➡️ that discovers your resources, - ➡️ finds useful troubleshooting commands from a community of engineers, and - ➡️ combines them into a (locally stored) copy & paste-able cheat sheet - - -## Discovery Status: - -
-

-
- - \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/javascript/extra.js b/src/cheat-sheet-docs/docs/javascript/extra.js deleted file mode 100644 index 1af5b3354..000000000 --- a/src/cheat-sheet-docs/docs/javascript/extra.js +++ /dev/null @@ -1,35 +0,0 @@ -// // Wait for the DOM to be ready -// document.addEventListener('DOMContentLoaded', function() { -// // Get the header element -// var header = document.querySelector('.md-header'); - -// // Check the current path -// var path = window.location.pathname; - -// // Fetch no-search header -// var customContentPath = 'overrides/partials/header-no-search.html'; -// fetch(customContentPath) -// .then(function(response) { -// if (response.ok) { -// return response.text(); -// } else { -// throw new Error('Failed to fetch custom content'); -// } -// }) -// .then(function(customContent) { -// if (path === '/') { -// customContent = customContent; -// } else { -// customContent = header.innerHTML.trim(); -// } -// // Combine the custom content with the original header content -// // var originalContent = header.innerHTML.trim(); -// // var updatedContent = originalContent + ' ' + customContent; - -// // Update the header content -// header.innerHTML = customContent; -// }) -// .catch(function(error) { -// console.error(error); -// }); -// }); \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/list.md b/src/cheat-sheet-docs/docs/list.md deleted file mode 100644 index 0f0e187fe..000000000 --- a/src/cheat-sheet-docs/docs/list.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Site Index -template: list.html ---- diff --git a/src/cheat-sheet-docs/docs/overrides/list.html b/src/cheat-sheet-docs/docs/overrides/list.html deleted file mode 100644 index ea810353e..000000000 --- a/src/cheat-sheet-docs/docs/overrides/list.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "main.html" %} - -{% block content %} -

Site Index

- - -{% endblock %} \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/overrides/main.html b/src/cheat-sheet-docs/docs/overrides/main.html deleted file mode 100644 index 553a04d67..000000000 --- a/src/cheat-sheet-docs/docs/overrides/main.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "base.html" %} - -{% block announce %} - -{% endblock %} - -{% block content %} - -{% if page.url_to_print_page %} - - {% include ".icons/material/printer.svg" %} - -{% endif %} -{{ super() }} -{% endblock content %} diff --git a/src/cheat-sheet-docs/docs/overrides/partials/build_details.html b/src/cheat-sheet-docs/docs/overrides/partials/build_details.html deleted file mode 100644 index c2af6a71c..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/build_details.html +++ /dev/null @@ -1,15 +0,0 @@ - \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/overrides/partials/content.html b/src/cheat-sheet-docs/docs/overrides/partials/content.html deleted file mode 100644 index 256df3947..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/content.html +++ /dev/null @@ -1,21 +0,0 @@ -{#- - This file was automatically generated - do not edit - -#} - {% if "material/tags" in config.plugins %} - {% include "partials/tags.html" %} - {% endif %} - {% include "partials/actions.html" %} - {% if "\x3ch1" not in page.content %} - {% if page.file.src_path != 'index.md' %} -

{{ page.title | d(config.site_name, true)}}

- {% endif %} - {% endif %} - {{ page.content }} - {% if page.meta and ( - page.meta.git_revision_date_localized or - page.meta.revision_date - ) %} - {% include "partials/source-file.html" %} - {% endif %} - {% include "partials/feedback.html" %} - {% include "partials/comments.html" %} \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/overrides/partials/copyright.html b/src/cheat-sheet-docs/docs/overrides/partials/copyright.html deleted file mode 100644 index b714a5a72..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/copyright.html +++ /dev/null @@ -1,15 +0,0 @@ - diff --git a/src/cheat-sheet-docs/docs/overrides/partials/footer.html b/src/cheat-sheet-docs/docs/overrides/partials/footer.html deleted file mode 100644 index e1ff66359..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/footer.html +++ /dev/null @@ -1,12 +0,0 @@ -
- -
diff --git a/src/cheat-sheet-docs/docs/overrides/partials/header.html b/src/cheat-sheet-docs/docs/overrides/partials/header.html deleted file mode 100644 index 922372519..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/header.html +++ /dev/null @@ -1,115 +0,0 @@ - - {% set class = "md-header" %} - {% if "navigation.tabs.sticky" in features %} - {% set class = class ~ " md-header--shadow md-header--lifted" %} - {% elif "navigation.tabs" not in features %} - {% set class = class ~ " md-header--shadow" %} - {% endif %} - - - -
- - {% if "navigation.tabs.sticky" in features %} - {% if "navigation.tabs" in features %} - {% include "partials/tabs.html" %} - {% endif %} - {% endif %} -
- - \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/overrides/partials/links.html b/src/cheat-sheet-docs/docs/overrides/partials/links.html deleted file mode 100644 index 87efa7da6..000000000 --- a/src/cheat-sheet-docs/docs/overrides/partials/links.html +++ /dev/null @@ -1,22 +0,0 @@ - - \ No newline at end of file diff --git a/src/cheat-sheet-docs/docs/tags.md b/src/cheat-sheet-docs/docs/tags.md deleted file mode 100644 index 3b49781ca..000000000 --- a/src/cheat-sheet-docs/docs/tags.md +++ /dev/null @@ -1,5 +0,0 @@ -# Tags - -Following is a list of relevant tags: - - \ No newline at end of file diff --git a/src/cheat-sheet-docs/mkdocs.yml b/src/cheat-sheet-docs/mkdocs.yml deleted file mode 100644 index dcb2cd673..000000000 --- a/src/cheat-sheet-docs/mkdocs.yml +++ /dev/null @@ -1,90 +0,0 @@ -site_name: Troubleshooting Cheat Sheet -site_author: RunWhen Inc. -dev_addr: 0.0.0.0:8081 -theme: - name: material - logo: assets/white_runwhen_logo_transparent_bg.png - favicon: assets/icon.png - palette: - - media: "(prefers-color-scheme: blue)" - scheme: default - toggle: - icon: material/toggle-switch-off-outline - name: Switch to dark mode - primary: blue - accent: "deep-orange" - - media: "(prefers-color-scheme: dark)" - scheme: slate - toggle: - icon: material/toggle-switch - name: Switch to light mode - primary: blue - accent: "deep-orange" - font: - text: "Roboto" - code: "Roboto Mono" - color: "blue" - icon: - admonition: - note: fontawesome/solid/note-sticky - abstract: fontawesome/solid/book - info: octicons/info-16 - tip: fontawesome/solid/bullhorn - success: fontawesome/solid/check - question: fontawesome/solid/circle-question - warning: fontawesome/solid/triangle-exclamation - failure: fontawesome/solid/bomb - danger: fontawesome/solid/skull - bug: fontawesome/solid/robot - example: fontawesome/solid/flask - quote: fontawesome/solid/quote-left - features: - - content.code.copy - - search.suggest - - search.share - - content.action.edit - - content.action.view - # - navigation.instant # Disabled as it breaks search on the main page - - navigation.tracking - - toc.integrate - custom_dir: docs/overrides - demo: true -build: - date: today - version: 0.1 - scan_date: today - demo: false - terminal_disabled: false -extra_css: - - css/custom.css -# extra_javascript: -# - javascript/server.js -docs_dir: docs -exclude_docs: | - /output/* - output/ - *.yaml -plugins: - - search - - tags: - tags_file: tags.md - -markdown_extensions: - # Python Markdown - - abbr - - admonition - - attr_list - - def_list - - footnotes - - md_in_html - - toc: - permalink: true - - pymdownx.superfences: - preserve_tabs: true - - pymdownx.tabbed: - alternate_style: true - - pymdownx.details - - pymdownx.inlinehilite - - pymdownx.tasklist - - pymdownx.extra - - codehilite diff --git a/src/cheat-sheet-docs/templates/doc-template.j2 b/src/cheat-sheet-docs/templates/doc-template.j2 deleted file mode 100644 index 9c66e4cbc..000000000 --- a/src/cheat-sheet-docs/templates/doc-template.j2 +++ /dev/null @@ -1,150 +0,0 @@ ---- -title: "{{ slx_hints['nice_name'] }}" -{% set has_tags = slx_hints.tags or parsed_robot['support_tags'] %} -{% if has_tags %} -tags: - {% if slx_hints.tags %} - {% for key, value in slx_hints.tags.items() %} - {% if value is not none %} - - "{{ key }}: {{ value }}" - {% endif %} - {% endfor %} - {% endif %} - {% if parsed_robot['support_tags'] %} - {% for tag in parsed_robot['support_tags'] %} - - "{{ tag }}" - {% endfor %} - {% endif %} -{% endif %} ---- - - - - - - - -
- Icon -
-# {{ slx_hints["nice_name"] }} -
- Profile Avatar -
-

- Icon 1 - {{ command_count }} Troubleshooting Commands

-

- Icon 1 - Last updated {{ commit_age }}

-

- Icon 1 - Contributed by {{ author_details["username"] }}

-
-
- - -

-
- -
- -### Troubleshooting Commands - -{% for command in interesting_commands %} - -!!! note "" -
- #### {{ command["name"] }} -
- !!! warning "" -
- What does it do? -
-

- {{ command["explanation"] | string | indent(4) }}

-
- Command -
-
-
```{{ command["command"]["private"] }}```
-
-
- IconCopy to clipboard - Copied to clipboard -
- ???- info "Learn more" - This multi-line content is auto-generated and used for educational purposes. Copying and pasting the multi-line text might not function as expected. - - {{ command["multi_line_details"] | string | indent(12) }} - - ???- abstract "Helpful Links" - - {{ command["doc_links"] | string | indent(8) }} - - - - -{% endfor %} - - - - - ---- \ No newline at end of file diff --git a/src/cheat-sheet-docs/templates/home-template.j2 b/src/cheat-sheet-docs/templates/home-template.j2 deleted file mode 100644 index 7199359dc..000000000 --- a/src/cheat-sheet-docs/templates/home-template.j2 +++ /dev/null @@ -1,74 +0,0 @@ -{% raw %} -{% extends "main.html" %} - -{% block header %} - {{ super() }} -{% endblock %} - -{% block tabs %} -{{ super() }} -{% import "partials/language.html" as lang with context %} - -{% endraw %} -
-
-

- RunWhen | Troubleshooting Cheat Sheet -
-
-

Discover the Troubleshooting Tasks from an Expert Community
- that powers the brain of the RunWhen Platform Digital Assistants -
-
- -
- {% if summarized_resources.get("clusters", 0) > 0 %} - - {% endif %} - - {% if summarized_resources.get("resource_groups", 0) > 0 %} - - {% endif %} - - {% if summarized_resources.get("aws_resources", 0) > 0 %} - - {% endif %} - - {% if summarized_resources.get("gcp_resources", 0) > 0 %} - - {% endif %} - - {% if summarized_resources.get("groups", 0) > 0 %} - - {% endif %} -
- -

- {{ command_generation_summary_stats["total_interesting_commands"] }} customized tasks across - {{ total_codebundles }} codebundles from - {{ command_generation_summary_stats["num_unique_authors"] }} contributors -

-
-
- - -{% raw %} -{% endblock %} - -{% block content %} -{{ page.content }} -{% endblock %} -{% block footer %} -{{ super() }} -{% endblock %} -{% endraw %} diff --git a/src/cheat-sheet-docs/templates/index-template.j2 b/src/cheat-sheet-docs/templates/index-template.j2 deleted file mode 100644 index 3ff10581c..000000000 --- a/src/cheat-sheet-docs/templates/index-template.j2 +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Home -template: home.html -hide: - - navigation - - toc ---- - -
-

Top Troubleshooting Categories


-

Troubleshooting tasks that matched your environment also support the following technologies:

-
- {% for tag in tags_with_icons %} - - {% endfor %} - -
-
-

- \ No newline at end of file diff --git a/src/cheat-sheet-docs/templates/index-template.j2.orig b/src/cheat-sheet-docs/templates/index-template.j2.orig deleted file mode 100644 index 70ea0fecf..000000000 --- a/src/cheat-sheet-docs/templates/index-template.j2.orig +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: Home -template: home.html ---- - - \ No newline at end of file diff --git a/src/cheat-sheet-docs/templates/platform-upload.j2 b/src/cheat-sheet-docs/templates/platform-upload.j2 deleted file mode 100644 index 4a44beb18..000000000 --- a/src/cheat-sheet-docs/templates/platform-upload.j2 +++ /dev/null @@ -1,202 +0,0 @@ ---- -search: - exclude: true ---- -# RunWhen Platform Upload -The Workspace Builder component of RunWhen Local generates configuration files that can be used in the RunWhen Platform, which provides: - -- Collaborative & interactive operational maps -- Collaborative & automated troubleshooting sessions -- Troubleshooting Digital Assistants -- Automated workflows - -> Workspace Builder generated a total of: **{{ slx_count }} SLXs** - - - -## QuickStart -
-
-

Don't have a RunWhen Workspace yet?

- Create a Workspace -
-
-

Want to use a self-hosted private runner?

- Register a Runner -
-
-

Attach your Workspace Configuration and Upload

- - -
-
-

Add Your Secrets (see section below for more details)

- Learn about Secrets -
-
-

Check out the Workspace Map (it may take a few minutes to populate)

- Learn about Workspace Maps -
-
- -!!! info "Documentation" - For complete documentation on integrating RunWhen Local with the RunWhen Platform, see [this link](https://docs.runwhen.com/public/getting-started/getting-started-with-runwhen-platform) - - - - - diff --git a/src/cheatsheet.py b/src/cheatsheet.py deleted file mode 100644 index 0787d6c1c..000000000 --- a/src/cheatsheet.py +++ /dev/null @@ -1,1033 +0,0 @@ -""" -Utility file to parse robot file, runbook config, -and perform variable substitution to produce clean -shell cmd output. -parse_robot_file written by Kyle Forster - -Author: Shea Stewart -""" -import sys -import os -import fnmatch -import re -import shutil -import jinja2 -import requests -import yaml -import json -import datetime -import time -import ruamel.yaml -import subprocess -import logging -import tempfile -from utils import get_proxy_config, get_request_verify -from robot.api import TestSuite -from tempfile import NamedTemporaryFile -from functools import lru_cache -from git import Repo, GitCommandError -from concurrent.futures import ThreadPoolExecutor -from urllib.parse import urlparse -from collections import Counter - -logger = logging.getLogger(__name__) -handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) - -# Tags -all_support_tags = [] -support_tags_to_remove = [] - -# Check for the environment variable and set the log level -if os.environ.get('DEBUG_LOGGING') == 'true': - logger.setLevel(logging.DEBUG) -else: - logger.setLevel(logging.INFO) - - -# ------------------------------------------------------------------ -# Shared git-mirror helpers (same semantics as code_collection.py) -# ------------------------------------------------------------------ -# ---------- resolve USE_LOCAL_GIT (precedence: env-var > workspaceInfo.yaml > default) ---------- -USE_LOCAL_GIT: bool = False # 1) default - -# 2) workspaceInfo.yaml (lowest override) -_ws_path = "/shared/workspaceInfo.yaml" -try: - with open(_ws_path, "r") as _f: - _cfg = yaml.safe_load(_f) or {} - _ws_val = _cfg.get("useLocalGit") - if isinstance(_ws_val, bool): - USE_LOCAL_GIT = _ws_val - elif isinstance(_ws_val, str): - USE_LOCAL_GIT = _ws_val.lower() == "true" -except Exception: - pass - -# 3) env-var override (highest precedence – wins over workspaceInfo) -_env_val = os.getenv("WB_USE_LOCAL_GIT") -if _env_val is not None: - USE_LOCAL_GIT = _env_val.lower() == "true" - -logger.info("USE_LOCAL_GIT resolved to %s (env=%s, yaml=%s)", - USE_LOCAL_GIT, os.getenv("WB_USE_LOCAL_GIT"), - _ws_path if os.path.isfile(_ws_path) else "") -# ---------- end USE_LOCAL_GIT resolution ---------- - -LOCAL_CACHE_ROOT = os.getenv("CODE_COLLECTION_CACHE_ROOT", - "/opt/runwhen/codecollection-cache") - -def mirror_path(owner: str, repo: str) -> str: - """Return the bare-mirror directory for owner/repo or '' if absent.""" - return os.path.join(LOCAL_CACHE_ROOT, f"{repo}.git") - -def ensure_worktree_from_mirror(owner: str, repo: str, ref: str, dest: str): - """ - Materialise (branch *or* tag) from the bare mirror into . - - * When USE_LOCAL_GIT is true, the mirror **must** exist. - * Creates the work-tree in **detached HEAD** mode so tags work. - """ - mpath = mirror_path(owner, repo) - - if USE_LOCAL_GIT: - if not os.path.isdir(mpath): - raise RuntimeError( - f"Local git mirror missing for {owner}/{repo}. " - "Rebuild the image with INCLUDE_CODE_COLLECTION_CACHE=true " - "or disable useLocalGit." - ) - - bare = Repo(mpath) - - # Resolve commitish (branch, tag, or SHA); raise if absent. - try: - commitish = bare.git.rev_parse("--verify", ref) - except GitCommandError: - # try explicit tag path - try: - commitish = bare.git.rev_parse("--verify", f"refs/tags/{ref}") - except GitCommandError as exc: - raise RuntimeError(f"Ref '{ref}' not found in mirror {owner}/{repo}") from exc - - # If dest exists and is already a worktree on the same commit, skip. - if os.path.isdir(dest): - try: - wt = Repo(dest) - if wt.head.commit.hexsha == commitish: - return - except Exception: - shutil.rmtree(dest) - - # Add work-tree in detached-HEAD mode so both branches & tags work. - bare.git.worktree("add", "--force", "--detach", dest, commitish) - return - - # -------- network-clone fallback -------- - Repo.clone_from( - f"https://github.com/{owner}/{repo}.git", - dest, - branch=ref, - depth=1 - ) -# ------------------------------------------------------------------ - - -@lru_cache(maxsize=2048) -def parse_robot_file(fpath): - """ - Parses a robot file into a python object that is - JSON-serializable, representing key items about the file contents. - """ - suite = TestSuite.from_file_system(fpath) - ret = {} - ret["doc"] = suite.doc # The doc string - ret["type"] = suite.name.lower() - ret["tags"] = [] - ret["supports"] = [] - - for k, v in suite.metadata.items(): - if k.lower() in ["author", "name"]: - ret[k.lower()] = v - if k.lower() in ["supports"]: - support_tags = re.split(r'\s*,\s*|\s+', v.strip().upper()) - ret["support_tags"] = support_tags - all_support_tags.extend(support_tags) - - tasks = [] - for task in suite.tests: - tags = [str(tag) for tag in task.tags if tag not in ["skipped"]] - tasks.append( - { - "id": task.id, - "name": task.name, - "doc": str(task.doc), - "keywords": task.body - } - ) - ret["tags"] = list(set(ret["tags"] + tags)) - ret["tasks"] = tasks - - resourcefile = suite.resource - ret["imports"] = [] - for i in resourcefile.imports: - ret["imports"].append(i.name) - return ret - -def strip_start_end_quotes(cmd): - if (cmd.startswith('"') and cmd.endswith('"')) or (cmd.startswith("'") and cmd.endswith("'")): - return cmd[1:-1] - else: - return cmd - -def remove_escape_chars(cmd): - # Cases where the command needs to use ${} but it's escaped from robot - cmd = cmd.replace(r'\\\$', '$') - cmd = cmd.replace(r'\\\%', '%') - cmd = cmd.replace('\\\n', '') - cmd = cmd.replace('\\\\', '\\') - # Cases where two spaces are escaped in robot - cmd = cmd.replace(' \\ ', ' ') - cmd = cmd.encode().decode('unicode_escape') - # Handle wrapped quotes - if len(cmd) > 1 and cmd[0] == cmd[-1] == '"': - cmd = cmd[1:-1] - if len(cmd) > 1 and cmd[0] == cmd[-1] == "'": - cmd = cmd[1:-1] - return cmd - -def safe_substitute(original_string, placeholder, replacement): - """ - Safely substitutes a placeholder in a string with the replacement value. - If the replacement is None or not a string, it substitutes with an empty string. - """ - replacement = str(replacement) if isinstance(replacement, str) else '' - return original_string.replace(placeholder, replacement) - -def cmd_expansion(keyword_arguments, parsed_runbook_config): - """ - Cleans up the command details from robot parsing, - attempts to make the command human readable, substituting - config-provided values. - """ - cmd = {} - cmd_components = str(keyword_arguments) - logger.debug(f"Pre-rendered command components: {cmd_components}") - logger.debug(f"Runbook config: {parsed_runbook_config}") - - cmd_components = cmd_components.lstrip('(').rstrip(')') - - logger.debug(f"Command Components: {cmd_components}") - split_regex = re.compile(r'''((?:[^,'"]|'(?:(?:\\')|[^'])*'|"(?:\\"|[^"])*")+)''') - cmd_components = split_regex.split(cmd_components)[1::2] - - # Basic check for position - logger.debug(f"Command: {cmd_components[0]}") - if len(cmd_components) > 1: - logger.debug(f"Arguments: {cmd_components[1]}") - - if cmd_components[0].startswith(("\'cmd=", "cmd=", "\"cmd=")): - cmd_components[0] = cmd_components[0].replace('cmd=', '') - cmd_str = cmd_components[0] - cmd_str = remove_auth_commands(cmd_str) - cmd_str = replace_env_key(cmd_str) - - service_name = "" - if parsed_runbook_config.get("spec", {}).get("servicesProvided"): - service_name = parsed_runbook_config["spec"]["servicesProvided"][0].get("name", "") - logger.debug(f"Safe substitution of {service_name}.") - cmd_str = safe_substitute(cmd_str, '${binary_name}', service_name) - cmd_str = safe_substitute(cmd_str, '${BINARY_USED}', service_name) - cmd_str = safe_substitute(cmd_str, '${KUBERNETES_DISTRIBUTION_BINARY}', service_name) - else: - logger.debug("No services provided in 'servicesProvided'; using empty string for service_name.") - - elif cmd_components[0].startswith('\'bash_file='): - logger.debug(f"Rendering bash file: {cmd_components[0]}") - script = cmd_components[0].replace('bash_file=', '') - codebundle_path_parts = parsed_runbook_config['spec']['codeBundle']['pathToRobot'].split('/') - codebundle_directory_path = '/'.join(codebundle_path_parts[:-1]) - file_path = f"{codebundle_directory_path}/{script}".replace("'", "") - logger.debug(f"Rendering bash file path: {file_path}") - - raw_script_url = generate_raw_git_url( - git_url=parsed_runbook_config["spec"]["codeBundle"]["repoUrl"], - ref=parsed_runbook_config["spec"]["codeBundle"]["ref"], - file_path=file_path - ) - env = "" - for var in parsed_runbook_config["spec"]["configProvided"]: - env += f"{var['name']}=\"{var['value']}\" " - - matched_cmd_override = None - for arg in cmd_components: - arg = arg.strip() - logger.debug(f"Bash File Arg: {arg}") - if arg.startswith("\'cmd_override"): - logger.debug(f"Command Override Detected: {arg}") - matched_cmd_override = arg - break - - if matched_cmd_override is not None: - cmd_parts = matched_cmd_override.split() - cmd_arguments = ' '.join(cmd_parts[1:]).strip("'") - cmd_components[0] = f"{env} bash -c \"$(curl -s {raw_script_url})\" _ {cmd_arguments}" - else: - cmd_components[0] = f"{env} bash -c \"$(curl -s {raw_script_url})\" _" - - logger.debug(f"Rendering bash file after split: {cmd_components}") - cmd_str = cmd_components[0] - else: - cmd_str = "Could not render command" - - cmd["public"] = remove_escape_chars(cmd_str) - - for var in parsed_runbook_config["spec"].get("configProvided", []): - placeholder = '${' + var["name"] + '}' - cmd_str = safe_substitute(cmd_str, placeholder, var["value"]) - - cmd["private"] = remove_escape_chars(cmd_str) - return cmd - -def generate_raw_git_url(git_url, file_path, ref): - """ - Returns a file:// URL when using local mirrors, falling back to raw.github - when network clones are allowed. - """ - parsed = urlparse(git_url) - owner, repo = parsed.path.lstrip("/").rstrip(".git").split("/")[:2] - - if USE_LOCAL_GIT: - mirror = mirror_path(owner, repo) - if not os.path.isdir(mirror): - logger.error("Mirror missing for %s/%s", owner, repo) - return None - return f"file://{mirror}/{file_path}" # consumed via 'bash -c "$(cat …)"' - - # original behaviour (network) - return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{file_path}" - - -def task_name_expansion(task_name, parsed_runbook_config): - """ - Cleans up the task title from robot parsing, - substituting dynamic vars from configProvided. - """ - logger.debug(f"Task Title Substitution: return {task_name}") - for var in parsed_runbook_config["spec"]["configProvided"]: - value_str = str(var["value"]) if var["value"] is not None else '' - if var["value"] is None: - logger.warning(f"Variable '{var['name']}' is None; substituting empty string.") - task_name = task_name.replace('${' + var["name"] + '}', value_str) - logger.debug(f"Tailored var name {var['name']} for {value_str}") - return task_name - -def remove_auth_commands(command): - """ - Removes common authentication patterns in CLI output, - assuming the user is already authenticated differently. - """ - auth_patterns = [ - "gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS && " - ] - for pattern in auth_patterns: - command = command.replace(pattern, '') - return command - -def replace_env_key(text): - """ - Replaces a pattern like $${JENKINS_SA_USERNAME.key} with ${JENKINS_SA_USERNAME}. - """ - pattern = r'\$\${([^}.]+)\.[^}]+\}' - replacement = r'${\1}' - return re.sub(pattern, replacement, text) - -def search_keywords(parsed_robot, parsed_runbook_config, search_list, meta): - """ - Search through the keywords in the robot file, - looking for interesting patterns from search_list. - """ - if meta is None: - meta = {"commands": []} - - def is_unique_command(cmd_list, cmd_dict): - return not any( - c['name'] == cmd_dict['name'] and c['command'] == cmd_dict['command'] - for c in cmd_list - ) - - commands = [] - for task in parsed_robot['tasks']: - for keyword in task['keywords']: - if hasattr(keyword, 'name'): - for item in search_list: - if item in keyword.args: - task_name = task_name_expansion(task["name"], parsed_runbook_config) - task_name_generalized = ( - task["name"].replace('`', '').replace('${', '').replace('}', '') - ) - name_snake_case = re.sub(r'\W+', '_', task_name_generalized.lower()) - command_info = { - "name": f"{task_name}", - "command": cmd_expansion(keyword.args, parsed_runbook_config) - } - logger.debug(f"Searching for command name in meta: {name_snake_case}") - for cmd_meta in meta['commands']: - if cmd_meta['name'] == name_snake_case: - logger.debug(f"Found meta for {name_snake_case}") - command_info['explanation'] = cmd_meta.get('explanation', "No command explanation available") - command_info['multi_line_details'] = cmd_meta.get('multi_line_details', "No multi-line explanation available") - command_info['doc_links'] = cmd_meta.get('doc_links', []) - break - else: - command_info['explanation'] = "No command explanation available" - command_info['multi_line_details'] = "No multi-line explanation available" - command_info['doc_links'] = [] - - if is_unique_command(commands, command_info): - commands.append(command_info) - - return commands - -def parse_yaml(fpath): - with open(fpath, 'r') as file: - return yaml.safe_load(file) - -def find_files(directory, pattern): - """ - Recursively search directory for files matching pattern. - """ - matches = [] - for root, dirnames, filenames in os.walk(directory): - for filename in fnmatch.filter(filenames, pattern): - matches.append(os.path.join(root, filename)) - return matches - -def fetch_robot_source(parsed_runbook_config, mkdocs_dir): - """ - Fetch raw robot file from the local cache - referenced by runbook.yaml. - """ - repo = parsed_runbook_config["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-1] - owner = parsed_runbook_config["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-2] - ref = parsed_runbook_config["spec"]["codeBundle"]["ref"] - robot_file_path = parsed_runbook_config["spec"]["codeBundle"]["pathToRobot"] - - cache_dir_name = f"{owner}_{repo}_{ref}-cache" - local_path = os.path.join(mkdocs_dir, cache_dir_name) - file_path = os.path.join(local_path, robot_file_path) - return file_path - -def generate_slx_hints(runbook_path): - """ - From runbook path, find the corresponding SLX - and generate hints from additionalContext, etc. - """ - parsed_slx = parse_yaml(runbook_path.replace('runbook', 'slx')) - slx_hints = { - "slug": f'{parsed_slx["spec"]["alias"]}'.replace(' ', '-'), - "icon": parsed_slx["spec"].get("imageURL", "https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/cloud_default.svg"), - "slx_short_name": f'{parsed_slx["metadata"]["name"]}'.split('--')[1].strip(), - "nice_name": f'{parsed_slx["spec"]["alias"]} ', - "statement": f'{parsed_slx["spec"]["statement"]}', - "as_measured_by": f'As Measured By: {parsed_slx["spec"]["asMeasuredBy"]}' - } - - allowed_tags = ["namespace", "cluster", "project", "resource_group"] - additional_context = parsed_slx.get("spec", {}).get("additionalContext", {}) - slx_hints["tags"] = {k: v for k, v in additional_context.items() if k in allowed_tags} - return slx_hints - -def find_group_name(groups, target_slx): - for group in groups: - if target_slx in group['slxs']: - return group['name'] - return 'ungrouped' - -def find_cluster_name(groups, target_slx): - for group in groups: - if target_slx in group['slxs']: - return group['name'] - return 'ungrouped' - -def update_last_scan_time(mkdocs_dir): - """ - Update the 'scan_date' field in mkdocs.yml at mkdocs_dir. - """ - file_path = os.path.join(mkdocs_dir, "mkdocs.yml") - - try: - with open(file_path, "r") as file: - content = file.readlines() - - current_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - updated_content = [] - for line in content: - if line.strip().startswith("scan_date:") or "scan_date:" in line: - # Loose check in case indentation changes - prefix_spaces = line.split("scan_date:")[0] - line = f"{prefix_spaces}scan_date: {current_date}\n" - updated_content.append(line) - - with open(file_path, "w") as file: - file.writelines(updated_content) - except Exception as e: - print(f"An error occurred while updating the file: {e}") - -def generate_index(all_support_tags_freq, - summarized_resources, - workspace_details, - command_generation_summary_stats, - slx_count, - mkdocs_dir): - """ - Generate an index.md and overrides/home.html in mkdocs_dir/docs, - using Jinja2 templates from mkdocs_dir/templates. - """ - import jinja2 - import datetime - import os - - # Paths where we'll render final output - index_path = os.path.join(mkdocs_dir, "docs", "index.md") - home_path = os.path.join(mkdocs_dir, "docs", "overrides", "home.html") - - # The directory where the .j2 templates live - templates_dir = os.path.join(mkdocs_dir, "templates") - - # Basenames for the Jinja templates - index_template_file = "index-template.j2" - home_template_file = "home-template.j2" - - # Create two separate Jinja environments (or reuse one if you prefer) - index_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_dir)) - home_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_dir)) - - # Get the actual Template objects by their names - index_template = index_env.get_template(index_template_file) - home_template = home_env.get_template(home_template_file) - - current_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - # Example data: cluster list -> string - cluster_list = [str(c) for c in summarized_resources.get("cluster_names", [])] - cluster_names = ', '.join(cluster_list) if cluster_list else None - - # Frequency of tags - top_10_support_tags = all_support_tags_freq.most_common(10) - top_10_support_tag_names = [tag for tag, _ in top_10_support_tags] - tag_icon_url_map = load_icon_urls_for_tags(top_10_support_tag_names) - - tags_with_icons = [{ - "name": tag, - "icon_url": tag_icon_url_map.get(tag, "default_icon_url") - } for tag in top_10_support_tag_names] - - # Summaries - resource_summary = { - "clusters": summarized_resources.get("clusters", 0), - "resource_groups": summarized_resources.get("resource_groups", 0), - "aws_resources": summarized_resources.get("aws_resources", 0), - "gcp_resources": summarized_resources.get("gcp_resources", 0), - "groups": summarized_resources.get("groups", 0) - } - resource_summary = {k: v for k, v in resource_summary.items() if v > 0} - - # Render index - index_output = index_template.render( - current_date=current_date, - summarized_resources=summarized_resources, - workspace_details=workspace_details, - cluster_names=cluster_names, - command_generation_summary_stats=command_generation_summary_stats, - slx_count=slx_count, - tags_with_icons=tags_with_icons - ) - - # Render home - home_output = home_template.render( - current_date=current_date, - summarized_resources=summarized_resources, - resource_summary=resource_summary, - workspace_details=workspace_details, - cluster_names=cluster_names, - command_generation_summary_stats=command_generation_summary_stats, - slx_count=slx_count - ) - - # Ensure directories exist - os.makedirs(os.path.dirname(index_path), exist_ok=True) - with open(index_path, 'w') as index_file: - index_file.write(index_output) - - os.makedirs(os.path.dirname(home_path), exist_ok=True) - with open(home_path, 'w') as home_file: - home_file.write(home_output) - - -def env_check(mkdocs_dir): - """ - Loads and re-saves mkdocs.yml in mkdocs_dir to preserve quotes, indentation, etc. - """ - config_file = os.path.join(mkdocs_dir, "mkdocs.yml") - if not os.path.exists(config_file): - logger.warning(f"No mkdocs.yml found at {config_file}; skipping env_check.") - return - - yaml_loader = ruamel.yaml.YAML() - yaml_loader.preserve_quotes = True - yaml_loader.indent(mapping=4, sequence=4, offset=2) - with open(config_file, "r") as f: - config = yaml_loader.load(f) - - with open(config_file, "w") as f: - yaml_loader.dump(config, f) - -cache = {} -command_generation_summary_stats = { - "total_interesting_commands": 0, - "unique_authors": [], - "num_unique_authors": 0 -} - -def load_cache(mkdocs_dir): - """ - Loads a JSON cache from mkdocs_dir/docs/github_profile_cache/cache.json - if present. - """ - cache_directory = os.path.join(mkdocs_dir, "docs", "github_profile_cache") - cache_file = os.path.join(cache_directory, "cache.json") - if os.path.exists(cache_file): - with open(cache_file, "r") as file: - return json.load(file) - return {} - -def save_cache(mkdocs_dir): - """ - Saves the global 'cache' to mkdocs_dir/docs/github_profile_cache/cache.json - """ - cache_directory = os.path.join(mkdocs_dir, "docs", "github_profile_cache") - cache_file = os.path.join(cache_directory, "cache.json") - os.makedirs(cache_directory, exist_ok=True) - with open(cache_file, "w") as file: - json.dump(cache, file) - -def fetch_github_profile_icon(identifier, mkdocs_dir=None): - """ - Look up GitHub user info for 'identifier' (user or email), - caching results in mkdocs_dir/docs/github_profile_cache/cache.json. - """ - if mkdocs_dir is None: - mkdocs_dir = "cheat-sheet-docs" # fallback if none provided - - global cache - if not cache: - cache = load_cache(mkdocs_dir) - - if identifier in cache: - return cache[identifier] - - author_details = {} - try: - response = requests.get( - f"https://api.github.com/search/users?q={identifier}", - verify=get_request_verify() - ) - data = response.json() - if data['total_count'] == 0: - return None - - username = data['items'][0]['login'] - - cache_directory = os.path.join(mkdocs_dir, "docs", "github_profile_cache") - os.makedirs(cache_directory, exist_ok=True) - - profile_icon_file = os.path.join(cache_directory, f"{username}_icon.png") - if os.path.exists(profile_icon_file): - author_details["username"] = username - author_details["profile_icon_path"] = profile_icon_file - author_details["url"] = f"https://github.com/{username}" - else: - # fetch user info - resp_user = requests.get(f"https://api.github.com/users/{username}", verify=get_request_verify()) - user_data = resp_user.json() - - author_details["username"] = user_data["login"] - author_details["profile_icon_path"] = profile_icon_file - author_details["url"] = user_data["html_url"] - - # download and cache icon - profile_icon_url = user_data["avatar_url"] - icon_resp = requests.get(profile_icon_url, verify=get_request_verify()) - with open(profile_icon_file, "wb") as file: - file.write(icon_resp.content) - - cache[identifier] = author_details - save_cache(mkdocs_dir) - return author_details - - except requests.RequestException as e: - logging.error(f"Error occurred: {str(e)}") - return "Not Available" - except requests.HTTPError as e: - if e.response.status_code == 403: - author_details["username"] = "apiLimitReached" - author_details["profile_icon_path"] = "apiLimitReached" - author_details["url"] = "apiLimitReached" - return author_details - logging.error(f"HTTP error occurred: {str(e)}") - return "Not Available" - except KeyError: - logging.warning("KeyError occurred: Required data not found in the API response.") - return "Not Available" - -def get_last_commit_age(owner, repo, ref, path, mkdocs_dir): - local_path = os.path.join(mkdocs_dir, f'{owner}_{repo}_{ref}-cache') - if not os.path.exists(local_path): - logging.debug(f"The repository for {owner}/{repo} with reference {ref} is not found in the cache.") - return None - - repo_obj = Repo(local_path) - if ref not in repo_obj.heads: - logging.debug(f"Reference {ref} not found in the cached repository for {owner}/{repo}.") - return None - - commits_touching_path = list(repo_obj.iter_commits(paths=path)) - if not commits_touching_path: - return None - - last_commit_timestamp = commits_touching_path[0].committed_date - last_commit_datetime = datetime.datetime.fromtimestamp(last_commit_timestamp) - current_datetime = datetime.datetime.now() - commit_age = current_datetime - last_commit_datetime - - age_in_hours = commit_age.total_seconds() / 3600 - if 1 <= age_in_hours < 48: - return f"{int(age_in_hours)} hours ago" - age_in_days = age_in_hours / 24 - if 2 <= age_in_days < 14: - return f"{int(age_in_days)} days ago" - age_in_weeks = age_in_days / 7 - if age_in_weeks >= 2: - return f"{int(age_in_weeks)} weeks ago" - return None - -def fetch_meta(owner, repo, path, mkdocs_dir, ref="main"): - cache_dir_name = f"{owner}_{repo}_{ref}-cache" - meta_path = path.rsplit('/', 1)[0] + '/meta.yaml' - local_path = os.path.join(mkdocs_dir, cache_dir_name) - local_meta_path = os.path.join(local_path, meta_path) - if os.path.exists(local_meta_path): - try: - with open(local_meta_path, 'r') as f: - yaml_data = yaml.safe_load(f) - return yaml_data - except IOError as e: - print(f"Failed to read local meta.yaml file due to: {str(e)}") - return None - -def find_group_path(group_name): - """ - Looks for a pattern like "group-name (cluster-name)" - to handle nested doc paths. - """ - cluster_separator = r'^(.*?)\s+\((.*?)\)$' - cluster_match = re.match(cluster_separator, group_name) - if cluster_match: - gname = cluster_match.group(1) - cname = cluster_match.group(2) - return f'{cname}/{gname}' - return group_name - -def warm_git_cache(runbook_files, mkdocs_dir): - """ - Build a unique set of repos from runbook_files and materialise them - from the local mirror (or network, if allowed) into mkdocs_dir. - """ - unique_repos = { - ( - cfg["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-2], # owner - cfg["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-1], # repo - cfg["spec"]["codeBundle"]["ref"] # ref - ) - for cfg in map(parse_yaml, runbook_files) - } - - for owner, repo, ref in unique_repos: - worktree_path = os.path.join(mkdocs_dir, f"{owner}_{repo}_{ref}-cache") - if os.path.isdir(worktree_path): - # optional: fast-forward to latest mirror state - try: - Repo(worktree_path).git.reset("--hard", f"origin/{ref}") - except GitCommandError: - shutil.rmtree(worktree_path) - - if not os.path.isdir(worktree_path): - logger.info("Materialising %s/%s@%s", owner, repo, ref) - ensure_worktree_from_mirror(owner, repo, ref, worktree_path) - - # collect authors for summary - for rb in find_files(worktree_path, 'runbook.robot'): - author = ''.join(parse_robot_file(rb).get("author", "").split()) - if author: - fetch_github_profile_icon(author, mkdocs_dir) - - -def clean_path(path): - """ - Deletes the specified path (file or directory). - """ - if os.path.exists(path): - if os.path.isdir(path): - shutil.rmtree(path) - print(f"Directory '{path}' has been removed along with its contents.") - else: - os.remove(path) - print(f"File '{path}' has been removed.") - else: - print(f"The path '{path}' does not exist.") - -def process_runbook(runbook, groups, search_list, template, mkdocs_dir): - """ - Renders a single runbook, writing the results into mkdocs_dir/docs-tmp. - """ - try: - parsed_runbook_config = parse_yaml(runbook) - robot_file = fetch_robot_source(parsed_runbook_config, mkdocs_dir) - runbook_url = ( - f'{parsed_runbook_config["spec"]["codeBundle"]["repoUrl"].rstrip(".git")}' - f'/tree/{parsed_runbook_config["spec"]["codeBundle"]["ref"]}/' - f'{parsed_runbook_config["spec"]["codeBundle"]["pathToRobot"]}' - ) - owner = parsed_runbook_config["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-2] - repo = parsed_runbook_config["spec"]["codeBundle"]["repoUrl"].rstrip(".git").split("/")[-1] - path = parsed_runbook_config["spec"]["codeBundle"]["pathToRobot"].rstrip('runbook.robot') - ref = parsed_runbook_config["spec"]["codeBundle"]["ref"] - - commit_age = get_last_commit_age(owner, repo, ref, path, mkdocs_dir) - parsed_robot = parse_robot_file(robot_file) - slx_hints = generate_slx_hints(runbook) - doc = ''.join(parsed_robot.get("doc", "").split('\n')) - author = ''.join(parsed_robot.get("author", "").split('\n')) - group_name = find_group_name(groups, slx_hints["slx_short_name"]) - group_path = find_group_path(group_name) - meta = fetch_meta(owner=owner, repo=repo, path=path, mkdocs_dir=mkdocs_dir, ref=ref) - - interesting_commands = search_keywords(parsed_robot, parsed_runbook_config, search_list, meta) - command_generation_summary_stats["total_interesting_commands"] += len(interesting_commands) - command_generation_summary_stats["unique_authors"].append(author) - - output = template.render( - runbook=runbook.split("/output", 1)[-1], - author=author, - slx_hints=slx_hints, - doc=doc, - runbook_url=runbook_url, - interesting_commands=interesting_commands, - command_count=len(interesting_commands), - author_details=fetch_github_profile_icon(author, mkdocs_dir), - commit_age=commit_age, - parsed_robot=parsed_robot - ) - - docs_tmp_dir = os.path.join(mkdocs_dir, 'docs-tmp', group_path) - os.makedirs(docs_tmp_dir, exist_ok=True) - md_output_path = os.path.join(docs_tmp_dir, f'{slx_hints["slug"]}.md') - with open(md_output_path, 'w') as md_file: - md_file.write(output) - - except Exception as e: - logger.error(f"Failed to process runbook {runbook}: {e}") - - -def load_icon_urls_for_tags(tags, filename="map-tag-icons.yaml", default_url="https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/tag.svg"): - """ - Load icon URLs for given tags from a YAML file, with a default URL for unmapped tags. - - :param tags: A single tag or a list of tags to find icon URLs for. - :param filename: The path to the YAML file. - :param default_url: The default icon URL to use for tags not found in the map. - :return: A dictionary of tags to their icon URLs. - """ - # Ensure tags is a list - if isinstance(tags, str): - tags = [tags] - - tag_icon_url_map = {} - try: - with open(filename, "r") as file: - data = yaml.safe_load(file) - icons = data.get("icons", []) - for tag in tags: - # Initialize each tag with a default URL - tag_icon_url_map[tag] = default_url - for icon in icons: - if tag in icon.get("tags", []): - # Update with specific URL if found - tag_icon_url_map[tag] = icon.get("url") - break - except FileNotFoundError: - print(f"File {filename} not found.") - except yaml.YAMLError as exc: - print(f"Error parsing YAML file: {exc}") - - return tag_icon_url_map - -def remove_custom_tags(file_path): - """ - Strips custom YAML tags like !Something from the file content. - """ - with open(file_path, 'r') as file: - content = file.read() - content = re.sub(r'!\w+', '', content) - return yaml.safe_load(content) - -def parse_and_summarize_resource_dump(file_path): - summarized_resources = { - "clusters": 0, - "namespaces": 0, - "resource_groups": 0, - "azure_resources": 0, - "aws_resources": 0, - "gcp_resources": 0, - } - - try: - data = remove_custom_tags(file_path) - # Summarize clusters - k8s_clusters = data.get("platforms", {}).get("kubernetes", {}).get("resourceTypes", {}).get("cluster", {}).get("instances", []) - summarized_resources["clusters"] = len(k8s_clusters) - - # Summarize namespaces - namespaces = data.get("platforms", {}).get("kubernetes", {}).get("resourceTypes", {}).get("custom", {}).get("instances", []) - summarized_resources["namespaces"] = len(namespaces) - - # Azure - azure_rgs = data.get("platforms", {}).get("azure", {}).get("resourceTypes", {}).get("resource_group", {}).get("instances", []) - azure_vmss = data.get("platforms", {}).get("azure", {}).get("resourceTypes", {}).get("azure_compute_virtual_machine_scale_sets", {}).get("instances", []) - summarized_resources["resource_groups"] = len(azure_rgs) - summarized_resources["azure_resources"] = len(azure_vmss) + len(azure_rgs) - - # AWS - aws_resources = data.get("platforms", {}).get("aws", {}).get("resourceTypes", {}) - summarized_resources["aws_resources"] = sum(len(res.get("instances", [])) for res in aws_resources.values()) - - # GCP - gcp_resources = data.get("platforms", {}).get("gcp", {}).get("resourceTypes", {}) - summarized_resources["gcp_resources"] = sum(len(res.get("instances", [])) for res in gcp_resources.values()) - - except FileNotFoundError: - print(f"File {file_path} not found.") - except yaml.YAMLError as e: - print(f"Error parsing YAML: {e}") - - return summarized_resources - -def cheat_sheet(directory_path, mkdocs_dir): - """ - Main function to parse runbooks in directory_path, - then generate docs inside mkdocs_dir for mkdocs usage. - """ - env_check(mkdocs_dir) - update_last_scan_time(mkdocs_dir) - - search_list = ['render_in_commandlist=true', 'show_in_rwl_cheatsheet=true'] - runbook_files = find_files(directory_path, 'runbook.yaml') - workspace_files = find_files(directory_path, 'workspace.yaml') - - with open("/shared/workspaceInfo.yaml", 'r') as workspace_info_file: - workspace_info = yaml.safe_load(workspace_info_file) - - slx_files = find_files(directory_path, 'slx.yaml') - slx_count = len(slx_files) - - warm_git_cache(runbook_files, mkdocs_dir) - - resource_dump_file = os.path.join(directory_path, 'resource-dump.yaml') - if workspace_files: - workspace_details = parse_yaml(workspace_files[0]) - else: - workspace_details = {} - - # Clear any leftover docs-tmp - docs_tmp_path = os.path.join(mkdocs_dir, 'docs-tmp') - if os.path.exists(docs_tmp_path): - try: - shutil.rmtree(docs_tmp_path) - except Exception as e: - print(f"An error occurred while removing docs-tmp: {e}") - - os.makedirs(docs_tmp_path, exist_ok=True) - os.makedirs(os.path.join(docs_tmp_path, 'ungrouped'), exist_ok=True) - - # Build group directories if present - if "spec" in workspace_details and "slxGroups" in workspace_details["spec"]: - groups = workspace_details["spec"]["slxGroups"] - for group in groups: - doc_group_dir_path = find_group_path(group['name']) - full_path = os.path.join(docs_tmp_path, doc_group_dir_path) - os.makedirs(full_path, exist_ok=True) - else: - groups = [] - - # Prepare Jinja - templates_dir = os.path.join(mkdocs_dir, "templates") - template_file = "doc-template.j2" - - env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_dir)) - template = env.get_template(template_file) - - # Process runbooks in parallel - with ThreadPoolExecutor() as executor: - executor.map( - lambda rb: process_runbook(rb, groups, search_list, template, mkdocs_dir), - runbook_files - ) - - # Move from docs-tmp to docs/ - source_dir = os.path.join(mkdocs_dir, 'docs-tmp') - destination_dir = os.path.join(mkdocs_dir, 'docs') - for root, dirs, files in os.walk(source_dir): - relative_path = os.path.relpath(root, source_dir) - dest_path = os.path.join(destination_dir, relative_path) - os.makedirs(dest_path, exist_ok=True) - for f in files: - src_file = os.path.join(root, f) - dst_file = os.path.join(dest_path, f) - shutil.copy2(src_file, dst_file) - - # Unique authors - command_generation_summary_stats["unique_authors"] = set(command_generation_summary_stats["unique_authors"]) - command_generation_summary_stats["num_unique_authors"] = len(command_generation_summary_stats["unique_authors"]) - - all_support_tags_freq = Counter(all_support_tags) - - # Summarize resources - summarized_resources = { - "groups": len(groups), - "kubernetes_clusters": [], - "azure_resources": [], - "aws_resources": [], - "gcp_resources": [] - } - - # If resourceDumpPath in workspace info, use that; else fallback - resource_dump_path = workspace_info.get("resourceDumpPath", resource_dump_file) - summarized_resources = parse_and_summarize_resource_dump(resource_dump_path) - summarized_resources["groups"] = len(groups) - - generate_index( - all_support_tags_freq, - summarized_resources, - workspace_details, - command_generation_summary_stats, - slx_count, - mkdocs_dir - ) - -if __name__ == "__main__": - # Usage: python cheatsheet.py - cheat_sheet(sys.argv[1], sys.argv[2]) diff --git a/src/component.py b/src/component.py index 5dd857f11..83d24db99 100644 --- a/src/component.py +++ b/src/component.py @@ -248,7 +248,7 @@ def init_components(): # be added here, which is less than ideal, although practically may not be # a huge deal. component_stages_init = ( - (Stage.INDEXER, ["load_resources", "kubeapi", "cloudquery", "azure_devops"]), + (Stage.INDEXER, ["load_resources", "kubeapi", "azureapi", "gcpapi", "awsapi", "cloudquery", "azure_devops"]), (Stage.ENRICHER, ["generation_rules"]), (Stage.RENDERER, ["render_output_items", "dump_resources"]) ) diff --git a/src/config/__init__.py b/src/config/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/config/asgi.py b/src/config/asgi.py deleted file mode 100644 index 2f61c6cb7..000000000 --- a/src/config/asgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -ASGI config for workspace builder project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ -""" - -import os - -from django.core.asgi import get_asgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') - -application = get_asgi_application() diff --git a/src/config/settings.py b/src/config/settings.py deleted file mode 100644 index 8bdd661f8..000000000 --- a/src/config/settings.py +++ /dev/null @@ -1,226 +0,0 @@ -""" -Django settings for workspace builder project. - -Generated by 'django-admin startproject' using Django 4.1.4. - -For more information on this file, see -https://docs.djangoproject.com/en/4.1/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/4.1/ref/settings/ -""" - -from pathlib import Path -import logging.config -import os -import secrets -import sys - -# Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent - - -# ─────────────────────────────────────────────────────────────────────────── -# Secret key -# • Users deploying in production set DJANGO_SECRET_KEY -# • Otherwise we generate a random key at start-up (good enough for -# local / ephemeral usage; it changes every start so session cookies -# become invalid on restart — fine for CLI tool) -# ─────────────────────────────────────────────────────────────────────────── -SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", default=secrets.token_urlsafe(50)) - - - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = ["localhost"] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - "rest_framework", - 'workspace_builder', -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'config.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [BASE_DIR / 'templates'] - , - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'config.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/4.1/ref/settings/#databases - -# Database configuration - use /shared if available (Docker), otherwise local -import os -from pathlib import Path - -def get_database_path(): - """Get the appropriate database path based on environment.""" - shared_path = Path('/shared/db.sqlite3') - local_path = BASE_DIR / 'db.sqlite3' - - # Check if /shared directory exists and is writable - try: - shared_dir = Path('/shared') - if shared_dir.exists() and os.access(shared_dir, os.W_OK): - return shared_path - except (OSError, PermissionError): - pass - - return local_path - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': get_database_path(), - } -} - - - -# Password validation -# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/4.1/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.1/howto/static-files/ - -STATIC_URL = 'static/' - -# Default primary key field type -# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field - -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' - -REST_FRAMEWORK = { - "EXCEPTION_HANDLER": "workspace_builder.exceptions.handle", -} - -# debug_logging = os.environ.get('DEBUG_LOGGING', "false").lower() -# root_log_level = "DEBUG" if debug_logging == 'true' or debug_logging == "1" else "INFO" - -# LOGGING = { -# "version": 1, -# "disable_existing_loggers": False, -# "handlers": { -# "console": { -# "class": "logging.StreamHandler", -# }, -# }, -# "loggers": { -# "root": { -# "handlers": ["console"], -# "level": root_log_level, -# }, -# }, -# } - -# ─────────────────────────────────────────────────────────────────────────── -# Logging -# • DEBUG_LOGGING=true|1 → root level DEBUG -# • otherwise → root level INFO -# • single console handler on stdout so Docker & K8s both collect it -# ─────────────────────────────────────────────────────────────────────────── -DEBUG_LOGGING = os.getenv("DEBUG_LOGGING", "false").lower() in ("true", "1") -ROOT_LOG_LEVEL = "DEBUG" if DEBUG_LOGGING else "INFO" - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "simple": { - "format": "[%(levelname)s] %(name)s: %(message)s", - }, - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "stream": "ext://sys.stdout", - "formatter": "simple", - }, - }, - # ← this is the **actual** root logger, outside "loggers" - "root": { - "handlers": ["console"], - "level": ROOT_LOG_LEVEL, - }, - "loggers": { - # keep Django chatter low - "django": { - "handlers": ["console"], - "level": "INFO", - "propagate": False, - }, - # workspace-builder package follows the root level - "workspace_builder": { - "handlers": ["console"], - "level": ROOT_LOG_LEVEL, - "propagate": False, - }, - }, -} - -# Apply the dict-based logging config immediately (helps when unit-testing) -logging.config.dictConfig(LOGGING) \ No newline at end of file diff --git a/src/config/urls.py b/src/config/urls.py deleted file mode 100644 index 44907a457..000000000 --- a/src/config/urls.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Workspace Builder URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/4.1/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" -from django.contrib import admin -from django.urls import include, path - -urlpatterns = [ - path('admin/', admin.site.urls), - path('', include('workspace_builder.urls')), -] diff --git a/src/config/wsgi.py b/src/config/wsgi.py deleted file mode 100644 index 5e12213b6..000000000 --- a/src/config/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for workspace builder project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') - -application = get_wsgi_application() diff --git a/src/enrichers/code_collection.py b/src/enrichers/code_collection.py index 72193061c..8dd793b05 100644 --- a/src/enrichers/code_collection.py +++ b/src/enrichers/code_collection.py @@ -422,6 +422,72 @@ def get_template_text(self, ref_name: str, code_bundle_name: str, template_name: template_text = template_bytes.decode('utf-8') return template_text + def get_code_bundle_file_text( + self, + ref_name: str, + code_bundle_name: str, + relative_path: str, + case_insensitive: bool = False, + ) -> Optional[str]: + """Read a file at ``codebundles//`` from the git ref. + + Returns the file contents as text when present, or ``None`` when the file is + not in the tree. Used to overlay companion artifacts that ship with a + codebundle (for example ``Skill.md``) onto every SLX rendered from it. + + When ``case_insensitive`` is True and ``relative_path`` is a single filename + (no path separators), the codebundle root is scanned for the first blob whose + name matches case-insensitively. Useful for files like ``Skill.md`` whose + upstream casing varies (``SKILL.md``, ``Skill.md``, ``skill.md``). + """ + resolved = self.find_code_bundle_file( + ref_name, code_bundle_name, relative_path, case_insensitive=case_insensitive + ) + if resolved is None: + return None + _, text = resolved + return text + + def find_code_bundle_file( + self, + ref_name: str, + code_bundle_name: str, + relative_path: str, + case_insensitive: bool = False, + ) -> Optional[tuple[str, str]]: + """Resolve a codebundle file and return ``(actual_filename, text)``. + + Returns ``None`` if the file is missing or not UTF-8 decodable. The + ``actual_filename`` reflects the on-disk casing in the git tree, which is + useful when ``case_insensitive=True`` and the caller wants to preserve the + upstream filename in its output. + """ + try: + code_bundles_tree = self.get_code_bundles_tree(ref_name) + code_bundle_tree = self.resolve_path(code_bundles_tree, code_bundle_name) + if case_insensitive and "/" not in relative_path and "\\" not in relative_path: + target = relative_path.lower() + blob = None + for item in code_bundle_tree: + if isinstance(item, Blob) and item.name.lower() == target: + blob = item + break + if blob is None: + return None + resolved_name = blob.name + else: + blob = self.resolve_path(code_bundle_tree, relative_path) + if not isinstance(blob, Blob): + return None + resolved_name = relative_path + except WorkspaceBuilderObjectNotFoundException: + return None + try: + text = blob.data_stream.read().decode("utf-8") + except UnicodeDecodeError: + return None + return resolved_name, text + code_collection_cache_temp_dir: Optional[TemporaryDirectory] = None code_collection_cache_dir: Optional[str] = None diff --git a/src/enrichers/generation_rules.py b/src/enrichers/generation_rules.py index 7ad470b14..90089c65d 100644 --- a/src/enrichers/generation_rules.py +++ b/src/enrichers/generation_rules.py @@ -868,6 +868,69 @@ def generate_output_item(generation_rule_info: GenerationRuleInfo, logger.error(f"Unhandled error in generate_output_item: {e}", exc_info=True) return False +# Canonical filename used when looking up a codebundle's Skill overlay. The +# lookup is case-insensitive (Git is case-sensitive on Linux but in practice +# codebundles ship the file as ``SKILL.md``, ``Skill.md``, or ``skill.md``). +# The upstream filename casing is preserved when we write the overlay into the +# SLX directory so the on-disk filename matches what the codebundle author +# published. +SKILL_OVERLAY_FILENAME = "Skill.md" +_SKILL_OVERLAY_CACHE_PROPERTY = "_skill_overlay_cache" + + +def _emit_skill_overlay(generation_rule_info: GenerationRuleInfo, + slx_directory_path: str, + slx_base_template_variables: dict[str, Any], + renderer_output_items: dict[str, RendererOutputItem], + context: Context) -> None: + """Copy a codebundle's Skill markdown (if present) into the rendered SLX directory. + + A CodeBundle defines a Skill (template); each SLX rendered from that CodeBundle + is an *instance* of the Skill. Carrying the Skill markdown alongside the SLX lets + an AI agent (or MCP) read what the skill does and decide when to invoke it. + The file is copied verbatim — no Jinja substitution. The lookup matches the + filename case-insensitively (``SKILL.md`` / ``Skill.md`` / ``skill.md``). + """ + code_collection = generation_rule_info.code_collection + if code_collection is None: + return + ref_name = generation_rule_info.generation_rule_file_spec.ref_name + code_bundle_name = generation_rule_info.generation_rule_file_spec.code_bundle_name + cache_key = (code_collection.repo_url, ref_name, code_bundle_name) + cache: Optional[dict] = context.get_property(_SKILL_OVERLAY_CACHE_PROPERTY) + if cache is None: + cache = {} + context.set_property(_SKILL_OVERLAY_CACHE_PROPERTY, cache) + if cache_key in cache: + resolved = cache[cache_key] + else: + resolved = code_collection.find_code_bundle_file( + ref_name, code_bundle_name, SKILL_OVERLAY_FILENAME, case_insensitive=True + ) + cache[cache_key] = resolved + if resolved is None: + logger.debug( + f"No Skill.md (any case) found in codebundle '{code_bundle_name}' at " + f"ref '{ref_name}' of {code_collection.repo_url}; skipping overlay." + ) + return + actual_filename, skill_text = resolved + overlay_path = f"{slx_directory_path}/{actual_filename}" + if overlay_path in renderer_output_items: + return + renderer_output_items[overlay_path] = RendererOutputItem( + overlay_path, + actual_filename, + slx_base_template_variables, + template_loader_func=None, + raw_content=skill_text, + ) + logger.debug( + f"Overlaying {actual_filename} from codebundle '{code_bundle_name}' " + f"onto SLX directory {slx_directory_path}" + ) + + def collect_emitted_slxs(generation_rule_info: GenerationRuleInfo, resource: Resource, level_of_detail: LevelOfDetail, @@ -1026,6 +1089,15 @@ def generate_slx_output_items(slx_info: SLXInfo, logger.error(f"Error generating output item: {e}") # Continue with next output item + try: + _emit_skill_overlay(generation_rule_info, + slx_directory_path, + slx_base_template_variables, + renderer_output_items, + context) + except Exception as e: + logger.warning(f"Error emitting Skill.md overlay for SLX {slx_info.full_name}: {e}") + try: customization_variables = { "resource": resource, diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 2f690ee47..00b580d95 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -77,28 +77,12 @@ fi # Use TMPDIR if set, or fall back to /tmp TMPDIR="${TMPDIR:-/tmp}" -# Only start mkdocs when cheat sheet is enabled (WB_DEBUG_SUPPRESS_CHEAT_SHEET defaults to "true") -WB_SUPPRESS="${WB_DEBUG_SUPPRESS_CHEAT_SHEET:-true}" -if [ "${WB_SUPPRESS,,}" = "false" ] || [ "$WB_SUPPRESS" = "0" ]; then - MKDOCS_TMP="$TMPDIR/mkdocs-temp" - rm -rf "$MKDOCS_TMP" - mkdir -p "$MKDOCS_TMP" - cp -r /workspace-builder/cheat-sheet-docs/* "$MKDOCS_TMP" - cd "$MKDOCS_TMP" - mkdocs serve -f mkdocs.yml & - echo "MkDocs serve started in the background, serving from $MKDOCS_TMP" -else - echo "Cheat sheet disabled (WB_DEBUG_SUPPRESS_CHEAT_SHEET=${WB_SUPPRESS}) — skipping MkDocs server" -fi - ## Clean stale lock files rm $TMPDIR/.wb_lock || true ## Execute main discovery process cd $RUNWHEN_HOME -# Run Django in the background -python manage.py migrate echo Starting workspace builder REST server # Check if AUTORUN_WORKSPACE_BUILDER_INTERVAL environment variable is set @@ -107,9 +91,7 @@ echo Starting workspace builder REST server if [ -n "$AUTORUN_WORKSPACE_BUILDER_INTERVAL" ]; then echo "AUTORUN_WORKSPACE_BUILDER_INTERVAL is set. Running workspace-builder" - python manage.py runserver 0.0.0.0:8000 & - # Put this back after testing - # python manage.py runserver 0.0.0.0:8000 --noreload & + uvicorn workspace_builder.api:app --host 0.0.0.0 --port 8000 & sleep 60 # Configure which files to watch for changes (inclusive list) @@ -217,7 +199,5 @@ then done fi else - python manage.py runserver 0.0.0.0:8000 - # Put this back after testing - # python manage.py runserver 0.0.0.0:8000 --noreload + uvicorn workspace_builder.api:app --host 0.0.0.0 --port 8000 fi diff --git a/src/indexers/aws_common.py b/src/indexers/aws_common.py new file mode 100644 index 000000000..f4cb26102 --- /dev/null +++ b/src/indexers/aws_common.py @@ -0,0 +1,183 @@ +""" +Shared AWS helper functions used by the native AWS SDK indexer (``awsapi.py``). + +This is the lowest layer of AWS-specific logic the native indexer relies on: +credential / scope resolution, label-based tag filtering, and region resolution. +Nothing here knows about CloudQuery internals. + +Authentication reuses the resolution order the existing AWS code already uses +(``aws_utils.get_aws_credential`` -- explicit keys, K8s secret, IRSA / Pod +Identity, assume-role, or the default credential chain) and returns a +``boto3.Session`` suitable for the Cloud Control client and the typed boto3 +service clients, plus the resolved account id / alias / name map and the region +list. + +``boto3`` / ``aws_utils`` are imported lazily inside the functions that need +them so this module (and its filter / region helpers) stays importable in +environments without ``boto3`` installed -- the same lazy-import discipline the +GCP and Azure typed collectors follow. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Label / tag filtering helpers +# --------------------------------------------------------------------------- +# +# CloudQuery rows expose user tags under ``tags`` (the indexer copies AWS tags +# into ``tags`` during normalization so the same include/exclude matchers work +# across every cloud). These helpers therefore read ``tags`` and match exactly +# like their Azure / GCP counterparts. + +def has_included_tags(resource_data: dict, include_tags: dict[str, str]) -> bool: + """Return True if any of ``include_tags`` is present in ``resource_data``.""" + tags = resource_data.get("tags", {}) or {} + return any(tags.get(key) == value for key, value in include_tags.items()) + + +def has_excluded_tags(resource_data: dict, exclude_tags: dict[str, str]) -> bool: + """Return True if any of ``exclude_tags`` is present in ``resource_data``.""" + tags = resource_data.get("tags", {}) or {} + for key, value in exclude_tags.items(): + if tags.get(key) == value: + logger.info( + f"Excluding resource {resource_data.get('name', 'unknown')} " + f"due to tag '{key}: {value}'" + ) + return True + return False + + +# --------------------------------------------------------------------------- +# Region resolution +# --------------------------------------------------------------------------- + +def resolve_regions(platform_cfg: dict[str, Any], default_region: Optional[str] = None) -> list[str]: + """Resolve the list of AWS regions to discover. + + Resolution order: + 1. ``regions`` (list, or comma-separated string) in the aws config. + 2. ``region`` / ``defaultRegion`` (single). + 3. ``default_region`` arg (e.g. the credential's region). + 4. ``AWS_REGION`` / ``AWS_DEFAULT_REGION`` env vars. + 5. ``us-east-1`` as a last resort. + + De-dupes while preserving order. + """ + raw_regions: list[str] = [] + cfg_regions = platform_cfg.get("regions") + if isinstance(cfg_regions, str): + raw_regions = [r.strip() for r in cfg_regions.split(",") if r.strip()] + elif isinstance(cfg_regions, (list, tuple)): + raw_regions = [str(r).strip() for r in cfg_regions if str(r).strip()] + + if not raw_regions: + single = platform_cfg.get("region") or platform_cfg.get("defaultRegion") + if single: + raw_regions = [str(single).strip()] + + if not raw_regions and default_region: + raw_regions = [str(default_region).strip()] + + if not raw_regions: + env_region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") + if env_region: + raw_regions = [env_region.strip()] + + if not raw_regions: + raw_regions = ["us-east-1"] + + seen: set[str] = set() + out: list[str] = [] + for region in raw_regions: + if region and region not in seen: + seen.add(region) + out.append(region) + return out + + +# --------------------------------------------------------------------------- +# Credentials + scope (account + regions) +# --------------------------------------------------------------------------- + +def aws_has_discovery_config(platform_cfg: dict[str, Any]) -> bool: + """Return True when the aws config block is present. + + AWS always discovers at least the authenticated account, and credentials + can come from the ambient credential chain (instance profile / IRSA / Pod + Identity), so the mere presence of an ``aws`` config section is enough; we + don't require explicit keys. + """ + return bool(platform_cfg) + + +def aws_get_session_and_scope(platform_cfg: dict[str, Any]) -> dict[str, Any]: + """Resolve an AWS session + the discovery scope (account + regions). + + Returns a dict with: + session - a ``boto3.Session`` for the Cloud Control + typed + service clients. + account_id - the authenticated account id (str or None). + account_alias - IAM account alias if set (str or None). + account_name - human-readable account name (falls back to id). + account_names - {account_id: account_name} map (consumed by the + AWSPlatformHandler via ``platform_config_data``). + regions - list[str] of regions to discover. + auth_type - resolved auth type (e.g. ``aws_explicit``). + auth_secret - K8s secret name if secret-based auth was used. + region - the primary (credential) region. + """ + # Lazy import so this module stays importable without boto3. + from aws_utils import ( # noqa: WPS433 + get_account_alias, + get_account_id, + get_account_name, + get_aws_credential, + ) + + workspace_info = {"cloudConfig": {"aws": platform_cfg}} + session, region, _akid, _sak, _stkn, auth_type, auth_secret = get_aws_credential( + workspace_info + ) + + account_id = get_account_id(session) + account_alias = get_account_alias(session) + account_name = get_account_name( + session, account_id=account_id, account_alias=account_alias + ) + + account_names: dict[str, str] = {} + if account_id: + account_names[str(account_id)] = account_name + + # Resolve names for any additional configured accounts (multi-account). + for acct_cfg in platform_cfg.get("accounts", []) or []: + if not isinstance(acct_cfg, dict): + continue + extra_id = str(acct_cfg.get("id") or acct_cfg.get("accountId") or "").strip() + if extra_id and extra_id not in account_names: + try: + account_names[extra_id] = get_account_name(session, account_id=extra_id) + except Exception: # pragma: no cover - best-effort cross-account name + account_names[extra_id] = extra_id + + regions = resolve_regions(platform_cfg, default_region=region) + + return { + "session": session, + "account_id": account_id, + "account_alias": account_alias, + "account_name": account_name, + "account_names": account_names, + "regions": regions, + "auth_type": auth_type, + "auth_secret": auth_secret, + "region": region, + } diff --git a/src/indexers/aws_resource_type_registry.py b/src/indexers/aws_resource_type_registry.py new file mode 100644 index 000000000..96c6205d5 --- /dev/null +++ b/src/indexers/aws_resource_type_registry.py @@ -0,0 +1,243 @@ +""" +Loader for the AWS resource-type registry. + +The registry maps every CloudQuery AWS table name to its AWS Cloud Control API +resource type -- the CloudFormation resource type name -- plus metadata used by +the native ``awsapi`` indexer: + +* canonical name = the CloudQuery table name (e.g. ``aws_ec2_instances``) +* CFN type for Cloud Control / generic discovery (e.g. ``AWS::EC2::Instance``) +* aliases for backward compatibility with legacy RWL ``resource_type_name`` + values (e.g. ``account`` -> ``aws_iam_accounts``, ``ec2_instance`` -> + ``aws_ec2_instances``) +* ``typed_collector`` flag indicating whether ``awsapi_resource_types`` ships a + hand-written boto3 collector for this table +* ``mandatory`` flag indicating whether the indexer must always emit the type + (today only ``aws_iam_accounts``, the account anchor everything is scoped + under) + +The data lives in ``aws_resource_type_registry.yaml`` next to this module. That +YAML is generated by ``scripts/aws/sync_aws_resource_type_registry.py``; +hand-edits to the YAML get overwritten on the next sync. To change behavior for +a specific table, edit ``scripts/aws/aws_resource_type_overrides.yaml`` and +re-run the sync script. + +This module is intentionally read-only: it loads, caches, and exposes the +registry. It does not own collector callables or generation-rule semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +_DEFAULT_REGISTRY_PATH = Path(__file__).with_name("aws_resource_type_registry.yaml") + + +@dataclass(frozen=True) +class AwsResourceTypeEntry: + """One row of the registry. + + ``cloudquery_table_name`` is the canonical key. Lookups by alias resolve to + the same entry (no duplicates). ``cfn_type`` may be ``None`` for tables that + have no Cloud Control resource type (reports, metrics, cost rollups, the + synthesized account anchor...); such tables are skipped by generic + discovery. + """ + + cloudquery_table_name: str + cfn_type: Optional[str] + cfn_type_source: Optional[str] + category: Optional[str] + aliases: tuple[str, ...] + typed_collector: bool + mandatory: bool + + def known_names(self) -> tuple[str, ...]: + """Every name (canonical + aliases) that resolves to this entry.""" + return (self.cloudquery_table_name, *self.aliases) + + +@dataclass(frozen=True) +class AwsRegistryMetadata: + source: Optional[str] = None + snapshot_date: Optional[str] = None + total_tables: int = 0 + typed_collectors: int = 0 + cfn_types_assigned: int = 0 + generator: Optional[str] = None + notes: Optional[str] = None + + +@dataclass +class AwsResourceTypeRegistry: + metadata: AwsRegistryMetadata + entries: tuple[AwsResourceTypeEntry, ...] + _by_canonical: dict[str, AwsResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_alias: dict[str, AwsResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_cfn_type_lower: dict[str, AwsResourceTypeEntry] = field(default_factory=dict, repr=False) + + def __post_init__(self) -> None: + for entry in self.entries: + self._by_canonical[entry.cloudquery_table_name] = entry + for alias in entry.aliases: + if not alias: + continue + # Aliases must not collide with another canonical name or alias; + # the sync script enforces this, but verify defensively in case + # someone hand-edits the YAML. + if alias in self._by_canonical and self._by_canonical[alias] is not entry: + raise ValueError( + f"Alias {alias!r} collides with canonical table name " + f"belonging to a different entry" + ) + if alias in self._by_alias and self._by_alias[alias] is not entry: + raise ValueError( + f"Alias {alias!r} is registered for both " + f"{self._by_alias[alias].cloudquery_table_name!r} and " + f"{entry.cloudquery_table_name!r}" + ) + self._by_alias[alias] = entry + if entry.cfn_type: + # CFN types are case-sensitive on the wire but we index + # lower-cased for robust matching. Multiple registry entries can + # occasionally share a CFN type; the first one wins, which is + # deterministic because entries arrive sorted by canonical name. + key = entry.cfn_type.lower() + self._by_cfn_type_lower.setdefault(key, entry) + + def find(self, name: str) -> Optional[AwsResourceTypeEntry]: + """Return the entry for a canonical table name or alias, else None.""" + if not name: + return None + entry = self._by_canonical.get(name) + if entry is not None: + return entry + return self._by_alias.get(name) + + def find_by_cfn_type(self, cfn_type: Optional[str]) -> Optional[AwsResourceTypeEntry]: + """Look up the registry entry whose ``cfn_type`` matches this string. + + Used by the Cloud Control generic-discovery pass to route each + ``TypeName`` value (e.g. ``AWS::S3::Bucket``) back to the registry entry + whose ``cloudquery_table_name`` is the canonical RWL identifier for that + type. + """ + if not cfn_type: + return None + return self._by_cfn_type_lower.get(cfn_type.lower()) + + def __contains__(self, name: object) -> bool: + return isinstance(name, str) and self.find(name) is not None + + def __iter__(self) -> Iterable[AwsResourceTypeEntry]: + return iter(self.entries) + + def __len__(self) -> int: + return len(self.entries) + + def all_canonical_names(self) -> tuple[str, ...]: + return tuple(entry.cloudquery_table_name for entry in self.entries) + + def all_cfn_types(self) -> tuple[str, ...]: + return tuple( + entry.cfn_type for entry in self.entries if entry.cfn_type + ) + + def typed_collector_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.typed_collector + ) + + def mandatory_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.mandatory + ) + + +def _coerce_aliases(value: object) -> tuple[str, ...]: + if not value: + return () + if isinstance(value, (list, tuple)): + return tuple(str(v) for v in value if v) + return (str(value),) + + +def _build_metadata(payload: Mapping[str, object]) -> AwsRegistryMetadata: + return AwsRegistryMetadata( + source=_optional_str(payload.get("source")), + snapshot_date=_optional_str(payload.get("snapshot_date")), + total_tables=int(payload.get("total_tables") or 0), + typed_collectors=int(payload.get("typed_collectors") or 0), + cfn_types_assigned=int(payload.get("cfn_types_assigned") or 0), + generator=_optional_str(payload.get("generator")), + notes=_optional_str(payload.get("notes")), + ) + + +def _optional_str(value: object) -> Optional[str]: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _build_entries(types_payload: Mapping[str, Mapping[str, object]]) -> tuple[AwsResourceTypeEntry, ...]: + entries: list[AwsResourceTypeEntry] = [] + for table_name, body in sorted(types_payload.items()): + if not isinstance(body, Mapping): + raise ValueError(f"Registry entry {table_name!r} is not a mapping") + entries.append( + AwsResourceTypeEntry( + cloudquery_table_name=str(table_name), + cfn_type=_optional_str(body.get("cfn_type")), + cfn_type_source=_optional_str(body.get("cfn_type_source")), + category=_optional_str(body.get("category")), + aliases=_coerce_aliases(body.get("aliases")), + typed_collector=bool(body.get("typed_collector")), + mandatory=bool(body.get("mandatory")), + ) + ) + return tuple(entries) + + +def load_registry_from_path(path: Path) -> AwsResourceTypeRegistry: + """Load a registry from an explicit YAML path. Bypasses the cache.""" + if not path.exists(): + raise FileNotFoundError(f"AWS resource-type registry not found at {path}") + with path.open("r", encoding="utf-8") as fh: + payload = yaml.safe_load(fh) or {} + if not isinstance(payload, Mapping): + raise ValueError(f"Registry YAML at {path} did not parse to a mapping") + + metadata = _build_metadata(payload.get("metadata") or {}) + types_payload = payload.get("types") or {} + if not isinstance(types_payload, Mapping): + raise ValueError(f"Registry YAML at {path} has non-mapping 'types' section") + + entries = _build_entries(types_payload) + return AwsResourceTypeRegistry(metadata=metadata, entries=entries) + + +@lru_cache(maxsize=1) +def load_registry() -> AwsResourceTypeRegistry: + """Load and cache the registry from the default location.""" + return load_registry_from_path(_DEFAULT_REGISTRY_PATH) + + +def find_entry(name: str) -> Optional[AwsResourceTypeEntry]: + """Convenience wrapper around the cached registry's ``find()``.""" + return load_registry().find(name) + + +def reset_cache() -> None: + """Clear the cached registry. Intended for tests.""" + load_registry.cache_clear() diff --git a/src/indexers/aws_resource_type_registry.yaml b/src/indexers/aws_resource_type_registry.yaml new file mode 100644 index 000000000..a76c57137 --- /dev/null +++ b/src/indexers/aws_resource_type_registry.yaml @@ -0,0 +1,7845 @@ +metadata: + source: https://www.cloudquery.io/hub/plugins/source/cloudquery/aws/latest/tables + snapshot_date: '2026-05-29' + total_tables: 1119 + typed_collectors: 3 + cfn_types_assigned: 1102 + generator: scripts/aws/sync_aws_resource_type_registry.py + notes: Generated file. To change the CloudFormation resource type for a table, edit scripts/aws/aws_resource_type_overrides.yaml and re-run the sync script. Hand-edits to this file will be overwritten. +types: + aws_accessanalyzer_analyzer_archive_rules: + cfn_type: AWS::AccessAnalyzer::AnalyzerArchiveRule + cfn_type_source: heuristic + category: accessanalyzer + aliases: [] + typed_collector: false + mandatory: false + aws_accessanalyzer_analyzer_findings: + cfn_type: AWS::AccessAnalyzer::AnalyzerFinding + cfn_type_source: heuristic + category: accessanalyzer + aliases: [] + typed_collector: false + mandatory: false + aws_accessanalyzer_analyzer_findings_v2: + cfn_type: AWS::AccessAnalyzer::AnalyzerFindingsV2 + cfn_type_source: heuristic + category: accessanalyzer + aliases: [] + typed_collector: false + mandatory: false + aws_accessanalyzer_analyzers: + cfn_type: AWS::AccessAnalyzer::Analyzer + cfn_type_source: heuristic + category: accessanalyzer + aliases: [] + typed_collector: false + mandatory: false + aws_account_alternate_contacts: + cfn_type: AWS::Account::AlternateContact + cfn_type_source: heuristic + category: account + aliases: [] + typed_collector: false + mandatory: false + aws_account_contacts: + cfn_type: AWS::Account::Contact + cfn_type_source: heuristic + category: account + aliases: [] + typed_collector: false + mandatory: false + aws_acm_certificates: + cfn_type: AWS::CertificateManager::Certificate + cfn_type_source: override + category: acm + aliases: [] + typed_collector: false + mandatory: false + aws_acmpca_certificate_authorities: + cfn_type: AWS::ACMPCA::CertificateAuthority + cfn_type_source: heuristic + category: acmpca + aliases: [] + typed_collector: false + mandatory: false + aws_amp_rule_groups_namespaces: + cfn_type: AWS::APS::RuleGroupsNamespace + cfn_type_source: heuristic + category: amp + aliases: [] + typed_collector: false + mandatory: false + aws_amp_workspaces: + cfn_type: AWS::APS::Workspace + cfn_type_source: heuristic + category: amp + aliases: [] + typed_collector: false + mandatory: false + aws_amplify_apps: + cfn_type: AWS::Amplify::App + cfn_type_source: heuristic + category: amplify + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_accounts: + cfn_type: AWS::ApiGateway::Account + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_api_keys: + cfn_type: AWS::ApiGateway::ApiKey + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_client_certificates: + cfn_type: AWS::ApiGateway::ClientCertificate + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_domain_name_base_path_mappings: + cfn_type: AWS::ApiGateway::DomainNameBasePathMapping + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_domain_names: + cfn_type: AWS::ApiGateway::DomainName + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_authorizers: + cfn_type: AWS::ApiGateway::RestApiAuthorizer + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_deployments: + cfn_type: AWS::ApiGateway::RestApiDeployment + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_documentation_parts: + cfn_type: AWS::ApiGateway::RestApiDocumentationPart + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_documentation_versions: + cfn_type: AWS::ApiGateway::RestApiDocumentationVersion + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_gateway_responses: + cfn_type: AWS::ApiGateway::RestApiGatewayRespons + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_models: + cfn_type: AWS::ApiGateway::RestApiModel + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_request_validators: + cfn_type: AWS::ApiGateway::RestApiRequestValidator + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_resource_method_integrations: + cfn_type: AWS::ApiGateway::RestApiResourceMethodIntegration + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_resource_methods: + cfn_type: AWS::ApiGateway::RestApiResourceMethod + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_resources: + cfn_type: AWS::ApiGateway::RestApiResource + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_api_stages: + cfn_type: AWS::ApiGateway::RestApiStage + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_rest_apis: + cfn_type: AWS::ApiGateway::RestApi + cfn_type_source: override + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_usage_plan_keys: + cfn_type: AWS::ApiGateway::UsagePlanKey + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_usage_plans: + cfn_type: AWS::ApiGateway::UsagePlan + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigateway_vpc_links: + cfn_type: AWS::ApiGateway::VpcLink + cfn_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_authorizers: + cfn_type: AWS::ApiGatewayV2::ApiAuthorizer + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_deployments: + cfn_type: AWS::ApiGatewayV2::ApiDeployment + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_integration_responses: + cfn_type: AWS::ApiGatewayV2::ApiIntegrationRespons + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_integrations: + cfn_type: AWS::ApiGatewayV2::ApiIntegration + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_models: + cfn_type: AWS::ApiGatewayV2::ApiModel + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_route_responses: + cfn_type: AWS::ApiGatewayV2::ApiRouteRespons + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_routes: + cfn_type: AWS::ApiGatewayV2::ApiRoute + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_api_stages: + cfn_type: AWS::ApiGatewayV2::ApiStage + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_apis: + cfn_type: AWS::ApiGatewayV2::Api + cfn_type_source: override + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_domain_name_rest_api_mappings: + cfn_type: AWS::ApiGatewayV2::DomainNameRestApiMapping + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_domain_names: + cfn_type: AWS::ApiGatewayV2::DomainName + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_apigatewayv2_vpc_links: + cfn_type: AWS::ApiGatewayV2::VpcLink + cfn_type_source: heuristic + category: apigatewayv2 + aliases: [] + typed_collector: false + mandatory: false + aws_appconfig_applications: + cfn_type: AWS::AppConfig::Application + cfn_type_source: heuristic + category: appconfig + aliases: [] + typed_collector: false + mandatory: false + aws_appconfig_configuration_profiles: + cfn_type: AWS::AppConfig::ConfigurationProfile + cfn_type_source: heuristic + category: appconfig + aliases: [] + typed_collector: false + mandatory: false + aws_appconfig_deployment_strategies: + cfn_type: AWS::AppConfig::DeploymentStrategy + cfn_type_source: heuristic + category: appconfig + aliases: [] + typed_collector: false + mandatory: false + aws_appconfig_environments: + cfn_type: AWS::AppConfig::Environment + cfn_type_source: heuristic + category: appconfig + aliases: [] + typed_collector: false + mandatory: false + aws_appconfig_hosted_configuration_versions: + cfn_type: AWS::AppConfig::HostedConfigurationVersion + cfn_type_source: heuristic + category: appconfig + aliases: [] + typed_collector: false + mandatory: false + aws_appflow_flows: + cfn_type: AWS::AppFlow::Flow + cfn_type_source: heuristic + category: appflow + aliases: [] + typed_collector: false + mandatory: false + aws_applicationautoscaling_policies: + cfn_type: AWS::ApplicationAutoScaling::Policy + cfn_type_source: heuristic + category: applicationautoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_applicationautoscaling_scalable_targets: + cfn_type: AWS::ApplicationAutoScaling::ScalableTarget + cfn_type_source: heuristic + category: applicationautoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_applicationautoscaling_scaling_activities: + cfn_type: AWS::ApplicationAutoScaling::ScalingActivity + cfn_type_source: heuristic + category: applicationautoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_applicationautoscaling_scheduled_actions: + cfn_type: AWS::ApplicationAutoScaling::ScheduledAction + cfn_type_source: heuristic + category: applicationautoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_appmesh_meshes: + cfn_type: AWS::AppMesh::Mesh + cfn_type_source: heuristic + category: appmesh + aliases: [] + typed_collector: false + mandatory: false + aws_appmesh_virtual_gateways: + cfn_type: AWS::AppMesh::VirtualGateway + cfn_type_source: heuristic + category: appmesh + aliases: [] + typed_collector: false + mandatory: false + aws_appmesh_virtual_nodes: + cfn_type: AWS::AppMesh::VirtualNode + cfn_type_source: heuristic + category: appmesh + aliases: [] + typed_collector: false + mandatory: false + aws_appmesh_virtual_routers: + cfn_type: AWS::AppMesh::VirtualRouter + cfn_type_source: heuristic + category: appmesh + aliases: [] + typed_collector: false + mandatory: false + aws_appmesh_virtual_services: + cfn_type: AWS::AppMesh::VirtualService + cfn_type_source: heuristic + category: appmesh + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_auto_scaling_configurations: + cfn_type: AWS::AppRunner::AutoScalingConfiguration + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_connections: + cfn_type: AWS::AppRunner::Connection + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_custom_domains: + cfn_type: AWS::AppRunner::CustomDomain + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_observability_configurations: + cfn_type: AWS::AppRunner::ObservabilityConfiguration + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_operations: + cfn_type: AWS::AppRunner::Operation + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_services: + cfn_type: AWS::AppRunner::Service + cfn_type_source: override + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_vpc_connectors: + cfn_type: AWS::AppRunner::VpcConnector + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_apprunner_vpc_ingress_connections: + cfn_type: AWS::AppRunner::VpcIngressConnection + cfn_type_source: heuristic + category: apprunner + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_app_blocks: + cfn_type: AWS::AppStream::AppBlock + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_application_fleet_associations: + cfn_type: AWS::AppStream::ApplicationFleetAssociation + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_applications: + cfn_type: AWS::AppStream::Application + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_directory_configs: + cfn_type: AWS::AppStream::DirectoryConfig + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_fleets: + cfn_type: AWS::AppStream::Fleet + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_image_builders: + cfn_type: AWS::AppStream::ImageBuilder + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_images: + cfn_type: AWS::AppStream::Image + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_stack_entitlements: + cfn_type: AWS::AppStream::StackEntitlement + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_stack_user_associations: + cfn_type: AWS::AppStream::StackUserAssociation + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_stacks: + cfn_type: AWS::AppStream::Stack + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_usage_report_subscriptions: + cfn_type: AWS::AppStream::UsageReportSubscription + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appstream_users: + cfn_type: AWS::AppStream::User + cfn_type_source: heuristic + category: appstream + aliases: [] + typed_collector: false + mandatory: false + aws_appsync_graphql_apis: + cfn_type: AWS::AppSync::GraphqlApi + cfn_type_source: heuristic + category: appsync + aliases: [] + typed_collector: false + mandatory: false + aws_athena_data_catalog_database_tables: + cfn_type: AWS::Athena::DataCatalogDatabaseTable + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_data_catalog_databases: + cfn_type: AWS::Athena::DataCatalogDatabas + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_data_catalogs: + cfn_type: AWS::Athena::DataCatalog + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_work_group_named_queries: + cfn_type: AWS::Athena::WorkGroupNamedQuery + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_work_group_prepared_statements: + cfn_type: AWS::Athena::WorkGroupPreparedStatement + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_work_group_query_executions: + cfn_type: AWS::Athena::WorkGroupQueryExecution + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_athena_work_groups: + cfn_type: AWS::Athena::WorkGroup + cfn_type_source: heuristic + category: athena + aliases: [] + typed_collector: false + mandatory: false + aws_auditmanager_assessments: + cfn_type: AWS::AuditManager::Assessment + cfn_type_source: heuristic + category: auditmanager + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_group_lifecycle_hooks: + cfn_type: AWS::AutoScaling::GroupLifecycleHook + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_group_scaling_policies: + cfn_type: AWS::AutoScaling::GroupScalingPolicy + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_groups: + cfn_type: AWS::AutoScaling::AutoScalingGroup + cfn_type_source: override + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_launch_configurations: + cfn_type: AWS::AutoScaling::LaunchConfiguration + cfn_type_source: override + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_plan_resources: + cfn_type: AWS::AutoScaling::PlanResource + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_plans: + cfn_type: AWS::AutoScaling::Plan + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_scheduled_actions: + cfn_type: AWS::AutoScaling::ScheduledAction + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_autoscaling_warm_pools: + cfn_type: AWS::AutoScaling::WarmPool + cfn_type_source: heuristic + category: autoscaling + aliases: [] + typed_collector: false + mandatory: false + aws_availability_zones: + cfn_type: null + cfn_type_source: override + category: availability + aliases: [] + typed_collector: false + mandatory: false + aws_backup_frameworks: + cfn_type: AWS::Backup::Framework + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_global_settings: + cfn_type: AWS::Backup::GlobalSetting + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_jobs: + cfn_type: AWS::Backup::Job + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_plan_selections: + cfn_type: AWS::Backup::PlanSelection + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_plans: + cfn_type: AWS::Backup::BackupPlan + cfn_type_source: override + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_protected_resources: + cfn_type: AWS::Backup::ProtectedResource + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_region_settings: + cfn_type: AWS::Backup::RegionSetting + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_report_plans: + cfn_type: AWS::Backup::ReportPlan + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_restore_testing_plans: + cfn_type: AWS::Backup::RestoreTestingPlan + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_restore_testing_selections: + cfn_type: AWS::Backup::RestoreTestingSelection + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_tiering_configurations: + cfn_type: AWS::Backup::TieringConfiguration + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_vault_recovery_points: + cfn_type: AWS::Backup::VaultRecoveryPoint + cfn_type_source: heuristic + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backup_vaults: + cfn_type: AWS::Backup::BackupVault + cfn_type_source: override + category: backup + aliases: [] + typed_collector: false + mandatory: false + aws_backupgateway_gateways: + cfn_type: AWS::BackupGateway::Gateway + cfn_type_source: heuristic + category: backupgateway + aliases: [] + typed_collector: false + mandatory: false + aws_batch_compute_environments: + cfn_type: AWS::Batch::ComputeEnvironment + cfn_type_source: override + category: batch + aliases: [] + typed_collector: false + mandatory: false + aws_batch_job_definitions: + cfn_type: AWS::Batch::JobDefinition + cfn_type_source: override + category: batch + aliases: [] + typed_collector: false + mandatory: false + aws_batch_job_queues: + cfn_type: AWS::Batch::JobQueue + cfn_type_source: override + category: batch + aliases: [] + typed_collector: false + mandatory: false + aws_batch_jobs: + cfn_type: AWS::Batch::Job + cfn_type_source: heuristic + category: batch + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_agent_versions: + cfn_type: AWS::Bedrock::AgentVersion + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_agents: + cfn_type: AWS::Bedrock::Agent + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_custom_models: + cfn_type: AWS::Bedrock::CustomModel + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_evaluation_jobs: + cfn_type: AWS::Bedrock::EvaluationJob + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_foundation_models: + cfn_type: AWS::Bedrock::FoundationModel + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_guardrails: + cfn_type: AWS::Bedrock::Guardrail + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_inference_profiles: + cfn_type: AWS::Bedrock::InferenceProfile + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_model_copy_jobs: + cfn_type: AWS::Bedrock::ModelCopyJob + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_model_customization_jobs: + cfn_type: AWS::Bedrock::ModelCustomizationJob + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_bedrock_provisioned_model_throughputs: + cfn_type: AWS::Bedrock::ProvisionedModelThroughput + cfn_type_source: heuristic + category: bedrock + aliases: [] + typed_collector: false + mandatory: false + aws_budgets_actions: + cfn_type: AWS::Budgets::Action + cfn_type_source: heuristic + category: budgets + aliases: [] + typed_collector: false + mandatory: false + aws_budgets_budgets: + cfn_type: AWS::Budgets::Budget + cfn_type_source: heuristic + category: budgets + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_instance_resource_drifts: + cfn_type: AWS::CloudFormation::StackInstanceResourceDrift + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_instance_summaries: + cfn_type: AWS::CloudFormation::StackInstanceSummary + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_resources: + cfn_type: AWS::CloudFormation::StackResource + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_set_operation_results: + cfn_type: AWS::CloudFormation::StackSetOperationResult + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_set_operations: + cfn_type: AWS::CloudFormation::StackSetOperation + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_sets: + cfn_type: AWS::CloudFormation::StackSet + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stack_templates: + cfn_type: AWS::CloudFormation::StackTemplate + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_stacks: + cfn_type: AWS::CloudFormation::Stack + cfn_type_source: override + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudformation_template_summaries: + cfn_type: AWS::CloudFormation::TemplateSummary + cfn_type_source: heuristic + category: cloudformation + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_cache_policies: + cfn_type: AWS::CloudFront::CachePolicy + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_distributions: + cfn_type: AWS::CloudFront::Distribution + cfn_type_source: override + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_functions: + cfn_type: AWS::CloudFront::Function + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_key_value_stores: + cfn_type: AWS::CloudFront::KeyValueStore + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_origin_access_identities: + cfn_type: AWS::CloudFront::OriginAccessIdentity + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_origin_request_policies: + cfn_type: AWS::CloudFront::OriginRequestPolicy + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudfront_response_headers_policies: + cfn_type: AWS::CloudFront::ResponseHeadersPolicy + cfn_type_source: heuristic + category: cloudfront + aliases: [] + typed_collector: false + mandatory: false + aws_cloudhsmv2_backups: + cfn_type: AWS::CloudHSM::Backup + cfn_type_source: heuristic + category: cloudhsmv2 + aliases: [] + typed_collector: false + mandatory: false + aws_cloudhsmv2_clusters: + cfn_type: AWS::CloudHSM::Cluster + cfn_type_source: heuristic + category: cloudhsmv2 + aliases: [] + typed_collector: false + mandatory: false + aws_cloudtrail_channels: + cfn_type: AWS::CloudTrail::Channel + cfn_type_source: heuristic + category: cloudtrail + aliases: [] + typed_collector: false + mandatory: false + aws_cloudtrail_events: + cfn_type: AWS::CloudTrail::Event + cfn_type_source: heuristic + category: cloudtrail + aliases: [] + typed_collector: false + mandatory: false + aws_cloudtrail_imports: + cfn_type: AWS::CloudTrail::Import + cfn_type_source: heuristic + category: cloudtrail + aliases: [] + typed_collector: false + mandatory: false + aws_cloudtrail_trail_event_selectors: + cfn_type: AWS::CloudTrail::TrailEventSelector + cfn_type_source: heuristic + category: cloudtrail + aliases: [] + typed_collector: false + mandatory: false + aws_cloudtrail_trails: + cfn_type: AWS::CloudTrail::Trail + cfn_type_source: heuristic + category: cloudtrail + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatch_alarms: + cfn_type: AWS::CloudWatch::Alarm + cfn_type_source: override + category: cloudwatch + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatch_metric_data: + cfn_type: null + cfn_type_source: override + category: cloudwatch + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatch_metric_statistics: + cfn_type: null + cfn_type_source: override + category: cloudwatch + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatch_metric_streams: + cfn_type: AWS::CloudWatch::MetricStream + cfn_type_source: heuristic + category: cloudwatch + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatch_metrics: + cfn_type: null + cfn_type_source: override + category: cloudwatch + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_deliveries: + cfn_type: AWS::Logs::Delivery + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_delivery_destinations: + cfn_type: AWS::Logs::DeliveryDestination + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_delivery_sources: + cfn_type: AWS::Logs::DeliverySource + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_log_group_data_protection_policies: + cfn_type: AWS::Logs::LogGroupDataProtectionPolicy + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_log_group_subscription_filters: + cfn_type: AWS::Logs::LogGroupSubscriptionFilter + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_log_groups: + cfn_type: AWS::Logs::LogGroup + cfn_type_source: override + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_log_streams: + cfn_type: AWS::Logs::LogStream + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_metric_filters: + cfn_type: AWS::Logs::MetricFilter + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_cloudwatchlogs_resource_policies: + cfn_type: AWS::Logs::ResourcePolicy + cfn_type_source: heuristic + category: cloudwatchlogs + aliases: [] + typed_collector: false + mandatory: false + aws_codeartifact_domains: + cfn_type: AWS::CodeArtifact::Domain + cfn_type_source: heuristic + category: codeartifact + aliases: [] + typed_collector: false + mandatory: false + aws_codeartifact_repositories: + cfn_type: AWS::CodeArtifact::Repository + cfn_type_source: heuristic + category: codeartifact + aliases: [] + typed_collector: false + mandatory: false + aws_codebuild_builds: + cfn_type: AWS::CodeBuild::Build + cfn_type_source: heuristic + category: codebuild + aliases: [] + typed_collector: false + mandatory: false + aws_codebuild_projects: + cfn_type: AWS::CodeBuild::Project + cfn_type_source: heuristic + category: codebuild + aliases: [] + typed_collector: false + mandatory: false + aws_codebuild_source_credentials: + cfn_type: AWS::CodeBuild::SourceCredential + cfn_type_source: heuristic + category: codebuild + aliases: [] + typed_collector: false + mandatory: false + aws_codecommit_repositories: + cfn_type: AWS::CodeCommit::Repository + cfn_type_source: heuristic + category: codecommit + aliases: [] + typed_collector: false + mandatory: false + aws_codedeploy_applications: + cfn_type: AWS::CodeDeploy::Application + cfn_type_source: heuristic + category: codedeploy + aliases: [] + typed_collector: false + mandatory: false + aws_codedeploy_deployment_configs: + cfn_type: AWS::CodeDeploy::DeploymentConfig + cfn_type_source: heuristic + category: codedeploy + aliases: [] + typed_collector: false + mandatory: false + aws_codedeploy_deployment_groups: + cfn_type: AWS::CodeDeploy::DeploymentGroup + cfn_type_source: heuristic + category: codedeploy + aliases: [] + typed_collector: false + mandatory: false + aws_codedeploy_deployments: + cfn_type: AWS::CodeDeploy::Deployment + cfn_type_source: heuristic + category: codedeploy + aliases: [] + typed_collector: false + mandatory: false + aws_codegurureviewer_repository_associations: + cfn_type: AWS::CodeGuruReviewer::RepositoryAssociation + cfn_type_source: heuristic + category: codegurureviewer + aliases: [] + typed_collector: false + mandatory: false + aws_codepipeline_pipelines: + cfn_type: AWS::CodePipeline::Pipeline + cfn_type_source: heuristic + category: codepipeline + aliases: [] + typed_collector: false + mandatory: false + aws_codepipeline_webhooks: + cfn_type: AWS::CodePipeline::Webhook + cfn_type_source: heuristic + category: codepipeline + aliases: [] + typed_collector: false + mandatory: false + aws_codestar_connections_managed: + cfn_type: AWS::CodeStarConnections::ConnectionsManaged + cfn_type_source: heuristic + category: codestar + aliases: [] + typed_collector: false + mandatory: false + aws_cognito_identity_pools: + cfn_type: AWS::Cognito::IdentityPool + cfn_type_source: heuristic + category: cognito + aliases: [] + typed_collector: false + mandatory: false + aws_cognito_user_pool_identity_providers: + cfn_type: AWS::Cognito::UserPoolIdentityProvider + cfn_type_source: heuristic + category: cognito + aliases: [] + typed_collector: false + mandatory: false + aws_cognito_user_pools: + cfn_type: AWS::Cognito::UserPool + cfn_type_source: override + category: cognito + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_document_classification_jobs: + cfn_type: AWS::Comprehend::DocumentClassificationJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_document_classifiers: + cfn_type: AWS::Comprehend::DocumentClassifier + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_dominant_language_detection_jobs: + cfn_type: AWS::Comprehend::DominantLanguageDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_endpoints: + cfn_type: AWS::Comprehend::Endpoint + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_entities_detection_jobs: + cfn_type: AWS::Comprehend::EntitiesDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_entity_recognizers: + cfn_type: AWS::Comprehend::EntityRecognizer + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_events_detection_jobs: + cfn_type: AWS::Comprehend::EventsDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_flywheel_datasets: + cfn_type: AWS::Comprehend::FlywheelDataset + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_flywheel_iteration_histories: + cfn_type: AWS::Comprehend::FlywheelIterationHistory + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_flywheels: + cfn_type: AWS::Comprehend::Flywheel + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_keyphrases_detection_jobs: + cfn_type: AWS::Comprehend::KeyphrasesDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_pii_entities_etection_jobs: + cfn_type: AWS::Comprehend::PiiEntitiesEtectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_sentiment_detection_jobs: + cfn_type: AWS::Comprehend::SentimentDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_targeted_sentiment_detection_jobs: + cfn_type: AWS::Comprehend::TargetedSentimentDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_comprehend_topics_detection_jobs: + cfn_type: AWS::Comprehend::TopicsDetectionJob + cfn_type_source: heuristic + category: comprehend + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_autoscaling_group_recommendations: + cfn_type: AWS::Computeoptimizer::AutoscalingGroupRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_ebs_volume_recommendations: + cfn_type: AWS::Computeoptimizer::EbsVolumeRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_ec2_instance_recommendations: + cfn_type: AWS::Computeoptimizer::Ec2InstanceRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_ecs_service_recommendations: + cfn_type: AWS::Computeoptimizer::EcsServiceRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_enrollment_statuses: + cfn_type: AWS::Computeoptimizer::EnrollmentStatus + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_lambda_function_recommendations: + cfn_type: AWS::Computeoptimizer::LambdaFunctionRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizer_rds_database_recommendations: + cfn_type: AWS::Computeoptimizer::RdsDatabaseRecommendation + cfn_type_source: heuristic + category: computeoptimizer + aliases: [] + typed_collector: false + mandatory: false + aws_computeoptimizerautomation_accounts: + cfn_type: AWS::Computeoptimizerautomation::Account + cfn_type_source: heuristic + category: computeoptimizerautomation + aliases: [] + typed_collector: false + mandatory: false + aws_config_config_rule_compliance_details: + cfn_type: AWS::Config::ConfigRuleComplianceDetail + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_config_rule_compliances: + cfn_type: AWS::Config::ConfigRuleCompliance + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_config_rules: + cfn_type: AWS::Config::ConfigRule + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_configuration_aggregators: + cfn_type: AWS::Config::ConfigurationAggregator + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_configuration_recorders: + cfn_type: AWS::Config::ConfigurationRecorder + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_conformance_pack_rule_compliances: + cfn_type: AWS::Config::ConformancePackRuleCompliance + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_conformance_packs: + cfn_type: AWS::Config::ConformancePack + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_delivery_channel_statuses: + cfn_type: AWS::Config::DeliveryChannelStatus + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_delivery_channels: + cfn_type: AWS::Config::DeliveryChannel + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_remediation_configurations: + cfn_type: AWS::Config::RemediationConfiguration + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_config_retention_configurations: + cfn_type: AWS::Config::RetentionConfiguration + cfn_type_source: heuristic + category: config + aliases: [] + typed_collector: false + mandatory: false + aws_connect_agent_queues: + cfn_type: AWS::Connect::AgentQueue + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_agent_statuses: + cfn_type: AWS::Connect::AgentStatus + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_approved_origins: + cfn_type: AWS::Connect::ApprovedOrigin + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_authentication_profiles: + cfn_type: AWS::Connect::AuthenticationProfile + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_contact_evaluations: + cfn_type: AWS::Connect::ContactEvaluation + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_contact_flow_modules: + cfn_type: AWS::Connect::ContactFlowModule + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_contact_references: + cfn_type: AWS::Connect::ContactReference + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_contacts: + cfn_type: AWS::Connect::Contact + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_default_vocabularies: + cfn_type: AWS::Connect::DefaultVocabulary + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_evaluation_form_versions: + cfn_type: AWS::Connect::EvaluationFormVersion + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_evaluation_forms: + cfn_type: AWS::Connect::EvaluationForm + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_flow_associations: + cfn_type: AWS::Connect::FlowAssociation + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_hours_of_operations: + cfn_type: AWS::Connect::HoursOfOperation + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_instance_storage_configs: + cfn_type: AWS::Connect::InstanceStorageConfig + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_instances: + cfn_type: AWS::Connect::Instance + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_integration_associations: + cfn_type: AWS::Connect::IntegrationAssociation + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lambda_functions: + cfn_type: AWS::Connect::LambdaFunction + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lex_bots: + cfn_type: AWS::Connect::LexBot + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lex_bots_v1: + cfn_type: AWS::Connect::LexBotsV1 + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lex_bots_v2: + cfn_type: AWS::Connect::LexBotsV2 + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lex_v1_bots: + cfn_type: AWS::Connect::LexV1Bot + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_lex_v2_bots: + cfn_type: AWS::Connect::LexV2Bot + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_phone_numbers: + cfn_type: AWS::Connect::PhoneNumber + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_prompts: + cfn_type: AWS::Connect::Prompt + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_queue_quick_connects: + cfn_type: AWS::Connect::QueueQuickConnect + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_queues: + cfn_type: AWS::Connect::Queue + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_quick_connects: + cfn_type: AWS::Connect::QuickConnect + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_routing_profile_queues: + cfn_type: AWS::Connect::RoutingProfileQueue + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_routing_profiles: + cfn_type: AWS::Connect::RoutingProfile + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_rules: + cfn_type: AWS::Connect::Rule + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_security_keys: + cfn_type: AWS::Connect::SecurityKey + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_security_profiles: + cfn_type: AWS::Connect::SecurityProfile + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_task_templates: + cfn_type: AWS::Connect::TaskTemplate + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_traffic_distribution_group_users: + cfn_type: AWS::Connect::TrafficDistributionGroupUser + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_traffic_distribution_groups: + cfn_type: AWS::Connect::TrafficDistributionGroup + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_use_cases: + cfn_type: AWS::Connect::UseCas + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_user_hierarchy_groups: + cfn_type: AWS::Connect::UserHierarchyGroup + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_user_proficiencies: + cfn_type: AWS::Connect::UserProficiency + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_users: + cfn_type: AWS::Connect::User + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_view_versions: + cfn_type: AWS::Connect::ViewVersion + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_connect_views: + cfn_type: AWS::Connect::View + cfn_type_source: heuristic + category: connect + aliases: [] + typed_collector: false + mandatory: false + aws_costexplorer_cost_30d: + cfn_type: null + cfn_type_source: override + category: costexplorer + aliases: [] + typed_collector: false + mandatory: false + aws_costexplorer_cost_custom: + cfn_type: null + cfn_type_source: override + category: costexplorer + aliases: [] + typed_collector: false + mandatory: false + aws_costexplorer_cost_forecast_30d: + cfn_type: null + cfn_type_source: override + category: costexplorer + aliases: [] + typed_collector: false + mandatory: false + aws_costexplorer_reservation_coverages: + cfn_type: null + cfn_type_source: override + category: costexplorer + aliases: [] + typed_collector: false + mandatory: false + aws_costexplorer_reservation_utilizations: + cfn_type: null + cfn_type_source: override + category: costexplorer + aliases: [] + typed_collector: false + mandatory: false + aws_costoptimizationhub_recommendations: + cfn_type: null + cfn_type_source: override + category: costoptimizationhub + aliases: [] + typed_collector: false + mandatory: false + aws_datapipeline_pipelines: + cfn_type: AWS::DataPipeline::Pipeline + cfn_type_source: heuristic + category: datapipeline + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_agents: + cfn_type: AWS::DataSync::Agent + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_azureblob_locations: + cfn_type: AWS::DataSync::AzureblobLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_efs_locations: + cfn_type: AWS::DataSync::EfsLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_fsxlustre_locations: + cfn_type: AWS::DataSync::FsxlustreLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_fsxontap_locations: + cfn_type: AWS::DataSync::FsxontapLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_fsxopenzfs_locations: + cfn_type: AWS::DataSync::FsxopenzfsLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_fsxwindows_locations: + cfn_type: AWS::DataSync::FsxwindowsLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_hdfs_locations: + cfn_type: AWS::DataSync::HdfsLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_locations: + cfn_type: AWS::DataSync::Location + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_nfs_locations: + cfn_type: AWS::DataSync::NfsLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_objectstorage_locations: + cfn_type: AWS::DataSync::ObjectstorageLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_s3_locations: + cfn_type: AWS::DataSync::S3Location + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_datasync_smb_locations: + cfn_type: AWS::DataSync::SmbLocation + cfn_type_source: heuristic + category: datasync + aliases: [] + typed_collector: false + mandatory: false + aws_dax_clusters: + cfn_type: AWS::DAX::Cluster + cfn_type_source: heuristic + category: dax + aliases: [] + typed_collector: false + mandatory: false + aws_detective_graph_members: + cfn_type: AWS::Detective::GraphMember + cfn_type_source: heuristic + category: detective + aliases: [] + typed_collector: false + mandatory: false + aws_detective_graphs: + cfn_type: AWS::Detective::Graph + cfn_type_source: heuristic + category: detective + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_anomalies: + cfn_type: AWS::DevOpsGuru::Anomaly + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_events: + cfn_type: AWS::DevOpsGuru::Event + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_insights: + cfn_type: AWS::DevOpsGuru::Insight + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_log_groups: + cfn_type: AWS::DevOpsGuru::LogGroup + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_monitored_resources: + cfn_type: AWS::DevOpsGuru::MonitoredResource + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_notification_channels: + cfn_type: AWS::DevOpsGuru::NotificationChannel + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_devopsguru_recommendations: + cfn_type: AWS::DevOpsGuru::Recommendation + cfn_type_source: heuristic + category: devopsguru + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_connections: + cfn_type: AWS::DirectConnect::Connection + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_gateway_associations: + cfn_type: AWS::DirectConnect::GatewayAssociation + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_gateway_attachments: + cfn_type: AWS::DirectConnect::GatewayAttachment + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_gateways: + cfn_type: AWS::DirectConnect::Gateway + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_lags: + cfn_type: AWS::DirectConnect::Lag + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_locations: + cfn_type: AWS::DirectConnect::Location + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_virtual_gateways: + cfn_type: AWS::DirectConnect::VirtualGateway + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directconnect_virtual_interfaces: + cfn_type: AWS::DirectConnect::VirtualInterface + cfn_type_source: heuristic + category: directconnect + aliases: [] + typed_collector: false + mandatory: false + aws_directoryservice_directories: + cfn_type: AWS::DirectoryService::Directory + cfn_type_source: heuristic + category: directoryservice + aliases: [] + typed_collector: false + mandatory: false + aws_dlm_lifecycle_policies: + cfn_type: AWS::DLM::LifecyclePolicy + cfn_type_source: heuristic + category: dlm + aliases: [] + typed_collector: false + mandatory: false + aws_dms_certificates: + cfn_type: AWS::DMS::Certificate + cfn_type_source: heuristic + category: dms + aliases: [] + typed_collector: false + mandatory: false + aws_dms_event_subscriptions: + cfn_type: AWS::DMS::EventSubscription + cfn_type_source: heuristic + category: dms + aliases: [] + typed_collector: false + mandatory: false + aws_dms_replication_instances: + cfn_type: AWS::DMS::ReplicationInstance + cfn_type_source: heuristic + category: dms + aliases: [] + typed_collector: false + mandatory: false + aws_dms_replication_subnet_groups: + cfn_type: AWS::DMS::ReplicationSubnetGroup + cfn_type_source: heuristic + category: dms + aliases: [] + typed_collector: false + mandatory: false + aws_dms_replication_tasks: + cfn_type: AWS::DMS::ReplicationTask + cfn_type_source: heuristic + category: dms + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_certificates: + cfn_type: AWS::DocDB::Certificate + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_cluster_parameter_groups: + cfn_type: AWS::DocDB::ClusterParameterGroup + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_cluster_parameters: + cfn_type: AWS::DocDB::ClusterParameter + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_cluster_snapshots: + cfn_type: AWS::DocDB::ClusterSnapshot + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_clusters: + cfn_type: AWS::DocDB::DBCluster + cfn_type_source: override + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_engine_versions: + cfn_type: AWS::DocDB::EngineVersion + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_event_categories: + cfn_type: AWS::DocDB::EventCategory + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_event_subscriptions: + cfn_type: AWS::DocDB::EventSubscription + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_events: + cfn_type: AWS::DocDB::Event + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_global_clusters: + cfn_type: AWS::DocDB::GlobalCluster + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_instances: + cfn_type: AWS::DocDB::DBInstance + cfn_type_source: override + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_orderable_db_instance_options: + cfn_type: AWS::DocDB::OrderableDbInstanceOption + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_pending_maintenance_actions: + cfn_type: AWS::DocDB::PendingMaintenanceAction + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_docdb_subnet_groups: + cfn_type: AWS::DocDB::SubnetGroup + cfn_type_source: heuristic + category: docdb + aliases: [] + typed_collector: false + mandatory: false + aws_dsql_cluster_policies: + cfn_type: AWS::DSQL::ClusterPolicy + cfn_type_source: heuristic + category: dsql + aliases: [] + typed_collector: false + mandatory: false + aws_dsql_clusters: + cfn_type: AWS::DSQL::Cluster + cfn_type_source: heuristic + category: dsql + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_backups: + cfn_type: AWS::DynamoDB::Backup + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_exports: + cfn_type: AWS::DynamoDB::Export + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_global_tables: + cfn_type: AWS::DynamoDB::GlobalTable + cfn_type_source: override + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_table_continuous_backups: + cfn_type: AWS::DynamoDB::TableContinuousBackup + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_table_replica_auto_scalings: + cfn_type: AWS::DynamoDB::TableReplicaAutoScaling + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_table_resource_policies: + cfn_type: AWS::DynamoDB::TableResourcePolicy + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_table_stream_resource_policies: + cfn_type: AWS::DynamoDB::TableStreamResourcePolicy + cfn_type_source: heuristic + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodb_tables: + cfn_type: AWS::DynamoDB::Table + cfn_type_source: override + category: dynamodb + aliases: [] + typed_collector: false + mandatory: false + aws_dynamodbstreams_streams: + cfn_type: AWS::DynamoDB::Stream + cfn_type_source: heuristic + category: dynamodbstreams + aliases: [] + typed_collector: false + mandatory: false + aws_ebs_default_kms_key_ids: + cfn_type: AWS::EC2::DefaultKmsKeyId + cfn_type_source: heuristic + category: ebs + aliases: [] + typed_collector: false + mandatory: false + aws_ebs_encryption_by_defaults: + cfn_type: AWS::EC2::EncryptionByDefault + cfn_type_source: heuristic + category: ebs + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_account_attributes: + cfn_type: null + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_byoip_cidrs: + cfn_type: AWS::EC2::ByoipCidr + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_capacity_reservation_topologies: + cfn_type: AWS::EC2::CapacityReservationTopology + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_capacity_reservations: + cfn_type: AWS::EC2::CapacityReservation + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_customer_gateways: + cfn_type: AWS::EC2::CustomerGateway + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_dhcp_options: + cfn_type: AWS::EC2::DHCPOptions + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ebs_snapshot_attributes: + cfn_type: AWS::EC2::EbsSnapshotAttribute + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ebs_snapshots: + cfn_type: AWS::EC2::Snapshot + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ebs_volume_statuses: + cfn_type: AWS::EC2::EbsVolumeStatus + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ebs_volumes: + cfn_type: AWS::EC2::Volume + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_egress_only_internet_gateways: + cfn_type: AWS::EC2::EgressOnlyInternetGateway + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_eips: + cfn_type: AWS::EC2::EIP + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_flow_logs: + cfn_type: AWS::EC2::FlowLog + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_hosts: + cfn_type: AWS::EC2::Host + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_image_block_public_access_states: + cfn_type: AWS::EC2::ImageBlockPublicAccessState + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_image_last_launched_times: + cfn_type: AWS::EC2::ImageLastLaunchedTime + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_image_launch_permissions: + cfn_type: AWS::EC2::ImageLaunchPermission + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_image_references: + cfn_type: AWS::EC2::ImageReference + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_images: + cfn_type: AWS::EC2::Image + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_connect_endpoints: + cfn_type: AWS::EC2::InstanceConnectEndpoint + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_credit_specifications: + cfn_type: AWS::EC2::InstanceCreditSpecification + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_disable_api_stop: + cfn_type: AWS::EC2::InstanceDisableApiStop + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_disable_api_termination: + cfn_type: AWS::EC2::InstanceDisableApiTermination + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_statuses: + cfn_type: null + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_topologies: + cfn_type: AWS::EC2::InstanceTopology + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_types: + cfn_type: null + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instance_user_data: + cfn_type: AWS::EC2::InstanceUserData + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_instances: + cfn_type: AWS::EC2::Instance + cfn_type_source: override + category: ec2 + aliases: + - ec2_instance + typed_collector: true + mandatory: false + aws_ec2_internet_gateways: + cfn_type: AWS::EC2::InternetGateway + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_address_history: + cfn_type: AWS::EC2::IpamAddressHistory + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_byoasns: + cfn_type: AWS::EC2::IpamByoasn + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_discovered_accounts: + cfn_type: AWS::EC2::IpamDiscoveredAccount + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_discovered_public_addresses: + cfn_type: AWS::EC2::IpamDiscoveredPublicAddress + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_discovered_resource_cidrs: + cfn_type: AWS::EC2::IpamDiscoveredResourceCidr + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_pool_allocations: + cfn_type: AWS::EC2::IpamPoolAllocation + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_pool_cidrs: + cfn_type: AWS::EC2::IpamPoolCidr + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_pools: + cfn_type: AWS::EC2::IpamPool + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_resource_cidrs: + cfn_type: AWS::EC2::IpamResourceCidr + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_resource_discoveries: + cfn_type: AWS::EC2::IpamResourceDiscovery + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_resource_discovery_associations: + cfn_type: AWS::EC2::IpamResourceDiscoveryAssociation + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipam_scopes: + cfn_type: AWS::EC2::IpamScope + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_ipams: + cfn_type: AWS::EC2::Ipam + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_key_pairs: + cfn_type: AWS::EC2::KeyPair + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_launch_template_versions: + cfn_type: AWS::EC2::LaunchTemplateVersion + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_launch_templates: + cfn_type: AWS::EC2::LaunchTemplate + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_managed_prefix_list_entries: + cfn_type: AWS::EC2::ManagedPrefixListEntry + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_managed_prefix_lists: + cfn_type: AWS::EC2::ManagedPrefixList + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_nat_gateways: + cfn_type: AWS::EC2::NatGateway + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_network_acls: + cfn_type: AWS::EC2::NetworkAcl + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_network_interfaces: + cfn_type: AWS::EC2::NetworkInterface + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_prefix_lists: + cfn_type: AWS::EC2::PrefixList + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_regional_configs: + cfn_type: AWS::EC2::RegionalConfig + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_replace_root_volume_tasks: + cfn_type: AWS::EC2::ReplaceRootVolumeTask + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_reserved_instances: + cfn_type: AWS::EC2::ReservedInstance + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_route_tables: + cfn_type: AWS::EC2::RouteTable + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_security_group_rules: + cfn_type: AWS::EC2::SecurityGroupRule + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_security_groups: + cfn_type: AWS::EC2::SecurityGroup + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_serial_console_access_statuses: + cfn_type: AWS::EC2::SerialConsoleAccessStatus + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_snapshot_block_public_access_states: + cfn_type: AWS::EC2::SnapshotBlockPublicAccessState + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_spot_fleet_instances: + cfn_type: AWS::EC2::SpotFleetInstance + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_spot_fleet_requests: + cfn_type: AWS::EC2::SpotFleetRequest + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_spot_instance_requests: + cfn_type: AWS::EC2::SpotInstanceRequest + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_subnets: + cfn_type: AWS::EC2::Subnet + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_traffic_mirror_filters: + cfn_type: AWS::EC2::TrafficMirrorFilter + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_traffic_mirror_sessions: + cfn_type: AWS::EC2::TrafficMirrorSession + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_traffic_mirror_targets: + cfn_type: AWS::EC2::TrafficMirrorTarget + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_attachments: + cfn_type: AWS::EC2::TransitGatewayAttachment + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_connect_peers: + cfn_type: AWS::EC2::TransitGatewayConnectPeer + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_multicast_domains: + cfn_type: AWS::EC2::TransitGatewayMulticastDomain + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_peering_attachments: + cfn_type: AWS::EC2::TransitGatewayPeeringAttachment + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_route_tables: + cfn_type: AWS::EC2::TransitGatewayRouteTable + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_routes: + cfn_type: AWS::EC2::TransitGatewayRoute + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateway_vpc_attachments: + cfn_type: AWS::EC2::TransitGatewayVpcAttachment + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_transit_gateways: + cfn_type: AWS::EC2::TransitGateway + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_endpoint_connections: + cfn_type: AWS::EC2::VpcEndpointConnection + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_endpoint_service_configurations: + cfn_type: AWS::EC2::VpcEndpointServiceConfiguration + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_endpoint_service_permissions: + cfn_type: AWS::EC2::VpcEndpointServicePermission + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_endpoint_services: + cfn_type: AWS::EC2::VpcEndpointService + cfn_type_source: heuristic + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_endpoints: + cfn_type: AWS::EC2::VPCEndpoint + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpc_peering_connections: + cfn_type: AWS::EC2::VPCPeeringConnection + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpcs: + cfn_type: AWS::EC2::VPC + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpn_connections: + cfn_type: AWS::EC2::VPNConnection + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ec2_vpn_gateways: + cfn_type: AWS::EC2::VPNGateway + cfn_type_source: override + category: ec2 + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_pull_through_cache_rules: + cfn_type: AWS::ECR::PullThroughCacheRule + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_registries: + cfn_type: AWS::ECR::Registry + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_registry_policies: + cfn_type: AWS::ECR::RegistryPolicy + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_repositories: + cfn_type: AWS::ECR::Repository + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_repository_image_scan_findings: + cfn_type: AWS::ECR::RepositoryImageScanFinding + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_repository_images: + cfn_type: AWS::ECR::RepositoryImage + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_repository_lifecycle_policies: + cfn_type: AWS::ECR::RepositoryLifecyclePolicy + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecr_repository_policies: + cfn_type: AWS::ECR::RepositoryPolicy + cfn_type_source: heuristic + category: ecr + aliases: [] + typed_collector: false + mandatory: false + aws_ecrpublic_repositories: + cfn_type: AWS::Ecrpublic::Repository + cfn_type_source: heuristic + category: ecrpublic + aliases: [] + typed_collector: false + mandatory: false + aws_ecrpublic_repository_images: + cfn_type: AWS::Ecrpublic::RepositoryImage + cfn_type_source: heuristic + category: ecrpublic + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_cluster_container_instances: + cfn_type: AWS::ECS::ClusterContainerInstance + cfn_type_source: heuristic + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_cluster_services: + cfn_type: AWS::ECS::Service + cfn_type_source: override + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_cluster_task_sets: + cfn_type: AWS::ECS::ClusterTaskSet + cfn_type_source: heuristic + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_cluster_tasks: + cfn_type: AWS::ECS::ClusterTask + cfn_type_source: heuristic + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_clusters: + cfn_type: AWS::ECS::Cluster + cfn_type_source: override + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_ecs_task_definitions: + cfn_type: AWS::ECS::TaskDefinition + cfn_type_source: override + category: ecs + aliases: [] + typed_collector: false + mandatory: false + aws_efs_access_points: + cfn_type: AWS::EFS::AccessPoint + cfn_type_source: heuristic + category: efs + aliases: [] + typed_collector: false + mandatory: false + aws_efs_filesystems: + cfn_type: AWS::EFS::FileSystem + cfn_type_source: override + category: efs + aliases: [] + typed_collector: false + mandatory: false + aws_eks_access_policies: + cfn_type: AWS::EKS::AccessPolicy + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_access_entries: + cfn_type: AWS::EKS::ClusterAccessEntry + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_addons: + cfn_type: AWS::EKS::ClusterAddon + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_associated_access_policies: + cfn_type: AWS::EKS::ClusterAssociatedAccessPolicy + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_node_groups: + cfn_type: AWS::EKS::Nodegroup + cfn_type_source: override + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_oidc_identity_provider_configs: + cfn_type: AWS::EKS::ClusterOidcIdentityProviderConfig + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_cluster_versions: + cfn_type: AWS::EKS::ClusterVersion + cfn_type_source: heuristic + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_clusters: + cfn_type: AWS::EKS::Cluster + cfn_type_source: override + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_eks_fargate_profiles: + cfn_type: AWS::EKS::FargateProfile + cfn_type_source: override + category: eks + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_clusters: + cfn_type: AWS::ElastiCache::CacheCluster + cfn_type_source: override + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_engine_versions: + cfn_type: AWS::ElastiCache::EngineVersion + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_events: + cfn_type: AWS::ElastiCache::Event + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_global_replication_groups: + cfn_type: AWS::ElastiCache::GlobalReplicationGroup + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_parameter_groups: + cfn_type: AWS::ElastiCache::ParameterGroup + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_replication_groups: + cfn_type: AWS::ElastiCache::ReplicationGroup + cfn_type_source: override + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_reserved_cache_nodes: + cfn_type: AWS::ElastiCache::ReservedCacheNode + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_reserved_cache_nodes_offerings: + cfn_type: AWS::ElastiCache::ReservedCacheNodesOffering + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_serverless_cache_snapshots: + cfn_type: AWS::ElastiCache::ServerlessCacheSnapshot + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_serverless_caches: + cfn_type: AWS::ElastiCache::ServerlessCach + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_service_updates: + cfn_type: AWS::ElastiCache::ServiceUpdate + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_snapshots: + cfn_type: AWS::ElastiCache::Snapshot + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_subnet_groups: + cfn_type: AWS::ElastiCache::SubnetGroup + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_update_actions: + cfn_type: AWS::ElastiCache::UpdateAction + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_user_groups: + cfn_type: AWS::ElastiCache::UserGroup + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticache_users: + cfn_type: AWS::ElastiCache::User + cfn_type_source: heuristic + category: elasticache + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_application_versions: + cfn_type: AWS::ElasticBeanstalk::ApplicationVersion + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_applications: + cfn_type: AWS::ElasticBeanstalk::Application + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_configuration_options: + cfn_type: AWS::ElasticBeanstalk::ConfigurationOption + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_configuration_settings: + cfn_type: AWS::ElasticBeanstalk::ConfigurationSetting + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_environments: + cfn_type: AWS::ElasticBeanstalk::Environment + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticbeanstalk_platform_versions: + cfn_type: AWS::ElasticBeanstalk::PlatformVersion + cfn_type_source: heuristic + category: elasticbeanstalk + aliases: [] + typed_collector: false + mandatory: false + aws_elasticsearch_domains: + cfn_type: AWS::Elasticsearch::Domain + cfn_type_source: heuristic + category: elasticsearch + aliases: [] + typed_collector: false + mandatory: false + aws_elasticsearch_packages: + cfn_type: AWS::Elasticsearch::Package + cfn_type_source: heuristic + category: elasticsearch + aliases: [] + typed_collector: false + mandatory: false + aws_elasticsearch_reserved_instances: + cfn_type: AWS::Elasticsearch::ReservedInstance + cfn_type_source: heuristic + category: elasticsearch + aliases: [] + typed_collector: false + mandatory: false + aws_elasticsearch_versions: + cfn_type: AWS::Elasticsearch::Version + cfn_type_source: heuristic + category: elasticsearch + aliases: [] + typed_collector: false + mandatory: false + aws_elasticsearch_vpc_endpoints: + cfn_type: AWS::Elasticsearch::VpcEndpoint + cfn_type_source: heuristic + category: elasticsearch + aliases: [] + typed_collector: false + mandatory: false + aws_elbv1_load_balancer_policies: + cfn_type: AWS::ElasticLoadBalancing::LoadBalancerPolicy + cfn_type_source: heuristic + category: elbv1 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv1_load_balancers: + cfn_type: AWS::ElasticLoadBalancing::LoadBalancer + cfn_type_source: override + category: elbv1 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_listener_certificates: + cfn_type: AWS::ElasticLoadBalancingV2::ListenerCertificate + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_listener_rules: + cfn_type: AWS::ElasticLoadBalancingV2::ListenerRule + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_listeners: + cfn_type: AWS::ElasticLoadBalancingV2::Listener + cfn_type_source: override + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_load_balancer_attributes: + cfn_type: AWS::ElasticLoadBalancingV2::LoadBalancerAttribute + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_load_balancer_capacity_reservations: + cfn_type: AWS::ElasticLoadBalancingV2::LoadBalancerCapacityReservation + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_load_balancer_web_acls: + cfn_type: AWS::ElasticLoadBalancingV2::LoadBalancerWebAcl + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_load_balancers: + cfn_type: AWS::ElasticLoadBalancingV2::LoadBalancer + cfn_type_source: override + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_target_group_attributes: + cfn_type: AWS::ElasticLoadBalancingV2::TargetGroupAttribute + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_target_group_target_health_descriptions: + cfn_type: AWS::ElasticLoadBalancingV2::TargetGroupTargetHealthDescription + cfn_type_source: heuristic + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_elbv2_target_groups: + cfn_type: AWS::ElasticLoadBalancingV2::TargetGroup + cfn_type_source: override + category: elbv2 + aliases: [] + typed_collector: false + mandatory: false + aws_emr_block_public_access_configs: + cfn_type: AWS::EMR::BlockPublicAccessConfig + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_cluster_instance_fleets: + cfn_type: AWS::EMR::ClusterInstanceFleet + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_cluster_instance_groups: + cfn_type: AWS::EMR::ClusterInstanceGroup + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_cluster_instances: + cfn_type: AWS::EMR::ClusterInstance + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_clusters: + cfn_type: AWS::EMR::Cluster + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_notebook_executions: + cfn_type: AWS::EMR::NotebookExecution + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_release_labels: + cfn_type: AWS::EMR::ReleaseLabel + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_security_configurations: + cfn_type: AWS::EMR::SecurityConfiguration + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_steps: + cfn_type: AWS::EMR::Step + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_studio_session_mappings: + cfn_type: AWS::EMR::StudioSessionMapping + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_studios: + cfn_type: AWS::EMR::Studio + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_emr_supported_instance_types: + cfn_type: AWS::EMR::SupportedInstanceType + cfn_type_source: heuristic + category: emr + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_api_destinations: + cfn_type: AWS::Events::ApiDestination + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_archives: + cfn_type: AWS::Events::Archive + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_connections: + cfn_type: AWS::Events::Connection + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_endpoints: + cfn_type: AWS::Events::Endpoint + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_event_bus_rules: + cfn_type: AWS::Events::EventBusRule + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_event_bus_targets: + cfn_type: AWS::Events::EventBusTarget + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_event_buses: + cfn_type: AWS::Events::EventBus + cfn_type_source: override + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_event_sources: + cfn_type: AWS::Events::EventSource + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_eventbridge_replays: + cfn_type: AWS::Events::Replay + cfn_type_source: heuristic + category: eventbridge + aliases: [] + typed_collector: false + mandatory: false + aws_firehose_delivery_streams: + cfn_type: AWS::KinesisFirehose::DeliveryStream + cfn_type_source: override + category: firehose + aliases: [] + typed_collector: false + mandatory: false + aws_fis_actions: + cfn_type: AWS::Fis::Action + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_fis_experiment_resolved_targets: + cfn_type: AWS::Fis::ExperimentResolvedTarget + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_fis_experiment_templates: + cfn_type: AWS::Fis::ExperimentTemplate + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_fis_experiments: + cfn_type: AWS::Fis::Experiment + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_fis_target_account_configurations: + cfn_type: AWS::Fis::TargetAccountConfiguration + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_fis_target_resource_types: + cfn_type: AWS::Fis::TargetResourceType + cfn_type_source: heuristic + category: fis + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_batch_imports: + cfn_type: AWS::FraudDetector::BatchImport + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_batch_predictions: + cfn_type: AWS::FraudDetector::BatchPrediction + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_detectors: + cfn_type: AWS::FraudDetector::Detector + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_entity_types: + cfn_type: AWS::FraudDetector::EntityType + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_event_types: + cfn_type: AWS::FraudDetector::EventType + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_external_models: + cfn_type: AWS::FraudDetector::ExternalModel + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_labels: + cfn_type: AWS::FraudDetector::Label + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_model_versions: + cfn_type: AWS::FraudDetector::ModelVersion + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_models: + cfn_type: AWS::FraudDetector::Model + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_outcomes: + cfn_type: AWS::FraudDetector::Outcome + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_rules: + cfn_type: AWS::FraudDetector::Rule + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_frauddetector_variables: + cfn_type: AWS::FraudDetector::Variable + cfn_type_source: heuristic + category: frauddetector + aliases: [] + typed_collector: false + mandatory: false + aws_freetier_usages: + cfn_type: AWS::Freetier::Usage + cfn_type_source: heuristic + category: freetier + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_backups: + cfn_type: AWS::FSx::Backup + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_data_repository_associations: + cfn_type: AWS::FSx::DataRepositoryAssociation + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_data_repository_tasks: + cfn_type: AWS::FSx::DataRepositoryTask + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_file_caches: + cfn_type: AWS::FSx::FileCach + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_file_systems: + cfn_type: AWS::FSx::FileSystem + cfn_type_source: override + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_snapshots: + cfn_type: AWS::FSx::Snapshot + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_storage_virtual_machines: + cfn_type: AWS::FSx::StorageVirtualMachine + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_fsx_volumes: + cfn_type: AWS::FSx::Volume + cfn_type_source: heuristic + category: fsx + aliases: [] + typed_collector: false + mandatory: false + aws_glacier_data_retrieval_policies: + cfn_type: AWS::Glacier::DataRetrievalPolicy + cfn_type_source: heuristic + category: glacier + aliases: [] + typed_collector: false + mandatory: false + aws_glacier_vault_access_policies: + cfn_type: AWS::Glacier::VaultAccessPolicy + cfn_type_source: heuristic + category: glacier + aliases: [] + typed_collector: false + mandatory: false + aws_glacier_vault_lock_policies: + cfn_type: AWS::Glacier::VaultLockPolicy + cfn_type_source: heuristic + category: glacier + aliases: [] + typed_collector: false + mandatory: false + aws_glacier_vault_notifications: + cfn_type: AWS::Glacier::VaultNotification + cfn_type_source: heuristic + category: glacier + aliases: [] + typed_collector: false + mandatory: false + aws_glacier_vaults: + cfn_type: AWS::Glacier::Vault + cfn_type_source: override + category: glacier + aliases: [] + typed_collector: false + mandatory: false + aws_globalaccelerator_accelerators: + cfn_type: AWS::GlobalAccelerator::Accelerator + cfn_type_source: override + category: globalaccelerator + aliases: [] + typed_collector: false + mandatory: false + aws_globalaccelerator_custom_routing_accelerators: + cfn_type: AWS::GlobalAccelerator::CustomRoutingAccelerator + cfn_type_source: heuristic + category: globalaccelerator + aliases: [] + typed_collector: false + mandatory: false + aws_globalaccelerator_endpoint_groups: + cfn_type: AWS::GlobalAccelerator::EndpointGroup + cfn_type_source: heuristic + category: globalaccelerator + aliases: [] + typed_collector: false + mandatory: false + aws_globalaccelerator_listeners: + cfn_type: AWS::GlobalAccelerator::Listener + cfn_type_source: heuristic + category: globalaccelerator + aliases: [] + typed_collector: false + mandatory: false + aws_glue_catalogs: + cfn_type: AWS::Glue::Catalog + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_classifiers: + cfn_type: AWS::Glue::Classifier + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_connections: + cfn_type: AWS::Glue::Connection + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_crawlers: + cfn_type: AWS::Glue::Crawler + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_database_table_indexes: + cfn_type: AWS::Glue::DatabaseTableIndex + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_database_table_storage_optimizers: + cfn_type: AWS::Glue::DatabaseTableStorageOptimizer + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_database_tables: + cfn_type: AWS::Glue::DatabaseTable + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_databases: + cfn_type: AWS::Glue::Databas + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_datacatalog_encryption_settings: + cfn_type: AWS::Glue::DatacatalogEncryptionSetting + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_dev_endpoints: + cfn_type: AWS::Glue::DevEndpoint + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_job_runs: + cfn_type: AWS::Glue::JobRun + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_jobs: + cfn_type: AWS::Glue::Job + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_ml_transform_task_runs: + cfn_type: AWS::Glue::MlTransformTaskRun + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_ml_transforms: + cfn_type: AWS::Glue::MlTransform + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_registries: + cfn_type: AWS::Glue::Registry + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_registry_schema_versions: + cfn_type: AWS::Glue::RegistrySchemaVersion + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_registry_schemas: + cfn_type: AWS::Glue::RegistrySchema + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_security_configurations: + cfn_type: AWS::Glue::SecurityConfiguration + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_triggers: + cfn_type: AWS::Glue::Trigger + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_glue_workflows: + cfn_type: AWS::Glue::Workflow + cfn_type_source: heuristic + category: glue + aliases: [] + typed_collector: false + mandatory: false + aws_grafana_permissions: + cfn_type: AWS::Grafana::Permission + cfn_type_source: heuristic + category: grafana + aliases: [] + typed_collector: false + mandatory: false + aws_grafana_versions: + cfn_type: AWS::Grafana::Version + cfn_type_source: heuristic + category: grafana + aliases: [] + typed_collector: false + mandatory: false + aws_grafana_workspace_service_account_tokens: + cfn_type: AWS::Grafana::WorkspaceServiceAccountToken + cfn_type_source: heuristic + category: grafana + aliases: [] + typed_collector: false + mandatory: false + aws_grafana_workspace_service_accounts: + cfn_type: AWS::Grafana::WorkspaceServiceAccount + cfn_type_source: heuristic + category: grafana + aliases: [] + typed_collector: false + mandatory: false + aws_grafana_workspaces: + cfn_type: AWS::Grafana::Workspace + cfn_type_source: heuristic + category: grafana + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_coverages: + cfn_type: AWS::GuardDuty::DetectorCoverage + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_filters: + cfn_type: AWS::GuardDuty::DetectorFilter + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_findings: + cfn_type: AWS::GuardDuty::DetectorFinding + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_intel_sets: + cfn_type: AWS::GuardDuty::DetectorIntelSet + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_ip_sets: + cfn_type: AWS::GuardDuty::DetectorIpSet + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_members: + cfn_type: AWS::GuardDuty::DetectorMember + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detector_publishing_destinations: + cfn_type: AWS::GuardDuty::DetectorPublishingDestination + cfn_type_source: heuristic + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_guardduty_detectors: + cfn_type: AWS::GuardDuty::Detector + cfn_type_source: override + category: guardduty + aliases: [] + typed_collector: false + mandatory: false + aws_health_affected_entities: + cfn_type: AWS::Health::AffectedEntity + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_health_event_details: + cfn_type: AWS::Health::EventDetail + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_health_events: + cfn_type: AWS::Health::Event + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_health_org_event_details: + cfn_type: AWS::Health::OrgEventDetail + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_health_organization_affected_entities: + cfn_type: AWS::Health::OrganizationAffectedEntity + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_health_organization_events: + cfn_type: AWS::Health::OrganizationEvent + cfn_type_source: heuristic + category: health + aliases: [] + typed_collector: false + mandatory: false + aws_healthlake_fhir_datastores: + cfn_type: AWS::HealthLake::FhirDatastore + cfn_type_source: heuristic + category: healthlake + aliases: [] + typed_collector: false + mandatory: false + aws_iam_account_authorization_details: + cfn_type: null + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_accounts: + cfn_type: null + cfn_type_source: override + category: iam + aliases: + - account + - aws_account + typed_collector: true + mandatory: true + aws_iam_credential_reports: + cfn_type: null + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_group_attached_policies: + cfn_type: AWS::IAM::GroupAttachedPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_group_last_accessed_details: + cfn_type: AWS::IAM::GroupLastAccessedDetail + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_group_policies: + cfn_type: AWS::IAM::GroupPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_groups: + cfn_type: AWS::IAM::Group + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_instance_profiles: + cfn_type: AWS::IAM::InstanceProfile + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_mfa_devices: + cfn_type: AWS::IAM::MfaDevice + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_openid_connect_identity_providers: + cfn_type: AWS::IAM::OpenidConnectIdentityProvider + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_outbound_web_identity_federations: + cfn_type: AWS::IAM::OutboundWebIdentityFederation + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_password_policies: + cfn_type: AWS::IAM::PasswordPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_policies: + cfn_type: AWS::IAM::ManagedPolicy + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_policy_default_versions: + cfn_type: AWS::IAM::PolicyDefaultVersion + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_policy_last_accessed_details: + cfn_type: AWS::IAM::PolicyLastAccessedDetail + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_policy_versions: + cfn_type: AWS::IAM::PolicyVersion + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_role_attached_policies: + cfn_type: AWS::IAM::RoleAttachedPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_role_last_accessed_details: + cfn_type: AWS::IAM::RoleLastAccessedDetail + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_role_policies: + cfn_type: AWS::IAM::RolePolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_roles: + cfn_type: AWS::IAM::Role + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_saml_identity_providers: + cfn_type: AWS::IAM::SamlIdentityProvider + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_server_certificates: + cfn_type: AWS::IAM::ServerCertificate + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_signing_certificates: + cfn_type: AWS::IAM::SigningCertificate + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_ssh_public_keys: + cfn_type: AWS::IAM::SshPublicKey + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_user_access_keys: + cfn_type: AWS::IAM::UserAccessKey + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_user_attached_policies: + cfn_type: AWS::IAM::UserAttachedPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_user_groups: + cfn_type: AWS::IAM::UserGroup + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_user_last_accessed_details: + cfn_type: AWS::IAM::UserLastAccessedDetail + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_user_policies: + cfn_type: AWS::IAM::UserPolicy + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_users: + cfn_type: AWS::IAM::User + cfn_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_iam_virtual_mfa_devices: + cfn_type: AWS::IAM::VirtualMfaDevice + cfn_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + aws_identitystore_group_memberships: + cfn_type: AWS::IdentityStore::GroupMembership + cfn_type_source: heuristic + category: identitystore + aliases: [] + typed_collector: false + mandatory: false + aws_identitystore_groups: + cfn_type: AWS::IdentityStore::Group + cfn_type_source: heuristic + category: identitystore + aliases: [] + typed_collector: false + mandatory: false + aws_identitystore_users: + cfn_type: AWS::IdentityStore::User + cfn_type_source: heuristic + category: identitystore + aliases: [] + typed_collector: false + mandatory: false + aws_imagebuilder_distribution_configurations: + cfn_type: AWS::ImageBuilder::DistributionConfiguration + cfn_type_source: heuristic + category: imagebuilder + aliases: [] + typed_collector: false + mandatory: false + aws_imagebuilder_images: + cfn_type: AWS::ImageBuilder::Image + cfn_type_source: heuristic + category: imagebuilder + aliases: [] + typed_collector: false + mandatory: false + aws_imagebuilder_workflows: + cfn_type: AWS::ImageBuilder::Workflow + cfn_type_source: heuristic + category: imagebuilder + aliases: [] + typed_collector: false + mandatory: false + aws_inspector2_cis_scan_result_details: + cfn_type: AWS::InspectorV2::CisScanResultDetail + cfn_type_source: heuristic + category: inspector2 + aliases: [] + typed_collector: false + mandatory: false + aws_inspector2_cis_scans: + cfn_type: AWS::InspectorV2::CisScan + cfn_type_source: heuristic + category: inspector2 + aliases: [] + typed_collector: false + mandatory: false + aws_inspector2_cis_target_resource_aggregations: + cfn_type: AWS::InspectorV2::CisTargetResourceAggregation + cfn_type_source: heuristic + category: inspector2 + aliases: [] + typed_collector: false + mandatory: false + aws_inspector2_covered_resources: + cfn_type: AWS::InspectorV2::CoveredResource + cfn_type_source: heuristic + category: inspector2 + aliases: [] + typed_collector: false + mandatory: false + aws_inspector2_findings: + cfn_type: AWS::InspectorV2::Finding + cfn_type_source: heuristic + category: inspector2 + aliases: [] + typed_collector: false + mandatory: false + aws_inspector_findings: + cfn_type: AWS::Inspector::Finding + cfn_type_source: heuristic + category: inspector + aliases: [] + typed_collector: false + mandatory: false + aws_invoicing_invoice_units: + cfn_type: AWS::Invoicing::InvoiceUnit + cfn_type_source: heuristic + category: invoicing + aliases: [] + typed_collector: false + mandatory: false + aws_iot_billing_groups: + cfn_type: AWS::IoT::BillingGroup + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_ca_certificates: + cfn_type: AWS::IoT::CaCertificate + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_certificates: + cfn_type: AWS::IoT::Certificate + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_jobs: + cfn_type: AWS::IoT::Job + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_policies: + cfn_type: AWS::IoT::Policy + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_security_profiles: + cfn_type: AWS::IoT::SecurityProfile + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_streams: + cfn_type: AWS::IoT::Stream + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_thing_groups: + cfn_type: AWS::IoT::ThingGroup + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_thing_types: + cfn_type: AWS::IoT::ThingType + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_things: + cfn_type: AWS::IoT::Thing + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_iot_topic_rules: + cfn_type: AWS::IoT::TopicRule + cfn_type_source: heuristic + category: iot + aliases: [] + typed_collector: false + mandatory: false + aws_kafka_cluster_operations: + cfn_type: AWS::MSK::ClusterOperation + cfn_type_source: heuristic + category: kafka + aliases: [] + typed_collector: false + mandatory: false + aws_kafka_cluster_policies: + cfn_type: AWS::MSK::ClusterPolicy + cfn_type_source: heuristic + category: kafka + aliases: [] + typed_collector: false + mandatory: false + aws_kafka_clusters: + cfn_type: AWS::MSK::Cluster + cfn_type_source: override + category: kafka + aliases: [] + typed_collector: false + mandatory: false + aws_kafka_configurations: + cfn_type: AWS::MSK::Configuration + cfn_type_source: heuristic + category: kafka + aliases: [] + typed_collector: false + mandatory: false + aws_kafka_nodes: + cfn_type: AWS::MSK::Node + cfn_type_source: heuristic + category: kafka + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_access_control_configurations: + cfn_type: AWS::Kendra::AccessControlConfiguration + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_data_source_sync_jobs: + cfn_type: AWS::Kendra::DataSourceSyncJob + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_data_sources: + cfn_type: AWS::Kendra::DataSource + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_experience_entities: + cfn_type: AWS::Kendra::ExperienceEntity + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_experience_entity_personas: + cfn_type: AWS::Kendra::ExperienceEntityPersona + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_experiences: + cfn_type: AWS::Kendra::Experience + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_faqs: + cfn_type: AWS::Kendra::Faq + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_featured_results_sets: + cfn_type: AWS::Kendra::FeaturedResultsSet + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_indices: + cfn_type: AWS::Kendra::Index + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_query_suggestions_block_lists: + cfn_type: AWS::Kendra::QuerySuggestionsBlockList + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_query_suggestions_configs: + cfn_type: AWS::Kendra::QuerySuggestionsConfig + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_kendra_thesauri: + cfn_type: AWS::Kendra::Thesaurus + cfn_type_source: heuristic + category: kendra + aliases: [] + typed_collector: false + mandatory: false + aws_keyspaces_keyspaces: + cfn_type: AWS::Cassandra::Keyspace + cfn_type_source: heuristic + category: keyspaces + aliases: [] + typed_collector: false + mandatory: false + aws_keyspaces_tables: + cfn_type: AWS::Cassandra::Table + cfn_type_source: heuristic + category: keyspaces + aliases: [] + typed_collector: false + mandatory: false + aws_kinesis_stream_consumers: + cfn_type: AWS::Kinesis::StreamConsumer + cfn_type_source: heuristic + category: kinesis + aliases: [] + typed_collector: false + mandatory: false + aws_kinesis_stream_shards: + cfn_type: AWS::Kinesis::StreamShard + cfn_type_source: heuristic + category: kinesis + aliases: [] + typed_collector: false + mandatory: false + aws_kinesis_streams: + cfn_type: AWS::Kinesis::Stream + cfn_type_source: override + category: kinesis + aliases: [] + typed_collector: false + mandatory: false + aws_kinesisanalytics_application_operations: + cfn_type: AWS::KinesisAnalyticsV2::ApplicationOperation + cfn_type_source: heuristic + category: kinesisanalytics + aliases: [] + typed_collector: false + mandatory: false + aws_kinesisanalytics_application_snapshots: + cfn_type: AWS::KinesisAnalyticsV2::ApplicationSnapshot + cfn_type_source: heuristic + category: kinesisanalytics + aliases: [] + typed_collector: false + mandatory: false + aws_kinesisanalytics_application_versions: + cfn_type: AWS::KinesisAnalyticsV2::ApplicationVersion + cfn_type_source: heuristic + category: kinesisanalytics + aliases: [] + typed_collector: false + mandatory: false + aws_kinesisanalytics_applications: + cfn_type: AWS::KinesisAnalyticsV2::Application + cfn_type_source: heuristic + category: kinesisanalytics + aliases: [] + typed_collector: false + mandatory: false + aws_kinesisvideo_streams: + cfn_type: AWS::KinesisVideo::Stream + cfn_type_source: heuristic + category: kinesisvideo + aliases: [] + typed_collector: false + mandatory: false + aws_kms_aliases: + cfn_type: AWS::KMS::Alias + cfn_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_kms_key_grants: + cfn_type: AWS::KMS::KeyGrant + cfn_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_kms_key_policies: + cfn_type: AWS::KMS::KeyPolicy + cfn_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_kms_key_rotation_statuses: + cfn_type: AWS::KMS::KeyRotationStatus + cfn_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_kms_key_rotations: + cfn_type: AWS::KMS::KeyRotation + cfn_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_kms_keys: + cfn_type: AWS::KMS::Key + cfn_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_data_cells_filters: + cfn_type: AWS::LakeFormation::DataCellsFilter + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_opt_ins: + cfn_type: AWS::LakeFormation::OptIn + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_permissions: + cfn_type: AWS::LakeFormation::Permission + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_resource_tags: + cfn_type: AWS::LakeFormation::ResourceTag + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_resources: + cfn_type: AWS::LakeFormation::Resource + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_tags: + cfn_type: AWS::LakeFormation::Tag + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lakeformation_transactions: + cfn_type: AWS::LakeFormation::Transaction + cfn_type_source: heuristic + category: lakeformation + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_aliases: + cfn_type: AWS::Lambda::FunctionAlias + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_concurrency_configs: + cfn_type: AWS::Lambda::FunctionConcurrencyConfig + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_event_invoke_configs: + cfn_type: AWS::Lambda::FunctionEventInvokeConfig + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_event_source_mappings: + cfn_type: AWS::Lambda::FunctionEventSourceMapping + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_url_configs: + cfn_type: AWS::Lambda::FunctionUrlConfig + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_function_versions: + cfn_type: AWS::Lambda::FunctionVersion + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_functions: + cfn_type: AWS::Lambda::Function + cfn_type_source: override + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_layer_version_policies: + cfn_type: AWS::Lambda::LayerVersionPolicy + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_layer_versions: + cfn_type: AWS::Lambda::LayerVersion + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_layers: + cfn_type: AWS::Lambda::LayerVersion + cfn_type_source: override + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lambda_runtimes: + cfn_type: AWS::Lambda::Runtime + cfn_type_source: heuristic + category: lambda + aliases: [] + typed_collector: false + mandatory: false + aws_lex_bot_aliases: + cfn_type: AWS::Lex::BotAlias + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_bot_channel_associations: + cfn_type: AWS::Lex::BotChannelAssociation + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_bot_version_utterances_views: + cfn_type: AWS::Lex::BotVersionUtterancesView + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_bot_versions: + cfn_type: AWS::Lex::BotVersion + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_bots: + cfn_type: AWS::Lex::Bot + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_builtin_intents: + cfn_type: AWS::Lex::BuiltinIntent + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_builtin_slot_types: + cfn_type: AWS::Lex::BuiltinSlotType + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_intent_versions: + cfn_type: AWS::Lex::IntentVersion + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_intents: + cfn_type: AWS::Lex::Intent + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_migrations: + cfn_type: AWS::Lex::Migration + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_slot_type_versions: + cfn_type: AWS::Lex::SlotTypeVersion + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lex_slot_types: + cfn_type: AWS::Lex::SlotType + cfn_type_source: heuristic + category: lex + aliases: [] + typed_collector: false + mandatory: false + aws_lexv2_bot_aliases: + cfn_type: AWS::LexV2::BotAlias + cfn_type_source: heuristic + category: lexv2 + aliases: [] + typed_collector: false + mandatory: false + aws_lexv2_bots: + cfn_type: AWS::LexV2::Bot + cfn_type_source: heuristic + category: lexv2 + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_alarms: + cfn_type: AWS::Lightsail::Alarm + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_bucket_access_keys: + cfn_type: AWS::Lightsail::BucketAccessKey + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_buckets: + cfn_type: AWS::Lightsail::Bucket + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_certificates: + cfn_type: AWS::Lightsail::Certificate + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_container_service_deployments: + cfn_type: AWS::Lightsail::ContainerServiceDeployment + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_container_service_images: + cfn_type: AWS::Lightsail::ContainerServiceImage + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_container_services: + cfn_type: AWS::Lightsail::ContainerService + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_database_events: + cfn_type: AWS::Lightsail::DatabaseEvent + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_database_log_events: + cfn_type: AWS::Lightsail::DatabaseLogEvent + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_database_parameters: + cfn_type: AWS::Lightsail::DatabaseParameter + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_database_snapshots: + cfn_type: AWS::Lightsail::DatabaseSnapshot + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_databases: + cfn_type: AWS::Lightsail::Databas + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_disk_snapshots: + cfn_type: AWS::Lightsail::DiskSnapshot + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_disks: + cfn_type: AWS::Lightsail::Disk + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_distributions: + cfn_type: AWS::Lightsail::Distribution + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_instance_port_states: + cfn_type: AWS::Lightsail::InstancePortState + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_instance_snapshots: + cfn_type: AWS::Lightsail::InstanceSnapshot + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_instances: + cfn_type: AWS::Lightsail::Instance + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_load_balancer_tls_certificates: + cfn_type: AWS::Lightsail::LoadBalancerTlsCertificate + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_load_balancers: + cfn_type: AWS::Lightsail::LoadBalancer + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_lightsail_static_ips: + cfn_type: AWS::Lightsail::StaticIp + cfn_type_source: heuristic + category: lightsail + aliases: [] + typed_collector: false + mandatory: false + aws_location_keys: + cfn_type: AWS::Location::Key + cfn_type_source: heuristic + category: location + aliases: [] + typed_collector: false + mandatory: false + aws_location_maps: + cfn_type: AWS::Location::Map + cfn_type_source: heuristic + category: location + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_allow_lists: + cfn_type: AWS::Macie::AllowList + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_automated_discovery_accounts: + cfn_type: AWS::Macie::AutomatedDiscoveryAccount + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_classification_jobs: + cfn_type: AWS::Macie::ClassificationJob + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_classification_scopes: + cfn_type: AWS::Macie::ClassificationScope + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_custom_data_identifiers: + cfn_type: AWS::Macie::CustomDataIdentifier + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_findings: + cfn_type: AWS::Macie::Finding + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_invitations: + cfn_type: AWS::Macie::Invitation + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_managed_data_identifiers: + cfn_type: AWS::Macie::ManagedDataIdentifier + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_members: + cfn_type: AWS::Macie::Member + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_sensitivity_inspection_templates: + cfn_type: AWS::Macie::SensitivityInspectionTemplate + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_macie2_usage_totals: + cfn_type: AWS::Macie::UsageTotal + cfn_type_source: heuristic + category: macie2 + aliases: [] + typed_collector: false + mandatory: false + aws_memorydb_reserved_nodes: + cfn_type: AWS::MemoryDB::Cluster + cfn_type_source: override + category: memorydb + aliases: [] + typed_collector: false + mandatory: false + aws_mpa_teams: + cfn_type: AWS::Mpa::Team + cfn_type_source: heuristic + category: mpa + aliases: [] + typed_collector: false + mandatory: false + aws_mq_broker_configuration_revisions: + cfn_type: AWS::AmazonMQ::BrokerConfigurationRevision + cfn_type_source: heuristic + category: mq + aliases: [] + typed_collector: false + mandatory: false + aws_mq_broker_configurations: + cfn_type: AWS::AmazonMQ::BrokerConfiguration + cfn_type_source: heuristic + category: mq + aliases: [] + typed_collector: false + mandatory: false + aws_mq_broker_users: + cfn_type: AWS::AmazonMQ::BrokerUser + cfn_type_source: heuristic + category: mq + aliases: [] + typed_collector: false + mandatory: false + aws_mq_brokers: + cfn_type: AWS::AmazonMQ::Broker + cfn_type_source: override + category: mq + aliases: [] + typed_collector: false + mandatory: false + aws_mwaa_environments: + cfn_type: AWS::MWAA::Environment + cfn_type_source: heuristic + category: mwaa + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_cluster_parameter_group_parameters: + cfn_type: AWS::Neptune::ClusterParameterGroupParameter + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_cluster_parameter_groups: + cfn_type: AWS::Neptune::ClusterParameterGroup + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_cluster_snapshots: + cfn_type: AWS::Neptune::ClusterSnapshot + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_clusters: + cfn_type: AWS::Neptune::DBCluster + cfn_type_source: override + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_db_parameter_group_db_parameters: + cfn_type: AWS::Neptune::DbParameterGroupDbParameter + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_db_parameter_groups: + cfn_type: AWS::Neptune::DbParameterGroup + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_event_subscriptions: + cfn_type: AWS::Neptune::EventSubscription + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_global_clusters: + cfn_type: AWS::Neptune::GlobalCluster + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_instances: + cfn_type: AWS::Neptune::DBInstance + cfn_type_source: override + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_neptune_subnet_groups: + cfn_type: AWS::Neptune::SubnetGroup + cfn_type_source: heuristic + category: neptune + aliases: [] + typed_collector: false + mandatory: false + aws_networkfirewall_firewall_policies: + cfn_type: AWS::NetworkFirewall::FirewallPolicy + cfn_type_source: heuristic + category: networkfirewall + aliases: [] + typed_collector: false + mandatory: false + aws_networkfirewall_firewalls: + cfn_type: AWS::NetworkFirewall::Firewall + cfn_type_source: heuristic + category: networkfirewall + aliases: [] + typed_collector: false + mandatory: false + aws_networkfirewall_rule_groups: + cfn_type: AWS::NetworkFirewall::RuleGroup + cfn_type_source: heuristic + category: networkfirewall + aliases: [] + typed_collector: false + mandatory: false + aws_networkfirewall_tls_inspection_configurations: + cfn_type: AWS::NetworkFirewall::TlsInspectionConfiguration + cfn_type_source: heuristic + category: networkfirewall + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_attachments: + cfn_type: AWS::NetworkManager::Attachment + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_core_network_policy_versions: + cfn_type: AWS::NetworkManager::CoreNetworkPolicyVersion + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_core_networks: + cfn_type: AWS::NetworkManager::CoreNetwork + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_global_networks: + cfn_type: AWS::NetworkManager::GlobalNetwork + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_links: + cfn_type: AWS::NetworkManager::Link + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_sites: + cfn_type: AWS::NetworkManager::Site + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_networkmanager_transit_gateway_registrations: + cfn_type: AWS::NetworkManager::TransitGatewayRegistration + cfn_type_source: heuristic + category: networkmanager + aliases: [] + typed_collector: false + mandatory: false + aws_odb_autonomous_virtual_machines: + cfn_type: AWS::Odb::AutonomousVirtualMachine + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_cloud_autonomous_vm_clusters: + cfn_type: AWS::Odb::CloudAutonomousVmCluster + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_cloud_exadata_infrastructures: + cfn_type: AWS::Odb::CloudExadataInfrastructure + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_cloud_vm_clusters: + cfn_type: AWS::Odb::CloudVmCluster + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_db_nodes: + cfn_type: AWS::Odb::DbNode + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_db_servers: + cfn_type: AWS::Odb::DbServer + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_db_system_shapes: + cfn_type: AWS::Odb::DbSystemShape + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_gi_versions: + cfn_type: AWS::Odb::GiVersion + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_networks: + cfn_type: AWS::Odb::Network + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_peering_connections: + cfn_type: AWS::Odb::PeeringConnection + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_odb_system_versions: + cfn_type: AWS::Odb::SystemVersion + cfn_type_source: heuristic + category: odb + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_auto_tunes: + cfn_type: AWS::OpenSearchService::DomainAutoTune + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_configs: + cfn_type: AWS::OpenSearchService::DomainConfig + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_data_sources: + cfn_type: AWS::OpenSearchService::DomainDataSource + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_health: + cfn_type: AWS::OpenSearchService::DomainHealth + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_maintenances: + cfn_type: AWS::OpenSearchService::DomainMaintenance + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_nodes: + cfn_type: AWS::OpenSearchService::DomainNode + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_packages: + cfn_type: AWS::OpenSearchService::DomainPackage + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domain_scheduled_actions: + cfn_type: AWS::OpenSearchService::DomainScheduledAction + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_domains: + cfn_type: AWS::OpenSearchService::Domain + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_inbound_connections: + cfn_type: AWS::OpenSearchService::InboundConnection + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_outbound_connections: + cfn_type: AWS::OpenSearchService::OutboundConnection + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_reserved_instances: + cfn_type: AWS::OpenSearchService::ReservedInstance + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_versions: + cfn_type: AWS::OpenSearchService::Version + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_opensearch_vpc_endpoints: + cfn_type: AWS::OpenSearchService::VpcEndpoint + cfn_type_source: heuristic + category: opensearch + aliases: [] + typed_collector: false + mandatory: false + aws_organization_resource_policies: + cfn_type: AWS::Organization::ResourcePolicy + cfn_type_source: heuristic + category: organization + aliases: [] + typed_collector: false + mandatory: false + aws_organizations: + cfn_type: AWS::Organizations::Organization + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_account_parents: + cfn_type: AWS::Organizations::AccountParent + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_accounts: + cfn_type: AWS::Organizations::Account + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_delegated_administrators: + cfn_type: AWS::Organizations::DelegatedAdministrator + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_delegated_services: + cfn_type: AWS::Organizations::DelegatedService + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_organizational_unit_parents: + cfn_type: AWS::Organizations::OrganizationalUnitParent + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_organizational_units: + cfn_type: AWS::Organizations::OrganizationalUnit + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_policies: + cfn_type: AWS::Organizations::Policy + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_policy_targets: + cfn_type: AWS::Organizations::PolicyTarget + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_organizations_roots: + cfn_type: AWS::Organizations::Root + cfn_type_source: heuristic + category: organizations + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_apps: + cfn_type: AWS::Pinpoint::App + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_campaign_versions: + cfn_type: AWS::Pinpoint::CampaignVersion + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_campaigns: + cfn_type: AWS::Pinpoint::Campaign + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_export_jobs: + cfn_type: AWS::Pinpoint::ExportJob + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_import_jobs: + cfn_type: AWS::Pinpoint::ImportJob + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_recommender_configurations: + cfn_type: AWS::Pinpoint::RecommenderConfiguration + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_segments: + cfn_type: AWS::Pinpoint::Segment + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_template_versions: + cfn_type: AWS::Pinpoint::TemplateVersion + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_pinpoint_templates: + cfn_type: AWS::Pinpoint::Template + cfn_type_source: heuristic + category: pinpoint + aliases: [] + typed_collector: false + mandatory: false + aws_polly_lexicons: + cfn_type: AWS::Polly::Lexicon + cfn_type_source: heuristic + category: polly + aliases: [] + typed_collector: false + mandatory: false + aws_polly_speech_synthesis_tasks: + cfn_type: AWS::Polly::SpeechSynthesisTask + cfn_type_source: heuristic + category: polly + aliases: [] + typed_collector: false + mandatory: false + aws_polly_voices: + cfn_type: AWS::Polly::Voice + cfn_type_source: heuristic + category: polly + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_analyses: + cfn_type: AWS::QuickSight::Analysis + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_dashboards: + cfn_type: AWS::QuickSight::Dashboard + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_data_sets: + cfn_type: AWS::QuickSight::DataSet + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_data_sources: + cfn_type: AWS::QuickSight::DataSource + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_folders: + cfn_type: AWS::QuickSight::Folder + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_group_members: + cfn_type: AWS::QuickSight::GroupMember + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_groups: + cfn_type: AWS::QuickSight::Group + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_ingestions: + cfn_type: AWS::QuickSight::Ingestion + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_templates: + cfn_type: AWS::QuickSight::Template + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_quicksight_users: + cfn_type: AWS::QuickSight::User + cfn_type_source: heuristic + category: quicksight + aliases: [] + typed_collector: false + mandatory: false + aws_ram_principals: + cfn_type: AWS::Ram::Principal + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resource_share_associations: + cfn_type: AWS::Ram::ResourceShareAssociation + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resource_share_invitations: + cfn_type: AWS::Ram::ResourceShareInvitation + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resource_share_permissions: + cfn_type: AWS::Ram::ResourceSharePermission + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resource_shares: + cfn_type: AWS::Ram::ResourceShare + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resource_types: + cfn_type: AWS::Ram::ResourceType + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_ram_resources: + cfn_type: AWS::Ram::Resource + cfn_type_source: heuristic + category: ram + aliases: [] + typed_collector: false + mandatory: false + aws_rds_certificates: + cfn_type: AWS::RDS::Certificate + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_cluster_backtracks: + cfn_type: AWS::RDS::ClusterBacktrack + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_cluster_parameter_group_parameters: + cfn_type: AWS::RDS::ClusterParameterGroupParameter + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_cluster_parameter_groups: + cfn_type: AWS::RDS::ClusterParameterGroup + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_cluster_parameters: + cfn_type: AWS::RDS::ClusterParameter + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_cluster_snapshots: + cfn_type: AWS::RDS::ClusterSnapshot + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_clusters: + cfn_type: AWS::RDS::DBCluster + cfn_type_source: override + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_parameter_group_db_parameters: + cfn_type: AWS::RDS::DbParameterGroupDbParameter + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_parameter_groups: + cfn_type: AWS::RDS::DbParameterGroup + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_proxies: + cfn_type: AWS::RDS::DBProxy + cfn_type_source: override + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_proxy_endpoints: + cfn_type: AWS::RDS::DbProxyEndpoint + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_proxy_target_groups: + cfn_type: AWS::RDS::DbProxyTargetGroup + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_proxy_targets: + cfn_type: AWS::RDS::DbProxyTarget + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_security_groups: + cfn_type: AWS::RDS::DbSecurityGroup + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_db_snapshots: + cfn_type: AWS::RDS::DbSnapshot + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_engine_versions: + cfn_type: AWS::RDS::EngineVersion + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_event_subscriptions: + cfn_type: AWS::RDS::EventSubscription + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_events: + cfn_type: AWS::RDS::Event + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_global_clusters: + cfn_type: AWS::RDS::GlobalCluster + cfn_type_source: override + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_instance_resource_metrics: + cfn_type: AWS::RDS::InstanceResourceMetric + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_instances: + cfn_type: AWS::RDS::DBInstance + cfn_type_source: override + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_major_engine_versions: + cfn_type: AWS::RDS::MajorEngineVersion + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_option_groups: + cfn_type: AWS::RDS::OptionGroup + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_pending_maintenance_actions: + cfn_type: AWS::RDS::PendingMaintenanceAction + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_reserved_instances: + cfn_type: AWS::RDS::ReservedInstance + cfn_type_source: heuristic + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_rds_subnet_groups: + cfn_type: AWS::RDS::DBSubnetGroup + cfn_type_source: override + category: rds + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_cluster_parameter_groups: + cfn_type: AWS::Redshift::ClusterParameterGroup + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_cluster_parameters: + cfn_type: AWS::Redshift::ClusterParameter + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_clusters: + cfn_type: AWS::Redshift::Cluster + cfn_type_source: override + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_data_shares: + cfn_type: AWS::Redshift::DataShare + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_endpoint_accesses: + cfn_type: AWS::Redshift::EndpointAccess + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_endpoint_authorizations: + cfn_type: AWS::Redshift::EndpointAuthorization + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_event_subscriptions: + cfn_type: AWS::Redshift::EventSubscription + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_events: + cfn_type: AWS::Redshift::Event + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_reserved_nodes: + cfn_type: AWS::Redshift::ReservedNode + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_snapshots: + cfn_type: AWS::Redshift::Snapshot + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_redshift_subnet_groups: + cfn_type: AWS::Redshift::SubnetGroup + cfn_type_source: heuristic + category: redshift + aliases: [] + typed_collector: false + mandatory: false + aws_regions: + cfn_type: null + cfn_type_source: override + category: regions + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_collection_faces: + cfn_type: AWS::Rekognition::CollectionFace + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_collections: + cfn_type: AWS::Rekognition::Collection + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_media_analysis_jobs: + cfn_type: AWS::Rekognition::MediaAnalysisJob + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_project_versions: + cfn_type: AWS::Rekognition::ProjectVersion + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_projects: + cfn_type: AWS::Rekognition::Project + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_rekognition_stream_processors: + cfn_type: AWS::Rekognition::StreamProcessor + cfn_type_source: heuristic + category: rekognition + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_alarm_recommendations: + cfn_type: AWS::ResilienceHub::AlarmRecommendation + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_app_assessments: + cfn_type: AWS::ResilienceHub::AppAssessment + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_app_component_compliances: + cfn_type: AWS::ResilienceHub::AppComponentCompliance + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_app_version_resource_mappings: + cfn_type: AWS::ResilienceHub::AppVersionResourceMapping + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_app_version_resources: + cfn_type: AWS::ResilienceHub::AppVersionResource + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_app_versions: + cfn_type: AWS::ResilienceHub::AppVersion + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_apps: + cfn_type: AWS::ResilienceHub::App + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_component_recommendations: + cfn_type: AWS::ResilienceHub::ComponentRecommendation + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_recommendation_templates: + cfn_type: AWS::ResilienceHub::RecommendationTemplate + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_resiliency_policies: + cfn_type: AWS::ResilienceHub::ResiliencyPolicy + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_sop_recommendations: + cfn_type: AWS::ResilienceHub::SopRecommendation + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_suggested_resiliency_policies: + cfn_type: AWS::ResilienceHub::SuggestedResiliencyPolicy + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resiliencehub_test_recommendations: + cfn_type: AWS::ResilienceHub::TestRecommendation + cfn_type_source: heuristic + category: resiliencehub + aliases: [] + typed_collector: false + mandatory: false + aws_resourcegroups_resource_groups: + cfn_type: AWS::ResourceGroups::ResourceGroup + cfn_type_source: heuristic + category: resourcegroups + aliases: [] + typed_collector: false + mandatory: false + aws_route53_delegation_sets: + cfn_type: AWS::Route53::DelegationSet + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_domains: + cfn_type: AWS::Route53::Domain + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_health_checks: + cfn_type: AWS::Route53::HealthCheck + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_hosted_zone_dnssecs: + cfn_type: AWS::Route53::HostedZoneDnssec + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_hosted_zone_query_logging_configs: + cfn_type: AWS::Route53::HostedZoneQueryLoggingConfig + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_hosted_zone_resource_record_sets: + cfn_type: AWS::Route53::HostedZoneResourceRecordSet + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_hosted_zone_traffic_policy_instances: + cfn_type: AWS::Route53::HostedZoneTrafficPolicyInstance + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_hosted_zones: + cfn_type: AWS::Route53::HostedZone + cfn_type_source: override + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_operations: + cfn_type: AWS::Route53::Operation + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_profiles: + cfn_type: AWS::Route53::Profile + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_traffic_policies: + cfn_type: AWS::Route53::TrafficPolicy + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53_traffic_policy_versions: + cfn_type: AWS::Route53::TrafficPolicyVersion + cfn_type_source: heuristic + category: route53 + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoverycontrolconfig_clusters: + cfn_type: AWS::Route53RecoveryControl::Cluster + cfn_type_source: heuristic + category: route53recoverycontrolconfig + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoverycontrolconfig_control_panels: + cfn_type: AWS::Route53RecoveryControl::ControlPanel + cfn_type_source: heuristic + category: route53recoverycontrolconfig + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoverycontrolconfig_routing_controls: + cfn_type: AWS::Route53RecoveryControl::RoutingControl + cfn_type_source: heuristic + category: route53recoverycontrolconfig + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoverycontrolconfig_safety_rules: + cfn_type: AWS::Route53RecoveryControl::SafetyRule + cfn_type_source: heuristic + category: route53recoverycontrolconfig + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoveryreadiness_cells: + cfn_type: AWS::Route53RecoveryReadiness::Cell + cfn_type_source: heuristic + category: route53recoveryreadiness + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoveryreadiness_readiness_checks: + cfn_type: AWS::Route53RecoveryReadiness::ReadinessCheck + cfn_type_source: heuristic + category: route53recoveryreadiness + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoveryreadiness_recovery_groups: + cfn_type: AWS::Route53RecoveryReadiness::RecoveryGroup + cfn_type_source: heuristic + category: route53recoveryreadiness + aliases: [] + typed_collector: false + mandatory: false + aws_route53recoveryreadiness_resource_sets: + cfn_type: AWS::Route53RecoveryReadiness::ResourceSet + cfn_type_source: heuristic + category: route53recoveryreadiness + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_firewall_configs: + cfn_type: AWS::Route53Resolver::FirewallConfig + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_firewall_domain_lists: + cfn_type: AWS::Route53Resolver::FirewallDomainList + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_firewall_rule_group_associations: + cfn_type: AWS::Route53Resolver::FirewallRuleGroupAssociation + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_firewall_rule_groups: + cfn_type: AWS::Route53Resolver::FirewallRuleGroup + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_resolver_endpoints: + cfn_type: AWS::Route53Resolver::ResolverEndpoint + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_resolver_query_log_config_associations: + cfn_type: AWS::Route53Resolver::ResolverQueryLogConfigAssociation + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_resolver_query_log_configs: + cfn_type: AWS::Route53Resolver::ResolverQueryLogConfig + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_resolver_rule_associations: + cfn_type: AWS::Route53Resolver::ResolverRuleAssociation + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_route53resolver_resolver_rules: + cfn_type: AWS::Route53Resolver::ResolverRule + cfn_type_source: heuristic + category: route53resolver + aliases: [] + typed_collector: false + mandatory: false + aws_s3_access_grant_instances: + cfn_type: AWS::S3::AccessGrantInstance + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_access_grants: + cfn_type: AWS::S3::AccessGrant + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_access_points: + cfn_type: AWS::S3::AccessPoint + cfn_type_source: override + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_accounts: + cfn_type: AWS::S3::Account + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_cors_rules: + cfn_type: AWS::S3::BucketCorsRule + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_encryption_rules: + cfn_type: AWS::S3::BucketEncryptionRule + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_grants: + cfn_type: AWS::S3::BucketGrant + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_lifecycles: + cfn_type: AWS::S3::BucketLifecycle + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_loggings: + cfn_type: AWS::S3::BucketLogging + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_notification_configurations: + cfn_type: AWS::S3::BucketNotificationConfiguration + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_object_grants: + cfn_type: AWS::S3::BucketObjectGrant + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_object_heads: + cfn_type: AWS::S3::BucketObjectHead + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_object_lock_configurations: + cfn_type: AWS::S3::BucketObjectLockConfiguration + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_objects: + cfn_type: AWS::S3::BucketObject + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_ownership_controls: + cfn_type: AWS::S3::BucketOwnershipControl + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_policies: + cfn_type: AWS::S3::BucketPolicy + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_public_access_blocks: + cfn_type: AWS::S3::BucketPublicAccessBlock + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_replications: + cfn_type: AWS::S3::BucketReplication + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_versionings: + cfn_type: AWS::S3::BucketVersioning + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_bucket_websites: + cfn_type: AWS::S3::BucketWebsite + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_buckets: + cfn_type: AWS::S3::Bucket + cfn_type_source: override + category: s3 + aliases: [] + typed_collector: true + mandatory: false + aws_s3_directory_buckets: + cfn_type: AWS::S3::DirectoryBucket + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_multi_region_access_points: + cfn_type: AWS::S3::MultiRegionAccessPoint + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_storage_lens_configurations: + cfn_type: AWS::S3::StorageLensConfiguration + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3_storage_lens_groups: + cfn_type: AWS::S3::StorageLensGroup + cfn_type_source: heuristic + category: s3 + aliases: [] + typed_collector: false + mandatory: false + aws_s3tables_bucket_policies: + cfn_type: AWS::S3tables::BucketPolicy + cfn_type_source: heuristic + category: s3tables + aliases: [] + typed_collector: false + mandatory: false + aws_s3tables_buckets: + cfn_type: AWS::S3tables::Bucket + cfn_type_source: heuristic + category: s3tables + aliases: [] + typed_collector: false + mandatory: false + aws_s3tables_namespaces: + cfn_type: AWS::S3tables::Namespace + cfn_type_source: heuristic + category: s3tables + aliases: [] + typed_collector: false + mandatory: false + aws_s3vectors_bucket_policies: + cfn_type: AWS::S3vectors::BucketPolicy + cfn_type_source: heuristic + category: s3vectors + aliases: [] + typed_collector: false + mandatory: false + aws_s3vectors_buckets: + cfn_type: AWS::S3vectors::Bucket + cfn_type_source: heuristic + category: s3vectors + aliases: [] + typed_collector: false + mandatory: false + aws_s3vectors_indexes: + cfn_type: AWS::S3vectors::Index + cfn_type_source: heuristic + category: s3vectors + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_apps: + cfn_type: AWS::SageMaker::App + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_domains: + cfn_type: AWS::SageMaker::Domain + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_endpoint_configurations: + cfn_type: AWS::SageMaker::EndpointConfiguration + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_endpoints: + cfn_type: AWS::SageMaker::Endpoint + cfn_type_source: override + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_hyperparameter_tuning_jobs: + cfn_type: AWS::SageMaker::HyperparameterTuningJob + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_image_versions: + cfn_type: AWS::SageMaker::ImageVersion + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_images: + cfn_type: AWS::SageMaker::Image + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_mlflow_apps: + cfn_type: AWS::SageMaker::MlflowApp + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_mlflow_tracking_servers: + cfn_type: AWS::SageMaker::MlflowTrackingServer + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_models: + cfn_type: AWS::SageMaker::Model + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_notebook_instance_lifecycle_configs: + cfn_type: AWS::SageMaker::NotebookInstanceLifecycleConfig + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_notebook_instances: + cfn_type: AWS::SageMaker::NotebookInstance + cfn_type_source: override + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_processing_jobs: + cfn_type: AWS::SageMaker::ProcessingJob + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_spaces: + cfn_type: AWS::SageMaker::Space + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_studio_lifecycle_configs: + cfn_type: AWS::SageMaker::StudioLifecycleConfig + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_training_jobs: + cfn_type: AWS::SageMaker::TrainingJob + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_transform_jobs: + cfn_type: AWS::SageMaker::TransformJob + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_sagemaker_user_profiles: + cfn_type: AWS::SageMaker::UserProfile + cfn_type_source: heuristic + category: sagemaker + aliases: [] + typed_collector: false + mandatory: false + aws_savingsplans_plans: + cfn_type: AWS::SavingsPlans::Plan + cfn_type_source: heuristic + category: savingsplans + aliases: [] + typed_collector: false + mandatory: false + aws_scheduler_schedule_groups: + cfn_type: AWS::Scheduler::ScheduleGroup + cfn_type_source: heuristic + category: scheduler + aliases: [] + typed_collector: false + mandatory: false + aws_scheduler_schedules: + cfn_type: AWS::Scheduler::Schedule + cfn_type_source: heuristic + category: scheduler + aliases: [] + typed_collector: false + mandatory: false + aws_secretsmanager_secret_versions: + cfn_type: AWS::SecretsManager::SecretVersion + cfn_type_source: heuristic + category: secretsmanager + aliases: [] + typed_collector: false + mandatory: false + aws_secretsmanager_secrets: + cfn_type: AWS::SecretsManager::Secret + cfn_type_source: override + category: secretsmanager + aliases: [] + typed_collector: false + mandatory: false + aws_securityhub_enabled_standards: + cfn_type: AWS::SecurityHub::EnabledStandard + cfn_type_source: heuristic + category: securityhub + aliases: [] + typed_collector: false + mandatory: false + aws_securityhub_findings: + cfn_type: AWS::SecurityHub::Finding + cfn_type_source: heuristic + category: securityhub + aliases: [] + typed_collector: false + mandatory: false + aws_securityhub_hubs: + cfn_type: AWS::SecurityHub::Hub + cfn_type_source: heuristic + category: securityhub + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_launch_paths: + cfn_type: AWS::ServiceCatalog::LaunchPath + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_portfolios: + cfn_type: AWS::ServiceCatalog::Portfolio + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_products: + cfn_type: AWS::ServiceCatalog::Product + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_provisioned_products: + cfn_type: AWS::ServiceCatalog::ProvisionedProduct + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_provisioning_artifacts: + cfn_type: AWS::ServiceCatalog::ProvisioningArtifact + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicecatalog_provisioning_parameters: + cfn_type: AWS::ServiceCatalog::ProvisioningParameter + cfn_type_source: heuristic + category: servicecatalog + aliases: [] + typed_collector: false + mandatory: false + aws_servicediscovery_instances: + cfn_type: AWS::ServiceDiscovery::Instance + cfn_type_source: heuristic + category: servicediscovery + aliases: [] + typed_collector: false + mandatory: false + aws_servicediscovery_namespaces: + cfn_type: AWS::ServiceDiscovery::Namespace + cfn_type_source: heuristic + category: servicediscovery + aliases: [] + typed_collector: false + mandatory: false + aws_servicediscovery_services: + cfn_type: AWS::ServiceDiscovery::Service + cfn_type_source: heuristic + category: servicediscovery + aliases: [] + typed_collector: false + mandatory: false + aws_servicequotas_awsdefaultservicequotas: + cfn_type: AWS::ServiceQuotas::Awsdefaultservicequota + cfn_type_source: heuristic + category: servicequotas + aliases: [] + typed_collector: false + mandatory: false + aws_servicequotas_quota_utilizations: + cfn_type: AWS::ServiceQuotas::QuotaUtilization + cfn_type_source: heuristic + category: servicequotas + aliases: [] + typed_collector: false + mandatory: false + aws_servicequotas_quotas: + cfn_type: AWS::ServiceQuotas::Quota + cfn_type_source: heuristic + category: servicequotas + aliases: [] + typed_collector: false + mandatory: false + aws_servicequotas_services: + cfn_type: AWS::ServiceQuotas::Service + cfn_type_source: heuristic + category: servicequotas + aliases: [] + typed_collector: false + mandatory: false + aws_ses_active_receipt_rule_sets: + cfn_type: AWS::SES::ActiveReceiptRuleSet + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_configuration_set_event_destinations: + cfn_type: AWS::SES::ConfigurationSetEventDestination + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_configuration_sets: + cfn_type: AWS::SES::ConfigurationSet + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_contact_lists: + cfn_type: AWS::SES::ContactList + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_custom_verification_email_templates: + cfn_type: AWS::SES::CustomVerificationEmailTemplate + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_identities: + cfn_type: AWS::SES::Identity + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_suppressed_destinations: + cfn_type: AWS::SES::SuppressedDestination + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_ses_templates: + cfn_type: AWS::SES::Template + cfn_type_source: heuristic + category: ses + aliases: [] + typed_collector: false + mandatory: false + aws_shield_attacks: + cfn_type: AWS::Shield::Attack + cfn_type_source: heuristic + category: shield + aliases: [] + typed_collector: false + mandatory: false + aws_shield_protection_groups: + cfn_type: AWS::Shield::ProtectionGroup + cfn_type_source: heuristic + category: shield + aliases: [] + typed_collector: false + mandatory: false + aws_shield_protections: + cfn_type: AWS::Shield::Protection + cfn_type_source: heuristic + category: shield + aliases: [] + typed_collector: false + mandatory: false + aws_shield_subscriptions: + cfn_type: AWS::Shield::Subscription + cfn_type_source: heuristic + category: shield + aliases: [] + typed_collector: false + mandatory: false + aws_signer_signing_profiles: + cfn_type: AWS::Signer::SigningProfile + cfn_type_source: heuristic + category: signer + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_addresses: + cfn_type: AWS::Snowball::Address + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_cluster_jobs: + cfn_type: AWS::Snowball::ClusterJob + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_clusters: + cfn_type: AWS::Snowball::Cluster + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_compatible_images: + cfn_type: AWS::Snowball::CompatibleImage + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_jobs: + cfn_type: AWS::Snowball::Job + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_long_term_pricing: + cfn_type: AWS::Snowball::LongTermPricing + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_snowball_pickup_locations: + cfn_type: AWS::Snowball::PickupLocation + cfn_type_source: heuristic + category: snowball + aliases: [] + typed_collector: false + mandatory: false + aws_sns_subscriptions: + cfn_type: AWS::SNS::Subscription + cfn_type_source: override + category: sns + aliases: [] + typed_collector: false + mandatory: false + aws_sns_topic_data_protection_policies: + cfn_type: AWS::SNS::TopicDataProtectionPolicy + cfn_type_source: heuristic + category: sns + aliases: [] + typed_collector: false + mandatory: false + aws_sns_topics: + cfn_type: AWS::SNS::Topic + cfn_type_source: override + category: sns + aliases: [] + typed_collector: false + mandatory: false + aws_sqs_queues: + cfn_type: AWS::SQS::Queue + cfn_type_source: override + category: sqs + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_associations: + cfn_type: AWS::SSM::Association + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_command_invocations: + cfn_type: AWS::SSM::CommandInvocation + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_compliance_summary_items: + cfn_type: AWS::SSM::ComplianceSummaryItem + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_document_contents: + cfn_type: AWS::SSM::DocumentContent + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_document_versions: + cfn_type: AWS::SSM::DocumentVersion + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_documents: + cfn_type: AWS::SSM::Document + cfn_type_source: override + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_instance_compliance_items: + cfn_type: AWS::SSM::InstanceComplianceItem + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_instance_patch_states: + cfn_type: AWS::SSM::InstancePatchState + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_instance_patches: + cfn_type: AWS::SSM::InstancePatch + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_instances: + cfn_type: AWS::SSM::Instance + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_inventories: + cfn_type: AWS::SSM::Inventory + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_inventory_entries: + cfn_type: AWS::SSM::InventoryEntry + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_inventory_schemas: + cfn_type: AWS::SSM::InventorySchema + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_maintenance_window_executions: + cfn_type: AWS::SSM::MaintenanceWindowExecution + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_maintenance_window_schedules: + cfn_type: AWS::SSM::MaintenanceWindowSchedule + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_maintenance_window_targets: + cfn_type: AWS::SSM::MaintenanceWindowTarget + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_maintenance_window_tasks: + cfn_type: AWS::SSM::MaintenanceWindowTask + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_maintenance_windows: + cfn_type: AWS::SSM::MaintenanceWindow + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_parameters: + cfn_type: AWS::SSM::Parameter + cfn_type_source: override + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_patch_baselines: + cfn_type: AWS::SSM::PatchBaseline + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssm_sessions: + cfn_type: AWS::SSM::Session + cfn_type_source: heuristic + category: ssm + aliases: [] + typed_collector: false + mandatory: false + aws_ssmincidents_incident_findings: + cfn_type: AWS::SSMIncidents::IncidentFinding + cfn_type_source: heuristic + category: ssmincidents + aliases: [] + typed_collector: false + mandatory: false + aws_ssmincidents_incident_related_items: + cfn_type: AWS::SSMIncidents::IncidentRelatedItem + cfn_type_source: heuristic + category: ssmincidents + aliases: [] + typed_collector: false + mandatory: false + aws_ssmincidents_incident_timeline_events: + cfn_type: AWS::SSMIncidents::IncidentTimelineEvent + cfn_type_source: heuristic + category: ssmincidents + aliases: [] + typed_collector: false + mandatory: false + aws_ssmincidents_incidents: + cfn_type: AWS::SSMIncidents::Incident + cfn_type_source: heuristic + category: ssmincidents + aliases: [] + typed_collector: false + mandatory: false + aws_ssmincidents_response_plans: + cfn_type: AWS::SSMIncidents::ResponsePlan + cfn_type_source: heuristic + category: ssmincidents + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_instances: + cfn_type: AWS::SSO::Instance + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_set_account_assignments: + cfn_type: AWS::SSO::PermissionSetAccountAssignment + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_set_customer_managed_policies: + cfn_type: AWS::SSO::PermissionSetCustomerManagedPolicy + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_set_inline_policies: + cfn_type: AWS::SSO::PermissionSetInlinePolicy + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_set_managed_policies: + cfn_type: AWS::SSO::PermissionSetManagedPolicy + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_set_permissions_boundaries: + cfn_type: AWS::SSO::PermissionSetPermissionsBoundary + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_permission_sets: + cfn_type: AWS::SSO::PermissionSet + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_ssoadmin_trusted_token_issuers: + cfn_type: AWS::SSO::TrustedTokenIssuer + cfn_type_source: heuristic + category: ssoadmin + aliases: [] + typed_collector: false + mandatory: false + aws_stepfunctions_activities: + cfn_type: AWS::Stepfunctions::Activity + cfn_type_source: heuristic + category: stepfunctions + aliases: [] + typed_collector: false + mandatory: false + aws_stepfunctions_executions: + cfn_type: AWS::Stepfunctions::Execution + cfn_type_source: heuristic + category: stepfunctions + aliases: [] + typed_collector: false + mandatory: false + aws_stepfunctions_map_run_executions: + cfn_type: AWS::Stepfunctions::MapRunExecution + cfn_type_source: heuristic + category: stepfunctions + aliases: [] + typed_collector: false + mandatory: false + aws_stepfunctions_map_runs: + cfn_type: AWS::Stepfunctions::MapRun + cfn_type_source: heuristic + category: stepfunctions + aliases: [] + typed_collector: false + mandatory: false + aws_stepfunctions_state_machines: + cfn_type: AWS::StepFunctions::StateMachine + cfn_type_source: override + category: stepfunctions + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_automatic_tape_creation_policies: + cfn_type: AWS::StorageGateway::AutomaticTapeCreationPolicy + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_cache_reports: + cfn_type: AWS::StorageGateway::CacheReport + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_file_shares: + cfn_type: AWS::StorageGateway::FileShare + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_file_system_associations: + cfn_type: AWS::StorageGateway::FileSystemAssociation + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_gateways: + cfn_type: AWS::StorageGateway::Gateway + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_local_disks: + cfn_type: AWS::StorageGateway::LocalDisk + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_tape_pools: + cfn_type: AWS::StorageGateway::TapePool + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_tapes: + cfn_type: AWS::StorageGateway::Tape + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_volume_recovery_points: + cfn_type: AWS::StorageGateway::VolumeRecoveryPoint + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_storagegateway_volumes: + cfn_type: AWS::StorageGateway::Volume + cfn_type_source: heuristic + category: storagegateway + aliases: [] + typed_collector: false + mandatory: false + aws_support_case_communications: + cfn_type: AWS::Support::CaseCommunication + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_cases: + cfn_type: AWS::Support::Cas + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_services: + cfn_type: AWS::Support::Service + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_severity_levels: + cfn_type: AWS::Support::SeverityLevel + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_trusted_advisor_check_results: + cfn_type: AWS::Support::TrustedAdvisorCheckResult + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_trusted_advisor_check_summaries: + cfn_type: AWS::Support::TrustedAdvisorCheckSummary + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_support_trusted_advisor_checks: + cfn_type: AWS::Support::TrustedAdvisorCheck + cfn_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + aws_swf_activity_types: + cfn_type: AWS::SWF::ActivityType + cfn_type_source: heuristic + category: swf + aliases: [] + typed_collector: false + mandatory: false + aws_swf_closed_workflow_executions: + cfn_type: AWS::SWF::ClosedWorkflowExecution + cfn_type_source: heuristic + category: swf + aliases: [] + typed_collector: false + mandatory: false + aws_swf_domains: + cfn_type: AWS::SWF::Domain + cfn_type_source: heuristic + category: swf + aliases: [] + typed_collector: false + mandatory: false + aws_swf_open_workflow_executions: + cfn_type: AWS::SWF::OpenWorkflowExecution + cfn_type_source: heuristic + category: swf + aliases: [] + typed_collector: false + mandatory: false + aws_swf_workflow_types: + cfn_type: AWS::SWF::WorkflowType + cfn_type_source: heuristic + category: swf + aliases: [] + typed_collector: false + mandatory: false + aws_timestream_databases: + cfn_type: AWS::Timestream::Databas + cfn_type_source: heuristic + category: timestream + aliases: [] + typed_collector: false + mandatory: false + aws_timestream_tables: + cfn_type: AWS::Timestream::Table + cfn_type_source: heuristic + category: timestream + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_call_analytics_categories: + cfn_type: AWS::Transcribe::CallAnalyticsCategory + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_call_analytics_jobs: + cfn_type: AWS::Transcribe::CallAnalyticsJob + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_language_models: + cfn_type: AWS::Transcribe::LanguageModel + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_medical_scribe_jobs: + cfn_type: AWS::Transcribe::MedicalScribeJob + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_medical_transcription_jobs: + cfn_type: AWS::Transcribe::MedicalTranscriptionJob + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_medical_vocabularies: + cfn_type: AWS::Transcribe::MedicalVocabulary + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_transcription_jobs: + cfn_type: AWS::Transcribe::TranscriptionJob + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_vocabularies: + cfn_type: AWS::Transcribe::Vocabulary + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transcribe_vocabulary_filters: + cfn_type: AWS::Transcribe::VocabularyFilter + cfn_type_source: heuristic + category: transcribe + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_agreements: + cfn_type: AWS::Transfer::Agreement + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_certificates: + cfn_type: AWS::Transfer::Certificate + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_connectors: + cfn_type: AWS::Transfer::Connector + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_profiles: + cfn_type: AWS::Transfer::Profile + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_servers: + cfn_type: AWS::Transfer::Server + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_users: + cfn_type: AWS::Transfer::User + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_transfer_workflows: + cfn_type: AWS::Transfer::Workflow + cfn_type_source: heuristic + category: transfer + aliases: [] + typed_collector: false + mandatory: false + aws_trustedadvisor_organization_recommendation_accounts: + cfn_type: AWS::Trustedadvisor::OrganizationRecommendationAccount + cfn_type_source: heuristic + category: trustedadvisor + aliases: [] + typed_collector: false + mandatory: false + aws_trustedadvisor_organization_recommendation_resources: + cfn_type: AWS::Trustedadvisor::OrganizationRecommendationResource + cfn_type_source: heuristic + category: trustedadvisor + aliases: [] + typed_collector: false + mandatory: false + aws_trustedadvisor_organization_recommendations: + cfn_type: AWS::Trustedadvisor::OrganizationRecommendation + cfn_type_source: heuristic + category: trustedadvisor + aliases: [] + typed_collector: false + mandatory: false + aws_trustedadvisor_recommendation_resources: + cfn_type: AWS::Trustedadvisor::RecommendationResource + cfn_type_source: heuristic + category: trustedadvisor + aliases: [] + typed_collector: false + mandatory: false + aws_trustedadvisor_recommendations: + cfn_type: AWS::Trustedadvisor::Recommendation + cfn_type_source: heuristic + category: trustedadvisor + aliases: [] + typed_collector: false + mandatory: false + aws_vpc_lattice_resource_configurations: + cfn_type: AWS::Vpc::LatticeResourceConfiguration + cfn_type_source: heuristic + category: vpc + aliases: [] + typed_collector: false + mandatory: false + aws_vpc_lattice_resource_gateways: + cfn_type: AWS::Vpc::LatticeResourceGateway + cfn_type_source: heuristic + category: vpc + aliases: [] + typed_collector: false + mandatory: false + aws_vpc_lattice_service_networks: + cfn_type: AWS::Vpc::LatticeServiceNetwork + cfn_type_source: heuristic + category: vpc + aliases: [] + typed_collector: false + mandatory: false + aws_vpc_lattice_services: + cfn_type: AWS::Vpc::LatticeService + cfn_type_source: heuristic + category: vpc + aliases: [] + typed_collector: false + mandatory: false + aws_waf_ipsets: + cfn_type: AWS::WAF::Ipset + cfn_type_source: heuristic + category: waf + aliases: [] + typed_collector: false + mandatory: false + aws_waf_rule_groups: + cfn_type: AWS::WAF::RuleGroup + cfn_type_source: heuristic + category: waf + aliases: [] + typed_collector: false + mandatory: false + aws_waf_rules: + cfn_type: AWS::WAF::Rule + cfn_type_source: heuristic + category: waf + aliases: [] + typed_collector: false + mandatory: false + aws_waf_subscribed_rule_groups: + cfn_type: AWS::WAF::SubscribedRuleGroup + cfn_type_source: heuristic + category: waf + aliases: [] + typed_collector: false + mandatory: false + aws_waf_web_acls: + cfn_type: AWS::WAF::WebAcl + cfn_type_source: heuristic + category: waf + aliases: [] + typed_collector: false + mandatory: false + aws_wafregional_rate_based_rules: + cfn_type: AWS::WAFRegional::RateBasedRule + cfn_type_source: heuristic + category: wafregional + aliases: [] + typed_collector: false + mandatory: false + aws_wafregional_rule_groups: + cfn_type: AWS::WAFRegional::RuleGroup + cfn_type_source: heuristic + category: wafregional + aliases: [] + typed_collector: false + mandatory: false + aws_wafregional_rules: + cfn_type: AWS::WAFRegional::Rule + cfn_type_source: heuristic + category: wafregional + aliases: [] + typed_collector: false + mandatory: false + aws_wafregional_web_acls: + cfn_type: AWS::WAFRegional::WebAcl + cfn_type_source: heuristic + category: wafregional + aliases: [] + typed_collector: false + mandatory: false + aws_wafv2_ipsets: + cfn_type: AWS::WAFv2::Ipset + cfn_type_source: heuristic + category: wafv2 + aliases: [] + typed_collector: false + mandatory: false + aws_wafv2_managed_rule_groups: + cfn_type: AWS::WAFv2::ManagedRuleGroup + cfn_type_source: heuristic + category: wafv2 + aliases: [] + typed_collector: false + mandatory: false + aws_wafv2_regex_pattern_sets: + cfn_type: AWS::WAFv2::RegexPatternSet + cfn_type_source: heuristic + category: wafv2 + aliases: [] + typed_collector: false + mandatory: false + aws_wafv2_rule_groups: + cfn_type: AWS::WAFv2::RuleGroup + cfn_type_source: heuristic + category: wafv2 + aliases: [] + typed_collector: false + mandatory: false + aws_wafv2_web_acls: + cfn_type: AWS::WAFv2::WebACL + cfn_type_source: override + category: wafv2 + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_lens_review_improvements: + cfn_type: AWS::WellArchitected::LensReviewImprovement + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_lens_reviews: + cfn_type: AWS::WellArchitected::LensReview + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_lenses: + cfn_type: AWS::WellArchitected::Lens + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_share_invitations: + cfn_type: AWS::WellArchitected::ShareInvitation + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_workload_milestones: + cfn_type: AWS::WellArchitected::WorkloadMilestone + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_workload_shares: + cfn_type: AWS::WellArchitected::WorkloadShare + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_wellarchitected_workloads: + cfn_type: AWS::WellArchitected::Workload + cfn_type_source: heuristic + category: wellarchitected + aliases: [] + typed_collector: false + mandatory: false + aws_workspaces_connection_alias_permissions: + cfn_type: AWS::WorkSpaces::ConnectionAliasPermission + cfn_type_source: heuristic + category: workspaces + aliases: [] + typed_collector: false + mandatory: false + aws_workspaces_connection_aliases: + cfn_type: AWS::WorkSpaces::ConnectionAlias + cfn_type_source: heuristic + category: workspaces + aliases: [] + typed_collector: false + mandatory: false + aws_workspaces_directories: + cfn_type: AWS::WorkSpaces::Directory + cfn_type_source: heuristic + category: workspaces + aliases: [] + typed_collector: false + mandatory: false + aws_workspaces_workspaces: + cfn_type: AWS::WorkSpaces::Workspace + cfn_type_source: heuristic + category: workspaces + aliases: [] + typed_collector: false + mandatory: false + aws_xray_encryption_configs: + cfn_type: AWS::XRay::EncryptionConfig + cfn_type_source: heuristic + category: xray + aliases: [] + typed_collector: false + mandatory: false + aws_xray_groups: + cfn_type: AWS::XRay::Group + cfn_type_source: heuristic + category: xray + aliases: [] + typed_collector: false + mandatory: false + aws_xray_resource_policies: + cfn_type: AWS::XRay::ResourcePolicy + cfn_type_source: heuristic + category: xray + aliases: [] + typed_collector: false + mandatory: false + aws_xray_sampling_rules: + cfn_type: AWS::XRay::SamplingRule + cfn_type_source: heuristic + category: xray + aliases: [] + typed_collector: false + mandatory: false diff --git a/src/indexers/awsapi.py b/src/indexers/awsapi.py new file mode 100644 index 000000000..b7af7e319 --- /dev/null +++ b/src/indexers/awsapi.py @@ -0,0 +1,477 @@ +""" +Native AWS SDK indexer. + +Replaces the CloudQuery-based AWS path with direct AWS Cloud Control API and +boto3 service calls while keeping the registry output compatible: each resource +is normalized into the same flat dict shape ``AWSPlatformHandler.parse_resource_data`` +already accepts, and writes flow through the :class:`ResourceWriter` seam. + +Discovery model (the AWS scope dimension is account + region(s)): + +* The authenticated account is the discovery scope. Its LOD comes from + ``accountLevelOfDetails[]`` (falling back to the workspace + default); an account whose effective LOD is ``NONE`` is skipped entirely + (selective discovery), keeping the resource store focused on what gen rules + need. +* The account (``aws_iam_accounts``) is the mandatory anchor every other AWS + resource is scoped under; it is synthesized directly from the resolved + credentials and written first. +* The Cloud Control API is the parity workhorse: one ``list_resources`` call + per (region, CFN type) referenced by generation rules returns full-payload + resources, routed by their CloudFormation type back to the registry-mapped + ``resource_type_name``. +* A thin typed tier (EC2 instances, S3 buckets) uses native boto3 service + clients for richer payloads; those CFN types are excluded from the Cloud + Control pass so a resource is never written twice. + +Coexists with the CloudQuery indexer behind the ``AWS_INDEXER_BACKEND`` setting: + +* ``"cloudquery"`` (default): this indexer is a no-op; CloudQuery handles AWS. +* ``"awsapi"``: this indexer discovers AWS resources and the + CloudQuery indexer skips the AWS block. + +Component name: ``awsapi``. Stage: ``INDEXER``. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +from component import Context, Setting, SettingDependency +from enrichers.generation_rule_types import ( + PLATFORM_HANDLERS_PROPERTY_NAME, + LevelOfDetail, + PlatformHandler, +) +from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY +from exceptions import WorkspaceBuilderException +from resources import ResourceTypeSpec + +from .common import CLOUD_CONFIG_SETTING +from .aws_common import ( + aws_get_session_and_scope, + aws_has_discovery_config, + has_excluded_tags, + has_included_tags, +) +from .awsapi_normalizers import ( + make_account_resource_data, + normalize_aws_resource, + normalize_cloudcontrol_resource, +) +from .awsapi_resource_types import ( + ACCOUNTS_TABLE, + collect_cloudcontrol_resources, + find_spec, + find_spec_by_cfn_type, +) +from .resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_PATH_SETTING, + get_resource_writer, +) + +logger = logging.getLogger(__name__) + +AWS_PLATFORM = "aws" + +DOCUMENTATION = "Index AWS resources using the Cloud Control API and boto3 SDKs" + +# --------------------------------------------------------------------------- +# Settings +# --------------------------------------------------------------------------- + +AWS_INDEXER_BACKEND_SETTING = Setting( + "AWS_INDEXER_BACKEND", + "awsIndexerBackend", + Setting.Type.STRING, + "Selects the backend used to discover AWS resources. " + "'cloudquery' (default) uses the legacy CloudQuery-based path; " + "'awsapi' uses the native Cloud Control API + boto3 indexer.", + "cloudquery", +) + +SETTINGS = ( + SettingDependency(CLOUD_CONFIG_SETTING, False), + SettingDependency(AWS_INDEXER_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_PATH_SETTING, False), +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _resolve_default_lod(context: Context, platform_cfg: dict[str, Any]) -> LevelOfDetail: + raw = ( + platform_cfg.get("defaultLOD") + or platform_cfg.get("defaultLevelOfDetail") + or context.get_setting("DEFAULT_LOD") + ) + if raw is None: + return LevelOfDetail.BASIC + try: + return LevelOfDetail.construct_from_config(raw) + except Exception: + return LevelOfDetail.BASIC + + +def _account_lod( + platform_cfg: dict[str, Any], + account_id: str, + default_lod: LevelOfDetail, +) -> LevelOfDetail: + """Effective LOD for ``account_id`` (per-account override -> default).""" + cfg = platform_cfg.get("accountLevelOfDetails", {}) or {} + raw = cfg.get(account_id) + if raw is None: + return default_lod + try: + return LevelOfDetail.construct_from_config(raw) + except Exception: + return default_lod + + +def _accessed_aws_type_names(context: Context) -> set[str]: + """Return the set of AWS resource-type names referenced by loaded + generation rules. Both the registry name and the CloudQuery table name are + accepted as valid spec values; the result mixes them.""" + all_specs: Optional[dict[str, dict[ResourceTypeSpec, Any]]] = context.get_property( + RESOURCE_TYPE_SPECS_PROPERTY + ) + if not all_specs: + return set() + aws_specs = all_specs.get(AWS_PLATFORM, {}) + return {spec.resource_type_name for spec in aws_specs.keys()} + + +def _resolve_platform_handler(context: Context) -> PlatformHandler: + handlers: Optional[dict[str, PlatformHandler]] = context.get_property( + PLATFORM_HANDLERS_PROPERTY_NAME + ) + if handlers and AWS_PLATFORM in handlers: + return handlers[AWS_PLATFORM] + from enrichers.aws import AWSPlatformHandler + + return AWSPlatformHandler() + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def index(context: Context) -> None: + backend = context.get_setting(AWS_INDEXER_BACKEND_SETTING) + if backend != "awsapi": + logger.info( + f"AWS indexer backend: '{backend}' (awsIndexerBackend in " + f"workspaceInfo.yaml). Native awsapi indexer is a no-op; the " + f"CloudQuery indexer will handle AWS." + ) + return + + cloud_config = context.get_setting(CLOUD_CONFIG_SETTING) or {} + platform_cfg = cloud_config.get(AWS_PLATFORM) + if not platform_cfg: + logger.info( + "AWS indexer backend: 'awsapi' selected, but no 'aws' section in " + "cloudConfig; nothing to discover." + ) + return + + if not aws_has_discovery_config(platform_cfg): + logger.info( + "AWS indexer backend: 'awsapi' selected, but cloudConfig.aws is " + "empty. Skipping." + ) + return + + logger.info( + "AWS indexer backend: 'awsapi' (Cloud Control API + boto3 SDK). " + "Starting AWS resource discovery." + ) + + scope = aws_get_session_and_scope(platform_cfg) + session = scope["session"] + account_id = scope["account_id"] + regions: list[str] = scope["regions"] + auth_type = scope["auth_type"] + auth_secret = scope["auth_secret"] + + if not account_id: + logger.warning("AWS indexer: no account id resolved; nothing to discover.") + return + + # The handler resolves account_name from this map; persist it on the config + # the same way the CloudQuery path does. + platform_cfg["_account_names"] = scope["account_names"] + + # Mirror cloudquery.py so enrichers.aws (cached auth/account) and any other + # downstream code sees the same active credentials we resolved here. + try: + from enrichers.aws import set_aws_credentials + + set_aws_credentials( + session=session, + auth_type=auth_type, + account_id=account_id, + account_alias=scope.get("account_alias"), + assume_role_arn=platform_cfg.get("assumeRoleArn"), + auth_secret=auth_secret, + ) + except Exception as e: + logger.warning(f"Could not update enrichers.aws credentials: {e}") + + accessed_names = _accessed_aws_type_names(context) + logger.info( + f"AWS resource types referenced by generation rules: " + f"{sorted(accessed_names) if accessed_names else '(none)'}" + ) + + # Resolve accessed gen-rule names -> specs. Split into the typed (boto3) + # tier and the Cloud Control generic tier. Typed CFN types are excluded + # from the generic filter so each resource is written exactly once. + typed_specs_to_collect = [] + seen_typed: set[str] = set() + generic_specs: list = [] # specs served by the Cloud Control pass + seen_generic: set[str] = set() + + for accessed in accessed_names: + spec = find_spec(accessed) + if spec is None: + warning = ( + f'AWS indexer: gen-rule references unknown AWS resource type ' + f'"{accessed}". Verify the name or add it to ' + f'scripts/aws/aws_resource_type_overrides.yaml and rerun the ' + f'sync script.' + ) + logger.warning(warning) + context.add_warning(warning) + continue + if spec.cloudquery_table_name == ACCOUNTS_TABLE: + # The account anchor is always emitted below; no extra pass needed. + continue + if spec.collector is not None: + if spec.cloudquery_table_name not in seen_typed: + typed_specs_to_collect.append(spec) + seen_typed.add(spec.cloudquery_table_name) + elif spec.cfn_type: + if spec.cloudquery_table_name not in seen_generic: + generic_specs.append(spec) + seen_generic.add(spec.cloudquery_table_name) + + platform_handler = _resolve_platform_handler(context) + include_tags = platform_cfg.get("includeTags", {}) + exclude_tags = platform_cfg.get("excludeTags", {}) + writer = get_resource_writer(context) + default_lod = _resolve_default_lod(context, platform_cfg) + + # Determine whether the account is in scope (LOD != NONE). + account_lod = _account_lod(platform_cfg, str(account_id), default_lod) + if account_lod is LevelOfDetail.NONE: + logger.info( + f"AWS account '{account_id}': effective LOD is NONE; skipping." + ) + return + logger.info( + f"AWS account '{account_id}': in scope (LOD={account_lod}, " + f"regions={regions})." + ) + + stats = { + "discovered": 0, + "added": 0, + "added_accounts": 0, + "added_typed": 0, + "added_generic": 0, + "skipped_dedup": 0, + "skipped_tag_filter": 0, + "skipped_parse_error": 0, + "skipped_collector_error": 0, + } + # Dedup across regions so a global resource (S3 bucket, IAM role) listed + # from multiple regional endpoints is written once. + seen_arns: set[tuple[str, str]] = set() + + def _process(spec, region, resource_data, *, source: str) -> None: + stats["discovered"] += 1 + if exclude_tags and has_excluded_tags(resource_data, exclude_tags): + stats["skipped_tag_filter"] += 1 + return + if include_tags and not has_included_tags(resource_data, include_tags): + stats["skipped_tag_filter"] += 1 + return + + arn = resource_data.get("arn") + dedup_key = (spec.resource_type_name, str(arn)) + if arn and dedup_key in seen_arns: + stats["skipped_dedup"] += 1 + return + + try: + resource_name, qualified_name, resource_attributes = ( + platform_handler.parse_resource_data( + resource_data, spec.resource_type_name, platform_cfg, context + ) + ) + except WorkspaceBuilderException as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data rejected {spec.resource_type_name} in " + f"region {region}: {e}" + ) + return + except (KeyError, ValueError, TypeError, AttributeError) as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data raised {type(e).__name__} for " + f"{spec.resource_type_name} in region {region}: {e}" + ) + return + + if arn: + seen_arns.add(dedup_key) + + resource_attributes["resource"] = resource_data + resource_attributes.setdefault("auth_type", auth_type) + resource_attributes.setdefault("auth_secret", auth_secret) + + writer.add_resource( + AWS_PLATFORM, + spec.resource_type_name, + resource_name, + qualified_name, + resource_attributes, + ) + stats["added"] += 1 + if source == "account": + stats["added_accounts"] += 1 + elif source == "typed": + stats["added_typed"] += 1 + else: + stats["added_generic"] += 1 + + primary_region = regions[0] if regions else None + + # Phase 0: emit the account anchor first so it is present before children. + account_spec = find_spec(ACCOUNTS_TABLE) + _process( + account_spec, + "global", + make_account_resource_data( + account_id, + account_name=scope.get("account_name"), + account_alias=scope.get("account_alias"), + ), + source="account", + ) + + if not accessed_names: + writer.finalize() + logger.info( + f"AWS indexing complete (anchor only): " + f"added_accounts={stats['added_accounts']}." + ) + return + + # Phase 1: typed (boto3) collectors. Regional collectors run per region; + # global collectors (S3, ...) run once. + for spec in typed_specs_to_collect: + collect_regions = regions if spec.regional else [primary_region] + for region in collect_regions: + try: + payloads = list(spec.collector(session, account_id, region)) + except Exception as e: + stats["skipped_collector_error"] += 1 + logger.error( + f"Failed to collect {spec.resource_type_name} in region " + f"{region}: {e}" + ) + context.add_warning( + f"Failed to collect AWS {spec.resource_type_name} in " + f"region {region}: {e}" + ) + continue + logger.info( + f"Collected {len(payloads)} {spec.resource_type_name} (typed) " + f"from region {region}" + ) + # For global collectors the payload carries its own region; pass + # None so the normalizer keeps it. + norm_region = region if spec.regional else None + for payload in payloads: + try: + resource_data = normalize_aws_resource( + payload, + account_id=account_id, + region=norm_region, + resource_type_name=spec.resource_type_name, + cfn_type=spec.cfn_type, + identifier=(payload.get("id") or payload.get("Arn")), + ) + except Exception as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"Failed to normalize AWS {spec.resource_type_name} " + f"payload in region {region}: {e}" + ) + continue + _process(spec, region, resource_data, source="typed") + + # Phase 2: Cloud Control generic pass, one list_resources call per + # (region, CFN type) referenced by gen rules (minus the typed ones). + for region in regions: + for spec in generic_specs: + cfn_type = spec.cfn_type + try: + descriptions = list( + collect_cloudcontrol_resources(session, region, cfn_type) + ) + except Exception as e: + stats["skipped_collector_error"] += 1 + logger.error( + f"Cloud Control list_resources failed for {cfn_type} in " + f"region {region}: {e}" + ) + context.add_warning( + f"Failed to list AWS {cfn_type} in region {region}: {e}" + ) + continue + logger.info( + f"Collected {len(descriptions)} {cfn_type} (Cloud Control) " + f"from region {region}" + ) + for desc in descriptions: + # Route by CFN type for robustness, mirroring GCP's CAI dispatch. + routed = find_spec_by_cfn_type(cfn_type) or spec + try: + resource_data = normalize_cloudcontrol_resource( + desc, + account_id=account_id, + region=region, + resource_type_name=routed.resource_type_name, + cfn_type=cfn_type, + ) + except Exception as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"Failed to normalize Cloud Control resource " + f"(type={cfn_type}) in region {region}: {e}" + ) + continue + _process(routed, region, resource_data, source="generic") + + writer.finalize() + + logger.info( + f"AWS indexing complete: " + f"discovered={stats['discovered']}, added={stats['added']} " + f"(accounts={stats['added_accounts']}, typed={stats['added_typed']}, " + f"generic={stats['added_generic']}), " + f"skipped_dedup={stats['skipped_dedup']}, " + f"skipped_tag_filter={stats['skipped_tag_filter']}, " + f"skipped_parse_error={stats['skipped_parse_error']}, " + f"skipped_collector_error={stats['skipped_collector_error']}" + ) diff --git a/src/indexers/awsapi_normalizers.py b/src/indexers/awsapi_normalizers.py new file mode 100644 index 000000000..f4006293f --- /dev/null +++ b/src/indexers/awsapi_normalizers.py @@ -0,0 +1,308 @@ +""" +Normalize AWS Cloud Control API resources and typed boto3 service payloads into +the flat dict shape that ``AWSPlatformHandler.parse_resource_data`` already +understands. + +Goal: produce a ``resource_data`` dict behaviourally equivalent to a row from +the legacy CloudQuery SQLite intermediate. ``parse_resource_data`` relies on a +small handful of fields: + +* ``arn`` - REQUIRED. The handler parses it for account/region/service + and uses ``arn.resource_id`` as a name fallback. +* ``name`` - short resource name (falls back to the ARN resource id). +* ``account_id`` - owning account (falls back to the ARN account segment). +* ``region`` - placement (falls back to the ARN region segment). +* ``tags`` - dict, defaults to ``{}``. AWS tags (a list of + ``{"Key","Value"}`` pairs, or a dict) are normalized here so + cross-cloud include/exclude tag matchers work unchanged. + +Everything else (the full API representation) is passed through at the top level +of the dict so generation-rule path matching keeps working. + +This module must not import ``boto3``: it operates on plain dicts and duck-typed +objects so the test suite runs without the AWS SDK. +""" + +from __future__ import annotations + +import datetime +import enum +import json +import logging +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +def _sanitize(value: Any) -> Any: + """Recursively convert SDK values into YAML-friendly primitives. + + Mirrors ``gcpapi_normalizers._sanitize`` / ``azureapi_normalizers._sanitize``: + datetimes / enums become strings so PyYAML's safe dumper (used by the + resource store) is happy and the output matches what CloudQuery serialized. + """ + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, datetime.date): + return value.isoformat() + if isinstance(value, enum.Enum): + return value.value if isinstance(value.value, (str, int, float, bool)) else str(value) + if isinstance(value, dict): + return {k: _sanitize(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_sanitize(v) for v in value] + return value + + +def normalize_tags(raw: Any) -> dict[str, str]: + """Normalize AWS tags into a ``{key: value}`` dict. + + AWS APIs express tags either as a list of ``{"Key": ..., "Value": ...}`` + pairs (most services) or as an already-flat dict (a few). Both collapse to + the canonical dict the cross-cloud tag matchers expect. + """ + if isinstance(raw, dict): + return {str(k): ("" if v is None else str(v)) for k, v in raw.items()} + out: dict[str, str] = {} + if isinstance(raw, (list, tuple)): + for item in raw: + if not isinstance(item, dict): + continue + key = item.get("Key", item.get("key")) + value = item.get("Value", item.get("value")) + if key is not None: + out[str(key)] = "" if value is None else str(value) + return out + + +_ARN_KEYS = ("Arn", "ARN", "arn", "ResourceArn", "resource_arn") + + +def _find_arn(payload: dict[str, Any]) -> Optional[str]: + """Find an ARN anywhere obvious in a payload. + + Prefers well-known keys, then falls back to any top-level key whose name + ends in ``Arn``/``ARN`` and whose value looks like an ARN string. + """ + for key in _ARN_KEYS: + val = payload.get(key) + if isinstance(val, str) and val.startswith("arn:"): + return val + for key, val in payload.items(): + if ( + isinstance(key, str) + and (key.endswith("Arn") or key.endswith("ARN")) + and isinstance(val, str) + and val.startswith("arn:") + ): + return val + return None + + +def _service_from_cfn(cfn_type: Optional[str]) -> str: + """Lowercase service token from a CFN type (``AWS::EC2::Instance`` -> ``ec2``).""" + if not cfn_type: + return "cloudcontrol" + parts = cfn_type.split("::") + if len(parts) >= 2 and parts[1]: + return parts[1].lower() + return "cloudcontrol" + + +def _entity_from_cfn(cfn_type: Optional[str]) -> str: + """Lowercase entity token from a CFN type (``AWS::EC2::Instance`` -> ``instance``).""" + if not cfn_type: + return "resource" + parts = cfn_type.split("::") + if len(parts) >= 3 and parts[2]: + return parts[2].lower() + return "resource" + + +def _synthesize_arn( + *, + cfn_type: Optional[str], + region: Optional[str], + account_id: Optional[str], + identifier: Optional[str], +) -> str: + """Build a best-effort ARN when the payload carries none. + + Many Cloud Control resource types expose an ``Arn`` property, but some only + expose a primary identifier. The AWSPlatformHandler requires an ARN, so we + synthesize a structurally-valid one (6 colon-separated segments) from the + CFN type + scope. The handler parses it back into account/region/service. + """ + service = _service_from_cfn(cfn_type) + entity = _entity_from_cfn(cfn_type) + region = region or "" + account_id = str(account_id or "") + ident = identifier or "unknown" + return f"arn:aws:{service}:{region}:{account_id}:{entity}/{ident}" + + +_NAME_KEYS = ( + "Name", + "name", + "BucketName", + "FunctionName", + "ClusterName", + "DBInstanceIdentifier", + "DBClusterIdentifier", + "TableName", + "QueueName", + "TopicName", + "RoleName", + "UserName", + "GroupName", + "KeyId", + "InstanceId", + "LoadBalancerName", + "RepositoryName", +) + + +def _derive_name(payload: dict[str, Any], identifier: Optional[str]) -> Optional[str]: + for key in _NAME_KEYS: + val = payload.get(key) + if isinstance(val, str) and val: + return val + if identifier: + # Cloud Control identifiers can be composite (``a|b``); keep the leaf. + return str(identifier).split("|")[-1] + return None + + +def normalize_aws_resource( + raw: Any, + *, + account_id: Optional[str], + region: Optional[str], + resource_type_name: str = "", + cfn_type: Optional[str] = None, + identifier: Optional[str] = None, +) -> dict[str, Any]: + """Convert a plain AWS payload dict into a ``resource_data`` dict. + + Used by both the Cloud Control generic pass (after the JSON ``Properties`` + blob is parsed) and the typed boto3 collectors. The output shape is + identical for both so the two passes are interchangeable, exactly mirroring + the GCP CAI / typed split. + """ + if not isinstance(raw, dict): + raw = {} + out: dict[str, Any] = _sanitize(dict(raw)) + + out["tags"] = normalize_tags(raw.get("Tags") if "Tags" in raw else raw.get("tags")) + + if account_id: + out["account_id"] = str(account_id) + if region: + out["region"] = region + + name = _derive_name(out, identifier) + if name: + out["name"] = name + + arn = _find_arn(out) + if not arn: + arn = _synthesize_arn( + cfn_type=cfn_type, + region=region, + account_id=account_id, + identifier=identifier or name, + ) + out["arn"] = arn + + if not out.get("id"): + out["id"] = identifier or arn or name + + if cfn_type: + out.setdefault("cfn_type", cfn_type) + + return out + + +def normalize_cloudcontrol_resource( + resource_description: Any, + *, + account_id: Optional[str], + region: Optional[str], + resource_type_name: str = "", + cfn_type: Optional[str] = None, +) -> dict[str, Any]: + """Convert a Cloud Control ``ResourceDescription`` into a ``resource_data`` dict. + + A Cloud Control ``list_resources`` / ``get_resource`` item looks like:: + + {"Identifier": "i-0abc...", "Properties": "{\"InstanceId\": ...}"} + + ``Properties`` is a JSON-encoded string of the resource's CFN schema + properties. We parse it, hoist it to the top level, and stamp the + handler-read fields. + """ + if isinstance(resource_description, dict): + identifier = resource_description.get("Identifier") or resource_description.get( + "identifier" + ) + props_raw = resource_description.get("Properties") or resource_description.get( + "properties" + ) + else: + identifier = getattr(resource_description, "identifier", None) or getattr( + resource_description, "Identifier", None + ) + props_raw = getattr(resource_description, "properties", None) or getattr( + resource_description, "Properties", None + ) + + props: dict[str, Any] + if isinstance(props_raw, str): + try: + parsed = json.loads(props_raw) + props = parsed if isinstance(parsed, dict) else {} + except (ValueError, TypeError): + props = {} + elif isinstance(props_raw, dict): + props = dict(props_raw) + else: + props = {} + + return normalize_aws_resource( + props, + account_id=account_id, + region=region, + resource_type_name=resource_type_name, + cfn_type=cfn_type, + identifier=identifier, + ) + + +def make_account_resource_data( + account_id: str, + account_name: Optional[str] = None, + account_alias: Optional[str] = None, +) -> dict[str, Any]: + """Build the ``resource_data`` dict for a synthesized account anchor. + + ``aws_iam_accounts`` is the mandatory anchor every other AWS resource is + scoped under (account_id / account_name). We materialize it from the + resolved credentials alone (no Cloud Control call): the handler needs an + ARN, a name, an account id and a region, all of which we can supply for the + account root. + """ + aid = str(account_id) + data: dict[str, Any] = { + "account_id": aid, + "name": account_name or aid, + "id": aid, + "arn": f"arn:aws:iam::{aid}:root", + "region": "global", + "cfn_type": None, + "tags": {}, + } + if account_alias: + data["account_alias"] = account_alias + if account_name: + data["account_name"] = account_name + return data diff --git a/src/indexers/awsapi_resource_types.py b/src/indexers/awsapi_resource_types.py new file mode 100644 index 000000000..7950bf6b8 --- /dev/null +++ b/src/indexers/awsapi_resource_types.py @@ -0,0 +1,244 @@ +""" +AWS resource-type specs for the native AWS SDK indexer. + +Like GCP's Cloud Asset Inventory, AWS's **Cloud Control API** +(``cloudcontrol`` boto3 client, ``list_resources`` / ``get_resource``) is a +single broad API that can enumerate hundreds of resource types by their +CloudFormation type name (``AWS::::``). It is therefore the +parity workhorse: one ``list_resources`` call per (account, region, CFN type) +covers every registry type that has a CFN type. Typed boto3 service collectors +are a thin enrichment layer for a handful of high-value resources (and the +synthesized account anchor). + +Each :class:`AwsResourceTypeSpec` describes one collectable type. Specs are +materialized from the registry in ``aws_resource_type_registry.yaml`` (see +:mod:`aws_resource_type_registry`). To enable a richer typed collector for a +type: + +* Make sure the table is in the registry (regenerate via + ``scripts/aws/sync_aws_resource_type_registry.py`` if needed, and flag it as + a ``typed_collector`` in the overrides YAML). +* Implement ``_collect_`` below and register it in + :data:`_TYPED_COLLECTORS` keyed by canonical CloudQuery table name. + +The public surface (``AWS_RESOURCE_TYPE_SPECS``, ``AwsResourceTypeSpec``, +``find_spec``, ``find_spec_by_cfn_type``) is what ``indexers.awsapi`` consumes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Optional + +from .aws_resource_type_registry import AwsResourceTypeEntry, load_registry + +# A typed collector takes (session, account_id, region) and yields raw payload +# dicts (later run through ``normalize_aws_resource``). +AwsCollector = Callable[[Any, Optional[str], Optional[str]], Iterable[dict]] + +# The canonical table name for the synthesized account anchor. +ACCOUNTS_TABLE = "aws_iam_accounts" + + +@dataclass(frozen=True) +class AwsResourceTypeSpec: + resource_type_name: str + cloudquery_table_name: str + cfn_type: Optional[str] + mandatory: bool + # ``True`` when a hand-written boto3 collector ships for this type (rich/ + # extra payload, or the synthesized account anchor). ``False`` means the + # type is materialized from the Cloud Control generic pass. + typed: bool = False + # Hand-written collector callable; ``None`` for generic specs and for the + # synthesized account anchor (handled directly by the orchestrator). + collector: Optional[AwsCollector] = None + # ``True`` for region-scoped typed collectors (run once per region); + # ``False`` for global services (e.g. S3, IAM) run once per account. + regional: bool = True + + +# --------------------------------------------------------------------------- +# Typed collectors (rich-payload enrichment tier) +# --------------------------------------------------------------------------- +# +# All boto3 client construction is lazy (via the passed-in ``session``) so this +# module stays importable - and the test suite stays runnable - without the AWS +# SDK installed. + +def _name_from_aws_tags(tags: Any) -> Optional[str]: + """Return the value of the ``Name`` tag from an AWS tag list, if present.""" + if isinstance(tags, (list, tuple)): + for item in tags: + if isinstance(item, dict) and item.get("Key") == "Name": + return item.get("Value") + return None + + +def _collect_ec2_instances(session, account_id, region): + """Typed collector for EC2 instances (regional).""" + client = session.client("ec2", region_name=region) + paginator = client.get_paginator("describe_instances") + for page in paginator.paginate(): + for reservation in page.get("Reservations", []): + for instance in reservation.get("Instances", []): + inst = dict(instance) + instance_id = inst.get("InstanceId") + # describe_instances doesn't return an ARN; synthesize one so + # the handler has the account/region/service it needs. + inst.setdefault( + "Arn", + f"arn:aws:ec2:{region or ''}:{account_id or ''}:instance/{instance_id}", + ) + inst["name"] = _name_from_aws_tags(inst.get("Tags")) or instance_id + yield inst + + +def _collect_s3_buckets(session, account_id, region): + """Typed collector for S3 buckets (global; region resolved per bucket).""" + client = session.client("s3") + resp = client.list_buckets() + for bucket in resp.get("Buckets", []): + name = bucket.get("Name") + out = dict(bucket) + out["name"] = name + out["Arn"] = f"arn:aws:s3:::{name}" + # Resolve the bucket's home region (best-effort). + try: + loc = client.get_bucket_location(Bucket=name) + out["region"] = loc.get("LocationConstraint") or "us-east-1" + except Exception: # pragma: no cover - permission / network dependent + pass + # Resolve bucket tags (best-effort; untagged buckets raise). + try: + tagging = client.get_bucket_tagging(Bucket=name) + out["Tags"] = tagging.get("TagSet", []) + except Exception: # pragma: no cover - NoSuchTagSet etc. + out["Tags"] = [] + yield out + + +# Maps canonical CQ table name -> (collector callable, regional flag). +_TYPED_COLLECTORS: dict[str, tuple[AwsCollector, bool]] = { + "aws_ec2_instances": (_collect_ec2_instances, True), + "aws_s3_buckets": (_collect_s3_buckets, False), +} + + +# --------------------------------------------------------------------------- +# Cloud Control generic collector (the catch-all parity pass) +# --------------------------------------------------------------------------- + +def collect_cloudcontrol_resources(session, region: str, cfn_type: str): + """List Cloud Control resources of one CFN type in one region. + + Yields ``ResourceDescription`` dicts (each with ``Identifier`` + + ``Properties``) for ``cfn_type`` (e.g. ``AWS::S3::Bucket``). The + ``cloudcontrol`` client + paginator are constructed lazily from the passed + ``session`` so importing this module never requires boto3. + """ + client = session.client("cloudcontrol", region_name=region) + paginator = client.get_paginator("list_resources") + for page in paginator.paginate(TypeName=cfn_type): + for desc in page.get("ResourceDescriptions", []): + yield desc + + +# --------------------------------------------------------------------------- +# Spec materialization +# --------------------------------------------------------------------------- + +def _legacy_resource_type_name(entry: AwsResourceTypeEntry) -> str: + """Pick the ``resource_type_name`` surfaced to generation rules. + + Historically a few AWS types were referenced by short legacy names + (``account``, ``ec2_instance``) and everything else by the CloudQuery table + name. The registry encodes both via ``cloudquery_table_name`` + ``aliases``; + prefer the first alias (legacy short name) when present. + """ + if entry.aliases: + return entry.aliases[0] + return entry.cloudquery_table_name + + +def _make_spec(entry: AwsResourceTypeEntry) -> AwsResourceTypeSpec: + collector_info = _TYPED_COLLECTORS.get(entry.cloudquery_table_name) + collector = collector_info[0] if collector_info else None + regional = collector_info[1] if collector_info else True + return AwsResourceTypeSpec( + resource_type_name=_legacy_resource_type_name(entry), + cloudquery_table_name=entry.cloudquery_table_name, + cfn_type=entry.cfn_type, + mandatory=entry.mandatory, + typed=bool(entry.typed_collector or collector is not None), + collector=collector, + regional=regional, + ) + + +def _build_specs() -> tuple[AwsResourceTypeSpec, ...]: + """Materialize one ``AwsResourceTypeSpec`` per registry entry.""" + registry = load_registry() + specs: list[AwsResourceTypeSpec] = [] + + # Account anchor first (mandatory bootstrap everything is scoped under). + accounts_entry = registry.find(ACCOUNTS_TABLE) + if accounts_entry is not None: + specs.append(_make_spec(accounts_entry)) + + for entry in registry: + if entry.cloudquery_table_name == ACCOUNTS_TABLE: + continue + specs.append(_make_spec(entry)) + + return tuple(specs) + + +AWS_RESOURCE_TYPE_SPECS: tuple[AwsResourceTypeSpec, ...] = _build_specs() + +# Convenience subset for callers that only want the typed (rich/SDK) tier. +AWS_TYPED_RESOURCE_TYPE_SPECS: tuple[AwsResourceTypeSpec, ...] = tuple( + s for s in AWS_RESOURCE_TYPE_SPECS if s.collector is not None +) + + +# --------------------------------------------------------------------------- +# Lookup +# --------------------------------------------------------------------------- + +def find_spec(name_or_table: str) -> Optional[AwsResourceTypeSpec]: + """Look up an AWS resource type spec by registry name, alias, or CQ table. + + Returns a spec for any name the registry knows about. Returning ``None`` + means "this name is not a registered AWS resource type at all". + """ + if not name_or_table: + return None + for spec in AWS_RESOURCE_TYPE_SPECS: + if name_or_table in (spec.resource_type_name, spec.cloudquery_table_name): + return spec + entry = load_registry().find(name_or_table) + if entry is None: + return None + for spec in AWS_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None + + +def find_spec_by_cfn_type(cfn_type: Optional[str]) -> Optional[AwsResourceTypeSpec]: + """Look up the spec whose ``cfn_type`` matches this string. + + Used by the Cloud Control generic pass in ``awsapi.index`` to route each + returned resource back to the spec that owns its ``resource_type_name``. + Case-insensitive. + """ + if not cfn_type: + return None + entry = load_registry().find_by_cfn_type(cfn_type) + if entry is None: + return None + for spec in AWS_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None diff --git a/src/indexers/azure_common.py b/src/indexers/azure_common.py new file mode 100644 index 000000000..dfd81416c --- /dev/null +++ b/src/indexers/azure_common.py @@ -0,0 +1,382 @@ +""" +Shared Azure helper functions used by both the legacy CloudQuery-based Azure +indexer (``cloudquery.py``) and the native Azure SDK indexer (``azureapi.py``). + +These were originally defined inline in ``cloudquery.py``. They have been +relocated here verbatim so both indexers can call them while the CloudQuery +Azure path remains live behind the ``AZURE_INDEXER_BACKEND`` feature flag. + +Nothing in this module is supposed to know about CloudQuery internals -- it is +the lowest layer of Azure-specific logic that both indexers share. +""" + +import base64 +import logging +import os +import sys +from typing import Any + +import requests +from azure.identity import ClientSecretCredential, DefaultAzureCredential +from azure.mgmt.resource import ResourceManagementClient, SubscriptionClient + +from exceptions import WorkspaceBuilderException +from utils import mask_string + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from k8s_utils import get_secret # noqa: E402 + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Tag filtering helpers +# --------------------------------------------------------------------------- + +def has_included_tags(resource_data: dict, include_tags: dict[str, str]) -> bool: + """Returns True if any of the tags in ``include_tags`` are found in ``resource_data``.""" + tags = resource_data.get("tags", {}) + return any(tags.get(key) == value for key, value in include_tags.items()) + + +def has_excluded_tags(resource_data: dict, exclude_tags: dict[str, str]) -> bool: + """Returns True if any of the tags in ``exclude_tags`` are found in ``resource_data``.""" + tags = resource_data.get("tags", {}) + for key, value in exclude_tags.items(): + if tags.get(key) == value: + logger.info( + f"Excluding resource {resource_data.get('name', 'unknown')} " + f"due to tag '{key}: {value}'" + ) + return True + return False + + +# --------------------------------------------------------------------------- +# Authentication helpers +# --------------------------------------------------------------------------- + +def get_managed_identity_details() -> dict[str, Any]: + """Resolve subscription / tenant / client info from the IMDS endpoint.""" + credential = DefaultAzureCredential() + + subscription_client = SubscriptionClient(credential) + subscription = next(subscription_client.subscriptions.list()) + subscription_id = subscription.subscription_id + tenant_id = subscription.tenant_id + + imds_url = "http://169.254.169.254/metadata/identity/oauth2/token" + headers = {"Metadata": "true"} + params = { + "api-version": "2019-08-01", + "resource": "https://management.azure.com/", + } + + response = requests.get(imds_url, headers=headers, params=params) + response.raise_for_status() + + token_data = response.json() + client_id = token_data.get("client_id") + + return { + "AZURE_TENANT_ID": tenant_id, + "AZURE_CLIENT_ID": client_id, + "AZURE_SUBSCRIPTION_ID": subscription_id, + "credential": credential, + } + + +def az_get_credentials_and_subscription_id(platform_config_data: dict[str, Any]) -> dict[str, Any]: + """ + Resolve Azure authentication and return: + credential - azure-identity credential object + subscription_ids - list[str] (final list for downstream use) + AZURE_* keys - env-var values for SP / MI auth + """ + + sp_secret_name = platform_config_data.get("spSecretName") + client_id = client_secret = tenant_id = None + if sp_secret_name: + secret = get_secret(sp_secret_name) + tenant_id = base64.b64decode(secret.get("tenantId")).decode() + client_id = base64.b64decode(secret.get("clientId")).decode() + client_secret = base64.b64decode(secret.get("clientSecret")).decode() + + if not all([client_id, client_secret, tenant_id]): + client_id = platform_config_data.get("clientId") + client_secret = platform_config_data.get("clientSecret") + tenant_id = platform_config_data.get("tenantId") + + # Detect the common foot-gun where the user supplies a Service Principal + # but one of the three SP fields renders as an empty string (e.g. an + # un-substituted ``"${AZ_CLIENT_SECRET}"`` or a redacted ``tf.secret``). + # Without this guard we silently fall through to Managed Identity / + # DefaultAzureCredential and the user is left chasing a 30-line + # ChainedTokenCredential traceback. + sp_fields = { + "clientId": client_id, + "clientSecret": client_secret, + "tenantId": tenant_id, + } + populated = {k: v for k, v in sp_fields.items() if v} + empty = [k for k, v in sp_fields.items() if v == ""] + if populated and empty: + raise WorkspaceBuilderException( + "Azure Service Principal configuration is incomplete: " + f"{', '.join(sorted(populated))} provided but " + f"{', '.join(sorted(empty))} is empty. " + "Verify that all of cloudConfig.azure.{clientId,clientSecret,tenantId} " + "are set in workspaceInfo.yaml (or that the env vars referenced from " + "your secret file are populated before the YAML is rendered)." + ) + + explicit_sub_ids = [ + str(e["subscriptionId"]) + for e in platform_config_data.get("subscriptions", []) + if e.get("subscriptionId") + ] + + if explicit_sub_ids: + subscription_ids = explicit_sub_ids + else: + subscription_ids: list[str] = [] + legacy_sid = platform_config_data.get("subscriptionId") + if legacy_sid: + subscription_ids.append(str(legacy_sid)) + env_sid = os.getenv("AZURE_SUBSCRIPTION_ID") + if env_sid and env_sid not in subscription_ids: + subscription_ids.append(str(env_sid)) + + if not subscription_ids: + raise ValueError("No Azure subscriptionId supplied.") + + if all([client_id, client_secret, tenant_id]): + credential = ClientSecretCredential(tenant_id, client_id, client_secret) + else: + mi = get_managed_identity_details() + credential = mi["credential"] + client_id = client_id or mi.get("AZURE_CLIENT_ID") + tenant_id = tenant_id or mi.get("AZURE_TENANT_ID") + + result = { + "credential": credential, + "subscription_ids": subscription_ids, + "AZURE_SUBSCRIPTION_ID": subscription_ids[0], + } + if client_id: + result["AZURE_CLIENT_ID"] = client_id + if client_secret: + result["AZURE_CLIENT_SECRET"] = client_secret + if tenant_id: + result["AZURE_TENANT_ID"] = tenant_id + return result + + +def _azure_has_only_devops_config(platform_cfg: dict) -> bool: + """Return True when the azure config block has no cloud discovery credentials. + + This happens when the user only configures ``azure.devops`` (for the ADO + indexer) without providing a subscriptionId or service-principal / + managed-identity credentials needed for Azure resource discovery. + """ + has_sub = bool( + platform_cfg.get("subscriptionId") + or platform_cfg.get("subscriptions") + or os.getenv("AZURE_SUBSCRIPTION_ID") + ) + has_sp = bool( + platform_cfg.get("spSecretName") + or (platform_cfg.get("clientId") and platform_cfg.get("clientSecret")) + ) + has_devops = bool(platform_cfg.get("devops")) + return has_devops and not has_sub and not has_sp + + +def az_discover_resource_groups(credential, subscription_id) -> list[str]: + if not subscription_id: + raise ValueError("subscription_id cannot be None in az_discover_resource_groups.") + + logger.debug(f"Discovering resource groups for subscription_id: {mask_string(subscription_id)}") + + resource_groups = [] + try: + resource_client = ResourceManagementClient(credential, subscription_id) + for rg in resource_client.resource_groups.list(): + resource_groups.append(rg.name) + logger.info(f"Discovered resource group: {rg.name}") + except Exception as e: + logger.error(f"Failed to discover resource groups: {str(e)}") + raise WorkspaceBuilderException(f"Error discovering resource groups: {str(e)}") + + if not resource_groups: + logger.warning("No resource groups were discovered.") + else: + logger.info(f"Total resource groups discovered: {len(resource_groups)}") + + return resource_groups + + +def az_validate_credential_access(credential, subscription_id): + try: + resource_client = ResourceManagementClient(credential, subscription_id) + resource_groups = list(resource_client.resource_groups.list()) + if resource_groups: + logger.info(f"Successfully accessed {len(resource_groups)} resource groups.") + else: + logger.warning("No resource groups found.") + except Exception as e: + logger.error(f"Failed to validate credential access: {str(e)}") + raise WorkspaceBuilderException("Credential validation failed.") + + +# --------------------------------------------------------------------------- +# Deferred RG resolution (post-index pass) +# --------------------------------------------------------------------------- + +def resolve_deferred_azure_relationships(registry, platform_handlers): + """ + Resolve deferred resource group relationships for Azure resources. + This handles cases where storage accounts (or any non-RG resource) were + processed before their resource groups. + + ``registry`` is a ``resources.Registry``; ``platform_handlers`` is a dict + keyed by platform name (we only look up ``"azure"``). + """ + logger.info("Starting deferred Azure relationship resolution...") + + azure_handler = platform_handlers.get("azure") + if not azure_handler: + logger.debug("No Azure platform handler found, skipping deferred relationship resolution") + return + + azure_platform = registry.platforms.get("azure") + if not azure_platform: + logger.debug("No Azure platform in registry, skipping deferred relationship resolution") + return + + rg_type = azure_platform.resource_types.get("resource_group") + if not rg_type: + logger.debug("No resource groups in registry, skipping deferred relationship resolution") + return + + resolved_count = 0 + failed_count = 0 + + for resource_type_name, resource_type in azure_platform.resource_types.items(): + if resource_type_name == "resource_group": + continue + + for resource_qualified_name, resource in list(resource_type.instances.items()): + deferred_info = getattr(resource, '_deferred_rg_lookup', None) + if not deferred_info: + continue + + rg_name = deferred_info.get('rg_name') + subscription_id = deferred_info.get('subscription_id') + + logger.debug( + f"Resolving deferred relationship for {resource.name}: " + f"looking for RG '{rg_name}' in subscription '{subscription_id}'" + ) + + rg_resource = None + for rg in rg_type.instances.values(): + if (rg.name.upper() == rg_name.upper() + and getattr(rg, 'subscription_id', None) == subscription_id): + rg_resource = rg + break + + if rg_resource: + setattr(resource, 'resource_group', rg_resource) + new_qualified_name = f"{rg_resource.name}/{resource.name}" + + old_qualified_name = resource.qualified_name + resource.qualified_name = new_qualified_name + + if old_qualified_name in resource_type.instances: + del resource_type.instances[old_qualified_name] + resource_type.instances[new_qualified_name] = resource + + delattr(resource, '_deferred_rg_lookup') + + resolved_count += 1 + logger.info( + f"SUCCESS: Resolved deferred relationship for '{resource.name}' -> " + f"resource group '{rg_resource.name}' " + f"(qualified name: {old_qualified_name} -> {new_qualified_name})" + ) + else: + failed_count += 1 + logger.warning( + f"FAILED: Could not resolve deferred relationship for '{resource.name}' - " + f"resource group '{rg_name}' in subscription '{subscription_id}' still not found" + ) + + logger.info( + f"Deferred relationship resolution completed: " + f"{resolved_count} resolved, {failed_count} failed" + ) + + +# --------------------------------------------------------------------------- +# Auth-type derivation (Azure + AWS branches) +# --------------------------------------------------------------------------- + +def get_auth_type(platform_name, platform_config_data: dict[str, Any]): + """ + Determine auth type from platform_config_data for use with auth templates. + + For Azure: azure-auth.yaml template + For AWS: aws-auth.yaml template + + Returns: + Tuple of (auth_type, auth_secret) + """ + auth_secret = None + auth_type = None + + if platform_name == "azure": + auth_secret = platform_config_data.get("clientId") + if auth_secret: + auth_type = "azure_explicit" + auth_secret = None + else: + auth_secret = platform_config_data.get("spSecretName") + if auth_secret: + auth_type = "azure_service_principal_secret" + else: + auth_type = "azure_identity" + auth_secret = None + + elif platform_name == "aws": + if platform_config_data.get("_auth_type"): + auth_type = platform_config_data.get("_auth_type") + auth_secret = platform_config_data.get("_auth_secret") + elif platform_config_data.get("awsAccessKeyId"): + auth_type = "aws_explicit" + elif platform_config_data.get("awsSecretName"): + auth_secret = platform_config_data.get("awsSecretName") + auth_type = "aws_secret" + elif (platform_config_data.get("useWorkloadIdentity") + or os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE') + or os.environ.get('AWS_CONTAINER_CREDENTIALS_FULL_URI')): + if os.environ.get('AWS_CONTAINER_CREDENTIALS_FULL_URI'): + auth_type = "aws_pod_identity" + else: + auth_type = "aws_workload_identity" + elif platform_config_data.get("assumeRoleArn"): + auth_type = "aws_assume_role" + else: + auth_type = "aws_default_chain" + + if (platform_config_data.get("assumeRoleArn") + and auth_type not in ( + "aws_assume_role", + "aws_explicit_assume_role", + "aws_secret_assume_role", + "aws_workload_identity_assume_role", + "aws_pod_identity_assume_role", + )): + auth_type = auth_type + "_assume_role" + + return auth_type, auth_secret diff --git a/src/indexers/azure_devops.py b/src/indexers/azure_devops.py index 394efc506..49518018a 100644 --- a/src/indexers/azure_devops.py +++ b/src/indexers/azure_devops.py @@ -1,22 +1,29 @@ -from azure.devops.connection import Connection -from azure.devops.v7_0.core.models import TeamProject -from azure.devops.v7_0.git.models import GitRepository -from azure.devops.v7_0.pipelines.models import Pipeline -from azure.devops.v7_0.release.models import ReleaseDefinition -from msrest.authentication import BasicAuthentication -from azure.identity import DefaultAzureCredential, ClientSecretCredential +from __future__ import annotations + import logging import os import re import requests import base64 -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from component import Context, SettingDependency from resources import Registry, REGISTRY_PROPERTY_NAME from .common import CLOUD_CONFIG_SETTING from k8s_utils import get_secret +if TYPE_CHECKING: + # Imported only for static type checking. The Azure DevOps SDK has an + # import-time filesystem side effect (azure/devops/_file_cache.py calls + # os.makedirs($HOME/.azure-devops)), so importing it at module scope can + # crash the whole REST service on startup when $HOME is not writable. + # Keep these imports lazy (inside the functions that instantiate them). + from azure.devops.connection import Connection + from azure.devops.v7_0.core.models import TeamProject + from azure.devops.v7_0.git.models import GitRepository + from azure.devops.v7_0.pipelines.models import Pipeline + from azure.devops.v7_0.release.models import ReleaseDefinition + logger = logging.getLogger(__name__) DOCUMENTATION = "Index Azure DevOps resources including projects, repositories, pipelines, and releases" @@ -100,6 +107,8 @@ def should_index_resource_type(self, resource_type: str, project_name: str) -> b def get_azure_devops_access_token_from_service_principal(tenant_id: str, client_id: str, client_secret: str) -> str: """Get an Azure DevOps access token using service principal credentials.""" + from azure.identity import ClientSecretCredential + try: credential = ClientSecretCredential(tenant_id, client_id, client_secret) token = credential.get_token("499b84ac-1321-427f-aa17-267ca6975798/.default") @@ -110,11 +119,17 @@ def get_azure_devops_access_token_from_service_principal(tenant_id: str, client_ def get_azure_devops_connection_with_pat(organization_url: str, personal_access_token: str) -> Connection: """Create an Azure DevOps connection using Personal Access Token.""" + from azure.devops.connection import Connection + from msrest.authentication import BasicAuthentication + credentials = BasicAuthentication('', personal_access_token) return Connection(base_url=organization_url, creds=credentials) def get_azure_devops_connection_with_service_principal(organization_url: str, access_token: str) -> Connection: """Create an Azure DevOps connection using Azure AD access token.""" + from azure.devops.connection import Connection + from msrest.authentication import BasicAuthentication + credentials = BasicAuthentication('', access_token) return Connection(base_url=organization_url, creds=credentials) @@ -244,6 +259,8 @@ def index(context: Context) -> None: if not connection: logger.info("Trying DefaultAzureCredential for Azure DevOps authentication") try: + from azure.identity import DefaultAzureCredential + credential = DefaultAzureCredential() token = credential.get_token("499b84ac-1321-427f-aa17-267ca6975798/.default") connection = get_azure_devops_connection_with_service_principal(organization_url, token.token) diff --git a/src/indexers/azure_resource_type_registry.py b/src/indexers/azure_resource_type_registry.py new file mode 100644 index 000000000..21120b48c --- /dev/null +++ b/src/indexers/azure_resource_type_registry.py @@ -0,0 +1,238 @@ +""" +Loader for the Azure resource-type registry. + +The registry maps every CloudQuery Azure table name to its ARM resource type +plus metadata used by the native ``azureapi`` indexer: + +* canonical name = the CloudQuery table name (e.g. ``azure_compute_virtual_machines``) +* ARM resource type for Resource Graph / generic discovery +* aliases for backward compatibility with legacy RWL ``resource_type_name`` + values (e.g. ``virtual_machine`` -> ``azure_compute_virtual_machines``, + ``azure_keyvault_vaults`` -> ``azure_keyvault_keyvaults``) +* ``typed_collector`` flag indicating whether ``azureapi_resource_types`` + ships a hand-written ``azure-mgmt-*`` collector for this table +* ``mandatory`` flag indicating whether the indexer must always list the + type (today only ``azure_resources_resource_groups``) + +The data lives in ``azure_resource_type_registry.yaml`` next to this module. +That YAML is generated by ``scripts/azure/sync_azure_resource_type_registry.py``; +hand-edits to the YAML get overwritten on the next sync. To change behavior +for a specific table, edit ``scripts/azure/azure_resource_type_overrides.yaml`` +and re-run the sync script. + +This module is intentionally read-only: it loads, caches, and exposes the +registry. It does not own collector callables or generation-rule semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +_DEFAULT_REGISTRY_PATH = Path(__file__).with_name("azure_resource_type_registry.yaml") + + +@dataclass(frozen=True) +class AzureResourceTypeEntry: + """One row of the registry. + + ``cloudquery_table_name`` is the canonical key. Lookups by alias resolve + to the same entry (no duplicates). + """ + + cloudquery_table_name: str + arm_type: Optional[str] + arm_type_source: Optional[str] + category: Optional[str] + aliases: tuple[str, ...] + typed_collector: bool + mandatory: bool + + def known_names(self) -> tuple[str, ...]: + """Every name (canonical + aliases) that resolves to this entry.""" + return (self.cloudquery_table_name, *self.aliases) + + +@dataclass(frozen=True) +class AzureRegistryMetadata: + source: Optional[str] = None + snapshot_date: Optional[str] = None + total_tables: int = 0 + typed_collectors: int = 0 + arm_types_assigned: int = 0 + generator: Optional[str] = None + notes: Optional[str] = None + + +@dataclass +class AzureResourceTypeRegistry: + metadata: AzureRegistryMetadata + entries: tuple[AzureResourceTypeEntry, ...] + _by_canonical: dict[str, AzureResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_alias: dict[str, AzureResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_arm_type_lower: dict[str, AzureResourceTypeEntry] = field(default_factory=dict, repr=False) + + def __post_init__(self) -> None: + for entry in self.entries: + self._by_canonical[entry.cloudquery_table_name] = entry + for alias in entry.aliases: + if not alias: + continue + # Aliases must not collide with another canonical name or alias; + # the sync script enforces this, but verify defensively in case + # someone hand-edits the YAML. + if alias in self._by_canonical and self._by_canonical[alias] is not entry: + raise ValueError( + f"Alias {alias!r} collides with canonical table name " + f"belonging to a different entry" + ) + if alias in self._by_alias and self._by_alias[alias] is not entry: + raise ValueError( + f"Alias {alias!r} is registered for both " + f"{self._by_alias[alias].cloudquery_table_name!r} and " + f"{entry.cloudquery_table_name!r}" + ) + self._by_alias[alias] = entry + if entry.arm_type: + # ARM types are case-insensitive on the wire; we lower-case for + # lookup. Multiple registry entries occasionally share an ARM + # type (e.g. legacy duplicates); the first one wins, which is + # deterministic because entries arrive sorted by canonical name. + key = entry.arm_type.lower() + self._by_arm_type_lower.setdefault(key, entry) + + def find(self, name: str) -> Optional[AzureResourceTypeEntry]: + """Return the entry for a canonical table name or alias, else None.""" + if not name: + return None + entry = self._by_canonical.get(name) + if entry is not None: + return entry + return self._by_alias.get(name) + + def find_by_arm_type(self, arm_type: Optional[str]) -> Optional[AzureResourceTypeEntry]: + """Look up the registry entry whose ``arm_type`` matches this string. + + Used by the generic-resources discovery pass to route each + ``GenericResource.type`` value (e.g. ``Microsoft.Storage/storageAccounts``) + back to the registry entry whose ``cloudquery_table_name`` is the + canonical RWL identifier for that type. + """ + if not arm_type: + return None + return self._by_arm_type_lower.get(arm_type.lower()) + + def __contains__(self, name: object) -> bool: + return isinstance(name, str) and self.find(name) is not None + + def __iter__(self) -> Iterable[AzureResourceTypeEntry]: + return iter(self.entries) + + def __len__(self) -> int: + return len(self.entries) + + def all_canonical_names(self) -> tuple[str, ...]: + return tuple(entry.cloudquery_table_name for entry in self.entries) + + def all_arm_types(self) -> tuple[str, ...]: + return tuple( + entry.arm_type for entry in self.entries if entry.arm_type + ) + + def typed_collector_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.typed_collector + ) + + def mandatory_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.mandatory + ) + + +def _coerce_aliases(value: object) -> tuple[str, ...]: + if not value: + return () + if isinstance(value, (list, tuple)): + return tuple(str(v) for v in value if v) + return (str(value),) + + +def _build_metadata(payload: Mapping[str, object]) -> AzureRegistryMetadata: + return AzureRegistryMetadata( + source=_optional_str(payload.get("source")), + snapshot_date=_optional_str(payload.get("snapshot_date")), + total_tables=int(payload.get("total_tables") or 0), + typed_collectors=int(payload.get("typed_collectors") or 0), + arm_types_assigned=int(payload.get("arm_types_assigned") or 0), + generator=_optional_str(payload.get("generator")), + notes=_optional_str(payload.get("notes")), + ) + + +def _optional_str(value: object) -> Optional[str]: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _build_entries(types_payload: Mapping[str, Mapping[str, object]]) -> tuple[AzureResourceTypeEntry, ...]: + entries: list[AzureResourceTypeEntry] = [] + for table_name, body in sorted(types_payload.items()): + if not isinstance(body, Mapping): + raise ValueError(f"Registry entry {table_name!r} is not a mapping") + entries.append( + AzureResourceTypeEntry( + cloudquery_table_name=str(table_name), + arm_type=_optional_str(body.get("arm_type")), + arm_type_source=_optional_str(body.get("arm_type_source")), + category=_optional_str(body.get("category")), + aliases=_coerce_aliases(body.get("aliases")), + typed_collector=bool(body.get("typed_collector")), + mandatory=bool(body.get("mandatory")), + ) + ) + return tuple(entries) + + +def load_registry_from_path(path: Path) -> AzureResourceTypeRegistry: + """Load a registry from an explicit YAML path. Bypasses the cache.""" + if not path.exists(): + raise FileNotFoundError(f"Azure resource-type registry not found at {path}") + with path.open("r", encoding="utf-8") as fh: + payload = yaml.safe_load(fh) or {} + if not isinstance(payload, Mapping): + raise ValueError(f"Registry YAML at {path} did not parse to a mapping") + + metadata = _build_metadata(payload.get("metadata") or {}) + types_payload = payload.get("types") or {} + if not isinstance(types_payload, Mapping): + raise ValueError(f"Registry YAML at {path} has non-mapping 'types' section") + + entries = _build_entries(types_payload) + return AzureResourceTypeRegistry(metadata=metadata, entries=entries) + + +@lru_cache(maxsize=1) +def load_registry() -> AzureResourceTypeRegistry: + """Load and cache the registry from the default location.""" + return load_registry_from_path(_DEFAULT_REGISTRY_PATH) + + +def find_entry(name: str) -> Optional[AzureResourceTypeEntry]: + """Convenience wrapper around the cached registry's ``find()``.""" + return load_registry().find(name) + + +def reset_cache() -> None: + """Clear the cached registry. Intended for tests.""" + load_registry.cache_clear() diff --git a/src/indexers/azure_resource_type_registry.yaml b/src/indexers/azure_resource_type_registry.yaml new file mode 100644 index 000000000..22cc93598 --- /dev/null +++ b/src/indexers/azure_resource_type_registry.yaml @@ -0,0 +1,4346 @@ +metadata: + source: https://www.cloudquery.io/hub/plugins/source/cloudquery/azure/latest/tables + snapshot_date: '2026-05-28' + total_tables: 619 + typed_collectors: 25 + arm_types_assigned: 618 + generator: scripts/azure/sync_azure_resource_type_registry.py + notes: Generated file. To change ARM type for a table, edit scripts/azure/azure_resource_type_overrides.yaml and re-run the sync script. Hand-edits to this file will be overwritten. +types: + azure_advisor_recommendation_metadata: + arm_type: Microsoft.Advisor/metadata + arm_type_source: override + category: advisor + aliases: [] + typed_collector: false + mandatory: false + azure_advisor_recommendations: + arm_type: Microsoft.Advisor/recommendations + arm_type_source: override + category: advisor + aliases: [] + typed_collector: false + mandatory: false + azure_advisor_suppressions: + arm_type: Microsoft.Advisor/suppressions + arm_type_source: override + category: advisor + aliases: [] + typed_collector: false + mandatory: false + azure_analysisservices_servers: + arm_type: Microsoft.AnalysisServices/servers + arm_type_source: heuristic + category: analysisservices + aliases: [] + typed_collector: false + mandatory: false + azure_apimanagement_service: + arm_type: Microsoft.ApiManagement/service + arm_type_source: heuristic + category: apimanagement + aliases: [] + typed_collector: true + mandatory: false + azure_appcomplianceautomation_reports: + arm_type: Microsoft.AppComplianceAutomation/reports + arm_type_source: heuristic + category: appcomplianceautomation + aliases: [] + typed_collector: false + mandatory: false + azure_appconfiguration_configuration_stores: + arm_type: Microsoft.AppConfiguration/configurationStores + arm_type_source: override + category: appconfiguration + aliases: [] + typed_collector: false + mandatory: false + azure_applicationinsights_components: + arm_type: Microsoft.Insights/components + arm_type_source: override + category: applicationinsights + aliases: [] + typed_collector: false + mandatory: false + azure_applicationinsights_web_tests: + arm_type: Microsoft.Insights/webtests + arm_type_source: override + category: applicationinsights + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_certificate_orders: + arm_type: Microsoft.CertificateRegistration/certificateOrders + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_certificates: + arm_type: Microsoft.Web/certificates + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_deleted_web_apps: + arm_type: Microsoft.Web/deletedSites + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_domains: + arm_type: Microsoft.DomainRegistration/domains + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_environments: + arm_type: Microsoft.Web/hostingEnvironments + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_plans: + arm_type: Microsoft.Web/serverFarms + arm_type_source: override + category: appservice + aliases: [] + typed_collector: true + mandatory: false + azure_appservice_recommendations: + arm_type: Microsoft.Web/recommendations + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_resource_health_metadata: + arm_type: Microsoft.Web/sites/resourceHealthMetadata + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_static_sites: + arm_type: Microsoft.Web/staticSites + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_top_level_domains: + arm_type: Microsoft.DomainRegistration/topLevelDomains + arm_type_source: override + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_web_app_auth_settings: + arm_type: Microsoft.Web/webAppAuthSettings + arm_type_source: heuristic + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_web_app_configurations: + arm_type: Microsoft.Web/webAppConfigurations + arm_type_source: heuristic + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_web_app_functions: + arm_type: Microsoft.Web/webAppFunctions + arm_type_source: heuristic + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_web_app_vnet_connections: + arm_type: Microsoft.Web/webAppVnetConnections + arm_type_source: heuristic + category: appservice + aliases: [] + typed_collector: false + mandatory: false + azure_appservice_web_apps: + arm_type: Microsoft.Web/sites + arm_type_source: override + category: appservice + aliases: [] + typed_collector: true + mandatory: false + azure_authorization_classic_administrators: + arm_type: Microsoft.Authorization/classicAdministrators + arm_type_source: override + category: authorization + aliases: [] + typed_collector: false + mandatory: false + azure_authorization_provider_operations_metadata: + arm_type: Microsoft.Authorization/providerOperations + arm_type_source: override + category: authorization + aliases: [] + typed_collector: false + mandatory: false + azure_authorization_role_assignments: + arm_type: Microsoft.Authorization/roleAssignments + arm_type_source: override + category: authorization + aliases: [] + typed_collector: false + mandatory: false + azure_authorization_role_definitions: + arm_type: Microsoft.Authorization/roleDefinitions + arm_type_source: override + category: authorization + aliases: [] + typed_collector: false + mandatory: false + azure_automation_account: + arm_type: Microsoft.Automation/account + arm_type_source: heuristic + category: automation + aliases: [] + typed_collector: false + mandatory: false + azure_azurearcdata_postgres_instances: + arm_type: Microsoft.AzureArcData/postgresInstances + arm_type_source: heuristic + category: azurearcdata + aliases: [] + typed_collector: false + mandatory: false + azure_azurearcdata_sql_managed_instances: + arm_type: Microsoft.AzureArcData/sqlManagedInstances + arm_type_source: heuristic + category: azurearcdata + aliases: [] + typed_collector: false + mandatory: false + azure_azurearcdata_sql_server_instances: + arm_type: Microsoft.AzureArcData/sqlServerInstances + arm_type_source: heuristic + category: azurearcdata + aliases: [] + typed_collector: true + mandatory: false + azure_batch_account: + arm_type: Microsoft.Batch/account + arm_type_source: heuristic + category: batch + aliases: [] + typed_collector: false + mandatory: false + azure_billing_accounts: + arm_type: Microsoft.Billing/billingAccounts + arm_type_source: override + category: billing + aliases: [] + typed_collector: false + mandatory: false + azure_billing_enrollment_accounts: + arm_type: Microsoft.Billing/enrollmentAccounts + arm_type_source: override + category: billing + aliases: [] + typed_collector: false + mandatory: false + azure_billing_periods: + arm_type: Microsoft.Billing/billingPeriods + arm_type_source: override + category: billing + aliases: [] + typed_collector: false + mandatory: false + azure_botservice_bots: + arm_type: Microsoft.Botservice/bots + arm_type_source: heuristic + category: botservice + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_edge_nodes: + arm_type: Microsoft.Cdn/edgeNodes + arm_type_source: heuristic + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_endpoints: + arm_type: Microsoft.Cdn/profiles/endpoints + arm_type_source: override + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_managed_rule_sets: + arm_type: Microsoft.Cdn/managedRuleSets + arm_type_source: heuristic + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_profiles: + arm_type: Microsoft.Cdn/profiles + arm_type_source: override + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_rule_sets: + arm_type: Microsoft.Cdn/ruleSets + arm_type_source: heuristic + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cdn_security_policies: + arm_type: Microsoft.Cdn/securityPolicies + arm_type_source: heuristic + category: cdn + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_capability_hosts: + arm_type: Microsoft.CognitiveServices/accountCapabilityHosts + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_connections: + arm_type: Microsoft.CognitiveServices/accountConnections + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_defender_for_ai_settings: + arm_type: Microsoft.CognitiveServices/accountDefenderForAiSettings + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_deployments: + arm_type: Microsoft.CognitiveServices/accountDeployments + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_encryption_scopes: + arm_type: Microsoft.CognitiveServices/accountEncryptionScopes + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_models: + arm_type: Microsoft.CognitiveServices/accountModels + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_network_security_perimeter_configurations: + arm_type: Microsoft.CognitiveServices/accountNetworkSecurityPerimeterConfigurations + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_private_endpoint_connections: + arm_type: Microsoft.CognitiveServices/accountPrivateEndpointConnections + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_private_link_resources: + arm_type: Microsoft.CognitiveServices/accountPrivateLinkResources + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_project_capability_hosts: + arm_type: Microsoft.CognitiveServices/accountProjectCapabilityHosts + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_project_connections: + arm_type: Microsoft.CognitiveServices/accountProjectConnections + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_projects: + arm_type: Microsoft.CognitiveServices/accountProjects + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_rai_blocklist_items: + arm_type: Microsoft.CognitiveServices/accountRaiBlocklistItems + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_rai_blocklists: + arm_type: Microsoft.CognitiveServices/accountRaiBlocklists + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_rai_policies: + arm_type: Microsoft.CognitiveServices/accountRaiPolicies + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_skus: + arm_type: Microsoft.CognitiveServices/accountSkus + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_account_usages: + arm_type: Microsoft.CognitiveServices/accountUsages + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_accounts: + arm_type: Microsoft.CognitiveServices/accounts + arm_type_source: override + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_commitment_plans: + arm_type: Microsoft.CognitiveServices/commitmentPlans + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_deleted_accounts: + arm_type: Microsoft.CognitiveServices/deletedAccounts + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_cognitiveservices_resource_skus: + arm_type: Microsoft.CognitiveServices/resourceSkus + arm_type_source: heuristic + category: cognitiveservices + aliases: [] + typed_collector: false + mandatory: false + azure_compute_availability_sets: + arm_type: Microsoft.Compute/availabilitySets + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_capacity_reservation_groups: + arm_type: Microsoft.Compute/capacityReservationGroups + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_capacity_reservations: + arm_type: Microsoft.Compute/capacityReservations + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_dedicated_host_groups: + arm_type: Microsoft.Compute/dedicatedHostGroups + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_dedicated_hosts: + arm_type: Microsoft.Compute/dedicatedHosts + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_disk_accesses: + arm_type: Microsoft.Compute/diskAccesses + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_disk_encryption_sets: + arm_type: Microsoft.Compute/diskEncryptionSets + arm_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_disks: + arm_type: Microsoft.Compute/disks + arm_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + azure_compute_galleries: + arm_type: Microsoft.Compute/galleries + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_gallery_image_versions: + arm_type: Microsoft.Compute/galleryImageVersions + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_gallery_images: + arm_type: Microsoft.Compute/galleryImages + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_images: + arm_type: Microsoft.Compute/images + arm_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_proximity_placement_groups: + arm_type: Microsoft.Compute/proximityPlacementGroups + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_restore_point_collections: + arm_type: Microsoft.Compute/restorePointCollections + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_skus: + arm_type: Microsoft.Compute/skus + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_snapshots: + arm_type: Microsoft.Compute/snapshots + arm_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + azure_compute_ssh_public_keys: + arm_type: Microsoft.Compute/sshPublicKeys + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machine_extensions: + arm_type: Microsoft.Compute/virtualMachineExtensions + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machine_patch_assessments: + arm_type: Microsoft.Compute/virtualMachinePatchAssessments + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machine_scale_set_network_interfaces: + arm_type: Microsoft.Compute/virtualMachineScaleSetNetworkInterfaces + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machine_scale_set_vms: + arm_type: Microsoft.Compute/virtualMachineScaleSetVms + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machine_scale_sets: + arm_type: Microsoft.Compute/virtualMachineScaleSets + arm_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + azure_compute_virtual_machine_software_inventories: + arm_type: Microsoft.Compute/virtualMachineSoftwareInventories + arm_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + azure_compute_virtual_machines: + arm_type: Microsoft.Compute/virtualMachines + arm_type_source: override + category: compute + aliases: + - virtual_machine + typed_collector: true + mandatory: false + azure_confidentialledger_ledgers: + arm_type: Microsoft.Confidentialledger/ledgers + arm_type_source: heuristic + category: confidentialledger + aliases: [] + typed_collector: false + mandatory: false + azure_confidentialledger_operations: + arm_type: Microsoft.Confidentialledger/operations + arm_type_source: heuristic + category: confidentialledger + aliases: [] + typed_collector: false + mandatory: false + azure_confluent_marketplace_agreements: + arm_type: Microsoft.Confluent/marketplaceAgreements + arm_type_source: heuristic + category: confluent + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_clusters: + arm_type: Microsoft.ConnectedVMware/clusters + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_datastores: + arm_type: Microsoft.ConnectedVMware/datastores + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_hosts: + arm_type: Microsoft.ConnectedVMware/hosts + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_resource_pools: + arm_type: Microsoft.ConnectedVMware/resourcePools + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_v_centers: + arm_type: Microsoft.ConnectedVMware/vCenters + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_virtual_machine_templates: + arm_type: Microsoft.ConnectedVMware/virtualMachineTemplates + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_virtual_machines: + arm_type: Microsoft.ConnectedVMware/virtualMachines + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_connectedvmware_virtual_networks: + arm_type: Microsoft.ConnectedVMware/virtualNetworks + arm_type_source: heuristic + category: connectedvmware + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_balances: + arm_type: Microsoft.Consumption/billingAccountBalances + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_budgets: + arm_type: Microsoft.Consumption/billingAccountBudgets + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_charges: + arm_type: Microsoft.Consumption/billingAccountCharges + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_events: + arm_type: Microsoft.Consumption/billingAccountEvents + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_legacy_usage_details: + arm_type: Microsoft.Consumption/billingAccountLegacyUsageDetails + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_lots: + arm_type: Microsoft.Consumption/billingAccountLots + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_marketplaces: + arm_type: Microsoft.Consumption/billingAccountMarketplaces + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_modern_usage_details: + arm_type: Microsoft.Consumption/billingAccountModernUsageDetails + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_profile_credits: + arm_type: Microsoft.Consumption/billingAccountProfileCredits + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_reservation_recommendations: + arm_type: Microsoft.Consumption/billingAccountReservationRecommendations + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_account_tags: + arm_type: Microsoft.Consumption/billingAccountTags + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_profile_reservation_details: + arm_type: Microsoft.Consumption/billingProfileReservationDetails + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_profile_reservation_recommendations: + arm_type: Microsoft.Consumption/billingProfileReservationRecommendations + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_profile_reservation_summaries: + arm_type: Microsoft.Consumption/billingProfileReservationSummaries + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_billing_profile_reservation_transactions: + arm_type: Microsoft.Consumption/billingProfileReservationTransactions + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_budgets: + arm_type: Microsoft.Consumption/subscriptionBudgets + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_legacy_usage_details: + arm_type: Microsoft.Consumption/subscriptionLegacyUsageDetails + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_marketplaces: + arm_type: Microsoft.Consumption/subscriptionMarketplaces + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_price_sheets: + arm_type: Microsoft.Consumption/subscriptionPriceSheets + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_reservation_recommendations: + arm_type: Microsoft.Consumption/subscriptionReservationRecommendations + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_consumption_subscription_tags: + arm_type: Microsoft.Consumption/subscriptionTags + arm_type_source: heuristic + category: consumption + aliases: [] + typed_collector: false + mandatory: false + azure_container_app_diagnostics_detectors: + arm_type: Microsoft.Container/appDiagnosticsDetectors + arm_type_source: heuristic + category: container + aliases: [] + typed_collector: false + mandatory: false + azure_container_app_source_controls: + arm_type: Microsoft.Container/appSourceControls + arm_type_source: heuristic + category: container + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_connected_environment_certificates: + arm_type: Microsoft.Containerapps/connectedEnvironmentCertificates + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_connected_environment_dapr_components: + arm_type: Microsoft.Containerapps/connectedEnvironmentDaprComponents + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_connected_environment_storages: + arm_type: Microsoft.Containerapps/connectedEnvironmentStorages + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_connected_environments: + arm_type: Microsoft.Containerapps/connectedEnvironments + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_container_app_auth_configs: + arm_type: Microsoft.Containerapps/containerAppAuthConfigs + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_container_app_detector_revisions: + arm_type: Microsoft.Containerapps/containerAppDetectorRevisions + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_container_app_revision_replicas: + arm_type: Microsoft.Containerapps/containerAppRevisionReplicas + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_container_app_revisions: + arm_type: Microsoft.Containerapps/containerAppRevisions + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_container_apps: + arm_type: Microsoft.Containerapps/containerApps + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_job_executions: + arm_type: Microsoft.Containerapps/jobExecutions + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_jobs: + arm_type: Microsoft.Containerapps/jobs + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_certificates: + arm_type: Microsoft.Containerapps/managedCertificates + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_environment_certificates: + arm_type: Microsoft.Containerapps/managedEnvironmentCertificates + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_environment_detectors: + arm_type: Microsoft.Containerapps/managedEnvironmentDetectors + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_environment_storages: + arm_type: Microsoft.Containerapps/managedEnvironmentStorages + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_environment_usages: + arm_type: Microsoft.Containerapps/managedEnvironmentUsages + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_managed_environments: + arm_type: Microsoft.Containerapps/managedEnvironments + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerapps_operations: + arm_type: Microsoft.Containerapps/operations + arm_type_source: heuristic + category: containerapps + aliases: [] + typed_collector: false + mandatory: false + azure_containerinstance_container_groups: + arm_type: Microsoft.ContainerInstance/containerGroups + arm_type_source: override + category: containerinstance + aliases: [] + typed_collector: false + mandatory: false + azure_containerregistry_registries: + arm_type: Microsoft.ContainerRegistry/registries + arm_type_source: override + category: containerregistry + aliases: [] + typed_collector: true + mandatory: false + azure_containerservice_managed_cluster_agent_pools: + arm_type: Microsoft.ContainerService/managedClusterAgentPools + arm_type_source: heuristic + category: containerservice + aliases: [] + typed_collector: false + mandatory: false + azure_containerservice_managed_cluster_upgrade_profiles: + arm_type: Microsoft.ContainerService/managedClusterUpgradeProfiles + arm_type_source: heuristic + category: containerservice + aliases: [] + typed_collector: false + mandatory: false + azure_containerservice_managed_clusters: + arm_type: Microsoft.ContainerService/managedClusters + arm_type_source: override + category: containerservice + aliases: [] + typed_collector: true + mandatory: false + azure_containerservice_snapshots: + arm_type: Microsoft.ContainerService/snapshots + arm_type_source: heuristic + category: containerservice + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_cassandra_clusters: + arm_type: Microsoft.Cosmos/cassandraClusters + arm_type_source: heuristic + category: cosmos + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_database_accounts: + arm_type: Microsoft.Cosmos/databaseAccounts + arm_type_source: heuristic + category: cosmos + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_locations: + arm_type: Microsoft.Cosmos/locations + arm_type_source: heuristic + category: cosmos + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_mongo_db_databases: + arm_type: Microsoft.Cosmos/mongoDbDatabases + arm_type_source: heuristic + category: cosmos + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_restorable_database_accounts: + arm_type: Microsoft.Cosmos/restorableDatabaseAccounts + arm_type_source: heuristic + category: cosmos + aliases: [] + typed_collector: false + mandatory: false + azure_cosmos_sql_databases: + arm_type: Microsoft.DocumentDB/databaseAccounts/sqlDatabases + arm_type_source: override + category: cosmos + aliases: [] + typed_collector: true + mandatory: false + azure_costmanagement_subscription_costs: + arm_type: Microsoft.CostManagement/subscriptionCosts + arm_type_source: heuristic + category: costmanagement + aliases: [] + typed_collector: false + mandatory: false + azure_costmanagement_view_queries: + arm_type: Microsoft.CostManagement/viewQueries + arm_type_source: heuristic + category: costmanagement + aliases: [] + typed_collector: false + mandatory: false + azure_costmanagement_views: + arm_type: Microsoft.CostManagement/views + arm_type_source: heuristic + category: costmanagement + aliases: [] + typed_collector: false + mandatory: false + azure_customerinsights_hubs: + arm_type: Microsoft.CustomerInsights/hubs + arm_type_source: heuristic + category: customerinsights + aliases: [] + typed_collector: false + mandatory: false + azure_dashboard_grafana: + arm_type: Microsoft.Dashboard/grafana + arm_type_source: heuristic + category: dashboard + aliases: [] + typed_collector: false + mandatory: false + azure_databox_jobs: + arm_type: Microsoft.Databox/jobs + arm_type_source: heuristic + category: databox + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_access_connectors: + arm_type: Microsoft.Databricks/accessConnectors + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_operations: + arm_type: Microsoft.Databricks/operations + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_outbound_network_dependencies_endpoints: + arm_type: Microsoft.Databricks/outboundNetworkDependenciesEndpoints + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_private_endpoint_connections: + arm_type: Microsoft.Databricks/privateEndpointConnections + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_private_link_resources: + arm_type: Microsoft.Databricks/privateLinkResources + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_virtual_network_peerings: + arm_type: Microsoft.Databricks/virtualNetworkPeerings + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_databricks_workspaces: + arm_type: Microsoft.Databricks/workspaces + arm_type_source: heuristic + category: databricks + aliases: [] + typed_collector: false + mandatory: false + azure_datacatalog_catalogs: + arm_type: Microsoft.Datacatalog/catalogs + arm_type_source: heuristic + category: datacatalog + aliases: [] + typed_collector: false + mandatory: false + azure_datadog_marketplace_agreements: + arm_type: Microsoft.Datadog/marketplaceAgreements + arm_type_source: heuristic + category: datadog + aliases: [] + typed_collector: false + mandatory: false + azure_datadog_monitors: + arm_type: Microsoft.Datadog/monitors + arm_type_source: heuristic + category: datadog + aliases: [] + typed_collector: false + mandatory: false + azure_datafactory_factories: + arm_type: Microsoft.DataFactory/factories + arm_type_source: override + category: datafactory + aliases: [] + typed_collector: true + mandatory: false + azure_datalakeanalytics_accounts: + arm_type: Microsoft.Datalakeanalytics/accounts + arm_type_source: heuristic + category: datalakeanalytics + aliases: [] + typed_collector: false + mandatory: false + azure_datalakestore_accounts: + arm_type: Microsoft.Datalakestore/accounts + arm_type_source: heuristic + category: datalakestore + aliases: [] + typed_collector: false + mandatory: false + azure_datamigration_services: + arm_type: Microsoft.Datamigration/services + arm_type_source: heuristic + category: datamigration + aliases: [] + typed_collector: false + mandatory: false + azure_datashare_accounts: + arm_type: Microsoft.Datashare/accounts + arm_type_source: heuristic + category: datashare + aliases: [] + typed_collector: false + mandatory: false + azure_desktopvirtualization_application_groups: + arm_type: Microsoft.DesktopVirtualization/applicationGroups + arm_type_source: heuristic + category: desktopvirtualization + aliases: [] + typed_collector: false + mandatory: false + azure_desktopvirtualization_host_pools: + arm_type: Microsoft.DesktopVirtualization/hostPools + arm_type_source: heuristic + category: desktopvirtualization + aliases: [] + typed_collector: false + mandatory: false + azure_desktopvirtualization_workspaces: + arm_type: Microsoft.DesktopVirtualization/workspaces + arm_type_source: heuristic + category: desktopvirtualization + aliases: [] + typed_collector: false + mandatory: false + azure_devhub_workflow: + arm_type: Microsoft.Devhub/workflow + arm_type_source: heuristic + category: devhub + aliases: [] + typed_collector: false + mandatory: false + azure_devops_pipeline_template_definitions: + arm_type: Microsoft.Devops/pipelineTemplateDefinitions + arm_type_source: heuristic + category: devops + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_global_schedules: + arm_type: Microsoft.Devtestlabs/globalSchedules + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_artifact_source_arm_templates: + arm_type: Microsoft.Devtestlabs/labArtifactSourceArmTemplates + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_artifact_source_artifact: + arm_type: Microsoft.Devtestlabs/labArtifactSourceArtifact + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_artifact_sources: + arm_type: Microsoft.Devtestlabs/labArtifactSources + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_custom_images: + arm_type: Microsoft.Devtestlabs/labCustomImages + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_environments: + arm_type: Microsoft.Devtestlabs/labEnvironments + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_formulas: + arm_type: Microsoft.Devtestlabs/labFormulas + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_galery_images: + arm_type: Microsoft.Devtestlabs/labGaleryImages + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_notification_channels: + arm_type: Microsoft.Devtestlabs/labNotificationChannels + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_schedules: + arm_type: Microsoft.Devtestlabs/labSchedules + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_user_disks: + arm_type: Microsoft.Devtestlabs/labUserDisks + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_user_secrets: + arm_type: Microsoft.Devtestlabs/labUserSecrets + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_users: + arm_type: Microsoft.Devtestlabs/labUsers + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_virtual_machine_schedules: + arm_type: Microsoft.Devtestlabs/labVirtualMachineSchedules + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_virtual_machines: + arm_type: Microsoft.Devtestlabs/labVirtualMachines + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_lab_virtual_networks: + arm_type: Microsoft.Devtestlabs/labVirtualNetworks + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_devtestlabs_labs: + arm_type: Microsoft.Devtestlabs/labs + arm_type_source: heuristic + category: devtestlabs + aliases: [] + typed_collector: false + mandatory: false + azure_dns_record_sets: + arm_type: Microsoft.Dns/recordSets + arm_type_source: heuristic + category: dns + aliases: [] + typed_collector: false + mandatory: false + azure_dns_zones: + arm_type: Microsoft.Dns/zones + arm_type_source: heuristic + category: dns + aliases: [] + typed_collector: false + mandatory: false + azure_dnsresolver_dns_forwarding_rulesets: + arm_type: Microsoft.Network/dnsForwardingRulesets + arm_type_source: heuristic + category: dnsresolver + aliases: [] + typed_collector: false + mandatory: false + azure_dnsresolver_dns_resolvers: + arm_type: Microsoft.Network/dnsResolvers + arm_type_source: heuristic + category: dnsresolver + aliases: [] + typed_collector: false + mandatory: false + azure_elastic_monitors: + arm_type: Microsoft.Elastic/monitors + arm_type_source: heuristic + category: elastic + aliases: [] + typed_collector: false + mandatory: false + azure_engagementfabric_accounts: + arm_type: Microsoft.Engagementfabric/accounts + arm_type_source: heuristic + category: engagementfabric + aliases: [] + typed_collector: false + mandatory: false + azure_eventgrid_topic_types: + arm_type: Microsoft.EventGrid/topicTypes + arm_type_source: heuristic + category: eventgrid + aliases: [] + typed_collector: false + mandatory: false + azure_eventhub_clusters: + arm_type: Microsoft.EventHub/clusters + arm_type_source: heuristic + category: eventhub + aliases: [] + typed_collector: false + mandatory: false + azure_eventhub_namespace_network_rule_sets: + arm_type: Microsoft.EventHub/namespaceNetworkRuleSets + arm_type_source: heuristic + category: eventhub + aliases: [] + typed_collector: false + mandatory: false + azure_eventhub_namespaces: + arm_type: Microsoft.EventHub/namespaces + arm_type_source: override + category: eventhub + aliases: [] + typed_collector: false + mandatory: false + azure_frontdoor_front_doors: + arm_type: Microsoft.Network/frontDoors + arm_type_source: heuristic + category: frontdoor + aliases: [] + typed_collector: false + mandatory: false + azure_frontdoor_managed_rule_sets: + arm_type: Microsoft.Network/managedRuleSets + arm_type_source: heuristic + category: frontdoor + aliases: [] + typed_collector: false + mandatory: false + azure_frontdoor_network_experiment_profiles: + arm_type: Microsoft.Network/networkExperimentProfiles + arm_type_source: heuristic + category: frontdoor + aliases: [] + typed_collector: false + mandatory: false + azure_hanaonazure_sap_monitors: + arm_type: Microsoft.Hanaonazure/sapMonitors + arm_type_source: heuristic + category: hanaonazure + aliases: [] + typed_collector: false + mandatory: false + azure_hdinsight_clusters: + arm_type: Microsoft.HDInsight/clusters + arm_type_source: heuristic + category: hdinsight + aliases: [] + typed_collector: false + mandatory: false + azure_healthbot_bots: + arm_type: Microsoft.Healthbot/bots + arm_type_source: heuristic + category: healthbot + aliases: [] + typed_collector: false + mandatory: false + azure_healthcareapis_services: + arm_type: Microsoft.HealthcareApis/services + arm_type_source: heuristic + category: healthcareapis + aliases: [] + typed_collector: false + mandatory: false + azure_hybridcompute_private_link_scopes: + arm_type: Microsoft.HybridCompute/privateLinkScopes + arm_type_source: heuristic + category: hybridcompute + aliases: [] + typed_collector: false + mandatory: false + azure_hybriddatamanager_data_managers: + arm_type: Microsoft.Hybriddatamanager/dataManagers + arm_type_source: heuristic + category: hybriddatamanager + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_certificate_issuers: + arm_type: Microsoft.KeyVault/vaults/certificates/issuers + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_certificate_policies: + arm_type: Microsoft.KeyVault/vaults/certificates/policies + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_certificates: + arm_type: Microsoft.KeyVault/vaults/certificates + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_key_rotation_policies: + arm_type: Microsoft.KeyVault/vaults/keys/rotationPolicies + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_keys: + arm_type: Microsoft.KeyVault/vaults/keys + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_keyvaults: + arm_type: Microsoft.KeyVault/vaults + arm_type_source: override + category: keyvault + aliases: + - azure_keyvault_vaults + - azure_keyvault_keyvault + typed_collector: true + mandatory: false + azure_keyvault_managed_hsms: + arm_type: Microsoft.KeyVault/managedHSMs + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_keyvault_secrets: + arm_type: Microsoft.KeyVault/vaults/secrets + arm_type_source: override + category: keyvault + aliases: [] + typed_collector: false + mandatory: false + azure_kusto_clusters: + arm_type: Microsoft.Kusto/clusters + arm_type_source: heuristic + category: kusto + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_plan_images: + arm_type: Microsoft.LabServices/labPlanImages + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_plans: + arm_type: Microsoft.LabServices/labPlans + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_schedules: + arm_type: Microsoft.LabServices/labSchedules + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_usages: + arm_type: Microsoft.LabServices/labUsages + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_users: + arm_type: Microsoft.LabServices/labUsers + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_lab_virtual_machines: + arm_type: Microsoft.LabServices/labVirtualMachines + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_labs: + arm_type: Microsoft.LabServices/labs + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_operations: + arm_type: Microsoft.LabServices/operations + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_labservices_skus: + arm_type: Microsoft.LabServices/skus + arm_type_source: heuristic + category: labservices + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_agreements: + arm_type: Microsoft.Logic/integrationAccountAgreements + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_assemblies: + arm_type: Microsoft.Logic/integrationAccountAssemblies + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_certificates: + arm_type: Microsoft.Logic/integrationAccountCertificates + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_maps: + arm_type: Microsoft.Logic/integrationAccountMaps + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_partners: + arm_type: Microsoft.Logic/integrationAccountPartners + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_schemas: + arm_type: Microsoft.Logic/integrationAccountSchemas + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_account_sessions: + arm_type: Microsoft.Logic/integrationAccountSessions + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_integration_accounts: + arm_type: Microsoft.Logic/integrationAccounts + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_run_action_request_histories: + arm_type: Microsoft.Logic/workflowRunActionRequestHistories + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_run_actions: + arm_type: Microsoft.Logic/workflowRunActions + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_runs: + arm_type: Microsoft.Logic/workflowRuns + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_trigger_histories: + arm_type: Microsoft.Logic/workflowTriggerHistories + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_triggers: + arm_type: Microsoft.Logic/workflowTriggers + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflow_versions: + arm_type: Microsoft.Logic/workflowVersions + arm_type_source: heuristic + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_logic_workflows: + arm_type: Microsoft.Logic/workflows + arm_type_source: override + category: logic + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_batch_deployments: + arm_type: Microsoft.MachineLearningServices/batchDeployments + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_batch_endpoints: + arm_type: Microsoft.MachineLearningServices/batchEndpoints + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_component_containers: + arm_type: Microsoft.MachineLearningServices/componentContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_component_versions: + arm_type: Microsoft.MachineLearningServices/componentVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_computes: + arm_type: Microsoft.MachineLearningServices/computes + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_data_containers: + arm_type: Microsoft.MachineLearningServices/dataContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_data_versions: + arm_type: Microsoft.MachineLearningServices/dataVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_datastores: + arm_type: Microsoft.MachineLearningServices/datastores + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_environment_containers: + arm_type: Microsoft.MachineLearningServices/environmentContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_environment_versions: + arm_type: Microsoft.MachineLearningServices/environmentVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_features: + arm_type: Microsoft.MachineLearningServices/features + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_featureset_containers: + arm_type: Microsoft.MachineLearningServices/featuresetContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_featureset_versions: + arm_type: Microsoft.MachineLearningServices/featuresetVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_featurestore_entity_containers: + arm_type: Microsoft.MachineLearningServices/featurestoreEntityContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_featurestore_entity_versions: + arm_type: Microsoft.MachineLearningServices/featurestoreEntityVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_jobs: + arm_type: Microsoft.MachineLearningServices/jobs + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_managed_network_setting_rules: + arm_type: Microsoft.MachineLearningServices/managedNetworkSettingRules + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_marketplace_subscriptions: + arm_type: Microsoft.MachineLearningServices/marketplaceSubscriptions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_model_containers: + arm_type: Microsoft.MachineLearningServices/modelContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_model_versions: + arm_type: Microsoft.MachineLearningServices/modelVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_online_deployments: + arm_type: Microsoft.MachineLearningServices/onlineDeployments + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_online_endpoints: + arm_type: Microsoft.MachineLearningServices/onlineEndpoints + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_operations: + arm_type: Microsoft.MachineLearningServices/operations + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_private_endpoint_connections: + arm_type: Microsoft.MachineLearningServices/privateEndpointConnections + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_private_link_resources: + arm_type: Microsoft.MachineLearningServices/privateLinkResources + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_quotas: + arm_type: Microsoft.MachineLearningServices/quotas + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registries: + arm_type: Microsoft.MachineLearningServices/registries + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_code_containers: + arm_type: Microsoft.MachineLearningServices/registryCodeContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_code_versions: + arm_type: Microsoft.MachineLearningServices/registryCodeVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_component_containers: + arm_type: Microsoft.MachineLearningServices/registryComponentContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_component_versions: + arm_type: Microsoft.MachineLearningServices/registryComponentVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_data_containers: + arm_type: Microsoft.MachineLearningServices/registryDataContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_data_versions: + arm_type: Microsoft.MachineLearningServices/registryDataVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_environment_containers: + arm_type: Microsoft.MachineLearningServices/registryEnvironmentContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_environment_versions: + arm_type: Microsoft.MachineLearningServices/registryEnvironmentVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_model_containers: + arm_type: Microsoft.MachineLearningServices/registryModelContainers + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_registry_model_versions: + arm_type: Microsoft.MachineLearningServices/registryModelVersions + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_schedules: + arm_type: Microsoft.MachineLearningServices/schedules + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_serverless_endpoints: + arm_type: Microsoft.MachineLearningServices/serverlessEndpoints + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_usages: + arm_type: Microsoft.MachineLearningServices/usages + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_virtual_machine_sizes: + arm_type: Microsoft.MachineLearningServices/virtualMachineSizes + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_workspace_connections: + arm_type: Microsoft.MachineLearningServices/workspaceConnections + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_workspace_features: + arm_type: Microsoft.MachineLearningServices/workspaceFeatures + arm_type_source: heuristic + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_machinelearning_workspaces: + arm_type: Microsoft.MachineLearningServices/workspaces + arm_type_source: override + category: machinelearning + aliases: [] + typed_collector: false + mandatory: false + azure_maintenance_configurations: + arm_type: Microsoft.Maintenance/configurations + arm_type_source: heuristic + category: maintenance + aliases: [] + typed_collector: false + mandatory: false + azure_maintenance_public_maintenance_configurations: + arm_type: Microsoft.Maintenance/publicMaintenanceConfigurations + arm_type_source: heuristic + category: maintenance + aliases: [] + typed_collector: false + mandatory: false + azure_managedapplications_applications: + arm_type: Microsoft.Managedapplications/applications + arm_type_source: heuristic + category: managedapplications + aliases: [] + typed_collector: false + mandatory: false + azure_management_locks: + arm_type: Microsoft.Management/locks + arm_type_source: heuristic + category: management + aliases: [] + typed_collector: false + mandatory: false + azure_managementgroups_entities: + arm_type: Microsoft.Managementgroups/entities + arm_type_source: heuristic + category: managementgroups + aliases: [] + typed_collector: false + mandatory: false + azure_managementgroups_management_groups: + arm_type: Microsoft.Managementgroups/managementGroups + arm_type_source: heuristic + category: managementgroups + aliases: [] + typed_collector: false + mandatory: false + azure_mariadb_server_configurations: + arm_type: Microsoft.DBforMariaDB/serverConfigurations + arm_type_source: heuristic + category: mariadb + aliases: [] + typed_collector: false + mandatory: false + azure_mariadb_servers: + arm_type: Microsoft.DBforMariaDB/servers + arm_type_source: override + category: mariadb + aliases: [] + typed_collector: false + mandatory: false + azure_marketplace_private_store: + arm_type: Microsoft.Marketplace/privateStore + arm_type_source: heuristic + category: marketplace + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_account_filters: + arm_type: Microsoft.Mediaservices/accountFilters + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_asset_filters: + arm_type: Microsoft.Mediaservices/assetFilters + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_asset_tracks: + arm_type: Microsoft.Mediaservices/assetTracks + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_assets: + arm_type: Microsoft.Mediaservices/assets + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_content_key_policies: + arm_type: Microsoft.Mediaservices/contentKeyPolicies + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_jobs: + arm_type: Microsoft.Mediaservices/jobs + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_live_event_outputs: + arm_type: Microsoft.Mediaservices/liveEventOutputs + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_live_events: + arm_type: Microsoft.Mediaservices/liveEvents + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_media_services: + arm_type: Microsoft.Mediaservices/mediaServices + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_private_endpoint_connections: + arm_type: Microsoft.Mediaservices/privateEndpointConnections + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_private_link_resources: + arm_type: Microsoft.Mediaservices/privateLinkResources + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_streaming_endpoints: + arm_type: Microsoft.Mediaservices/streamingEndpoints + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_streaming_locators: + arm_type: Microsoft.Mediaservices/streamingLocators + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_streaming_policies: + arm_type: Microsoft.Mediaservices/streamingPolicies + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_mediaservices_transforms: + arm_type: Microsoft.Mediaservices/transforms + arm_type_source: heuristic + category: mediaservices + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_action_groups: + arm_type: Microsoft.Insights/actionGroups + arm_type_source: override + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_activity_log_alerts: + arm_type: Microsoft.Monitor/activityLogAlerts + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_autoscale_settings: + arm_type: Microsoft.Insights/autoscaleSettings + arm_type_source: override + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_data_collection_rule_associations: + arm_type: Microsoft.Monitor/dataCollectionRuleAssociations + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_data_collection_rules: + arm_type: Microsoft.Monitor/dataCollectionRules + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_diagnostic_settings: + arm_type: Microsoft.Insights/diagnosticSettings + arm_type_source: override + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_log_profiles: + arm_type: Microsoft.Monitor/logProfiles + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_metric_alerts: + arm_type: Microsoft.Insights/metricAlerts + arm_type_source: override + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_metrics: + arm_type: Microsoft.Monitor/metrics + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_private_link_scopes: + arm_type: Microsoft.Monitor/privateLinkScopes + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_resources: + arm_type: Microsoft.Monitor/resources + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_scheduled_query_rules: + arm_type: Microsoft.Monitor/scheduledQueryRules + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_subscription_diagnostic_settings: + arm_type: Microsoft.Monitor/subscriptionDiagnosticSettings + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_tenant_activity_log_alerts: + arm_type: Microsoft.Monitor/tenantActivityLogAlerts + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_monitor_tenant_activity_logs: + arm_type: Microsoft.Monitor/tenantActivityLogs + arm_type_source: heuristic + category: monitor + aliases: [] + typed_collector: false + mandatory: false + azure_mysql_server_configurations: + arm_type: Microsoft.DBforMySQL/serverConfigurations + arm_type_source: heuristic + category: mysql + aliases: [] + typed_collector: false + mandatory: false + azure_mysql_server_databases: + arm_type: Microsoft.DBforMySQL/serverDatabases + arm_type_source: heuristic + category: mysql + aliases: [] + typed_collector: false + mandatory: false + azure_mysql_server_firewall_rules: + arm_type: Microsoft.DBforMySQL/serverFirewallRules + arm_type_source: heuristic + category: mysql + aliases: [] + typed_collector: false + mandatory: false + azure_mysql_servers: + arm_type: Microsoft.DBforMySQL/servers + arm_type_source: override + category: mysql + aliases: [] + typed_collector: true + mandatory: false + azure_mysqlflexibleservers_server_configurations: + arm_type: Microsoft.Mysqlflexibleservers/serverConfigurations + arm_type_source: heuristic + category: mysqlflexibleservers + aliases: [] + typed_collector: false + mandatory: false + azure_mysqlflexibleservers_server_firewall_rules: + arm_type: Microsoft.Mysqlflexibleservers/serverFirewallRules + arm_type_source: heuristic + category: mysqlflexibleservers + aliases: [] + typed_collector: false + mandatory: false + azure_mysqlflexibleservers_servers: + arm_type: Microsoft.DBforMySQL/flexibleServers + arm_type_source: override + category: mysqlflexibleservers + aliases: [] + typed_collector: true + mandatory: false + azure_netappfiles_account_backup_policies: + arm_type: Microsoft.NetApp/accountBackupPolicies + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_backup_vault_backups: + arm_type: Microsoft.NetApp/accountBackupVaultBackups + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_backup_vaults: + arm_type: Microsoft.NetApp/accountBackupVaults + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_pool_volume_quota_rules: + arm_type: Microsoft.NetApp/accountPoolVolumeQuotaRules + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_pool_volume_snapshots: + arm_type: Microsoft.NetApp/accountPoolVolumeSnapshots + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_pool_volume_subvolumes: + arm_type: Microsoft.NetApp/accountPoolVolumeSubvolumes + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_pool_volumes: + arm_type: Microsoft.NetApp/accountPoolVolumes + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_pools: + arm_type: Microsoft.NetApp/accountPools + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_snapshot_policies: + arm_type: Microsoft.NetApp/accountSnapshotPolicies + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_snapshot_policy_associated_volumes: + arm_type: Microsoft.NetApp/accountSnapshotPolicyAssociatedVolumes + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_account_volume_groups: + arm_type: Microsoft.NetApp/accountVolumeGroups + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_accounts: + arm_type: Microsoft.NetApp/accounts + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_quota_limits: + arm_type: Microsoft.NetApp/quotaLimits + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_netappfiles_region_infos: + arm_type: Microsoft.NetApp/regionInfos + arm_type_source: heuristic + category: netappfiles + aliases: [] + typed_collector: false + mandatory: false + azure_network_application_gateways: + arm_type: Microsoft.Network/applicationGateways + arm_type_source: override + category: network + aliases: [] + typed_collector: true + mandatory: false + azure_network_application_security_groups: + arm_type: Microsoft.Network/applicationSecurityGroups + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_azure_firewall_fqdn_tags: + arm_type: Microsoft.Network/azureFirewallFqdnTags + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_azure_firewalls: + arm_type: Microsoft.Network/azureFirewalls + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_bastion_hosts: + arm_type: Microsoft.Network/bastionHosts + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_bgp_service_communities: + arm_type: Microsoft.Network/bgpServiceCommunities + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_custom_ip_prefixes: + arm_type: Microsoft.Network/customIpPrefixes + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_ddos_protection_plans: + arm_type: Microsoft.Network/ddosProtectionPlans + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_dscp_configuration: + arm_type: Microsoft.Network/dscpConfiguration + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_circuit_authorizations: + arm_type: Microsoft.Network/expressRouteCircuitAuthorizations + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_circuit_peerings: + arm_type: Microsoft.Network/expressRouteCircuitPeerings + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_circuits: + arm_type: Microsoft.Network/expressRouteCircuits + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_gateways: + arm_type: Microsoft.Network/expressRouteGateways + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_ports: + arm_type: Microsoft.Network/expressRoutePorts + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_ports_locations: + arm_type: Microsoft.Network/expressRoutePortsLocations + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_express_route_service_providers: + arm_type: Microsoft.Network/expressRouteServiceProviders + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_firewall_policies: + arm_type: Microsoft.Network/firewallPolicies + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_interface_effective_route_tables: + arm_type: Microsoft.Network/interfaceEffectiveRouteTables + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_interface_ip_configurations: + arm_type: Microsoft.Network/interfaceIpConfigurations + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_interfaces: + arm_type: Microsoft.Network/interfaces + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_ip_allocations: + arm_type: Microsoft.Network/ipAllocations + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_ip_groups: + arm_type: Microsoft.Network/ipGroups + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_load_balancers: + arm_type: Microsoft.Network/loadBalancers + arm_type_source: override + category: network + aliases: [] + typed_collector: true + mandatory: false + azure_network_nat_gateways: + arm_type: Microsoft.Network/natGateways + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_peering_route_tables: + arm_type: Microsoft.Network/peeringRouteTables + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_private_endpoints: + arm_type: Microsoft.Network/privateEndpoints + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_private_link_services: + arm_type: Microsoft.Network/privateLinkServices + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_profiles: + arm_type: Microsoft.Network/profiles + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_public_ip_addresses: + arm_type: Microsoft.Network/publicIPAddresses + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_public_ip_prefixes: + arm_type: Microsoft.Network/publicIpPrefixes + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_route_filters: + arm_type: Microsoft.Network/routeFilters + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_route_tables: + arm_type: Microsoft.Network/routeTables + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_security_groups: + arm_type: Microsoft.Network/networkSecurityGroups + arm_type_source: override + category: network + aliases: [] + typed_collector: true + mandatory: false + azure_network_security_partner_providers: + arm_type: Microsoft.Network/securityPartnerProviders + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_service_endpoint_policies: + arm_type: Microsoft.Network/serviceEndpointPolicies + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_subscription_network_manager_connections: + arm_type: Microsoft.Network/subscriptionNetworkManagerConnections + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_appliances: + arm_type: Microsoft.Network/virtualAppliances + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_hubs: + arm_type: Microsoft.Network/virtualHubs + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_network_gateway_connections: + arm_type: Microsoft.Network/virtualNetworkGatewayConnections + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_network_gateways: + arm_type: Microsoft.Network/virtualNetworkGateways + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_network_subnets: + arm_type: Microsoft.Network/virtualNetworkSubnets + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_network_taps: + arm_type: Microsoft.Network/virtualNetworkTaps + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_networks: + arm_type: Microsoft.Network/virtualNetworks + arm_type_source: override + category: network + aliases: [] + typed_collector: true + mandatory: false + azure_network_virtual_routers: + arm_type: Microsoft.Network/virtualRouters + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_virtual_wans: + arm_type: Microsoft.Network/virtualWans + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_vpn_gateways: + arm_type: Microsoft.Network/vpnGateways + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_vpn_server_configurations: + arm_type: Microsoft.Network/vpnServerConfigurations + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_vpn_sites: + arm_type: Microsoft.Network/vpnSites + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_watcher_flow_logs: + arm_type: Microsoft.Network/watcherFlowLogs + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_watchers: + arm_type: Microsoft.Network/networkWatchers + arm_type_source: override + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_network_web_application_firewall_policies: + arm_type: Microsoft.Network/webApplicationFirewallPolicies + arm_type_source: heuristic + category: network + aliases: [] + typed_collector: false + mandatory: false + azure_networkfunction_azure_traffic_collectors_by_subscription: + arm_type: Microsoft.Networkfunction/azureTrafficCollectorsBySubscription + arm_type_source: heuristic + category: networkfunction + aliases: [] + typed_collector: false + mandatory: false + azure_nginx_deployments: + arm_type: Microsoft.Nginx/deployments + arm_type_source: heuristic + category: nginx + aliases: [] + typed_collector: false + mandatory: false + azure_notificationhubs_namespaces: + arm_type: Microsoft.NotificationHubs/namespaces + arm_type_source: override + category: notificationhubs + aliases: [] + typed_collector: false + mandatory: false + azure_operationalinsights_clusters: + arm_type: Microsoft.OperationalInsights/clusters + arm_type_source: heuristic + category: operationalinsights + aliases: [] + typed_collector: false + mandatory: false + azure_operationalinsights_workspaces: + arm_type: Microsoft.OperationalInsights/workspaces + arm_type_source: override + category: operationalinsights + aliases: [] + typed_collector: false + mandatory: false + azure_peering_service_countries: + arm_type: Microsoft.Peering/serviceCountries + arm_type_source: heuristic + category: peering + aliases: [] + typed_collector: false + mandatory: false + azure_peering_service_locations: + arm_type: Microsoft.Peering/serviceLocations + arm_type_source: heuristic + category: peering + aliases: [] + typed_collector: false + mandatory: false + azure_peering_service_providers: + arm_type: Microsoft.Peering/serviceProviders + arm_type_source: heuristic + category: peering + aliases: [] + typed_collector: false + mandatory: false + azure_policy_assignments: + arm_type: Microsoft.Policy/assignments + arm_type_source: heuristic + category: policy + aliases: [] + typed_collector: false + mandatory: false + azure_policy_definition_versions: + arm_type: Microsoft.Policy/definitionVersions + arm_type_source: heuristic + category: policy + aliases: [] + typed_collector: false + mandatory: false + azure_policy_definitions: + arm_type: Microsoft.Policy/definitions + arm_type_source: heuristic + category: policy + aliases: [] + typed_collector: false + mandatory: false + azure_policy_set_definition_versions: + arm_type: Microsoft.Policy/setDefinitionVersions + arm_type_source: heuristic + category: policy + aliases: [] + typed_collector: false + mandatory: false + azure_policy_set_definitions: + arm_type: Microsoft.Policy/setDefinitions + arm_type_source: heuristic + category: policy + aliases: [] + typed_collector: false + mandatory: false + azure_policyinsights_attestations: + arm_type: Microsoft.PolicyInsights/attestations + arm_type_source: heuristic + category: policyinsights + aliases: [] + typed_collector: false + mandatory: false + azure_policyinsights_policy_events: + arm_type: Microsoft.PolicyInsights/policyEvents + arm_type_source: heuristic + category: policyinsights + aliases: [] + typed_collector: false + mandatory: false + azure_policyinsights_policy_states: + arm_type: Microsoft.PolicyInsights/policyStates + arm_type_source: heuristic + category: policyinsights + aliases: [] + typed_collector: false + mandatory: false + azure_policyinsights_policy_tracked_resources: + arm_type: Microsoft.PolicyInsights/policyTrackedResources + arm_type_source: heuristic + category: policyinsights + aliases: [] + typed_collector: false + mandatory: false + azure_portal_list_tenant_configuration_violations: + arm_type: Microsoft.Portal/listTenantConfigurationViolations + arm_type_source: heuristic + category: portal + aliases: [] + typed_collector: false + mandatory: false + azure_portal_tenant_configurations: + arm_type: Microsoft.Portal/tenantConfigurations + arm_type_source: heuristic + category: portal + aliases: [] + typed_collector: false + mandatory: false + azure_postgresql_databases: + arm_type: Microsoft.DBforPostgreSQL/servers/databases + arm_type_source: override + category: postgresql + aliases: [] + typed_collector: true + mandatory: false + azure_postgresql_server_configurations: + arm_type: Microsoft.DBforPostgreSQL/serverConfigurations + arm_type_source: heuristic + category: postgresql + aliases: [] + typed_collector: false + mandatory: false + azure_postgresql_server_firewall_rules: + arm_type: Microsoft.DBforPostgreSQL/serverFirewallRules + arm_type_source: heuristic + category: postgresql + aliases: [] + typed_collector: false + mandatory: false + azure_postgresql_servers: + arm_type: Microsoft.DBforPostgreSQL/servers + arm_type_source: override + category: postgresql + aliases: [] + typed_collector: false + mandatory: false + azure_postgresqlflexibleservers_server_configurations: + arm_type: Microsoft.Postgresqlflexibleservers/serverConfigurations + arm_type_source: heuristic + category: postgresqlflexibleservers + aliases: [] + typed_collector: false + mandatory: false + azure_postgresqlflexibleservers_server_firewall_rules: + arm_type: Microsoft.Postgresqlflexibleservers/serverFirewallRules + arm_type_source: heuristic + category: postgresqlflexibleservers + aliases: [] + typed_collector: false + mandatory: false + azure_postgresqlflexibleservers_servers: + arm_type: Microsoft.Postgresqlflexibleservers/servers + arm_type_source: heuristic + category: postgresqlflexibleservers + aliases: [] + typed_collector: false + mandatory: false + azure_postgresqlhsc_server_groups: + arm_type: Microsoft.Postgresqlhsc/serverGroups + arm_type_source: heuristic + category: postgresqlhsc + aliases: [] + typed_collector: false + mandatory: false + azure_powerbidedicated_capacities: + arm_type: Microsoft.PowerBIDedicated/capacities + arm_type_source: heuristic + category: powerbidedicated + aliases: [] + typed_collector: false + mandatory: false + azure_privatedns_private_zone_record_sets: + arm_type: Microsoft.Network/privateZoneRecordSets + arm_type_source: heuristic + category: privatedns + aliases: [] + typed_collector: false + mandatory: false + azure_privatedns_private_zone_virtual_network_links: + arm_type: Microsoft.Network/privateZoneVirtualNetworkLinks + arm_type_source: heuristic + category: privatedns + aliases: [] + typed_collector: false + mandatory: false + azure_privatedns_private_zones: + arm_type: Microsoft.Network/privateZones + arm_type_source: heuristic + category: privatedns + aliases: [] + typed_collector: false + mandatory: false + azure_providerhub_provider_registrations: + arm_type: Microsoft.Providerhub/providerRegistrations + arm_type_source: heuristic + category: providerhub + aliases: [] + typed_collector: false + mandatory: false + azure_purview_account_keys: + arm_type: Microsoft.Purview/accountKeys + arm_type_source: heuristic + category: purview + aliases: [] + typed_collector: false + mandatory: false + azure_purview_account_private_endpoint_connections: + arm_type: Microsoft.Purview/accountPrivateEndpointConnections + arm_type_source: heuristic + category: purview + aliases: [] + typed_collector: false + mandatory: false + azure_purview_account_private_link_resources: + arm_type: Microsoft.Purview/accountPrivateLinkResources + arm_type_source: heuristic + category: purview + aliases: [] + typed_collector: false + mandatory: false + azure_purview_accounts: + arm_type: Microsoft.Purview/accounts + arm_type_source: heuristic + category: purview + aliases: [] + typed_collector: false + mandatory: false + azure_purview_operations: + arm_type: Microsoft.Purview/operations + arm_type_source: heuristic + category: purview + aliases: [] + typed_collector: false + mandatory: false + azure_quota_quotas: + arm_type: Microsoft.Quota/quotas + arm_type_source: heuristic + category: quota + aliases: [] + typed_collector: false + mandatory: false + azure_quota_usages: + arm_type: Microsoft.Quota/usages + arm_type_source: heuristic + category: quota + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_engines: + arm_type: Microsoft.RecoveryServices/backupEngines + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_jobs: + arm_type: Microsoft.RecoveryServices/backupJobs + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_policies: + arm_type: Microsoft.RecoveryServices/backupPolicies + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_protected_items: + arm_type: Microsoft.RecoveryServices/backupProtectedItems + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_protection_containers: + arm_type: Microsoft.RecoveryServices/backupProtectionContainers + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_protection_intents: + arm_type: Microsoft.RecoveryServices/backupProtectionIntents + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_backup_usage_summaries: + arm_type: Microsoft.RecoveryServices/backupUsageSummaries + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_deleted_protection_containers: + arm_type: Microsoft.RecoveryServices/deletedProtectionContainers + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_operations: + arm_type: Microsoft.RecoveryServices/operations + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_private_link_resources: + arm_type: Microsoft.RecoveryServices/privateLinkResources + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_alert_settings: + arm_type: Microsoft.RecoveryServices/replicationAlertSettings + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_events: + arm_type: Microsoft.RecoveryServices/replicationEvents + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_fabrics: + arm_type: Microsoft.RecoveryServices/replicationFabrics + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_jobs: + arm_type: Microsoft.RecoveryServices/replicationJobs + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_logical_networks: + arm_type: Microsoft.RecoveryServices/replicationLogicalNetworks + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_migration_items: + arm_type: Microsoft.RecoveryServices/replicationMigrationItems + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_network_mappings: + arm_type: Microsoft.RecoveryServices/replicationNetworkMappings + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_networks: + arm_type: Microsoft.RecoveryServices/replicationNetworks + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_policies: + arm_type: Microsoft.RecoveryServices/replicationPolicies + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_protectable_items: + arm_type: Microsoft.RecoveryServices/replicationProtectableItems + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_protected_items: + arm_type: Microsoft.RecoveryServices/replicationProtectedItems + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_protection_containers: + arm_type: Microsoft.RecoveryServices/replicationProtectionContainers + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_protection_ctnr_mappings: + arm_type: Microsoft.RecoveryServices/replicationProtectionCtnrMappings + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_protection_intents: + arm_type: Microsoft.RecoveryServices/replicationProtectionIntents + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_recovery_plans: + arm_type: Microsoft.RecoveryServices/replicationRecoveryPlans + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_recovery_service_providers: + arm_type: Microsoft.RecoveryServices/replicationRecoveryServiceProviders + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_storage_classif_mappings: + arm_type: Microsoft.RecoveryServices/replicationStorageClassifMappings + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_storage_classifications: + arm_type: Microsoft.RecoveryServices/replicationStorageClassifications + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_vault_settings: + arm_type: Microsoft.RecoveryServices/replicationVaultSettings + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_replication_vcenters: + arm_type: Microsoft.RecoveryServices/replicationVcenters + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_vault_replication_usages: + arm_type: Microsoft.RecoveryServices/vaultReplicationUsages + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_vault_usages: + arm_type: Microsoft.RecoveryServices/vaultUsages + arm_type_source: heuristic + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_recoveryservices_vaults: + arm_type: Microsoft.RecoveryServices/vaults + arm_type_source: override + category: recoveryservices + aliases: [] + typed_collector: false + mandatory: false + azure_redhatopenshift_open_shift_clusters: + arm_type: Microsoft.RedHatOpenShift/openShiftClusters + arm_type_source: heuristic + category: redhatopenshift + aliases: [] + typed_collector: false + mandatory: false + azure_redis_caches: + arm_type: Microsoft.Cache/Redis + arm_type_source: override + category: redis + aliases: [] + typed_collector: true + mandatory: false + azure_redis_firewall_rules: + arm_type: Microsoft.Redis/firewallRules + arm_type_source: heuristic + category: redis + aliases: [] + typed_collector: false + mandatory: false + azure_redis_patch_schedules: + arm_type: Microsoft.Redis/patchSchedules + arm_type_source: heuristic + category: redis + aliases: [] + typed_collector: false + mandatory: false + azure_relay_namespaces: + arm_type: Microsoft.Relay/namespaces + arm_type_source: heuristic + category: relay + aliases: [] + typed_collector: false + mandatory: false + azure_reservations_reservation: + arm_type: Microsoft.Capacity/reservation + arm_type_source: heuristic + category: reservations + aliases: [] + typed_collector: false + mandatory: false + azure_reservations_reservation_order: + arm_type: Microsoft.Capacity/reservationOrder + arm_type_source: heuristic + category: reservations + aliases: [] + typed_collector: false + mandatory: false + azure_resourcehealth_availability_statuses: + arm_type: Microsoft.ResourceHealth/availabilityStatuses + arm_type_source: heuristic + category: resourcehealth + aliases: [] + typed_collector: false + mandatory: false + azure_resourcehealth_emerging_issues: + arm_type: Microsoft.ResourceHealth/emergingIssues + arm_type_source: heuristic + category: resourcehealth + aliases: [] + typed_collector: false + mandatory: false + azure_resourcehealth_event_impacted_resources: + arm_type: Microsoft.ResourceHealth/eventImpactedResources + arm_type_source: heuristic + category: resourcehealth + aliases: [] + typed_collector: false + mandatory: false + azure_resourcehealth_events: + arm_type: Microsoft.ResourceHealth/events + arm_type_source: heuristic + category: resourcehealth + aliases: [] + typed_collector: false + mandatory: false + azure_resourcehealth_security_advisory_impacted_resources: + arm_type: Microsoft.ResourceHealth/securityAdvisoryImpactedResources + arm_type_source: heuristic + category: resourcehealth + aliases: [] + typed_collector: false + mandatory: false + azure_resources_links: + arm_type: Microsoft.Resources/links + arm_type_source: override + category: resources + aliases: [] + typed_collector: false + mandatory: false + azure_resources_providers: + arm_type: Microsoft.Resources/providers + arm_type_source: override + category: resources + aliases: [] + typed_collector: false + mandatory: false + azure_resources_resource_groups: + arm_type: Microsoft.Resources/resourceGroups + arm_type_source: override + category: resources + aliases: + - resource_group + typed_collector: true + mandatory: true + azure_resources_resources: + arm_type: null + arm_type_source: override + category: resources + aliases: [] + typed_collector: false + mandatory: false + azure_role_management_policy_assignments: + arm_type: Microsoft.Role/managementPolicyAssignments + arm_type_source: heuristic + category: role + aliases: [] + typed_collector: false + mandatory: false + azure_saas_resources: + arm_type: Microsoft.Saas/resources + arm_type_source: heuristic + category: saas + aliases: [] + typed_collector: false + mandatory: false + azure_search_services: + arm_type: Microsoft.Search/searchServices + arm_type_source: override + category: search + aliases: [] + typed_collector: false + mandatory: false + azure_security_adaptive_application_controls: + arm_type: Microsoft.Security/adaptiveApplicationControls + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_alerts: + arm_type: Microsoft.Security/alerts + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_alerts_suppression_rules: + arm_type: Microsoft.Security/alertsSuppressionRules + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_allowed_connections: + arm_type: Microsoft.Security/allowedConnections + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_applications: + arm_type: Microsoft.Security/applications + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_assessments: + arm_type: Microsoft.Security/assessments + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_assessments_metadata: + arm_type: Microsoft.Security/assessmentsMetadata + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_auto_provisioning_settings: + arm_type: Microsoft.Security/autoProvisioningSettings + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_automations: + arm_type: Microsoft.Security/automations + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_connectors: + arm_type: Microsoft.Security/connectors + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_contacts: + arm_type: Microsoft.Security/contacts + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_container_registry_vulnerability_details: + arm_type: Microsoft.Security/containerRegistryVulnerabilityDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_discovered_security_solutions: + arm_type: Microsoft.Security/discoveredSecuritySolutions + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_external_security_solutions: + arm_type: Microsoft.Security/externalSecuritySolutions + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_governance_rule: + arm_type: Microsoft.Security/governanceRule + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_jit_network_access_policies: + arm_type: Microsoft.Security/jitNetworkAccessPolicies + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_locations: + arm_type: Microsoft.Security/locations + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_pricings: + arm_type: Microsoft.Security/pricings + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_regulatory_compliance_assessments: + arm_type: Microsoft.Security/regulatoryComplianceAssessments + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_regulatory_compliance_controls: + arm_type: Microsoft.Security/regulatoryComplianceControls + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_regulatory_compliance_standards: + arm_type: Microsoft.Security/regulatoryComplianceStandards + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_secure_score_control_definitions: + arm_type: Microsoft.Security/secureScoreControlDefinitions + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_secure_score_controls: + arm_type: Microsoft.Security/secureScoreControls + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_secure_scores: + arm_type: Microsoft.Security/secureScores + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_server_vulnerability_details: + arm_type: Microsoft.Security/serverVulnerabilityDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_settings: + arm_type: Microsoft.Security/settings + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_solutions: + arm_type: Microsoft.Security/solutions + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_sql_server_vulnerability_details: + arm_type: Microsoft.Security/sqlServerVulnerabilityDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_sub_assessment_azure_resource_details: + arm_type: Microsoft.Security/subAssessmentAzureResourceDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_sub_assessment_on_premise_resource_details: + arm_type: Microsoft.Security/subAssessmentOnPremiseResourceDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_sub_assessment_on_premise_sql_resource_details: + arm_type: Microsoft.Security/subAssessmentOnPremiseSqlResourceDetails + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_sub_assessments: + arm_type: Microsoft.Security/subAssessments + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_tasks: + arm_type: Microsoft.Security/tasks + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_topology: + arm_type: Microsoft.Security/topology + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_security_workspace_settings: + arm_type: Microsoft.Security/workspaceSettings + arm_type_source: heuristic + category: security + aliases: [] + typed_collector: false + mandatory: false + azure_servicebus_namespace_topic_authorization_rules: + arm_type: Microsoft.ServiceBus/namespaceTopicAuthorizationRules + arm_type_source: heuristic + category: servicebus + aliases: [] + typed_collector: false + mandatory: false + azure_servicebus_namespace_topic_rule_access_keys: + arm_type: Microsoft.ServiceBus/namespaceTopicRuleAccessKeys + arm_type_source: heuristic + category: servicebus + aliases: [] + typed_collector: false + mandatory: false + azure_servicebus_namespace_topics: + arm_type: Microsoft.ServiceBus/namespaceTopics + arm_type_source: heuristic + category: servicebus + aliases: [] + typed_collector: false + mandatory: false + azure_servicebus_namespaces: + arm_type: Microsoft.ServiceBus/namespaces + arm_type_source: override + category: servicebus + aliases: [] + typed_collector: true + mandatory: false + azure_servicefabric_cluster_application_services: + arm_type: Microsoft.ServiceFabric/clusterApplicationServices + arm_type_source: heuristic + category: servicefabric + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabric_cluster_application_types: + arm_type: Microsoft.ServiceFabric/clusterApplicationTypes + arm_type_source: heuristic + category: servicefabric + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabric_cluster_applications: + arm_type: Microsoft.ServiceFabric/clusterApplications + arm_type_source: heuristic + category: servicefabric + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabric_clusters: + arm_type: Microsoft.ServiceFabric/clusters + arm_type_source: heuristic + category: servicefabric + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_cluster_application_services: + arm_type: Microsoft.Servicefabricmanaged/clusterApplicationServices + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_cluster_application_types: + arm_type: Microsoft.Servicefabricmanaged/clusterApplicationTypes + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_cluster_applications: + arm_type: Microsoft.Servicefabricmanaged/clusterApplications + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_cluster_node_type_skus: + arm_type: Microsoft.Servicefabricmanaged/clusterNodeTypeSkus + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_cluster_node_types: + arm_type: Microsoft.Servicefabricmanaged/clusterNodeTypes + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_servicefabricmanaged_clusters: + arm_type: Microsoft.Servicefabricmanaged/clusters + arm_type_source: heuristic + category: servicefabricmanaged + aliases: [] + typed_collector: false + mandatory: false + azure_sql_instance_pools: + arm_type: Microsoft.Sql/instancePools + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_managed_instance_encryption_protectors: + arm_type: Microsoft.Sql/managedInstanceEncryptionProtectors + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_managed_instance_vulnerability_assessments: + arm_type: Microsoft.Sql/managedInstanceVulnerabilityAssessments + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_managed_instances: + arm_type: Microsoft.Sql/managedInstances + arm_type_source: override + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_admins: + arm_type: Microsoft.Sql/serverAdmins + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_advanced_threat_protection_settings: + arm_type: Microsoft.Sql/serverAdvancedThreatProtectionSettings + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_blob_auditing_policies: + arm_type: Microsoft.Sql/serverBlobAuditingPolicies + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_database_blob_auditing_policies: + arm_type: Microsoft.Sql/serverDatabaseBlobAuditingPolicies + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_database_long_term_retention_policies: + arm_type: Microsoft.Sql/serverDatabaseLongTermRetentionPolicies + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_database_threat_protections: + arm_type: Microsoft.Sql/serverDatabaseThreatProtections + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_database_vulnerability_assessment_scans: + arm_type: Microsoft.Sql/serverDatabaseVulnerabilityAssessmentScans + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_database_vulnerability_assessments: + arm_type: Microsoft.Sql/serverDatabaseVulnerabilityAssessments + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_databases: + arm_type: Microsoft.Sql/serverDatabases + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_encryption_protectors: + arm_type: Microsoft.Sql/serverEncryptionProtectors + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_failover_groups: + arm_type: Microsoft.Sql/serverFailoverGroups + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_firewall_rules: + arm_type: Microsoft.Sql/serverFirewallRules + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_security_alert_policies: + arm_type: Microsoft.Sql/serverSecurityAlertPolicies + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_virtual_network_rules: + arm_type: Microsoft.Sql/serverVirtualNetworkRules + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_server_vulnerability_assessments: + arm_type: Microsoft.Sql/serverVulnerabilityAssessments + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_servers: + arm_type: Microsoft.Sql/servers + arm_type_source: override + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_transparent_data_encryptions: + arm_type: Microsoft.Sql/transparentDataEncryptions + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sql_virtual_clusters: + arm_type: Microsoft.Sql/virtualClusters + arm_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + azure_sqlvirtualmachine_groups: + arm_type: Microsoft.Sqlvirtualmachine/groups + arm_type_source: heuristic + category: sqlvirtualmachine + aliases: [] + typed_collector: false + mandatory: false + azure_sqlvirtualmachine_sql_virtual_machines: + arm_type: Microsoft.Sqlvirtualmachine/sqlVirtualMachines + arm_type_source: heuristic + category: sqlvirtualmachine + aliases: [] + typed_collector: false + mandatory: false + azure_storage_account_keys: + arm_type: Microsoft.Storage/accountKeys + arm_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_accounts: + arm_type: Microsoft.Storage/storageAccounts + arm_type_source: override + category: storage + aliases: [] + typed_collector: true + mandatory: false + azure_storage_blob_services: + arm_type: Microsoft.Storage/storageAccounts/blobServices + arm_type_source: override + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_containers: + arm_type: Microsoft.Storage/containers + arm_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_encryption_scopes: + arm_type: Microsoft.Storage/encryptionScopes + arm_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_file_shares: + arm_type: Microsoft.Storage/storageAccounts/fileServices/shares + arm_type_source: override + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_management_policies: + arm_type: Microsoft.Storage/managementPolicies + arm_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_queue_acl: + arm_type: Microsoft.Storage/queueAcl + arm_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_queue_services: + arm_type: Microsoft.Storage/storageAccounts/queueServices + arm_type_source: override + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_queues: + arm_type: Microsoft.Storage/storageAccounts/queueServices/queues + arm_type_source: override + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storage_tables: + arm_type: Microsoft.Storage/storageAccounts/tableServices/tables + arm_type_source: override + category: storage + aliases: [] + typed_collector: false + mandatory: false + azure_storagecache_caches: + arm_type: Microsoft.Storagecache/caches + arm_type_source: heuristic + category: storagecache + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_agents: + arm_type: Microsoft.Storagemover/agents + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_endpoints: + arm_type: Microsoft.Storagemover/endpoints + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_job_definitions: + arm_type: Microsoft.Storagemover/jobDefinitions + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_job_runs: + arm_type: Microsoft.Storagemover/jobRuns + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_operations: + arm_type: Microsoft.Storagemover/operations + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_projects: + arm_type: Microsoft.Storagemover/projects + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagemover_storagemovers: + arm_type: Microsoft.Storagemover/storagemovers + arm_type_source: heuristic + category: storagemover + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_service_private_endpoint_connections: + arm_type: Microsoft.StorageSync/servicePrivateEndpointConnections + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_service_private_link_resources: + arm_type: Microsoft.StorageSync/servicePrivateLinkResources + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_service_registered_servers: + arm_type: Microsoft.StorageSync/serviceRegisteredServers + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_service_sync_group_server_endpoints: + arm_type: Microsoft.StorageSync/serviceSyncGroupServerEndpoints + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_service_sync_groups: + arm_type: Microsoft.StorageSync/serviceSyncGroups + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_storagesync_services: + arm_type: Microsoft.StorageSync/services + arm_type_source: heuristic + category: storagesync + aliases: [] + typed_collector: false + mandatory: false + azure_streamanalytics_streaming_jobs: + arm_type: Microsoft.StreamAnalytics/streamingjobs + arm_type_source: override + category: streamanalytics + aliases: [] + typed_collector: false + mandatory: false + azure_subscription_subscription_locations: + arm_type: Microsoft.Subscription/subscriptionLocations + arm_type_source: heuristic + category: subscription + aliases: [] + typed_collector: false + mandatory: false + azure_subscription_subscriptions: + arm_type: Microsoft.Subscription/subscriptions + arm_type_source: heuristic + category: subscription + aliases: [] + typed_collector: true + mandatory: false + azure_subscription_tenants: + arm_type: Microsoft.Subscription/tenants + arm_type_source: heuristic + category: subscription + aliases: [] + typed_collector: false + mandatory: false + azure_support_services: + arm_type: Microsoft.Support/services + arm_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + azure_support_tickets: + arm_type: Microsoft.Support/tickets + arm_type_source: heuristic + category: support + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_ip_firewall_rules: + arm_type: Microsoft.Synapse/ipFirewallRules + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_keys: + arm_type: Microsoft.Synapse/keys + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_private_link_hubs: + arm_type: Microsoft.Synapse/privateLinkHubs + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_restorable_dropped_sql_pools: + arm_type: Microsoft.Synapse/restorableDroppedSqlPools + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_blob_auditing_policies: + arm_type: Microsoft.Synapse/sqlPoolBlobAuditingPolicies + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_current_sensitivity_labels: + arm_type: Microsoft.Synapse/sqlPoolCurrentSensitivityLabels + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_geo_backup_policies: + arm_type: Microsoft.Synapse/sqlPoolGeoBackupPolicies + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_operations: + arm_type: Microsoft.Synapse/sqlPoolOperations + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_recommended_sensitivity_labels: + arm_type: Microsoft.Synapse/sqlPoolRecommendedSensitivityLabels + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_replication_links: + arm_type: Microsoft.Synapse/sqlPoolReplicationLinks + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_restore_points: + arm_type: Microsoft.Synapse/sqlPoolRestorePoints + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_schema_table_columns: + arm_type: Microsoft.Synapse/sqlPoolSchemaTableColumns + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_schema_tables: + arm_type: Microsoft.Synapse/sqlPoolSchemaTables + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_schemas: + arm_type: Microsoft.Synapse/sqlPoolSchemas + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_security_alert_policies: + arm_type: Microsoft.Synapse/sqlPoolSecurityAlertPolicies + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_transparent_data_encryptions: + arm_type: Microsoft.Synapse/sqlPoolTransparentDataEncryptions + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pool_usages: + arm_type: Microsoft.Synapse/sqlPoolUsages + arm_type_source: heuristic + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_sql_pools: + arm_type: Microsoft.Synapse/workspaces/sqlPools + arm_type_source: override + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_synapse_workspaces: + arm_type: Microsoft.Synapse/workspaces + arm_type_source: override + category: synapse + aliases: [] + typed_collector: false + mandatory: false + azure_trafficmanager_profiles: + arm_type: Microsoft.Trafficmanager/profiles + arm_type_source: heuristic + category: trafficmanager + aliases: [] + typed_collector: false + mandatory: false + azure_windowsiot_services: + arm_type: Microsoft.WindowsIoT/services + arm_type_source: heuristic + category: windowsiot + aliases: [] + typed_collector: false + mandatory: false + azure_workloads_monitors: + arm_type: Microsoft.Workloads/monitors + arm_type_source: heuristic + category: workloads + aliases: [] + typed_collector: false + mandatory: false diff --git a/src/indexers/azureapi.py b/src/indexers/azureapi.py new file mode 100644 index 000000000..207dbf127 --- /dev/null +++ b/src/indexers/azureapi.py @@ -0,0 +1,881 @@ +""" +Native Azure SDK indexer. + +Replaces the CloudQuery-based Azure path with direct ``azure-mgmt-*`` calls +while keeping the registry output byte-compatible: each resource is normalized +into the same flat dict shape ``AzurePlatformHandler.parse_resource_data`` +already accepts, and writes flow through the :class:`ResourceWriter` seam so +the future local-DB / REST substrate is a drop-in swap. + +Coexists with the CloudQuery indexer behind the ``AZURE_INDEXER_BACKEND`` +setting: + +* ``"cloudquery"`` (default): this indexer is a no-op; CloudQuery handles + Azure as it does today. +* ``"azureapi"``: this indexer discovers Azure resources and the + CloudQuery indexer skips the Azure block. + +Component name: ``azureapi``. Stage: ``INDEXER``. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +from component import Context, Setting, SettingDependency +from enrichers.generation_rule_types import ( + PLATFORM_HANDLERS_PROPERTY_NAME, + LevelOfDetail, + PlatformHandler, +) +from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY +from exceptions import WorkspaceBuilderException +from resources import ResourceTypeSpec + +from .azure_common import ( + _azure_has_only_devops_config, + az_get_credentials_and_subscription_id, + get_auth_type, + has_excluded_tags, + has_included_tags, +) +from .azureapi_normalizers import normalize_azure_resource +from .azureapi_resource_types import ( + AZURE_RESOURCE_TYPE_SPECS, + AzureResourceTypeSpec, + find_spec, + find_spec_by_arm_type, +) +from .common import CLOUD_CONFIG_SETTING +from .resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_PATH_SETTING, + get_resource_writer, +) + +logger = logging.getLogger(__name__) + +AZURE_PLATFORM = "azure" + +DOCUMENTATION = "Index Azure resources using the Azure management SDK" + +# --------------------------------------------------------------------------- +# Settings +# --------------------------------------------------------------------------- + +AZURE_INDEXER_BACKEND_SETTING = Setting( + "AZURE_INDEXER_BACKEND", + "azureIndexerBackend", + Setting.Type.STRING, + "Selects the backend used to discover Azure resources. " + "'cloudquery' (default) uses the legacy CloudQuery-based path; " + "'azureapi' uses the native azure-mgmt-* SDK indexer.", + "cloudquery", +) + +SETTINGS = ( + SettingDependency(CLOUD_CONFIG_SETTING, False), + SettingDependency(AZURE_INDEXER_BACKEND_SETTING, False), + # Expose the ResourceWriter backend selection on the azureapi component + # (currently the only indexer that funnels through the writer seam) so + # the settings appear in the active schema and can be set via + # workspaceInfo.yaml. + SettingDependency(RESOURCE_STORE_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_PATH_SETTING, False), +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _build_subscription_rg_lod_map(platform_cfg: dict[str, Any]) -> None: + """Populate ``platform_cfg["subscriptionResourceGroupLevelOfDetails"]``. + + This is a lift of the LOD-map construction in ``cloudquery.init_cloudquery_config`` + so ``AzurePlatformHandler.parse_resource_data`` (which reads this nested map + to compute per-resource LOD) keeps producing the same values regardless of + which indexer ran. + """ + rg_lod_map: dict[str, dict[str, str]] = {} + global_default = ( + platform_cfg.get("defaultLOD") or platform_cfg.get("defaultLevelOfDetail") + ) + + for item in platform_cfg.get("subscriptions", []): + if not isinstance(item, dict): + continue + sid = str(item.get("subscriptionId", "")).strip() + if not sid: + continue + sub_default = ( + item.get("defaultLOD") + or item.get("defaultLevelOfDetail") + or global_default + ) + lod_dict: dict[str, str] = {} + if sub_default: + lod_dict["*"] = sub_default + lod_dict.update(item.get("resourceGroupLevelOfDetails", {})) + if lod_dict: + rg_lod_map[sid] = lod_dict + + if not platform_cfg.get("subscriptions"): + sid = platform_cfg.get("subscriptionId") + if sid: + lod_dict = platform_cfg.get("resourceGroupLevelOfDetails", {}) + if not lod_dict and global_default: + lod_dict = {"*": global_default} + if lod_dict: + rg_lod_map[str(sid)] = lod_dict + + platform_cfg["subscriptionResourceGroupLevelOfDetails"] = rg_lod_map + + +# --------------------------------------------------------------------------- +# Selective indexing +# --------------------------------------------------------------------------- +# +# The cloudquery-era pipeline indexed every Azure resource the SP could see +# and let render-time LOD checks decide what to emit. The native indexer is +# stricter: resources whose effective LOD is NONE are dropped before the +# resource writer ever sees them. This keeps the resource store focused on +# what generation rules actually need, makes the SQLite store dramatically +# smaller, and turns the workspaceInfo Azure block into the single source of +# truth for "what should be discovered". +# +# The per-resource decision mirrors the lookup chain that +# ``AzurePlatformHandler.parse_resource_data`` uses (sub-specific RG override +# -> per-subscription default -> legacy global RG override -> legacy global +# wildcard -> workspace DEFAULT_LOD), keeping selective-indexing decisions +# byte-compatible with what render-time enforcement would have done. + +_RG_MARKER = "resourcegroups/" +_SUB_MARKER = "/subscriptions/" + + +def _extract_rg_name_from_arm_id(arm_id: Optional[str]) -> Optional[str]: + """Extract the resource-group name from an ARM ID. Case-insensitive + marker match, but value casing is preserved so it round-trips against + the resource-group registry.""" + if not arm_id: + return None + lowered = arm_id.lower() + start = lowered.find(_RG_MARKER) + if start < 0: + return None + start += len(_RG_MARKER) + end = arm_id.find("/", start) + if end < 0: + end = len(arm_id) + rg = arm_id[start:end] + return rg or None + + +def _extract_subscription_id_from_arm_id(arm_id: Optional[str]) -> Optional[str]: + """Extract the subscription ID from an ARM ID, e.g. + ``/subscriptions//resourceGroups/...``.""" + if not arm_id: + return None + lowered = arm_id.lower() + start = lowered.find(_SUB_MARKER) + if start < 0: + return None + start += len(_SUB_MARKER) + end = arm_id.find("/", start) + if end < 0: + end = len(arm_id) + sub = arm_id[start:end] + return sub or None + + +def _compute_effective_lod( + platform_cfg: dict[str, Any], + subscription_id: Optional[str], + rg_name: Optional[str], + default_lod: LevelOfDetail, +) -> LevelOfDetail: + """Resolve the effective LOD for a resource living in ``rg_name`` under + ``subscription_id``. Mirrors the resolution chain in + ``AzurePlatformHandler.parse_resource_data``.""" + sub_map: dict[str, Any] = ( + platform_cfg.get("subscriptionResourceGroupLevelOfDetails", {}) or {} + ).get(subscription_id or "", {}) or {} + global_rg_map: dict[str, Any] = platform_cfg.get("resourceGroupLevelOfDetails", {}) or {} + + candidate = ( + (sub_map.get(rg_name) if rg_name else None) + or sub_map.get("*") + or (global_rg_map.get(rg_name) if rg_name else None) + or global_rg_map.get("*") + ) + if candidate is None: + return default_lod + try: + return LevelOfDetail.construct_from_config(candidate) + except Exception: + return default_lod + + +def _resource_is_in_scope( + platform_cfg: dict[str, Any], + resource_data: dict[str, Any], + resource_type_name: str, + subscription_id: Optional[str], + default_lod: LevelOfDetail, +) -> tuple[bool, Optional[LevelOfDetail]]: + """Return ``(in_scope, effective_lod)`` for a resource. + + ``in_scope`` is False iff the effective LOD is ``NONE`` - the indexer + drops these before they reach the writer. The effective LOD is also + returned so callers can log a reason or stash it for later. + """ + if resource_type_name == "azure_subscription_subscriptions": + # Subscription resources are indexed unconditionally so downstream + # code can attach metadata / qualified names. Their LOD is the + # workspace default. + return True, default_lod + + if resource_type_name == "resource_group": + rg_name = resource_data.get("name") + else: + rg_name = _extract_rg_name_from_arm_id(resource_data.get("id")) + + sub_id = subscription_id or _extract_subscription_id_from_arm_id(resource_data.get("id")) + effective = _compute_effective_lod(platform_cfg, sub_id, rg_name, default_lod) + return effective is not LevelOfDetail.NONE, effective + + +def _resolve_default_lod(context: Context, platform_cfg: dict[str, Any]) -> LevelOfDetail: + """Resolve the workspace default LOD. Order of precedence: + platform_cfg.defaultLOD -> context DEFAULT_LOD setting -> BASIC.""" + raw = ( + platform_cfg.get("defaultLOD") + or platform_cfg.get("defaultLevelOfDetail") + or context.get_setting("DEFAULT_LOD") + ) + if raw is None: + return LevelOfDetail.BASIC + try: + return LevelOfDetail.construct_from_config(raw) + except Exception: + return LevelOfDetail.BASIC + + +def _safe_lod(value: Any) -> Optional[LevelOfDetail]: + if value is None: + return None + try: + return LevelOfDetail.construct_from_config(value) + except Exception: + return None + + +def _rgs_in_scope_from_config( + platform_cfg: dict[str, Any], + subscription_id: str, + default_lod: LevelOfDetail, +) -> Optional[list[str]]: + """Return the explicit set of in-scope RG names for ``subscription_id``, + or ``None`` if the configuration permits unbounded discovery for that + subscription. + + Selective discovery mode (returns a finite list) is triggered when: + + * the per-subscription wildcard (``sub_map["*"]``) resolves to NONE + *or* is unset, **and** + * the legacy global wildcard (``resourceGroupLevelOfDetails["*"]``) + resolves to NONE *or* is unset, **and** + * the workspace ``defaultLOD`` resolves to NONE. + + When all three escape hatches yield NONE, the only resources in scope + are those with explicit non-NONE per-RG overrides; we return the list + of those RG names (call ``list_by_resource_group`` for each). + + Returning ``None`` means "use the subscription-wide list endpoint and + rely on post-filtering" - the workspace's defaultLOD or wildcard says + discovery should not be artificially scoped. + """ + sub_map: dict[str, Any] = ( + platform_cfg.get("subscriptionResourceGroupLevelOfDetails", {}) or {} + ).get(subscription_id, {}) or {} + global_rg_map: dict[str, Any] = platform_cfg.get("resourceGroupLevelOfDetails", {}) or {} + + sub_wildcard = _safe_lod(sub_map.get("*")) + if sub_wildcard is not None and sub_wildcard is not LevelOfDetail.NONE: + return None + global_wildcard = _safe_lod(global_rg_map.get("*")) + if global_wildcard is not None and global_wildcard is not LevelOfDetail.NONE: + return None + if default_lod is not LevelOfDetail.NONE: + return None + + in_scope: list[str] = [] + seen: set[str] = set() + for name, raw_lod in sub_map.items(): + if name == "*": + continue + lod = _safe_lod(raw_lod) + if lod is None or lod is LevelOfDetail.NONE: + continue + if name not in seen: + seen.add(name) + in_scope.append(name) + for name, raw_lod in global_rg_map.items(): + if name == "*": + continue + lod = _safe_lod(raw_lod) + if lod is None or lod is LevelOfDetail.NONE: + continue + if name not in seen: + seen.add(name) + in_scope.append(name) + return in_scope + + +def _accessed_azure_type_names(context: Context) -> set[str]: + """Return the set of Azure resource-type names referenced by loaded + generation rules. Both the registry name and the CloudQuery table name are + accepted as valid spec values; the result mixes them. + """ + all_specs: Optional[dict[str, dict[ResourceTypeSpec, Any]]] = context.get_property( + RESOURCE_TYPE_SPECS_PROPERTY + ) + if not all_specs: + return set() + azure_specs = all_specs.get(AZURE_PLATFORM, {}) + return {spec.resource_type_name for spec in azure_specs.keys()} + + +def _resolve_platform_handler(context: Context) -> PlatformHandler: + handlers: Optional[dict[str, PlatformHandler]] = context.get_property( + PLATFORM_HANDLERS_PROPERTY_NAME + ) + if handlers and AZURE_PLATFORM in handlers: + return handlers[AZURE_PLATFORM] + # Bootstrap a default handler if generation_rules.load wasn't run (e.g. test setups). + from enrichers.azure import AzurePlatformHandler + + return AzurePlatformHandler() + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def index(context: Context) -> None: + backend = context.get_setting(AZURE_INDEXER_BACKEND_SETTING) + if backend != "azureapi": + # Bumped from debug to info so a default-verbosity log makes it + # obvious which Azure backend will own discovery on this run. + logger.info( + f"Azure indexer backend: '{backend}' (azureIndexerBackend in " + f"workspaceInfo.yaml). Native azureapi indexer is a no-op; the " + f"CloudQuery indexer will handle Azure." + ) + return + + cloud_config = context.get_setting(CLOUD_CONFIG_SETTING) or {} + platform_cfg = cloud_config.get(AZURE_PLATFORM) + if not platform_cfg: + logger.info( + "Azure indexer backend: 'azureapi' selected, but no 'azure' section " + "in cloudConfig; nothing to discover." + ) + return + + if _azure_has_only_devops_config(platform_cfg): + logger.info( + "Azure indexer backend: 'azureapi' selected, but cloudConfig.azure " + "contains only DevOps settings (no subscriptionId / cloud " + "credentials). Skipping resource discovery; Azure DevOps indexer " + "will handle ADO resources." + ) + return + + logger.info( + "Azure indexer backend: 'azureapi' (native azure-mgmt-* SDK). " + "Starting Azure resource discovery." + ) + + az = az_get_credentials_and_subscription_id(platform_cfg) + credential = az["credential"] + subscription_ids: list[str] = az["subscription_ids"] + + # Mirror what cloudquery.py does so AzurePlatformHandler.parse_resource_data, + # get_subscription_name, and any other downstream code that reads global + # Azure credentials sees the same values we resolved here. + try: + from enrichers.azure import set_azure_credentials + + set_azure_credentials( + tenant_id=az.get("AZURE_TENANT_ID"), + client_id=az.get("AZURE_CLIENT_ID"), + client_secret=az.get("AZURE_CLIENT_SECRET"), + credential=credential, + ) + except Exception as e: + logger.warning(f"Could not update enrichers.azure credentials: {e}") + + _build_subscription_rg_lod_map(platform_cfg) + + accessed_names = _accessed_azure_type_names(context) + logger.info( + f"Azure resource types referenced by generation rules: " + f"{sorted(accessed_names) if accessed_names else '(none)'}" + ) + + # Resolve "what does the workspace want indexed" up-front. We keep + # two collections: + # + # * ``typed_specs_to_collect`` - typed (rich-payload) specs whose + # CQ table name OR any registry alias is referenced by a gen + # rule, OR which are mandatory (today: just resource_groups). + # * ``generic_specs_by_arm_type`` - lower-cased ARM type -> spec + # for non-typed registry entries referenced by a gen rule. + # These are fed by the generic-resources pass below. + # + # Resolve every accessed name via ``find_spec`` so we honor registry + # aliases (e.g. gen rules referencing ``azure_keyvault_keyvault`` + # need to map to the typed ``azure_keyvault_vaults`` collector). + typed_specs_to_collect: list[AzureResourceTypeSpec] = [] + seen_typed_specs: set[str] = set() + generic_specs_by_arm_type: dict[str, AzureResourceTypeSpec] = {} + + # Always pull in mandatory typed specs (resource_group bootstrap), + # whether or not any gen rule names them. + for spec in AZURE_RESOURCE_TYPE_SPECS: + if spec.mandatory and getattr(spec, "typed", True): + if spec.resource_type_name not in seen_typed_specs: + typed_specs_to_collect.append(spec) + seen_typed_specs.add(spec.resource_type_name) + + for accessed in accessed_names: + spec = find_spec(accessed) + if spec is None: + # ``find_spec`` returns None only for names that are not in + # the Azure resource-type registry at all. The gen rule has + # a typo or the registry needs a new entry. + warning = ( + f'Azure SDK indexer: gen-rule references unknown Azure resource ' + f'type "{accessed}". Verify the name or add it to ' + f'scripts/azure/azure_resource_type_overrides.yaml and rerun the ' + f'sync script.' + ) + logger.warning(warning) + context.add_warning(warning) + continue + if getattr(spec, "typed", True): + if spec.resource_type_name not in seen_typed_specs: + typed_specs_to_collect.append(spec) + seen_typed_specs.add(spec.resource_type_name) + elif spec.arm_type: + generic_specs_by_arm_type[spec.arm_type.lower()] = spec + + platform_handler = _resolve_platform_handler(context) + + include_tags = platform_cfg.get("includeTags", {}) + exclude_tags = platform_cfg.get("excludeTags", {}) + + auth_type, auth_secret = get_auth_type(AZURE_PLATFORM, platform_cfg) + + writer = get_resource_writer(context) + + default_lod = _resolve_default_lod(context, platform_cfg) + + # Resolve discovery scope per subscription. Each entry is either a + # finite list of in-scope RG names (selective discovery; we'll call + # ``list_by_resource_group`` for each) or ``None`` (subscription-wide + # discovery; we'll call ``list_all`` and post-filter LOD). + discovery_scope: dict[str, Optional[list[str]]] = { + sub_id: _rgs_in_scope_from_config(platform_cfg, sub_id, default_lod) + for sub_id in subscription_ids + } + for sub_id, scope in discovery_scope.items(): + if scope is None: + logger.info( + f"Azure subscription {sub_id}: subscription-wide discovery " + f"(workspace default LOD={default_lod}, no NONE-only wildcard)." + ) + else: + logger.info( + f"Azure subscription {sub_id}: selective discovery, " + f"in-scope RGs={sorted(scope) if scope else '(none)'}." + ) + + stats = { + "discovered": 0, + "added": 0, + "added_typed": 0, + "added_generic": 0, + "generic_unmatched_arm_type": 0, + "generic_already_typed": 0, + "skipped_tag_filter": 0, + "skipped_lod_filter": 0, + "skipped_parse_error": 0, + "skipped_collector_error": 0, + "skipped_rg_not_found": 0, + } + # Track ARM IDs the typed pass already emitted so the generic pass + # doesn't double-write the same resource with a sparser payload. + typed_arm_ids: set[str] = set() + + def _process_models(spec, subscription_id, models, *, lod_filter: bool, source: str = "typed") -> None: + """Normalize -> tag filter -> (optional) LOD filter -> parse -> write. + + ``source`` is either ``"typed"`` (the rich-payload pass) or + ``"generic"`` (the ARM-resources catch-all pass). It controls + per-source bookkeeping (``typed_arm_ids`` write set, generic-pass + skip-if-already-typed check, per-source stats). + """ + for model in models: + stats["discovered"] += 1 + try: + resource_data = normalize_azure_resource( + model, + subscription_id=subscription_id, + resource_type_name=spec.resource_type_name, + ) + except Exception as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"Failed to normalize Azure {spec.resource_type_name} " + f"model in subscription {subscription_id}: {e}" + ) + continue + + if exclude_tags and has_excluded_tags(resource_data, exclude_tags): + stats["skipped_tag_filter"] += 1 + continue + if include_tags and not has_included_tags(resource_data, include_tags): + stats["skipped_tag_filter"] += 1 + continue + + if lod_filter: + in_scope, effective_lod = _resource_is_in_scope( + platform_cfg, + resource_data, + spec.resource_type_name, + subscription_id, + default_lod, + ) + if not in_scope: + stats["skipped_lod_filter"] += 1 + logger.debug( + f"Skipping {spec.resource_type_name} " + f"'{resource_data.get('name')}' in subscription " + f"{subscription_id}: effective LOD is {effective_lod}" + ) + continue + + try: + resource_name, qualified_name, resource_attributes = ( + platform_handler.parse_resource_data( + resource_data, + spec.resource_type_name, + platform_cfg, + context, + ) + ) + except WorkspaceBuilderException as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data rejected {spec.resource_type_name} " + f"resource in subscription {subscription_id}: {e}" + ) + continue + except (KeyError, ValueError, TypeError, AttributeError) as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data raised {type(e).__name__} for " + f"{spec.resource_type_name} resource in subscription " + f"{subscription_id}: {e}" + ) + continue + + arm_id = resource_data.get("id") + if source == "generic" and arm_id and arm_id in typed_arm_ids: + # The typed pass already wrote a richer copy of this + # resource; don't overwrite with the basic generic payload. + stats["generic_already_typed"] += 1 + continue + + resource_attributes["resource"] = resource_data + resource_attributes["auth_type"] = auth_type + resource_attributes["auth_secret"] = auth_secret + + writer.add_resource( + AZURE_PLATFORM, + spec.resource_type_name, + resource_name, + qualified_name, + resource_attributes, + ) + stats["added"] += 1 + stats["added_typed" if source == "typed" else "added_generic"] += 1 + if source == "typed" and arm_id: + typed_arm_ids.add(arm_id) + + def _list_subscription_wide(spec, subscription_id): + try: + return list(spec.collector_all(credential, subscription_id)) + except Exception as e: + stats["skipped_collector_error"] += 1 + logger.error( + f"Failed to list {spec.resource_type_name} in subscription " + f"{subscription_id}: {e}" + ) + context.add_warning( + f"Failed to list Azure {spec.resource_type_name} in " + f"subscription {subscription_id}: {e}" + ) + return None + + def _list_in_rg(spec, subscription_id, rg_name): + try: + return list(spec.collector_in_rg(credential, subscription_id, rg_name)) + except Exception as e: + # 404 / RG-not-found is the most likely failure here: the + # workspace declared an RG in scope that doesn't (yet) exist. + # Demote to a warning + dedicated counter rather than aborting. + err_text = str(e) + if "ResourceGroupNotFound" in err_text or "ResourceNotFound" in err_text: + stats["skipped_rg_not_found"] += 1 + logger.warning( + f"Resource group '{rg_name}' not found in subscription " + f"{subscription_id} while listing {spec.resource_type_name}; " + f"skipping." + ) + return None + stats["skipped_collector_error"] += 1 + logger.error( + f"Failed to list {spec.resource_type_name} in RG '{rg_name}' " + f"under subscription {subscription_id}: {e}" + ) + context.add_warning( + f"Failed to list Azure {spec.resource_type_name} in RG " + f"'{rg_name}' under subscription {subscription_id}: {e}" + ) + return None + + # Phase 1: enumerate resource groups subscription-wide and post-filter. + # RGs themselves only have a subscription-wide list endpoint, so this + # is the one mandatory ``list_all`` call. Out-of-scope RGs are dropped + # via ``_resource_is_in_scope`` and never reach the writer. + rg_specs = [s for s in typed_specs_to_collect if s.resource_type_name == "resource_group"] + non_rg_specs = [s for s in typed_specs_to_collect if s.resource_type_name != "resource_group"] + + for spec in rg_specs: + for subscription_id in subscription_ids: + models = _list_subscription_wide(spec, subscription_id) + if models is None: + continue + logger.info( + f"Collected {len(models)} {spec.resource_type_name} " + f"from subscription {subscription_id}" + ) + _process_models(spec, subscription_id, models, lod_filter=True) + + # Phase 2: non-RG specs. Per subscription, decide between selective + # (per-RG) and subscription-wide enumeration. + for spec in non_rg_specs: + for subscription_id in subscription_ids: + scope = discovery_scope.get(subscription_id) + if scope is not None and spec.supports_in_rg: + logger.info( + f"Selective discovery: listing {spec.resource_type_name} " + f"per-RG in subscription {subscription_id} " + f"(in-scope RGs={sorted(scope) if scope else '(none)'})" + ) + for rg_name in scope: + models = _list_in_rg(spec, subscription_id, rg_name) + if models is None: + continue + logger.info( + f"Collected {len(models)} {spec.resource_type_name} " + f"from {subscription_id}/{rg_name}" + ) + # Per-RG enumeration already restricts to in-scope RGs, + # so the LOD post-filter is a no-op; keeping it on as a + # belt-and-suspenders check is cheap and protects against + # SDK quirks (e.g. a child resource pointing to a + # different RG via its ARM id). + _process_models(spec, subscription_id, models, lod_filter=True) + continue + + # Subscription-wide path: workspace declared unbounded discovery + # (defaultLOD non-NONE, or wildcard non-NONE), or the spec lacks + # a per-RG collector. Either way, list_all + LOD post-filter. + if scope is not None and not spec.supports_in_rg: + logger.warning( + f"Spec {spec.resource_type_name} has no per-RG collector; " + f"falling back to subscription-wide listing in " + f"{subscription_id}. Out-of-scope rows will be dropped " + f"via skipped_lod_filter." + ) + models = _list_subscription_wide(spec, subscription_id) + if models is None: + continue + logger.info( + f"Collected {len(models)} {spec.resource_type_name} " + f"from subscription {subscription_id}" + ) + _process_models(spec, subscription_id, models, lod_filter=True) + + # Phase 3: generic ARM-resources pass. + # + # One ``ResourceManagementClient.resources.list[_by_resource_group]`` + # call per subscription (or per in-scope RG) returns *every* + # top-level ARM resource the credential can see. We route each + # ``GenericResource`` by its ``type`` field through the registry, + # and emit it under the registry-mapped ``resource_type_name`` + # **only if** that type was referenced by an accessed gen rule and + # **only if** the typed pass didn't already write the same ARM ID + # (richer-typed-payload wins). + # + # This is what gives us coverage parity with the CloudQuery indexer: + # any ARM resource type a gen rule cares about is discoverable, even + # if we don't ship a hand-written collector for it. Types in + # ``generic_specs_by_arm_type`` get the basic envelope (``id``, + # ``name``, ``type``, ``location``, ``tags``, ``sku``, ``kind``, + # ``identity``, ``managed_by``); ``properties`` is empty (that's an + # ARM API limitation, not a workspace-builder one). + if generic_specs_by_arm_type: + from .azureapi_resource_types import ( + _collect_generic_resources_all, + _collect_generic_resources_in_rg, + ) + + # ARM types the typed pass already owns. We never emit a generic + # copy for these; the typed pass either already wrote them or was + # gated off (mandatory/accessed) by intent. + typed_arm_types = { + (s.arm_type or "").lower() + for s in typed_specs_to_collect + if s.arm_type + } + + def _route_generic(model) -> Optional[AzureResourceTypeSpec]: + """Return the spec to emit a ``GenericResource`` under, or + None if it should be dropped.""" + arm_type = getattr(model, "type", None) or ( + model.get("type") if isinstance(model, dict) else None + ) + if not arm_type: + return None + arm_lower = arm_type.lower() + if arm_lower in typed_arm_types: + # Owned by the typed pass; skip silently. + return None + spec = generic_specs_by_arm_type.get(arm_lower) + if spec is None: + # ARM type is real but no gen rule referenced it. Drop + # (the indexer is gen-rule-driven; we don't balloon the + # resource store with unused rows). + stats["generic_unmatched_arm_type"] += 1 + return None + return spec + + def _generic_pass_for_rg(subscription_id: str, rg_name: str) -> None: + try: + resources_iter = list( + _collect_generic_resources_in_rg( + credential, subscription_id, rg_name + ) + ) + except Exception as e: + err_text = str(e) + if ( + "ResourceGroupNotFound" in err_text + or "ResourceNotFound" in err_text + ): + stats["skipped_rg_not_found"] += 1 + logger.warning( + f"Resource group '{rg_name}' not found in subscription " + f"{subscription_id} during generic discovery; skipping." + ) + return + stats["skipped_collector_error"] += 1 + logger.error( + f"Generic ARM resources list failed for " + f"{subscription_id}/{rg_name}: {e}" + ) + context.add_warning( + f"Failed to list generic Azure resources in RG " + f"'{rg_name}' under subscription {subscription_id}: {e}" + ) + return + _emit_generic_models(subscription_id, resources_iter, scope_label=f"{subscription_id}/{rg_name}") + + def _generic_pass_subscription_wide(subscription_id: str) -> None: + try: + resources_iter = list( + _collect_generic_resources_all(credential, subscription_id) + ) + except Exception as e: + stats["skipped_collector_error"] += 1 + logger.error( + f"Generic ARM resources list failed for " + f"subscription {subscription_id}: {e}" + ) + context.add_warning( + f"Failed to list generic Azure resources in " + f"subscription {subscription_id}: {e}" + ) + return + _emit_generic_models(subscription_id, resources_iter, scope_label=f"subscription {subscription_id}") + + def _emit_generic_models( + subscription_id: str, models: list, *, scope_label: str + ) -> None: + routed: dict[str, list] = {} + for model in models: + spec = _route_generic(model) + if spec is None: + continue + routed.setdefault(spec.cloudquery_table_name, []).append(model) + for table_name, batch in routed.items(): + spec = find_spec(table_name) + if spec is None: + continue + logger.info( + f"Collected {len(batch)} {spec.resource_type_name} " + f"(generic) from {scope_label}" + ) + _process_models( + spec, subscription_id, batch, + lod_filter=True, source="generic", + ) + + for subscription_id in subscription_ids: + scope = discovery_scope.get(subscription_id) + if scope is not None: + logger.info( + f"Generic discovery (selective): listing ARM resources " + f"per-RG in subscription {subscription_id} " + f"(in-scope RGs={sorted(scope) if scope else '(none)'})" + ) + for rg_name in scope: + _generic_pass_for_rg(subscription_id, rg_name) + else: + logger.info( + f"Generic discovery (subscription-wide): listing ARM " + f"resources in subscription {subscription_id}" + ) + _generic_pass_subscription_wide(subscription_id) + + writer.finalize() + + logger.info( + f"Azure SDK indexing complete: " + f"discovered={stats['discovered']}, added={stats['added']} " + f"(typed={stats['added_typed']}, generic={stats['added_generic']}), " + f"generic_already_typed={stats['generic_already_typed']}, " + f"generic_unmatched_arm_type={stats['generic_unmatched_arm_type']}, " + f"skipped_tag_filter={stats['skipped_tag_filter']}, " + f"skipped_lod_filter={stats['skipped_lod_filter']}, " + f"skipped_rg_not_found={stats['skipped_rg_not_found']}, " + f"skipped_parse_error={stats['skipped_parse_error']}, " + f"skipped_collector_error={stats['skipped_collector_error']}" + ) diff --git a/src/indexers/azureapi_normalizers.py b/src/indexers/azureapi_normalizers.py new file mode 100644 index 000000000..5ea40b8ee --- /dev/null +++ b/src/indexers/azureapi_normalizers.py @@ -0,0 +1,162 @@ +""" +Normalize Azure SDK model objects into the flat dict shape that +``AzurePlatformHandler.parse_resource_data`` already understands. + +Goal: produce a ``resource_data`` dict that is *behaviourally* equivalent to a +row from the legacy CloudQuery SQLite intermediate. ``parse_resource_data`` +relies on a small handful of fields: + +* ``id`` - ARM path; used to extract subscription_id and the parent RG. +* ``name`` - top-level resource name. +* ``type`` - ARM type string. +* ``location`` - region. +* ``tags`` - dict, defaults to ``{}``. +* ``subscription_id`` - column-level; the parser prefers the value parsed + out of ``id`` but warns if this column disagrees, so + we always set it explicitly. + +Everything else (``properties``, ``sku``, ``identity``, etc.) is passed +through to ``resource_data["resource"]`` for generation-rule path matching. +We intentionally **do not** mirror the CloudQuery ``_cq_*`` metadata columns; +they exist only because CloudQuery wrote those rows. Tests diff +``resource-dump.yaml`` ignoring ``_cq_*`` keys. +""" + +from __future__ import annotations + +import datetime +import enum +import logging +import re +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# Top-level keys the CloudQuery sync emits in snake_case. The Azure SDK +# returns them in camelCase via ``as_dict()``; we rename so existing rules +# that path-match on these keys still work. +_TOP_LEVEL_SNAKE_KEYS = { + "extendedLocation": "extended_location", + "managedBy": "managed_by", + "managedByExtended": "managed_by_extended", + "subscriptionId": "subscription_id", +} + +_CAMEL_RE = re.compile(r"(? str: + return _CAMEL_RE.sub("_", name).lower() + + +def _sanitize(value: Any) -> Any: + """Recursively convert SDK-returned values into YAML-friendly primitives. + + The Azure SDK occasionally returns ``datetime`` instances and Enum values + inside ``properties``; both serialize awkwardly via PyYAML's safe dumper + that the registry uses. CloudQuery already serializes these as strings, so + matching that behaviour keeps the output stable. + """ + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, datetime.date): + return value.isoformat() + if isinstance(value, enum.Enum): + return value.value if isinstance(value.value, (str, int, float, bool)) else str(value) + if isinstance(value, dict): + return {k: _sanitize(v) for k, v in value.items()} + if isinstance(value, list): + return [_sanitize(v) for v in value] + if isinstance(value, tuple): + return [_sanitize(v) for v in value] + return value + + +def _model_to_dict(model: Any) -> dict[str, Any]: + """Convert an Azure SDK model into a plain dict. + + Most Azure SDK models expose ``.as_dict(keep_readonly=True)``. We fall back + to ``__dict__`` for the rare objects (or test doubles) that don't. + """ + if isinstance(model, dict): + return dict(model) + + as_dict_fn = getattr(model, "as_dict", None) + if callable(as_dict_fn): + try: + return as_dict_fn(keep_readonly=True) + except TypeError: + return as_dict_fn() + + if hasattr(model, "__dict__"): + return {k: v for k, v in vars(model).items() if not k.startswith("_")} + + raise TypeError(f"Cannot convert Azure SDK model of type {type(model).__name__} to dict") + + +def _rename_top_level_keys(data: dict[str, Any]) -> dict[str, Any]: + """Rename the small set of well-known Azure top-level fields from camelCase + to snake_case so generation rules that already path-match these keys (the + CloudQuery shape) keep working unchanged. + + Anything we don't explicitly know about is left in its native key form. + """ + out: dict[str, Any] = {} + for key, value in data.items(): + new_key = _TOP_LEVEL_SNAKE_KEYS.get(key, key) + out[new_key] = value + return out + + +def normalize_azure_resource( + model: Any, + *, + subscription_id: str, + resource_type_name: str, +) -> dict[str, Any]: + """Convert an Azure SDK model into the ``resource_data`` dict shape that + :class:`enrichers.azure.AzurePlatformHandler.parse_resource_data` accepts. + + Parameters + ---------- + model: + An Azure SDK model object (e.g. ``ResourceGroup``, + ``VirtualMachine``, ``StorageAccount``). + subscription_id: + The subscription this resource was discovered in. We always inject + this so the parser doesn't have to guess. + resource_type_name: + The runwhen-local registry resource type name (e.g. ``"resource_group"``, + ``"azure_storage_accounts"``). Used for diagnostic logging only. + """ + raw = _model_to_dict(model) + raw = _rename_top_level_keys(raw) + raw = _sanitize(raw) + + # ``id``/``name``/``type``/``location`` come straight off the SDK model. + # Most Azure SDK models include them, but a small number of older models + # don't expose ``type`` even though the REST payload does. We don't + # synthesize anything we can't read from the SDK - that's a normalizer + # bug, not something to paper over. + + # ``tags`` is the one field generation rules rely on existing as a dict + # at the top level of the raw payload. SDKs return ``None`` when no tags + # are set; normalize to an empty dict so the parser's + # ``resource_data.get("tags", {})`` and gen-rule tag matchers behave the + # same way they did under CloudQuery. + if raw.get("tags") is None: + raw["tags"] = {} + + # Always stamp the subscription. The parser prefers the value parsed from + # ``id`` and warns on mismatch, but we want to provide a value here for + # any code path that reads the column directly. + raw["subscription_id"] = str(subscription_id) + + if not raw.get("id"): + logger.warning( + f"Azure SDK model for resource_type_name={resource_type_name!r} " + f"in subscription {subscription_id} has no 'id' field; " + f"parse_resource_data will likely fail to link this resource." + ) + + return raw diff --git a/src/indexers/azureapi_resource_types.py b/src/indexers/azureapi_resource_types.py new file mode 100644 index 000000000..e6a62e3c3 --- /dev/null +++ b/src/indexers/azureapi_resource_types.py @@ -0,0 +1,865 @@ +""" +Azure resource-type specs for the native Azure SDK indexer. + +Each :class:`AzureResourceTypeSpec` describes one collectable Azure resource +type and ships *two* collector callables: + +* ``collector_all(credential, subscription_id)`` - subscription-wide listing, + used when the workspace permits unbounded discovery (any non-NONE wildcard + / global default). + +* ``collector_in_rg(credential, subscription_id, rg_name)`` - resource-group + scoped listing, used when the workspace declares a finite scope (e.g. + ``defaultLOD: none`` + an explicit per-RG whitelist). Setting this to + ``None`` opts a type out of selective discovery; it'll fall back to + ``collector_all`` even in scoped mode. + +Specs are derived from the registry in +``azure_resource_type_registry.yaml``; see +:mod:`azure_resource_type_registry`. The registry holds the metadata for +all 600+ CloudQuery Azure tables; this module only owns the small set of +hand-written ``azure-mgmt-*`` collectors. To enable a new type: + +* Make sure the table is in the registry (regenerate via + ``scripts/azure/sync_azure_resource_type_registry.py`` if needed). +* Implement ``_collect__all`` (and optionally ``_collect__in_rg``) + below and register them in :data:`_TYPED_COLLECTORS` keyed by canonical + CloudQuery table name. + +The public surface (``AZURE_RESOURCE_TYPE_SPECS``, ``AzureResourceTypeSpec``, +``find_spec``) is preserved so existing callers (notably ``indexers.azureapi``) +keep working. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Optional + +from .azure_resource_type_registry import ( + AzureResourceTypeEntry, + load_registry, +) + +AzureCollectorAll = Callable[[Any, str], Iterable[Any]] +AzureCollectorInRg = Callable[[Any, str, str], Iterable[Any]] + + +def _rg_from_arm_id(arm_id: str) -> Optional[str]: + """Pull the resource-group name out of an Azure ARM resource ID. + + ARM IDs are of the form + ``/subscriptions//resourceGroups//providers/...`` with case + variations (some emit lower-case ``resourcegroups``). Used by + child-resource collectors that walk parent → child (e.g. Postgres + databases under their parent server). + """ + if not arm_id: + return None + parts = arm_id.split("/") + for needle in ("resourceGroups", "resourcegroups"): + if needle in parts: + idx = parts.index(needle) + if idx + 1 < len(parts): + return parts[idx + 1] + return None + + +@dataclass(frozen=True) +class AzureResourceTypeSpec: + resource_type_name: str + cloudquery_table_name: str + mandatory: bool + collector_all: AzureCollectorAll + collector_in_rg: Optional[AzureCollectorInRg] = None + # ``True`` for hand-written azure-mgmt-* collectors that produce a rich + # payload (full ``properties``, sku, identity, etc.). ``False`` means the + # spec is materialized from the generic ARM-resources catch-all, which + # gives us the basic envelope (id / name / type / tags / location / sku / + # kind / identity) but **no** ``properties``. Used by the indexer's + # dispatch loop to decide which pass owns the row. + typed: bool = True + # The ARM type string this spec corresponds to. Set for both typed and + # generic specs; the generic-pass dispatcher uses this to route each + # ``GenericResource.type`` back to a spec. + arm_type: Optional[str] = None + + @property + def supports_in_rg(self) -> bool: + return self.collector_in_rg is not None + + # Back-compat alias for callers that hadn't migrated to the two-method + # split yet. ``collector(...)`` keeps the original (credential, sub_id) + # signature. + @property + def collector(self) -> AzureCollectorAll: # pragma: no cover - trivial + return self.collector_all + + +# --------------------------------------------------------------------------- +# Subscription-wide collectors (used in unbounded-discovery mode) +# --------------------------------------------------------------------------- + +def _collect_resource_groups_all(credential, subscription_id): + from azure.mgmt.resource import ResourceManagementClient + + client = ResourceManagementClient(credential, subscription_id) + return client.resource_groups.list() + + +def _collect_virtual_machines_all(credential, subscription_id): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.virtual_machines.list_all() + + +def _collect_storage_accounts_all(credential, subscription_id): + from azure.mgmt.storage import StorageManagementClient + + client = StorageManagementClient(credential, subscription_id) + return client.storage_accounts.list() + + +def _collect_virtual_networks_all(credential, subscription_id): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.virtual_networks.list_all() + + +def _collect_network_security_groups_all(credential, subscription_id): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.network_security_groups.list_all() + + +def _collect_keyvault_vaults_all(credential, subscription_id): + from azure.mgmt.keyvault import KeyVaultManagementClient + + client = KeyVaultManagementClient(credential, subscription_id) + return client.vaults.list_by_subscription() + + +def _collect_managed_clusters_all(credential, subscription_id): + from azure.mgmt.containerservice import ContainerServiceClient + + client = ContainerServiceClient(credential, subscription_id) + return client.managed_clusters.list() + + +# --- Compute (disks / snapshots / VMSS) --- + +def _collect_compute_disks_all(credential, subscription_id): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.disks.list() + + +def _collect_compute_snapshots_all(credential, subscription_id): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.snapshots.list() + + +def _collect_compute_vmss_all(credential, subscription_id): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.virtual_machine_scale_sets.list_all() + + +# --- Network (load balancers / application gateways) --- + +def _collect_network_load_balancers_all(credential, subscription_id): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.load_balancers.list_all() + + +def _collect_network_application_gateways_all(credential, subscription_id): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.application_gateways.list_all() + + +# --- Subscription as a resource --- + +def _collect_subscriptions_all(credential, subscription_id): + """Emit the configured subscription as a discovered resource. + + Uses ``SubscriptionClient.subscriptions.get(...)`` to fetch metadata + for the *single* subscription that the indexer is currently iterating + over (rather than ``subscriptions.list()`` which would return every + subscription the credential can see). The indexer wraps each call in + a per-subscription loop, so returning a one-element iterable here is + correct and yields exactly one ``azure_subscription_subscriptions`` + resource per workspaceInfo subscription. + """ + from azure.mgmt.resource import SubscriptionClient + + client = SubscriptionClient(credential) + return [client.subscriptions.get(subscription_id)] + + +# --- App Service (web apps + plans) --- + +def _collect_appservice_plans_all(credential, subscription_id): + from azure.mgmt.web import WebSiteManagementClient + + client = WebSiteManagementClient(credential, subscription_id) + return client.app_service_plans.list() + + +def _collect_appservice_web_apps_all(credential, subscription_id): + from azure.mgmt.web import WebSiteManagementClient + + client = WebSiteManagementClient(credential, subscription_id) + return client.web_apps.list() + + +# --- MySQL (single + flexible) --- + +def _collect_mysql_servers_all(credential, subscription_id): + from azure.mgmt.rdbms.mysql import MySQLManagementClient + + client = MySQLManagementClient(credential, subscription_id) + return client.servers.list() + + +def _collect_mysql_flexible_servers_all(credential, subscription_id): + from azure.mgmt.rdbms.mysql_flexibleservers import MySQLManagementClient + + client = MySQLManagementClient(credential, subscription_id) + return client.servers.list() + + +# --- PostgreSQL databases (child of Microsoft.DBforPostgreSQL/servers) --- + +def _collect_postgresql_databases_all(credential, subscription_id): + """Walk Postgres servers (subscription-wide) and yield each database. + + ``azure_postgresql_databases`` is a child resource type + (``Microsoft.DBforPostgreSQL/servers/databases``); there's no + subscription-wide list endpoint, so we have to iterate parent + servers first and call ``list_by_server`` for each. + """ + from azure.mgmt.rdbms.postgresql import PostgreSQLManagementClient + + client = PostgreSQLManagementClient(credential, subscription_id) + for server in client.servers.list(): + rg = _rg_from_arm_id(getattr(server, "id", "") or "") + if not rg or not getattr(server, "name", None): + continue + try: + yield from client.databases.list_by_server(rg, server.name) + except Exception: + # Per-server failure (auth scoping, transient outage, no DBs + # visible to this SP) shouldn't abort the whole subscription. + continue + + +# --- Redis --- + +def _collect_redis_caches_all(credential, subscription_id): + from azure.mgmt.redis import RedisManagementClient + + client = RedisManagementClient(credential, subscription_id) + # azure-mgmt-redis renamed the subscription-wide pager to list_by_subscription; + # there is no plain .list() on RedisOperations. + return client.redis.list_by_subscription() + + +# --- Service Bus --- + +def _collect_servicebus_namespaces_all(credential, subscription_id): + from azure.mgmt.servicebus import ServiceBusManagementClient + + client = ServiceBusManagementClient(credential, subscription_id) + return client.namespaces.list() + + +# --- Data Factory --- + +def _collect_datafactory_factories_all(credential, subscription_id): + from azure.mgmt.datafactory import DataFactoryManagementClient + + client = DataFactoryManagementClient(credential, subscription_id) + return client.factories.list() + + +# --- Container Registry (ACR) --- + +def _collect_containerregistry_registries_all(credential, subscription_id): + from azure.mgmt.containerregistry import ContainerRegistryManagementClient + + client = ContainerRegistryManagementClient(credential, subscription_id) + return client.registries.list() + + +# --- API Management --- + +def _collect_apimanagement_service_all(credential, subscription_id): + from azure.mgmt.apimanagement import ApiManagementClient + + client = ApiManagementClient(credential, subscription_id) + return client.api_management_service.list() + + +# --- Cosmos SQL databases (child of Microsoft.DocumentDB/databaseAccounts) --- + +def _collect_cosmos_sql_databases_all(credential, subscription_id): + """Walk Cosmos accounts and yield each SQL database. + + ``azure_cosmos_sql_databases`` is a child resource type + (``Microsoft.DocumentDB/databaseAccounts/sqlDatabases``); we iterate + parent accounts first, mirroring ``_collect_postgresql_databases_all``. + """ + from azure.mgmt.cosmosdb import CosmosDBManagementClient + + client = CosmosDBManagementClient(credential, subscription_id) + for account in client.database_accounts.list(): + rg = _rg_from_arm_id(getattr(account, "id", "") or "") + if not rg or not getattr(account, "name", None): + continue + try: + yield from client.sql_resources.list_sql_databases(rg, account.name) + except Exception: + continue + + +# --- Azure Arc-enabled SQL Server instances --- + +def _collect_arc_sql_server_instances_all(credential, subscription_id): + from azure.mgmt.azurearcdata import AzureArcDataManagementClient + + client = AzureArcDataManagementClient(credential, subscription_id) + return client.sql_server_instances.list() + + +# --------------------------------------------------------------------------- +# Per-RG collectors (used in selective-discovery mode) +# --------------------------------------------------------------------------- +# +# Resource groups themselves only have a subscription-wide list endpoint; +# we keep them on the ``collector_all`` path. Every other typed resource +# in this module exposes ``list(resource_group_name=...)`` (or its SDK- +# specific equivalent) so we can scope discovery to exactly the RGs the +# workspace asked about. + +def _collect_virtual_machines_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.virtual_machines.list(resource_group_name=rg_name) + + +def _collect_storage_accounts_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.storage import StorageManagementClient + + client = StorageManagementClient(credential, subscription_id) + return client.storage_accounts.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_virtual_networks_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.virtual_networks.list(resource_group_name=rg_name) + + +def _collect_network_security_groups_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.network_security_groups.list(resource_group_name=rg_name) + + +def _collect_keyvault_vaults_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.keyvault import KeyVaultManagementClient + + client = KeyVaultManagementClient(credential, subscription_id) + return client.vaults.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_managed_clusters_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.containerservice import ContainerServiceClient + + client = ContainerServiceClient(credential, subscription_id) + return client.managed_clusters.list_by_resource_group(resource_group_name=rg_name) + + +# --- Compute (disks / snapshots / VMSS) --- + +def _collect_compute_disks_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.disks.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_compute_snapshots_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.snapshots.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_compute_vmss_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.compute import ComputeManagementClient + + client = ComputeManagementClient(credential, subscription_id) + return client.virtual_machine_scale_sets.list(resource_group_name=rg_name) + + +# --- Network (load balancers / application gateways) --- + +def _collect_network_load_balancers_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.load_balancers.list(resource_group_name=rg_name) + + +def _collect_network_application_gateways_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.network import NetworkManagementClient + + client = NetworkManagementClient(credential, subscription_id) + return client.application_gateways.list(resource_group_name=rg_name) + + +# --- App Service (web apps + plans) --- + +def _collect_appservice_plans_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.web import WebSiteManagementClient + + client = WebSiteManagementClient(credential, subscription_id) + return client.app_service_plans.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_appservice_web_apps_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.web import WebSiteManagementClient + + client = WebSiteManagementClient(credential, subscription_id) + return client.web_apps.list_by_resource_group(resource_group_name=rg_name) + + +# --- MySQL (single + flexible) --- + +def _collect_mysql_servers_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.rdbms.mysql import MySQLManagementClient + + client = MySQLManagementClient(credential, subscription_id) + return client.servers.list_by_resource_group(resource_group_name=rg_name) + + +def _collect_mysql_flexible_servers_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.rdbms.mysql_flexibleservers import MySQLManagementClient + + client = MySQLManagementClient(credential, subscription_id) + return client.servers.list_by_resource_group(resource_group_name=rg_name) + + +# --- PostgreSQL databases (child of Microsoft.DBforPostgreSQL/servers) --- + +def _collect_postgresql_databases_in_rg(credential, subscription_id, rg_name): + """Walk Postgres servers in ``rg_name`` and yield each database.""" + from azure.mgmt.rdbms.postgresql import PostgreSQLManagementClient + + client = PostgreSQLManagementClient(credential, subscription_id) + for server in client.servers.list_by_resource_group(rg_name): + if not getattr(server, "name", None): + continue + try: + yield from client.databases.list_by_server(rg_name, server.name) + except Exception: + continue + + +# --- Redis --- + +def _collect_redis_caches_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.redis import RedisManagementClient + + client = RedisManagementClient(credential, subscription_id) + return client.redis.list_by_resource_group(resource_group_name=rg_name) + + +# --- Service Bus --- + +def _collect_servicebus_namespaces_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.servicebus import ServiceBusManagementClient + + client = ServiceBusManagementClient(credential, subscription_id) + return client.namespaces.list_by_resource_group(resource_group_name=rg_name) + + +# --- Data Factory --- + +def _collect_datafactory_factories_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.datafactory import DataFactoryManagementClient + + client = DataFactoryManagementClient(credential, subscription_id) + return client.factories.list_by_resource_group(resource_group_name=rg_name) + + +# --- Container Registry (ACR) --- + +def _collect_containerregistry_registries_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.containerregistry import ContainerRegistryManagementClient + + client = ContainerRegistryManagementClient(credential, subscription_id) + return client.registries.list_by_resource_group(resource_group_name=rg_name) + + +# --- API Management --- + +def _collect_apimanagement_service_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.apimanagement import ApiManagementClient + + client = ApiManagementClient(credential, subscription_id) + return client.api_management_service.list_by_resource_group(resource_group_name=rg_name) + + +# --- Cosmos SQL databases (child of Microsoft.DocumentDB/databaseAccounts) --- + +def _collect_cosmos_sql_databases_in_rg(credential, subscription_id, rg_name): + """Walk Cosmos accounts in ``rg_name`` and yield each SQL database.""" + from azure.mgmt.cosmosdb import CosmosDBManagementClient + + client = CosmosDBManagementClient(credential, subscription_id) + for account in client.database_accounts.list_by_resource_group(rg_name): + if not getattr(account, "name", None): + continue + try: + yield from client.sql_resources.list_sql_databases(rg_name, account.name) + except Exception: + continue + + +# --- Azure Arc-enabled SQL Server instances --- + +def _collect_arc_sql_server_instances_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.azurearcdata import AzureArcDataManagementClient + + client = AzureArcDataManagementClient(credential, subscription_id) + return client.sql_server_instances.list_by_resource_group(resource_group_name=rg_name) + + +# --------------------------------------------------------------------------- +# Generic ARM-resources collector (the catch-all) +# --------------------------------------------------------------------------- +# +# Calls ``ResourceManagementClient.resources.list[_by_resource_group]()``, +# which returns ``GenericResource`` rows for *every* top-level ARM resource +# type the credential can see. The native indexer dispatches a single +# generic pass per subscription / RG and routes each row through the +# registry (``find_by_arm_type``) to the correct ``resource_type_name``. +# +# Generic rows carry the basic envelope (``id``, ``name``, ``type``, +# ``location``, ``tags``, ``sku``, ``kind``, ``identity``, ``managed_by``, +# ``plan``) but **not** ``properties`` - the ARM resources API doesn't +# expand them. For richer payloads on a specific type, add a typed +# collector and the typed pass will run alongside the generic pass with +# the typed result winning. + +def _collect_generic_resources_all(credential, subscription_id): + from azure.mgmt.resource import ResourceManagementClient + + client = ResourceManagementClient(credential, subscription_id) + return client.resources.list() + + +def _collect_generic_resources_in_rg(credential, subscription_id, rg_name): + from azure.mgmt.resource import ResourceManagementClient + + client = ResourceManagementClient(credential, subscription_id) + return client.resources.list_by_resource_group(rg_name) + + +# --------------------------------------------------------------------------- +# Typed-collector binding +# --------------------------------------------------------------------------- + +# Maps canonical CQ table name -> (collector_all, collector_in_rg). +# ``collector_in_rg`` is None for resource groups (no per-RG endpoint exists). +_TYPED_COLLECTORS: dict[ + str, + tuple[AzureCollectorAll, Optional[AzureCollectorInRg]], +] = { + "azure_resources_resource_groups": ( + _collect_resource_groups_all, + None, + ), + "azure_compute_virtual_machines": ( + _collect_virtual_machines_all, + _collect_virtual_machines_in_rg, + ), + "azure_storage_accounts": ( + _collect_storage_accounts_all, + _collect_storage_accounts_in_rg, + ), + "azure_network_virtual_networks": ( + _collect_virtual_networks_all, + _collect_virtual_networks_in_rg, + ), + "azure_network_security_groups": ( + _collect_network_security_groups_all, + _collect_network_security_groups_in_rg, + ), + "azure_keyvault_keyvaults": ( + _collect_keyvault_vaults_all, + _collect_keyvault_vaults_in_rg, + ), + "azure_containerservice_managed_clusters": ( + _collect_managed_clusters_all, + _collect_managed_clusters_in_rg, + ), + # Compute (disks / snapshots / VMSS) - reuse azure-mgmt-compute. + "azure_compute_disks": ( + _collect_compute_disks_all, + _collect_compute_disks_in_rg, + ), + "azure_compute_snapshots": ( + _collect_compute_snapshots_all, + _collect_compute_snapshots_in_rg, + ), + "azure_compute_virtual_machine_scale_sets": ( + _collect_compute_vmss_all, + _collect_compute_vmss_in_rg, + ), + # Network (load balancers / app gateways) - reuse azure-mgmt-network. + "azure_network_load_balancers": ( + _collect_network_load_balancers_all, + _collect_network_load_balancers_in_rg, + ), + "azure_network_application_gateways": ( + _collect_network_application_gateways_all, + _collect_network_application_gateways_in_rg, + ), + # Subscription as a top-level resource (no per-RG variant: subscriptions + # don't live in resource groups). Selective mode falls back to + # ``collector_all`` which yields a single resource per configured sub. + "azure_subscription_subscriptions": ( + _collect_subscriptions_all, + None, + ), + # App Service (azure-mgmt-web). + "azure_appservice_plans": ( + _collect_appservice_plans_all, + _collect_appservice_plans_in_rg, + ), + "azure_appservice_web_apps": ( + _collect_appservice_web_apps_all, + _collect_appservice_web_apps_in_rg, + ), + # RDBMS (azure-mgmt-rdbms covers MySQL, Postgres, MariaDB). + "azure_mysql_servers": ( + _collect_mysql_servers_all, + _collect_mysql_servers_in_rg, + ), + "azure_mysqlflexibleservers_servers": ( + _collect_mysql_flexible_servers_all, + _collect_mysql_flexible_servers_in_rg, + ), + "azure_postgresql_databases": ( + _collect_postgresql_databases_all, + _collect_postgresql_databases_in_rg, + ), + # One-off SDKs (azure-mgmt-redis, -servicebus, -datafactory, + # -containerregistry, -apimanagement, -cosmosdb, -azurearcdata). + "azure_redis_caches": ( + _collect_redis_caches_all, + _collect_redis_caches_in_rg, + ), + "azure_servicebus_namespaces": ( + _collect_servicebus_namespaces_all, + _collect_servicebus_namespaces_in_rg, + ), + "azure_datafactory_factories": ( + _collect_datafactory_factories_all, + _collect_datafactory_factories_in_rg, + ), + "azure_containerregistry_registries": ( + _collect_containerregistry_registries_all, + _collect_containerregistry_registries_in_rg, + ), + "azure_apimanagement_service": ( + _collect_apimanagement_service_all, + _collect_apimanagement_service_in_rg, + ), + "azure_cosmos_sql_databases": ( + _collect_cosmos_sql_databases_all, + _collect_cosmos_sql_databases_in_rg, + ), + "azure_azurearcdata_sql_server_instances": ( + _collect_arc_sql_server_instances_all, + _collect_arc_sql_server_instances_in_rg, + ), +} + + +# --------------------------------------------------------------------------- +# Spec materialization +# --------------------------------------------------------------------------- + +def _legacy_resource_type_name(entry: AzureResourceTypeEntry) -> str: + """Pick the ``resource_type_name`` we surface to generation rules. + + Historically RWL generation rules referenced Azure types by short legacy + names (``resource_group``, ``virtual_machine``) for a few core types and + by the CloudQuery table name otherwise. The registry encodes both via + ``cloudquery_table_name`` + ``aliases``; we prefer the first alias if + one exists (the legacy short name) and fall back to the canonical CQ + name otherwise. + """ + if entry.aliases: + return entry.aliases[0] + return entry.cloudquery_table_name + + +def _make_typed_spec( + entry: AzureResourceTypeEntry, + collector_all: AzureCollectorAll, + collector_in_rg: Optional[AzureCollectorInRg], +) -> AzureResourceTypeSpec: + return AzureResourceTypeSpec( + resource_type_name=_legacy_resource_type_name(entry), + cloudquery_table_name=entry.cloudquery_table_name, + mandatory=entry.mandatory, + collector_all=collector_all, + collector_in_rg=collector_in_rg, + typed=True, + arm_type=entry.arm_type, + ) + + +def _make_generic_spec(entry: AzureResourceTypeEntry) -> AzureResourceTypeSpec: + """Build a generic-collector spec for a registry entry that has no + hand-written azure-mgmt-* collector. + + The generic spec uses the catch-all ARM-resources collector pair; it's + addressable via ``find_spec`` so generation rules can reference any + registered Azure type, but the indexer's main loop doesn't iterate + these specs directly - the generic pass routes ``GenericResource`` + rows to them in bulk. Setting ``collector_all`` / ``collector_in_rg`` + to the generic functions keeps the spec usable as a one-off if a + caller really wants to invoke it. + """ + return AzureResourceTypeSpec( + resource_type_name=_legacy_resource_type_name(entry), + cloudquery_table_name=entry.cloudquery_table_name, + mandatory=entry.mandatory, + collector_all=_collect_generic_resources_all, + collector_in_rg=_collect_generic_resources_in_rg, + typed=False, + arm_type=entry.arm_type, + ) + + +def _build_specs() -> tuple[AzureResourceTypeSpec, ...]: + """Materialize one ``AzureResourceTypeSpec`` per registry entry. + + Order: + 1. Resource groups first (always mandatory, RG enumeration is the + bootstrap step every other phase depends on). + 2. Remaining typed collectors next (the rich-payload tier). + 3. Every other registry entry as a generic-collector spec (the + basic-envelope tier). + + The result is what ``find_spec`` searches and what gen-rule reference + checks use to decide whether a given resource-type name is + discoverable. Indexer dispatch (``azureapi.index``) iterates the + typed slice for its rich-pass and runs the generic pass once globally + rather than once per generic spec. + """ + registry = load_registry() + specs: list[AzureResourceTypeSpec] = [] + seen_tables: set[str] = set() + + rg_table = "azure_resources_resource_groups" + if rg_table in _TYPED_COLLECTORS: + rg_entry = registry.find(rg_table) + if rg_entry is not None: + collector_all, collector_in_rg = _TYPED_COLLECTORS[rg_table] + specs.append(_make_typed_spec(rg_entry, collector_all, collector_in_rg)) + seen_tables.add(rg_table) + + for table_name, (collector_all, collector_in_rg) in _TYPED_COLLECTORS.items(): + if table_name in seen_tables: + continue + entry = registry.find(table_name) + if entry is None: + continue + specs.append(_make_typed_spec(entry, collector_all, collector_in_rg)) + seen_tables.add(table_name) + + for entry in registry: + if entry.cloudquery_table_name in seen_tables: + continue + specs.append(_make_generic_spec(entry)) + seen_tables.add(entry.cloudquery_table_name) + + return tuple(specs) + + +AZURE_RESOURCE_TYPE_SPECS: tuple[AzureResourceTypeSpec, ...] = _build_specs() + +# Convenience subset for callers that only care about the rich-payload +# tier (e.g. the indexer's typed-collector dispatch loop). Keeps callers +# from having to filter ``AZURE_RESOURCE_TYPE_SPECS`` themselves. +AZURE_TYPED_RESOURCE_TYPE_SPECS: tuple[AzureResourceTypeSpec, ...] = tuple( + s for s in AZURE_RESOURCE_TYPE_SPECS if s.typed +) + + +# --------------------------------------------------------------------------- +# Lookup +# --------------------------------------------------------------------------- + +def find_spec(name_or_table: str) -> Optional[AzureResourceTypeSpec]: + """Look up an Azure resource type spec by registry name, alias, or CQ table. + + Returns a spec for any name the registry knows about - typed when a + hand-written collector exists, otherwise a generic-collector spec + backed by the ARM-resources catch-all. Returning ``None`` now means + "this name is not a registered Azure resource type at all" rather + than "we don't have a collector for it". + """ + if not name_or_table: + return None + + for spec in AZURE_RESOURCE_TYPE_SPECS: + if name_or_table in (spec.resource_type_name, spec.cloudquery_table_name): + return spec + + entry = load_registry().find(name_or_table) + if entry is None: + return None + for spec in AZURE_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None + + +def find_spec_by_arm_type(arm_type: Optional[str]) -> Optional[AzureResourceTypeSpec]: + """Look up the spec whose ``arm_type`` matches this string. + + Used by the generic-resources pass in ``azureapi.index`` to route + each ``GenericResource.type`` back to the spec that owns its + ``resource_type_name``. Case-insensitive on the ARM-type string. + """ + if not arm_type: + return None + entry = load_registry().find_by_arm_type(arm_type) + if entry is None: + return None + for spec in AZURE_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None diff --git a/src/indexers/cloudquery.py b/src/indexers/cloudquery.py index 0fa7d7795..02e9cbe3e 100644 --- a/src/indexers/cloudquery.py +++ b/src/indexers/cloudquery.py @@ -31,6 +31,17 @@ from utils import read_file, write_file, mask_string from .common import CLOUD_CONFIG_SETTING from .airgap_support import get_airgap_manager +from .azure_common import ( + az_discover_resource_groups, + az_get_credentials_and_subscription_id, + az_validate_credential_access, + _azure_has_only_devops_config, + get_auth_type, + get_managed_identity_details, + has_excluded_tags, + has_included_tags, + resolve_deferred_azure_relationships, +) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from k8s_utils import get_secret from gcp_utils import get_gcp_credential, authenticate_gcloud, validate_gcp_credentials @@ -83,8 +94,27 @@ def _sqlite_template_vars(database_path: str) -> dict: DOCUMENTATION = "Index resources using CloudQuery" + +def _import_azure_indexer_backend_setting(): + # Module-level local import to avoid a circular import between cloudquery + # and azureapi: SETTINGS must reference the same Setting object the + # azureapi indexer registers so the backend selector is honored when + # users include cloudquery without explicitly including azureapi. + from .azureapi import AZURE_INDEXER_BACKEND_SETTING + return AZURE_INDEXER_BACKEND_SETTING + + +def _import_aws_indexer_backend_setting(): + # Same rationale as the Azure helper: honor awsIndexerBackend even when the + # awsapi indexer is not explicitly included alongside cloudquery. + from .awsapi import AWS_INDEXER_BACKEND_SETTING + return AWS_INDEXER_BACKEND_SETTING + + SETTINGS = ( SettingDependency(CLOUD_CONFIG_SETTING, False), + SettingDependency(_import_azure_indexer_backend_setting(), False), + SettingDependency(_import_aws_indexer_backend_setting(), False), ) @dataclass @@ -148,21 +178,6 @@ class CloudQueryPlatformSpec: ]), ] -def has_included_tags(resource_data: dict, include_tags: dict[str, str]) -> bool: - """Returns True if any of the tags in `include_tags` are found in `resource_data`.""" - tags = resource_data.get("tags", {}) - return any(tags.get(key) == value for key, value in include_tags.items()) - - -def has_excluded_tags(resource_data: dict, exclude_tags: dict[str, str]) -> bool: - """Returns True if any of the tags in `exclude_tags` are found in `resource_data`.""" - tags = resource_data.get("tags", {}) - for key, value in exclude_tags.items(): - if tags.get(key) == value: - logger.info(f"Excluding resource {resource_data.get('name', 'unknown')} due to tag '{key}: {value}'") - return True - return False - def is_rate_limited(stdout_text: str, stderr_text: str) -> bool: """ Detect if CloudQuery output indicates rate limiting. @@ -525,34 +540,6 @@ def init_cloudquery_table_info(): else: logger.info("No platforms available for table discovery in init phase") -def get_managed_identity_details(): - credential = DefaultAzureCredential() - - subscription_client = SubscriptionClient(credential) - subscription = next(subscription_client.subscriptions.list()) - subscription_id = subscription.subscription_id - tenant_id = subscription.tenant_id - - imds_url = "http://169.254.169.254/metadata/identity/oauth2/token" - headers = {"Metadata": "true"} - params = { - "api-version": "2019-08-01", - "resource": "https://management.azure.com/" - } - - response = requests.get(imds_url, headers=headers, params=params) - response.raise_for_status() - - token_data = response.json() - client_id = token_data.get("client_id") - - return { - "AZURE_TENANT_ID": tenant_id, - "AZURE_CLIENT_ID": client_id, - "AZURE_SUBSCRIPTION_ID": subscription_id, - "credential": credential - } - def gcp_get_credentials_and_project_ids(platform_config_data: dict[str, Any], temp_dir: str = None) -> dict[str, Any]: """ Resolve GCP authentication and return: @@ -652,92 +639,6 @@ def gcp_get_credentials_and_project_ids(platform_config_data: dict[str, Any], te "env_vars": env_vars } -def az_get_credentials_and_subscription_id(platform_config_data: dict[str, Any]) -> dict[str, Any]: - """ - Resolve Azure authentication and return: - credential – azure-identity credential object - subscription_ids – list[str] (final list for CloudQuery) - AZURE_* keys – env-var values for SP / MI auth - """ - - # ──────────────────────── 0. optional SP via K8s secret - sp_secret_name = platform_config_data.get("spSecretName") - client_id = client_secret = tenant_id = None - if sp_secret_name: - secret = get_secret(sp_secret_name) - tenant_id = base64.b64decode(secret.get("tenantId")).decode() - client_id = base64.b64decode(secret.get("clientId")).decode() - client_secret = base64.b64decode(secret.get("clientSecret")).decode() - - # ──────────────────────── 1. inline SP - if not all([client_id, client_secret, tenant_id]): - client_id = platform_config_data.get("clientId") - client_secret = platform_config_data.get("clientSecret") - tenant_id = platform_config_data.get("tenantId") - - # ──────────────────────── 2. collect subscription IDs - explicit_sub_ids = [ - str(e["subscriptionId"]) - for e in platform_config_data.get("subscriptions", []) - if e.get("subscriptionId") - ] - - if explicit_sub_ids: - subscription_ids = explicit_sub_ids # ← preferred - else: - # Legacy single field + env-var - subscription_ids: list[str] = [] - legacy_sid = platform_config_data.get("subscriptionId") - if legacy_sid: - subscription_ids.append(str(legacy_sid)) - env_sid = os.getenv("AZURE_SUBSCRIPTION_ID") - if env_sid and env_sid not in subscription_ids: - subscription_ids.append(str(env_sid)) - - if not subscription_ids: - raise ValueError("No Azure subscriptionId supplied.") - - # ──────────────────────── 3. credential object - if all([client_id, client_secret, tenant_id]): - credential = ClientSecretCredential(tenant_id, client_id, client_secret) - else: - mi = get_managed_identity_details() - credential = mi["credential"] - client_id = client_id or mi.get("AZURE_CLIENT_ID") - tenant_id = tenant_id or mi.get("AZURE_TENANT_ID") - - # ──────────────────────── 4. package result - result = { - "credential": credential, - "subscription_ids": subscription_ids, - "AZURE_SUBSCRIPTION_ID": subscription_ids[0], # env-var for SDKs - } - if client_id: result["AZURE_CLIENT_ID"] = client_id - if client_secret: result["AZURE_CLIENT_SECRET"] = client_secret - if tenant_id: result["AZURE_TENANT_ID"] = tenant_id - return result - - -def _azure_has_only_devops_config(platform_cfg: dict) -> bool: - """Return True when the azure config block has no cloud discovery credentials. - - This happens when the user only configures ``azure.devops`` (for the ADO - indexer) without providing a subscriptionId or service-principal / - managed-identity credentials needed for Azure resource discovery. - """ - has_sub = bool( - platform_cfg.get("subscriptionId") - or platform_cfg.get("subscriptions") - or os.getenv("AZURE_SUBSCRIPTION_ID") - ) - has_sp = bool( - platform_cfg.get("spSecretName") - or (platform_cfg.get("clientId") and platform_cfg.get("clientSecret")) - ) - has_devops = bool(platform_cfg.get("devops")) - return has_devops and not has_sub and not has_sp - - def init_cloudquery_config( context: Context, cloud_config_data: dict[str, Any], @@ -768,12 +669,42 @@ def init_cloudquery_config( ] = [] # ========================================================= per-platform + # Resolve the Azure backend selector once so we can short-circuit cleanly + # when the native Azure SDK indexer (azureapi) is responsible for Azure. + from .azureapi import AZURE_INDEXER_BACKEND_SETTING # local import to avoid cycles + azure_backend = context.get_setting(AZURE_INDEXER_BACKEND_SETTING) + from .gcpapi import GCP_INDEXER_BACKEND_SETTING # local import to avoid cycles + gcp_backend = context.get_setting(GCP_INDEXER_BACKEND_SETTING) + from .awsapi import AWS_INDEXER_BACKEND_SETTING # local import to avoid cycles + aws_backend = context.get_setting(AWS_INDEXER_BACKEND_SETTING) + for platform_spec in platform_specs: platform_name = platform_spec.name platform_cfg = cloud_config_data.get(platform_name) if platform_cfg is None: continue + if platform_name == "azure" and azure_backend == "azureapi": + logger.info( + "Azure indexer backend: 'azureapi' (native azure-mgmt-* SDK); " + "skipping Azure in CloudQuery." + ) + continue + + if platform_name == "gcp" and gcp_backend == "gcpapi": + logger.info( + "GCP indexer backend: 'gcpapi' (native Cloud Asset Inventory + " + "google-cloud-* SDK); skipping GCP in CloudQuery." + ) + continue + + if platform_name == "aws" and aws_backend == "awsapi": + logger.info( + "AWS indexer backend: 'awsapi' (native Cloud Control API + " + "boto3 SDK); skipping AWS in CloudQuery." + ) + continue + if platform_name == "azure" and _azure_has_only_devops_config(platform_cfg): logger.info( "Azure config contains only DevOps settings (no subscriptionId or cloud credentials). " @@ -781,6 +712,14 @@ def init_cloudquery_config( ) continue + if platform_name == "azure": + # Mirror the announcement the azureapi path makes when it owns + # Azure, so a default-verbosity log makes the choice obvious. + logger.info( + "Azure indexer backend: 'cloudquery' (legacy). Starting " + "Azure resource discovery via the CloudQuery Azure plugin." + ) + # ---------- mandatory specs/tables ---------- cq_resource_type_specs: list[CloudQueryResourceTypeSpec] = [] tables: list[str] = [] @@ -1050,29 +989,6 @@ def init_cloudquery_config( return cq_process_environment_vars, platform_tables -def az_discover_resource_groups(credential, subscription_id) -> list[str]: - if not subscription_id: - raise ValueError("subscription_id cannot be None in az_discover_resource_groups.") - - logger.debug(f"Discovering resource groups for subscription_id: {mask_string(subscription_id)}") - - resource_groups = [] - try: - resource_client = ResourceManagementClient(credential, subscription_id) - for rg in resource_client.resource_groups.list(): - resource_groups.append(rg.name) - logger.info(f"Discovered resource group: {rg.name}") - except Exception as e: - logger.error(f"Failed to discover resource groups: {str(e)}") - raise WorkspaceBuilderException(f"Error discovering resource groups: {str(e)}") - - if not resource_groups: - logger.warning("No resource groups were discovered.") - else: - logger.info(f"Total resource groups discovered: {len(resource_groups)}") - - return resource_groups - def transform_cloud_config(cloud_config: dict[str, Any], cq_temp_dir: str, platform_handlers: dict[str, PlatformHandler]) -> None: @@ -1083,18 +999,6 @@ def transform_cloud_config(cloud_config: dict[str, Any], platform_handler = platform_handlers[platform_name] platform_handler.transform_cloud_config(platform_cloud_config, cq_temp_dir) -def az_validate_credential_access(credential, subscription_id): - try: - resource_client = ResourceManagementClient(credential, subscription_id) - resource_groups = list(resource_client.resource_groups.list()) - if resource_groups: - logger.info(f"Successfully accessed {len(resource_groups)} resource groups.") - else: - logger.warning("No resource groups found.") - except Exception as e: - logger.error(f"Failed to validate credential access: {str(e)}") - raise WorkspaceBuilderException("Credential validation failed.") - def index(context: Context): logger.info("Starting CloudQuery indexing") @@ -1365,136 +1269,3 @@ def index(context: Context): for table_name, table_stats in platform_stats['tables'].items(): logger.debug(f" Table {table_name}: discovered={table_stats['discovered']}, added={table_stats['added_to_registry']}, skipped={table_stats['skipped']}") - -def resolve_deferred_azure_relationships(registry: Registry, platform_handlers: dict[str, PlatformHandler]): - """ - Resolve deferred resource group relationships for Azure resources. - This handles cases where storage accounts were processed before their resource groups. - """ - logger.info("Starting deferred Azure relationship resolution...") - - # Get Azure platform handler - azure_handler = platform_handlers.get("azure") - if not azure_handler: - logger.debug("No Azure platform handler found, skipping deferred relationship resolution") - return - - # Get Azure platform from registry - azure_platform = registry.platforms.get("azure") - if not azure_platform: - logger.debug("No Azure platform in registry, skipping deferred relationship resolution") - return - - # Get resource group type - rg_type = azure_platform.resource_types.get("resource_group") - if not rg_type: - logger.debug("No resource groups in registry, skipping deferred relationship resolution") - return - - resolved_count = 0 - failed_count = 0 - - # Process all resource types that might have deferred relationships - for resource_type_name, resource_type in azure_platform.resource_types.items(): - if resource_type_name == "resource_group": - continue # Skip resource groups themselves - - # Create a snapshot to avoid "dictionary changed size during iteration" error - for resource_qualified_name, resource in list(resource_type.instances.items()): - deferred_info = getattr(resource, '_deferred_rg_lookup', None) - if not deferred_info: - continue # No deferred lookup needed - - rg_name = deferred_info.get('rg_name') - subscription_id = deferred_info.get('subscription_id') - - logger.debug(f"Resolving deferred relationship for {resource.name}: looking for RG '{rg_name}' in subscription '{subscription_id}'") - - # Try to find the resource group now that all resources are loaded - rg_resource = None - for rg in rg_type.instances.values(): - if (rg.name.upper() == rg_name.upper() and - getattr(rg, 'subscription_id', None) == subscription_id): - rg_resource = rg - break - - if rg_resource: - # SUCCESS: Establish the relationship - setattr(resource, 'resource_group', rg_resource) - # Update qualified name to include resource group - new_qualified_name = f"{rg_resource.name}/{resource.name}" - - # Update the registry with the new qualified name - old_qualified_name = resource.qualified_name - resource.qualified_name = new_qualified_name - - # Update the instances dictionary - if old_qualified_name in resource_type.instances: - del resource_type.instances[old_qualified_name] - resource_type.instances[new_qualified_name] = resource - - # Clean up the deferred lookup info - delattr(resource, '_deferred_rg_lookup') - - resolved_count += 1 - logger.info(f"SUCCESS: Resolved deferred relationship for '{resource.name}' -> resource group '{rg_resource.name}' (qualified name: {old_qualified_name} -> {new_qualified_name})") - else: - failed_count += 1 - logger.warning(f"FAILED: Could not resolve deferred relationship for '{resource.name}' - resource group '{rg_name}' in subscription '{subscription_id}' still not found") - - logger.info(f"Deferred relationship resolution completed: {resolved_count} resolved, {failed_count} failed") - - -def get_auth_type(platform_name, platform_config_data: dict[str,Any]): - """ - Determine auth type from platform_config_data for use with auth templates. - - For Azure: azure-auth.yaml template - For AWS: aws-auth.yaml template - - Returns: - Tuple of (auth_type, auth_secret) - """ - auth_secret = None - auth_type = None - - if platform_name == "azure": - auth_secret = platform_config_data.get("clientId") - if auth_secret: - auth_type = "azure_explicit" - auth_secret = None - else: - auth_secret = platform_config_data.get("spSecretName") - if auth_secret: - auth_type = "azure_service_principal_secret" - else: - auth_type = "azure_identity" - auth_secret = None - - elif platform_name == "aws": - # Check for cached auth type from get_aws_credential - if platform_config_data.get("_auth_type"): - auth_type = platform_config_data.get("_auth_type") - auth_secret = platform_config_data.get("_auth_secret") - # Fallback to determining from config - elif platform_config_data.get("awsAccessKeyId"): - auth_type = "aws_explicit" - elif platform_config_data.get("awsSecretName"): - auth_secret = platform_config_data.get("awsSecretName") - auth_type = "aws_secret" - elif platform_config_data.get("useWorkloadIdentity") or os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE') or os.environ.get('AWS_CONTAINER_CREDENTIALS_FULL_URI'): - # Determine which workload identity method - if os.environ.get('AWS_CONTAINER_CREDENTIALS_FULL_URI'): - auth_type = "aws_pod_identity" - else: - auth_type = "aws_workload_identity" - elif platform_config_data.get("assumeRoleArn"): - auth_type = "aws_assume_role" - else: - auth_type = "aws_default_chain" - - # Check for assume role modifier (when combined with other auth methods) - if platform_config_data.get("assumeRoleArn") and auth_type not in ("aws_assume_role", "aws_explicit_assume_role", "aws_secret_assume_role", "aws_workload_identity_assume_role", "aws_pod_identity_assume_role"): - auth_type = auth_type + "_assume_role" - - return auth_type, auth_secret \ No newline at end of file diff --git a/src/indexers/gcp_common.py b/src/indexers/gcp_common.py new file mode 100644 index 000000000..e71edab42 --- /dev/null +++ b/src/indexers/gcp_common.py @@ -0,0 +1,267 @@ +""" +Shared GCP helper functions used by both the legacy CloudQuery-based GCP +indexer (``cloudquery.py``) and the native GCP SDK indexer (``gcpapi.py``). + +This is the lowest layer of GCP-specific logic that both indexers share: +credential / project resolution, label-based tag filtering, and per-project +level-of-detail resolution. Nothing here knows about CloudQuery internals. + +Authentication mirrors the resolution order CloudQuery already uses +(``gcp_get_credentials_and_project_ids`` in ``cloudquery.py``) but returns a +``google.auth`` credentials object suitable for the Cloud Asset Inventory and +typed ``google-cloud-*`` clients instead of shelling out to ``gcloud``: + + 1. Kubernetes secret (``saSecretName`` -> ``serviceAccountKey`` base64) . + 2. Inline service-account key (``serviceAccountKey``) or an already-decoded + credentials file path (``applicationCredentialsFile``). + 3. Application Default Credentials (ADC) - the env the pod/host already has. + +Project IDs come from ``projects`` (list or comma-separated string), +``projectId``, or the ``GOOGLE_CLOUD_PROJECT`` / ``GCP_PROJECT`` env vars. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import os +import sys +import tempfile +from typing import Any, Optional + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from k8s_utils import get_secret # noqa: E402 + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Label / tag filtering helpers +# --------------------------------------------------------------------------- +# +# CloudQuery rows expose user labels under ``tags`` (the indexer copies GCP +# ``labels`` into ``tags`` during normalization so the same include/exclude +# matchers work across every cloud). These helpers therefore read ``tags`` and +# match exactly like their Azure counterparts. + +def has_included_tags(resource_data: dict, include_tags: dict[str, str]) -> bool: + """Return True if any of ``include_tags`` is present in ``resource_data``.""" + tags = resource_data.get("tags", {}) or {} + return any(tags.get(key) == value for key, value in include_tags.items()) + + +def has_excluded_tags(resource_data: dict, exclude_tags: dict[str, str]) -> bool: + """Return True if any of ``exclude_tags`` is present in ``resource_data``.""" + tags = resource_data.get("tags", {}) or {} + for key, value in exclude_tags.items(): + if tags.get(key) == value: + logger.info( + f"Excluding resource {resource_data.get('name', 'unknown')} " + f"due to label '{key}: {value}'" + ) + return True + return False + + +# --------------------------------------------------------------------------- +# Credentials + projects +# --------------------------------------------------------------------------- + +def _normalize_projects(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [p.strip() for p in value.split(",") if p.strip()] + if isinstance(value, (list, tuple)): + out: list[str] = [] + for item in value: + if isinstance(item, dict): + pid = item.get("projectId") or item.get("project_id") or item.get("id") + if pid: + out.append(str(pid).strip()) + elif item: + out.append(str(item).strip()) + return out + return [str(value).strip()] + + +def _service_account_key_to_credentials(key_text: str): + """Build a ``google.oauth2.service_account.Credentials`` from JSON text. + + Imported lazily so the module (and its filter helpers / LOD helpers) stays + importable in environments without ``google-auth`` installed - the same + lazy-import discipline the Azure typed collectors follow. + """ + from google.oauth2 import service_account # noqa: WPS433 (lazy import) + + info = json.loads(key_text) + return service_account.Credentials.from_service_account_info( + info, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + + +def gcp_get_credentials_and_projects(platform_config_data: dict[str, Any]) -> dict[str, Any]: + """Resolve GCP credentials + the project IDs to discover. + + Returns a dict with: + credentials - a ``google.auth`` credentials object, or ``None`` when + falling back to Application Default Credentials (the + Cloud Asset client picks ADC up automatically). + project_ids - list[str] of project IDs to index. + quota_project - the project used for billing/quota (first project). + env - dict of env vars to set so other code paths + (``enrichers.gcp``, gcloud subprocesses) agree on the + active credentials/project. + """ + credentials = None + env: dict[str, str] = {} + + sa_secret_name = platform_config_data.get("saSecretName") + service_account_key = platform_config_data.get("serviceAccountKey") + project_id_hint = platform_config_data.get("projectId") + + if sa_secret_name: + secret = get_secret(sa_secret_name) + if secret: + if not service_account_key and secret.get("serviceAccountKey"): + service_account_key = base64.b64decode( + secret["serviceAccountKey"] + ).decode("utf-8") + if not project_id_hint and secret.get("projectId"): + project_id_hint = base64.b64decode(secret["projectId"]).decode("utf-8") + + # An already-decoded credentials file (GCPPlatformHandler.transform_cloud_config + # writes the base64 applicationCredentialsFile out to a temp path). + app_creds_file = platform_config_data.get("applicationCredentialsFile") + + if service_account_key: + # ``serviceAccountKey`` may arrive base64-encoded or as raw JSON. + key_text = service_account_key + if not key_text.lstrip().startswith("{"): + try: + key_text = base64.b64decode(service_account_key).decode("utf-8") + except Exception: + key_text = service_account_key + try: + credentials = _service_account_key_to_credentials(key_text) + except Exception as e: # pragma: no cover - defensive + logger.warning(f"Failed to build SA credentials from serviceAccountKey: {e}") + else: + # Write the key to a temp file so gcloud / ADC consumers agree. + fd, tmp_path = tempfile.mkstemp(prefix="gcp-sa-", suffix=".json") + with os.fdopen(fd, "w") as fh: + fh.write(key_text) + env["GOOGLE_APPLICATION_CREDENTIALS"] = tmp_path + elif app_creds_file: + # ``applicationCredentialsFile`` reaches the indexers in one of two + # shapes: + # * a real path on disk (e.g. a pre-existing ADC file), or + # * the credentials file *content*, base64-encoded - the exact shape + # CloudQuery's ``GCPPlatformHandler.transform_cloud_config`` decodes + # before it writes a temp file. The native indexer runs *before* + # that transform, so it must decode the content itself (raw JSON is + # accepted too for robustness). + key_text: Optional[str] = None + if os.path.exists(app_creds_file): + try: + with open(app_creds_file, "r", encoding="utf-8") as fh: + key_text = fh.read() + except Exception as e: # pragma: no cover - defensive + logger.warning( + f"Failed to read applicationCredentialsFile path: {e}" + ) + else: + env["GOOGLE_APPLICATION_CREDENTIALS"] = app_creds_file + else: + text = app_creds_file + if not text.lstrip().startswith("{"): + try: + text = base64.b64decode(app_creds_file).decode("utf-8") + except Exception: + text = app_creds_file + if text.lstrip().startswith("{"): + key_text = text + + if key_text: + try: + credentials = _service_account_key_to_credentials(key_text) + except Exception as e: # pragma: no cover - defensive + logger.warning( + f"Failed to load SA credentials from applicationCredentialsFile: {e}" + ) + else: + # When the value was inline content (not a path), persist it so + # gcloud / ADC consumers downstream agree on the same key. + if "GOOGLE_APPLICATION_CREDENTIALS" not in env: + fd, tmp_path = tempfile.mkstemp(prefix="gcp-sa-", suffix=".json") + with os.fdopen(fd, "w") as fh: + fh.write(key_text) + env["GOOGLE_APPLICATION_CREDENTIALS"] = tmp_path + + # Resolve the project list. + project_ids = _normalize_projects(platform_config_data.get("projects")) + if not project_ids and project_id_hint: + project_ids = [str(project_id_hint)] + if not project_ids: + for env_var in ("GOOGLE_CLOUD_PROJECT", "GCP_PROJECT"): + val = os.getenv(env_var) + if val: + project_ids = [val] + break + + # De-dupe, preserve order. + seen: set[str] = set() + deduped: list[str] = [] + for pid in project_ids: + if pid and pid not in seen: + seen.add(pid) + deduped.append(pid) + project_ids = deduped + + quota_project = project_ids[0] if project_ids else project_id_hint + if quota_project: + env.setdefault("GOOGLE_CLOUD_PROJECT", quota_project) + + return { + "credentials": credentials, + "project_ids": project_ids, + "quota_project": quota_project, + "env": env, + } + + +def gcp_has_discovery_config(platform_cfg: dict[str, Any]) -> bool: + """Return True when the gcp config block has enough to discover resources. + + We need at least one project to scope discovery to; credentials may come + from ADC, so their absence isn't disqualifying. + """ + if not platform_cfg: + return False + if _normalize_projects(platform_cfg.get("projects")): + return True + if platform_cfg.get("projectId"): + return True + if os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT"): + return True + return False + + +# --------------------------------------------------------------------------- +# Auth-type derivation (for auth templates) +# --------------------------------------------------------------------------- + +def get_gcp_auth_type(platform_config_data: dict[str, Any]) -> tuple[Optional[str], Optional[str]]: + """Determine the auth type + secret for use with the gcp-auth template. + + Returns ``(auth_type, auth_secret)``. + """ + if platform_config_data.get("saSecretName"): + return "gcp_service_account_secret", platform_config_data.get("saSecretName") + if platform_config_data.get("serviceAccountKey") or platform_config_data.get( + "applicationCredentialsFile" + ): + return "gcp_service_account", None + return "gcp_adc", None diff --git a/src/indexers/gcp_resource_type_registry.py b/src/indexers/gcp_resource_type_registry.py new file mode 100644 index 000000000..cb026f403 --- /dev/null +++ b/src/indexers/gcp_resource_type_registry.py @@ -0,0 +1,241 @@ +""" +Loader for the GCP resource-type registry. + +The registry maps every CloudQuery GCP table name to its Cloud Asset Inventory +(CAI) asset type plus metadata used by the native ``gcpapi`` indexer: + +* canonical name = the CloudQuery table name (e.g. ``gcp_compute_instances``) +* CAI asset type for Cloud Asset Inventory / generic discovery + (e.g. ``compute.googleapis.com/Instance``) +* aliases for backward compatibility with legacy RWL ``resource_type_name`` + values (e.g. ``project`` -> ``gcp_projects``, ``compute_instance`` -> + ``gcp_compute_instances``) +* ``typed_collector`` flag indicating whether ``gcpapi_resource_types`` ships a + hand-written ``google-cloud-*`` collector for this table +* ``mandatory`` flag indicating whether the indexer must always list the type + (today only ``gcp_projects``, the anchor that every other resource links to) + +The data lives in ``gcp_resource_type_registry.yaml`` next to this module. That +YAML is generated by ``scripts/gcp/sync_gcp_resource_type_registry.py``; +hand-edits to the YAML get overwritten on the next sync. To change behavior for +a specific table, edit ``scripts/gcp/gcp_resource_type_overrides.yaml`` and +re-run the sync script. + +This module is intentionally read-only: it loads, caches, and exposes the +registry. It does not own collector callables or generation-rule semantics. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +_DEFAULT_REGISTRY_PATH = Path(__file__).with_name("gcp_resource_type_registry.yaml") + + +@dataclass(frozen=True) +class GcpResourceTypeEntry: + """One row of the registry. + + ``cloudquery_table_name`` is the canonical key. Lookups by alias resolve to + the same entry (no duplicates). ``cai_asset_type`` may be ``None`` for tables + that have no Cloud Asset Inventory equivalent (IAM bindings, billing + rollups...); such tables are skipped by generic discovery. + """ + + cloudquery_table_name: str + cai_asset_type: Optional[str] + cai_asset_type_source: Optional[str] + category: Optional[str] + aliases: tuple[str, ...] + typed_collector: bool + mandatory: bool + + def known_names(self) -> tuple[str, ...]: + """Every name (canonical + aliases) that resolves to this entry.""" + return (self.cloudquery_table_name, *self.aliases) + + +@dataclass(frozen=True) +class GcpRegistryMetadata: + source: Optional[str] = None + snapshot_date: Optional[str] = None + total_tables: int = 0 + typed_collectors: int = 0 + cai_types_assigned: int = 0 + generator: Optional[str] = None + notes: Optional[str] = None + + +@dataclass +class GcpResourceTypeRegistry: + metadata: GcpRegistryMetadata + entries: tuple[GcpResourceTypeEntry, ...] + _by_canonical: dict[str, GcpResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_alias: dict[str, GcpResourceTypeEntry] = field(default_factory=dict, repr=False) + _by_cai_type_lower: dict[str, GcpResourceTypeEntry] = field(default_factory=dict, repr=False) + + def __post_init__(self) -> None: + for entry in self.entries: + self._by_canonical[entry.cloudquery_table_name] = entry + for alias in entry.aliases: + if not alias: + continue + # Aliases must not collide with another canonical name or alias; + # the sync script enforces this, but verify defensively in case + # someone hand-edits the YAML. + if alias in self._by_canonical and self._by_canonical[alias] is not entry: + raise ValueError( + f"Alias {alias!r} collides with canonical table name " + f"belonging to a different entry" + ) + if alias in self._by_alias and self._by_alias[alias] is not entry: + raise ValueError( + f"Alias {alias!r} is registered for both " + f"{self._by_alias[alias].cloudquery_table_name!r} and " + f"{entry.cloudquery_table_name!r}" + ) + self._by_alias[alias] = entry + if entry.cai_asset_type: + # CAI asset types are case-sensitive on the wire but we index + # lower-cased for robust matching. Multiple registry entries can + # occasionally share an asset type; the first one wins, which is + # deterministic because entries arrive sorted by canonical name. + key = entry.cai_asset_type.lower() + self._by_cai_type_lower.setdefault(key, entry) + + def find(self, name: str) -> Optional[GcpResourceTypeEntry]: + """Return the entry for a canonical table name or alias, else None.""" + if not name: + return None + entry = self._by_canonical.get(name) + if entry is not None: + return entry + return self._by_alias.get(name) + + def find_by_cai_type(self, cai_asset_type: Optional[str]) -> Optional[GcpResourceTypeEntry]: + """Look up the registry entry whose ``cai_asset_type`` matches this string. + + Used by the Cloud Asset Inventory generic-discovery pass to route each + ``asset.asset_type`` value (e.g. ``storage.googleapis.com/Bucket``) back + to the registry entry whose ``cloudquery_table_name`` is the canonical + RWL identifier for that type. + """ + if not cai_asset_type: + return None + return self._by_cai_type_lower.get(cai_asset_type.lower()) + + def __contains__(self, name: object) -> bool: + return isinstance(name, str) and self.find(name) is not None + + def __iter__(self) -> Iterable[GcpResourceTypeEntry]: + return iter(self.entries) + + def __len__(self) -> int: + return len(self.entries) + + def all_canonical_names(self) -> tuple[str, ...]: + return tuple(entry.cloudquery_table_name for entry in self.entries) + + def all_cai_types(self) -> tuple[str, ...]: + return tuple( + entry.cai_asset_type for entry in self.entries if entry.cai_asset_type + ) + + def typed_collector_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.typed_collector + ) + + def mandatory_tables(self) -> tuple[str, ...]: + return tuple( + entry.cloudquery_table_name + for entry in self.entries + if entry.mandatory + ) + + +def _coerce_aliases(value: object) -> tuple[str, ...]: + if not value: + return () + if isinstance(value, (list, tuple)): + return tuple(str(v) for v in value if v) + return (str(value),) + + +def _build_metadata(payload: Mapping[str, object]) -> GcpRegistryMetadata: + return GcpRegistryMetadata( + source=_optional_str(payload.get("source")), + snapshot_date=_optional_str(payload.get("snapshot_date")), + total_tables=int(payload.get("total_tables") or 0), + typed_collectors=int(payload.get("typed_collectors") or 0), + cai_types_assigned=int(payload.get("cai_types_assigned") or 0), + generator=_optional_str(payload.get("generator")), + notes=_optional_str(payload.get("notes")), + ) + + +def _optional_str(value: object) -> Optional[str]: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _build_entries(types_payload: Mapping[str, Mapping[str, object]]) -> tuple[GcpResourceTypeEntry, ...]: + entries: list[GcpResourceTypeEntry] = [] + for table_name, body in sorted(types_payload.items()): + if not isinstance(body, Mapping): + raise ValueError(f"Registry entry {table_name!r} is not a mapping") + entries.append( + GcpResourceTypeEntry( + cloudquery_table_name=str(table_name), + cai_asset_type=_optional_str(body.get("cai_asset_type")), + cai_asset_type_source=_optional_str(body.get("cai_asset_type_source")), + category=_optional_str(body.get("category")), + aliases=_coerce_aliases(body.get("aliases")), + typed_collector=bool(body.get("typed_collector")), + mandatory=bool(body.get("mandatory")), + ) + ) + return tuple(entries) + + +def load_registry_from_path(path: Path) -> GcpResourceTypeRegistry: + """Load a registry from an explicit YAML path. Bypasses the cache.""" + if not path.exists(): + raise FileNotFoundError(f"GCP resource-type registry not found at {path}") + with path.open("r", encoding="utf-8") as fh: + payload = yaml.safe_load(fh) or {} + if not isinstance(payload, Mapping): + raise ValueError(f"Registry YAML at {path} did not parse to a mapping") + + metadata = _build_metadata(payload.get("metadata") or {}) + types_payload = payload.get("types") or {} + if not isinstance(types_payload, Mapping): + raise ValueError(f"Registry YAML at {path} has non-mapping 'types' section") + + entries = _build_entries(types_payload) + return GcpResourceTypeRegistry(metadata=metadata, entries=entries) + + +@lru_cache(maxsize=1) +def load_registry() -> GcpResourceTypeRegistry: + """Load and cache the registry from the default location.""" + return load_registry_from_path(_DEFAULT_REGISTRY_PATH) + + +def find_entry(name: str) -> Optional[GcpResourceTypeEntry]: + """Convenience wrapper around the cached registry's ``find()``.""" + return load_registry().find(name) + + +def reset_cache() -> None: + """Clear the cached registry. Intended for tests.""" + load_registry.cache_clear() diff --git a/src/indexers/gcp_resource_type_registry.yaml b/src/indexers/gcp_resource_type_registry.yaml new file mode 100644 index 000000000..6fc6383d3 --- /dev/null +++ b/src/indexers/gcp_resource_type_registry.yaml @@ -0,0 +1,2839 @@ +metadata: + source: https://www.cloudquery.io/hub/plugins/source/cloudquery/gcp/latest/tables + snapshot_date: '2026-05-29' + total_tables: 404 + typed_collectors: 13 + cai_types_assigned: 403 + generator: scripts/gcp/sync_gcp_resource_type_registry.py + notes: Generated file. To change the CAI asset type for a table, edit scripts/gcp/gcp_resource_type_overrides.yaml and re-run the sync script. Hand-edits to this file will be overwritten. +types: + gcp_accessapproval_folder_approval_requests: + cai_asset_type: accessapproval.googleapis.com/FolderApprovalRequest + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_folder_service_accounts: + cai_asset_type: accessapproval.googleapis.com/FolderServiceAccount + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_folder_settings: + cai_asset_type: accessapproval.googleapis.com/FolderSetting + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_organization_approval_requests: + cai_asset_type: accessapproval.googleapis.com/OrganizationApprovalRequest + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_organization_service_accounts: + cai_asset_type: accessapproval.googleapis.com/OrganizationServiceAccount + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_organization_settings: + cai_asset_type: accessapproval.googleapis.com/OrganizationSetting + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_project_approval_requests: + cai_asset_type: accessapproval.googleapis.com/ProjectApprovalRequest + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_project_service_accounts: + cai_asset_type: accessapproval.googleapis.com/ProjectServiceAccount + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_accessapproval_project_settings: + cai_asset_type: accessapproval.googleapis.com/ProjectSetting + cai_asset_type_source: heuristic + category: accessapproval + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_batch_prediction_jobs: + cai_asset_type: aiplatform.googleapis.com/BatchPredictionJob + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_custom_jobs: + cai_asset_type: aiplatform.googleapis.com/CustomJob + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_dataset_locations: + cai_asset_type: aiplatform.googleapis.com/DatasetLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_datasets: + cai_asset_type: aiplatform.googleapis.com/Dataset + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_endpoint_locations: + cai_asset_type: aiplatform.googleapis.com/EndpointLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_endpoints: + cai_asset_type: aiplatform.googleapis.com/Endpoint + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_featurestore_locations: + cai_asset_type: aiplatform.googleapis.com/FeaturestoreLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_featurestores: + cai_asset_type: aiplatform.googleapis.com/Featurestore + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_hyperparameter_tuning_jobs: + cai_asset_type: aiplatform.googleapis.com/HyperparameterTuningJob + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_index_endpoints: + cai_asset_type: aiplatform.googleapis.com/IndexEndpoint + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_index_locations: + cai_asset_type: aiplatform.googleapis.com/IndexLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_indexendpoint_locations: + cai_asset_type: aiplatform.googleapis.com/IndexendpointLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_indexes: + cai_asset_type: aiplatform.googleapis.com/Index + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_job_locations: + cai_asset_type: aiplatform.googleapis.com/JobLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_metadata_locations: + cai_asset_type: aiplatform.googleapis.com/MetadataLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_metadata_stores: + cai_asset_type: aiplatform.googleapis.com/MetadataStore + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_model_deployment_monitoring_jobs: + cai_asset_type: aiplatform.googleapis.com/ModelDeploymentMonitoringJob + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_model_locations: + cai_asset_type: aiplatform.googleapis.com/ModelLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_models: + cai_asset_type: aiplatform.googleapis.com/Model + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_operations: + cai_asset_type: aiplatform.googleapis.com/Operation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_pipeline_jobs: + cai_asset_type: aiplatform.googleapis.com/PipelineJob + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_pipeline_locations: + cai_asset_type: aiplatform.googleapis.com/PipelineLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_specialist_pools: + cai_asset_type: aiplatform.googleapis.com/SpecialistPool + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_specialistpool_locations: + cai_asset_type: aiplatform.googleapis.com/SpecialistpoolLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_studies: + cai_asset_type: aiplatform.googleapis.com/Study + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_tensorboard_locations: + cai_asset_type: aiplatform.googleapis.com/TensorboardLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_tensorboards: + cai_asset_type: aiplatform.googleapis.com/Tensorboard + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_training_pipelines: + cai_asset_type: aiplatform.googleapis.com/TrainingPipeline + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_aiplatform_vizier_locations: + cai_asset_type: aiplatform.googleapis.com/VizierLocation + cai_asset_type_source: heuristic + category: aiplatform + aliases: [] + typed_collector: false + mandatory: false + gcp_alloydb_clusters: + cai_asset_type: alloydb.googleapis.com/Cluster + cai_asset_type_source: override + category: alloydb + aliases: [] + typed_collector: false + mandatory: false + gcp_alloydb_instances: + cai_asset_type: alloydb.googleapis.com/Instance + cai_asset_type_source: override + category: alloydb + aliases: [] + typed_collector: false + mandatory: false + gcp_apigateway_apis: + cai_asset_type: apigateway.googleapis.com/Api + cai_asset_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + gcp_apigateway_gateways: + cai_asset_type: apigateway.googleapis.com/Gateway + cai_asset_type_source: heuristic + category: apigateway + aliases: [] + typed_collector: false + mandatory: false + gcp_apikeys_keys: + cai_asset_type: apikeys.googleapis.com/Key + cai_asset_type_source: heuristic + category: apikeys + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_apps: + cai_asset_type: appengine.googleapis.com/Application + cai_asset_type_source: override + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_authorized_certificates: + cai_asset_type: appengine.googleapis.com/AuthorizedCertificate + cai_asset_type_source: heuristic + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_authorized_domains: + cai_asset_type: appengine.googleapis.com/AuthorizedDomain + cai_asset_type_source: heuristic + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_domain_mappings: + cai_asset_type: appengine.googleapis.com/DomainMapping + cai_asset_type_source: heuristic + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_firewall_ingress_rules: + cai_asset_type: appengine.googleapis.com/FirewallIngressRule + cai_asset_type_source: heuristic + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_instances: + cai_asset_type: appengine.googleapis.com/Instance + cai_asset_type_source: heuristic + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_services: + cai_asset_type: appengine.googleapis.com/Service + cai_asset_type_source: override + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_appengine_versions: + cai_asset_type: appengine.googleapis.com/Version + cai_asset_type_source: override + category: appengine + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_authconfigs: + cai_asset_type: applicationintegration.googleapis.com/Authconfig + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_certificates: + cai_asset_type: applicationintegration.googleapis.com/Certificate + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_integration_execution_suspensions: + cai_asset_type: applicationintegration.googleapis.com/IntegrationExecutionSuspension + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_integration_executions: + cai_asset_type: applicationintegration.googleapis.com/IntegrationExecution + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_integration_versions: + cai_asset_type: applicationintegration.googleapis.com/IntegrationVersion + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_integrations: + cai_asset_type: applicationintegration.googleapis.com/Integration + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_sfdc_channels: + cai_asset_type: applicationintegration.googleapis.com/SfdcChannel + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_applicationintegration_sfdc_instances: + cai_asset_type: applicationintegration.googleapis.com/SfdcInstance + cai_asset_type_source: heuristic + category: applicationintegration + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_docker_images: + cai_asset_type: artifactregistry.googleapis.com/DockerImage + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_files: + cai_asset_type: artifactregistry.googleapis.com/File + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_locations: + cai_asset_type: artifactregistry.googleapis.com/Location + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_packages: + cai_asset_type: artifactregistry.googleapis.com/Package + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_repositories: + cai_asset_type: artifactregistry.googleapis.com/Repository + cai_asset_type_source: override + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_tags: + cai_asset_type: artifactregistry.googleapis.com/Tag + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_artifactregistry_versions: + cai_asset_type: artifactregistry.googleapis.com/Version + cai_asset_type_source: heuristic + category: artifactregistry + aliases: [] + typed_collector: false + mandatory: false + gcp_baremetalsolution_instances: + cai_asset_type: baremetalsolution.googleapis.com/Instance + cai_asset_type_source: heuristic + category: baremetalsolution + aliases: [] + typed_collector: false + mandatory: false + gcp_baremetalsolution_networks: + cai_asset_type: baremetalsolution.googleapis.com/Network + cai_asset_type_source: heuristic + category: baremetalsolution + aliases: [] + typed_collector: false + mandatory: false + gcp_baremetalsolution_nfs_shares: + cai_asset_type: baremetalsolution.googleapis.com/NfsShare + cai_asset_type_source: heuristic + category: baremetalsolution + aliases: [] + typed_collector: false + mandatory: false + gcp_baremetalsolution_volume_luns: + cai_asset_type: baremetalsolution.googleapis.com/VolumeLun + cai_asset_type_source: heuristic + category: baremetalsolution + aliases: [] + typed_collector: false + mandatory: false + gcp_baremetalsolution_volumes: + cai_asset_type: baremetalsolution.googleapis.com/Volume + cai_asset_type_source: heuristic + category: baremetalsolution + aliases: [] + typed_collector: false + mandatory: false + gcp_batch_jobs: + cai_asset_type: batch.googleapis.com/Job + cai_asset_type_source: heuristic + category: batch + aliases: [] + typed_collector: false + mandatory: false + gcp_batch_task_groups: + cai_asset_type: batch.googleapis.com/TaskGroup + cai_asset_type_source: heuristic + category: batch + aliases: [] + typed_collector: false + mandatory: false + gcp_batch_tasks: + cai_asset_type: batch.googleapis.com/Task + cai_asset_type_source: heuristic + category: batch + aliases: [] + typed_collector: false + mandatory: false + gcp_beyondcorp_app_connections: + cai_asset_type: beyondcorp.googleapis.com/AppConnection + cai_asset_type_source: heuristic + category: beyondcorp + aliases: [] + typed_collector: false + mandatory: false + gcp_beyondcorp_app_connectors: + cai_asset_type: beyondcorp.googleapis.com/AppConnector + cai_asset_type_source: heuristic + category: beyondcorp + aliases: [] + typed_collector: false + mandatory: false + gcp_beyondcorp_app_gateways: + cai_asset_type: beyondcorp.googleapis.com/AppGateway + cai_asset_type_source: heuristic + category: beyondcorp + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquery_datasets: + cai_asset_type: bigquery.googleapis.com/Dataset + cai_asset_type_source: override + category: bigquery + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquery_tables: + cai_asset_type: bigquery.googleapis.com/Table + cai_asset_type_source: override + category: bigquery + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquerydatatransfer_configs: + cai_asset_type: bigquerydatatransfer.googleapis.com/Config + cai_asset_type_source: heuristic + category: bigquerydatatransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquerydatatransfer_datasources: + cai_asset_type: bigquerydatatransfer.googleapis.com/Datasource + cai_asset_type_source: heuristic + category: bigquerydatatransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquerydatatransfer_locations: + cai_asset_type: bigquerydatatransfer.googleapis.com/Location + cai_asset_type_source: heuristic + category: bigquerydatatransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquerydatatransfer_logs: + cai_asset_type: bigquerydatatransfer.googleapis.com/Log + cai_asset_type_source: heuristic + category: bigquerydatatransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_bigquerydatatransfer_runs: + cai_asset_type: bigquerydatatransfer.googleapis.com/Run + cai_asset_type_source: heuristic + category: bigquerydatatransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_bigtableadmin_app_profiles: + cai_asset_type: bigtableadmin.googleapis.com/AppProfile + cai_asset_type_source: override + category: bigtableadmin + aliases: [] + typed_collector: false + mandatory: false + gcp_bigtableadmin_backups: + cai_asset_type: bigtableadmin.googleapis.com/Backup + cai_asset_type_source: heuristic + category: bigtableadmin + aliases: [] + typed_collector: false + mandatory: false + gcp_bigtableadmin_clusters: + cai_asset_type: bigtableadmin.googleapis.com/Cluster + cai_asset_type_source: override + category: bigtableadmin + aliases: [] + typed_collector: false + mandatory: false + gcp_bigtableadmin_instances: + cai_asset_type: bigtableadmin.googleapis.com/Instance + cai_asset_type_source: override + category: bigtableadmin + aliases: [] + typed_collector: false + mandatory: false + gcp_bigtableadmin_tables: + cai_asset_type: bigtableadmin.googleapis.com/Table + cai_asset_type_source: override + category: bigtableadmin + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_billing_account_subaccounts: + cai_asset_type: billing.googleapis.com/BillingAccountSubaccount + cai_asset_type_source: heuristic + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_billing_accounts: + cai_asset_type: null + cai_asset_type_source: override + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_budgets: + cai_asset_type: billing.googleapis.com/Budget + cai_asset_type_source: heuristic + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_projects: + cai_asset_type: billing.googleapis.com/Project + cai_asset_type_source: heuristic + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_service_skus: + cai_asset_type: billing.googleapis.com/ServiceSku + cai_asset_type_source: heuristic + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_billing_services: + cai_asset_type: billing.googleapis.com/Service + cai_asset_type_source: heuristic + category: billing + aliases: [] + typed_collector: false + mandatory: false + gcp_binaryauthorization_assertors: + cai_asset_type: binaryauthorization.googleapis.com/Assertor + cai_asset_type_source: heuristic + category: binaryauthorization + aliases: [] + typed_collector: false + mandatory: false + gcp_certificatemanager_certificate_issuance_configs: + cai_asset_type: certificatemanager.googleapis.com/CertificateIssuanceConfig + cai_asset_type_source: heuristic + category: certificatemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_certificatemanager_certificate_map_entries: + cai_asset_type: certificatemanager.googleapis.com/CertificateMapEntry + cai_asset_type_source: heuristic + category: certificatemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_certificatemanager_certificate_maps: + cai_asset_type: certificatemanager.googleapis.com/CertificateMap + cai_asset_type_source: heuristic + category: certificatemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_certificatemanager_certificates: + cai_asset_type: certificatemanager.googleapis.com/Certificate + cai_asset_type_source: heuristic + category: certificatemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_certificatemanager_dns_authorizations: + cai_asset_type: certificatemanager.googleapis.com/DnsAuthorization + cai_asset_type_source: heuristic + category: certificatemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets: + cai_asset_type: cloudasset.googleapis.com/Asset + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_access_policies: + cai_asset_type: cloudasset.googleapis.com/AssetsAccessPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_history: + cai_asset_type: cloudasset.googleapis.com/AssetsHistory + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_iam_policies: + cai_asset_type: cloudasset.googleapis.com/AssetsIamPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_org_policies: + cai_asset_type: cloudasset.googleapis.com/AssetsOrgPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_os_inventories: + cai_asset_type: cloudasset.googleapis.com/AssetsOsInventory + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_relationships: + cai_asset_type: cloudasset.googleapis.com/AssetsRelationship + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_assets_resources: + cai_asset_type: cloudasset.googleapis.com/AssetsResource + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_effective_iam_policies: + cai_asset_type: cloudasset.googleapis.com/EffectiveIamPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_feeds: + cai_asset_type: cloudasset.googleapis.com/Feed + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets: + cai_asset_type: cloudasset.googleapis.com/OrgAsset + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_access_policies: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsAccessPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_iam_policies: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsIamPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_org_policies: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsOrgPolicy + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_os_inventories: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsOsInventory + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_relationships: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsRelationship + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_org_assets_resources: + cai_asset_type: cloudasset.googleapis.com/OrgAssetsResource + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudassetinventory_savedqueries: + cai_asset_type: cloudasset.googleapis.com/Savedquery + cai_asset_type_source: heuristic + category: cloudassetinventory + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudbuild_builds: + cai_asset_type: cloudbuild.googleapis.com/Build + cai_asset_type_source: heuristic + category: cloudbuild + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudbuild_connection_repositories: + cai_asset_type: cloudbuild.googleapis.com/ConnectionRepository + cai_asset_type_source: heuristic + category: cloudbuild + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudbuild_connections: + cai_asset_type: cloudbuild.googleapis.com/Connection + cai_asset_type_source: heuristic + category: cloudbuild + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudbuild_triggers: + cai_asset_type: cloudbuild.googleapis.com/Trigger + cai_asset_type_source: heuristic + category: cloudbuild + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudbuild_worker_pools: + cai_asset_type: cloudbuild.googleapis.com/WorkerPool + cai_asset_type_source: heuristic + category: cloudbuild + aliases: [] + typed_collector: false + mandatory: false + gcp_clouddeploy_delivery_pipelines: + cai_asset_type: clouddeploy.googleapis.com/DeliveryPipeline + cai_asset_type_source: heuristic + category: clouddeploy + aliases: [] + typed_collector: false + mandatory: false + gcp_clouddeploy_job_runs: + cai_asset_type: clouddeploy.googleapis.com/JobRun + cai_asset_type_source: heuristic + category: clouddeploy + aliases: [] + typed_collector: false + mandatory: false + gcp_clouddeploy_releases: + cai_asset_type: clouddeploy.googleapis.com/Releas + cai_asset_type_source: heuristic + category: clouddeploy + aliases: [] + typed_collector: false + mandatory: false + gcp_clouddeploy_rollouts: + cai_asset_type: clouddeploy.googleapis.com/Rollout + cai_asset_type_source: heuristic + category: clouddeploy + aliases: [] + typed_collector: false + mandatory: false + gcp_clouddeploy_targets: + cai_asset_type: clouddeploy.googleapis.com/Target + cai_asset_type_source: heuristic + category: clouddeploy + aliases: [] + typed_collector: false + mandatory: false + gcp_clouderrorreporting_error_events: + cai_asset_type: clouderrorreporting.googleapis.com/ErrorEvent + cai_asset_type_source: heuristic + category: clouderrorreporting + aliases: [] + typed_collector: false + mandatory: false + gcp_clouderrorreporting_error_group_stats: + cai_asset_type: clouderrorreporting.googleapis.com/ErrorGroupStat + cai_asset_type_source: heuristic + category: clouderrorreporting + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudresourcemanager_organizations: + cai_asset_type: cloudresourcemanager.googleapis.com/Organization + cai_asset_type_source: heuristic + category: cloudresourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudscheduler_jobs: + cai_asset_type: cloudscheduler.googleapis.com/Job + cai_asset_type_source: heuristic + category: cloudscheduler + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudscheduler_locations: + cai_asset_type: cloudscheduler.googleapis.com/Location + cai_asset_type_source: heuristic + category: cloudscheduler + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudsupport_cases: + cai_asset_type: cloudsupport.googleapis.com/Cas + cai_asset_type_source: heuristic + category: cloudsupport + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudtasks_locations: + cai_asset_type: cloudtasks.googleapis.com/Location + cai_asset_type_source: heuristic + category: cloudtasks + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudtasks_queues: + cai_asset_type: cloudtasks.googleapis.com/Queue + cai_asset_type_source: heuristic + category: cloudtasks + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudtasks_tasks: + cai_asset_type: cloudtasks.googleapis.com/Task + cai_asset_type_source: heuristic + category: cloudtasks + aliases: [] + typed_collector: false + mandatory: false + gcp_cloudtrace_traces: + cai_asset_type: cloudtrace.googleapis.com/Trace + cai_asset_type_source: heuristic + category: cloudtrace + aliases: [] + typed_collector: false + mandatory: false + gcp_composer_environments: + cai_asset_type: composer.googleapis.com/Environment + cai_asset_type_source: heuristic + category: composer + aliases: [] + typed_collector: false + mandatory: false + gcp_composer_image_versions: + cai_asset_type: composer.googleapis.com/ImageVersion + cai_asset_type_source: heuristic + category: composer + aliases: [] + typed_collector: false + mandatory: false + gcp_composer_operations: + cai_asset_type: composer.googleapis.com/Operation + cai_asset_type_source: heuristic + category: composer + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_addresses: + cai_asset_type: compute.googleapis.com/Address + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_autoscalers: + cai_asset_type: compute.googleapis.com/Autoscaler + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_backend_buckets: + cai_asset_type: compute.googleapis.com/BackendBucket + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_backend_services: + cai_asset_type: compute.googleapis.com/BackendService + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_disk_types: + cai_asset_type: compute.googleapis.com/DiskType + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_disks: + cai_asset_type: compute.googleapis.com/Disk + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_external_vpn_gateways: + cai_asset_type: compute.googleapis.com/ExternalVpnGateway + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_firewalls: + cai_asset_type: compute.googleapis.com/Firewall + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_forwarding_rules: + cai_asset_type: compute.googleapis.com/ForwardingRule + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_global_addresses: + cai_asset_type: compute.googleapis.com/GlobalAddress + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_global_forwarding_rules: + cai_asset_type: compute.googleapis.com/GlobalForwardingRule + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_health_checks: + cai_asset_type: compute.googleapis.com/HealthCheck + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_image_policies: + cai_asset_type: compute.googleapis.com/ImagePolicy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_images: + cai_asset_type: compute.googleapis.com/Image + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_group_instances: + cai_asset_type: compute.googleapis.com/InstanceGroupInstance + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_group_managers: + cai_asset_type: compute.googleapis.com/InstanceGroupManager + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_group_regional_instances: + cai_asset_type: compute.googleapis.com/InstanceGroupRegionalInstance + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_groups: + cai_asset_type: compute.googleapis.com/InstanceGroup + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_tag_bindings: + cai_asset_type: compute.googleapis.com/InstanceTagBinding + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instance_templates: + cai_asset_type: compute.googleapis.com/InstanceTemplate + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_instances: + cai_asset_type: compute.googleapis.com/Instance + cai_asset_type_source: override + category: compute + aliases: + - compute_instance + typed_collector: true + mandatory: false + gcp_compute_interconnect_attachments: + cai_asset_type: compute.googleapis.com/InterconnectAttachment + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_interconnect_locations: + cai_asset_type: compute.googleapis.com/InterconnectLocation + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_interconnect_remote_locations: + cai_asset_type: compute.googleapis.com/InterconnectRemoteLocation + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_interconnects: + cai_asset_type: compute.googleapis.com/Interconnect + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_machine_types: + cai_asset_type: compute.googleapis.com/MachineType + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_network_endpoint_groups: + cai_asset_type: compute.googleapis.com/NetworkEndpointGroup + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_network_firewall_policies: + cai_asset_type: compute.googleapis.com/NetworkFirewallPolicy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_networks: + cai_asset_type: compute.googleapis.com/Network + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_osconfig_inventories: + cai_asset_type: compute.googleapis.com/OsconfigInventory + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_patch_deployments: + cai_asset_type: compute.googleapis.com/OsconfigOsPatchDeployment + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_patch_jobs: + cai_asset_type: compute.googleapis.com/OsconfigOsPatchJob + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_patch_jobs_instance_details: + cai_asset_type: compute.googleapis.com/OsconfigOsPatchJobsInstanceDetail + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_policy_assignment_reports: + cai_asset_type: compute.googleapis.com/OsconfigOsPolicyAssignmentReport + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_policy_assignments: + cai_asset_type: compute.googleapis.com/OsconfigOsPolicyAssignment + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_osconfig_os_vulnerability_reports: + cai_asset_type: compute.googleapis.com/OsconfigOsVulnerabilityReport + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_packet_mirrorings: + cai_asset_type: compute.googleapis.com/PacketMirroring + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_projects: + cai_asset_type: compute.googleapis.com/Project + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_region_instance_templates: + cai_asset_type: compute.googleapis.com/RegionInstanceTemplate + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_region_network_firewall_policies: + cai_asset_type: compute.googleapis.com/RegionNetworkFirewallPolicy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_router_nat_mapping_infos: + cai_asset_type: compute.googleapis.com/RouterNatMappingInfo + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_routers: + cai_asset_type: compute.googleapis.com/Router + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_routes: + cai_asset_type: compute.googleapis.com/Route + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_security_policies: + cai_asset_type: compute.googleapis.com/SecurityPolicy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_snapshots: + cai_asset_type: compute.googleapis.com/Snapshot + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_ssl_certificates: + cai_asset_type: compute.googleapis.com/SslCertificate + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_ssl_policies: + cai_asset_type: compute.googleapis.com/SslPolicy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_subnetworks: + cai_asset_type: compute.googleapis.com/Subnetwork + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: true + mandatory: false + gcp_compute_target_grpc_proxies: + cai_asset_type: compute.googleapis.com/TargetGrpcProxy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_http_proxies: + cai_asset_type: compute.googleapis.com/TargetHttpProxy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_https_proxies: + cai_asset_type: compute.googleapis.com/TargetHttpsProxy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_instances: + cai_asset_type: compute.googleapis.com/TargetInstance + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_pools: + cai_asset_type: compute.googleapis.com/TargetPool + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_ssl_proxies: + cai_asset_type: compute.googleapis.com/TargetSslProxy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_tcp_proxies: + cai_asset_type: compute.googleapis.com/TargetTcpProxy + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_target_vpn_gateways: + cai_asset_type: compute.googleapis.com/TargetVpnGateway + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_url_maps: + cai_asset_type: compute.googleapis.com/UrlMap + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_vpn_gateways: + cai_asset_type: compute.googleapis.com/VpnGateway + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_vpn_tunnels: + cai_asset_type: compute.googleapis.com/VpnTunnel + cai_asset_type_source: override + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_compute_zones: + cai_asset_type: compute.googleapis.com/Zone + cai_asset_type_source: heuristic + category: compute + aliases: [] + typed_collector: false + mandatory: false + gcp_container_clusters: + cai_asset_type: container.googleapis.com/Cluster + cai_asset_type_source: override + category: container + aliases: [] + typed_collector: true + mandatory: false + gcp_container_node_pools: + cai_asset_type: container.googleapis.com/NodePool + cai_asset_type_source: override + category: container + aliases: [] + typed_collector: false + mandatory: false + gcp_containeranalysis_occurrences: + cai_asset_type: containeranalysis.googleapis.com/Occurrence + cai_asset_type_source: heuristic + category: containeranalysis + aliases: [] + typed_collector: false + mandatory: false + gcp_databasemigration_locations: + cai_asset_type: datamigration.googleapis.com/Location + cai_asset_type_source: heuristic + category: databasemigration + aliases: [] + typed_collector: false + mandatory: false + gcp_databasemigration_migration_jobs: + cai_asset_type: datamigration.googleapis.com/MigrationJob + cai_asset_type_source: heuristic + category: databasemigration + aliases: [] + typed_collector: false + mandatory: false + gcp_databasemigration_operations: + cai_asset_type: datamigration.googleapis.com/Operation + cai_asset_type_source: heuristic + category: databasemigration + aliases: [] + typed_collector: false + mandatory: false + gcp_dataflow_job_messages: + cai_asset_type: dataflow.googleapis.com/JobMessage + cai_asset_type_source: heuristic + category: dataflow + aliases: [] + typed_collector: false + mandatory: false + gcp_dataflow_job_metrics: + cai_asset_type: dataflow.googleapis.com/JobMetric + cai_asset_type_source: heuristic + category: dataflow + aliases: [] + typed_collector: false + mandatory: false + gcp_dataflow_jobs: + cai_asset_type: dataflow.googleapis.com/Job + cai_asset_type_source: heuristic + category: dataflow + aliases: [] + typed_collector: false + mandatory: false + gcp_dataflow_snapshots: + cai_asset_type: dataflow.googleapis.com/Snapshot + cai_asset_type_source: heuristic + category: dataflow + aliases: [] + typed_collector: false + mandatory: false + gcp_datafusion_available_versions: + cai_asset_type: datafusion.googleapis.com/AvailableVersion + cai_asset_type_source: heuristic + category: datafusion + aliases: [] + typed_collector: false + mandatory: false + gcp_datafusion_instance_operations: + cai_asset_type: datafusion.googleapis.com/InstanceOperation + cai_asset_type_source: heuristic + category: datafusion + aliases: [] + typed_collector: false + mandatory: false + gcp_datafusion_instances: + cai_asset_type: datafusion.googleapis.com/Instance + cai_asset_type_source: heuristic + category: datafusion + aliases: [] + typed_collector: false + mandatory: false + gcp_dataproc_autoscaling_policies: + cai_asset_type: dataproc.googleapis.com/AutoscalingPolicy + cai_asset_type_source: heuristic + category: dataproc + aliases: [] + typed_collector: false + mandatory: false + gcp_dataproc_cluster_nodegroups: + cai_asset_type: dataproc.googleapis.com/ClusterNodegroup + cai_asset_type_source: heuristic + category: dataproc + aliases: [] + typed_collector: false + mandatory: false + gcp_dataproc_clusters: + cai_asset_type: dataproc.googleapis.com/Cluster + cai_asset_type_source: heuristic + category: dataproc + aliases: [] + typed_collector: false + mandatory: false + gcp_dataproc_jobs: + cai_asset_type: dataproc.googleapis.com/Job + cai_asset_type_source: heuristic + category: dataproc + aliases: [] + typed_collector: false + mandatory: false + gcp_dataproc_regions: + cai_asset_type: dataproc.googleapis.com/Region + cai_asset_type_source: heuristic + category: dataproc + aliases: [] + typed_collector: false + mandatory: false + gcp_deploymentmanager_deployments: + cai_asset_type: deploymentmanager.googleapis.com/Deployment + cai_asset_type_source: heuristic + category: deploymentmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_deploymentmanager_manifests: + cai_asset_type: deploymentmanager.googleapis.com/Manifest + cai_asset_type_source: heuristic + category: deploymentmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_deploymentmanager_operations: + cai_asset_type: deploymentmanager.googleapis.com/Operation + cai_asset_type_source: heuristic + category: deploymentmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_deploymentmanager_resources: + cai_asset_type: deploymentmanager.googleapis.com/Resource + cai_asset_type_source: heuristic + category: deploymentmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_deploymentmanager_types: + cai_asset_type: deploymentmanager.googleapis.com/Type + cai_asset_type_source: heuristic + category: deploymentmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_dns_managed_zones: + cai_asset_type: dns.googleapis.com/ManagedZone + cai_asset_type_source: override + category: dns + aliases: [] + typed_collector: false + mandatory: false + gcp_dns_policies: + cai_asset_type: dns.googleapis.com/Policy + cai_asset_type_source: override + category: dns + aliases: [] + typed_collector: false + mandatory: false + gcp_dns_resource_record_sets: + cai_asset_type: dns.googleapis.com/ResourceRecordSet + cai_asset_type_source: heuristic + category: dns + aliases: [] + typed_collector: false + mandatory: false + gcp_domains_registrations: + cai_asset_type: domains.googleapis.com/Registration + cai_asset_type_source: heuristic + category: domains + aliases: [] + typed_collector: false + mandatory: false + gcp_essentialcontacts_folder_contacts: + cai_asset_type: essentialcontacts.googleapis.com/FolderContact + cai_asset_type_source: heuristic + category: essentialcontacts + aliases: [] + typed_collector: false + mandatory: false + gcp_essentialcontacts_organization_contacts: + cai_asset_type: essentialcontacts.googleapis.com/OrganizationContact + cai_asset_type_source: heuristic + category: essentialcontacts + aliases: [] + typed_collector: false + mandatory: false + gcp_essentialcontacts_project_contacts: + cai_asset_type: essentialcontacts.googleapis.com/ProjectContact + cai_asset_type_source: heuristic + category: essentialcontacts + aliases: [] + typed_collector: false + mandatory: false + gcp_eventarc_channels: + cai_asset_type: eventarc.googleapis.com/Channel + cai_asset_type_source: heuristic + category: eventarc + aliases: [] + typed_collector: false + mandatory: false + gcp_eventarc_providers: + cai_asset_type: eventarc.googleapis.com/Provider + cai_asset_type_source: heuristic + category: eventarc + aliases: [] + typed_collector: false + mandatory: false + gcp_eventarc_triggers: + cai_asset_type: eventarc.googleapis.com/Trigger + cai_asset_type_source: heuristic + category: eventarc + aliases: [] + typed_collector: false + mandatory: false + gcp_filestore_backups: + cai_asset_type: file.googleapis.com/Backup + cai_asset_type_source: override + category: filestore + aliases: [] + typed_collector: false + mandatory: false + gcp_filestore_instances: + cai_asset_type: file.googleapis.com/Instance + cai_asset_type_source: override + category: filestore + aliases: [] + typed_collector: false + mandatory: false + gcp_filestore_locations: + cai_asset_type: file.googleapis.com/Location + cai_asset_type_source: heuristic + category: filestore + aliases: [] + typed_collector: false + mandatory: false + gcp_filestore_operations: + cai_asset_type: file.googleapis.com/Operation + cai_asset_type_source: heuristic + category: filestore + aliases: [] + typed_collector: false + mandatory: false + gcp_filestore_snapshots: + cai_asset_type: file.googleapis.com/Snapshot + cai_asset_type_source: heuristic + category: filestore + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_hosting_site_channels: + cai_asset_type: firebase.googleapis.com/HostingSiteChannel + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_hosting_site_custom_domains: + cai_asset_type: firebase.googleapis.com/HostingSiteCustomDomain + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_hosting_site_releases: + cai_asset_type: firebase.googleapis.com/HostingSiteReleas + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_hosting_site_versions: + cai_asset_type: firebase.googleapis.com/HostingSiteVersion + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_hosting_sites: + cai_asset_type: firebase.googleapis.com/HostingSite + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_rules_releases: + cai_asset_type: firebase.googleapis.com/RulesReleas + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebase_rules_rulesets: + cai_asset_type: firebase.googleapis.com/RulesRuleset + cai_asset_type_source: heuristic + category: firebase + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_app_attest_configs: + cai_asset_type: firebaseappcheck.googleapis.com/AppAttestConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_device_check_configs: + cai_asset_type: firebaseappcheck.googleapis.com/DeviceCheckConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_play_integrity_configs: + cai_asset_type: firebaseappcheck.googleapis.com/PlayIntegrityConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_recaptcha_configs: + cai_asset_type: firebaseappcheck.googleapis.com/RecaptchaConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_recaptcha_enterprise_configs: + cai_asset_type: firebaseappcheck.googleapis.com/RecaptchaEnterpriseConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firebaseappcheck_safety_net_configs: + cai_asset_type: firebaseappcheck.googleapis.com/SafetyNetConfig + cai_asset_type_source: heuristic + category: firebaseappcheck + aliases: [] + typed_collector: false + mandatory: false + gcp_firestore_databases: + cai_asset_type: firestore.googleapis.com/Databas + cai_asset_type_source: heuristic + category: firestore + aliases: [] + typed_collector: false + mandatory: false + gcp_functions_function_policies: + cai_asset_type: cloudfunctions.googleapis.com/FunctionPolicy + cai_asset_type_source: heuristic + category: functions + aliases: [] + typed_collector: false + mandatory: false + gcp_functions_functions: + cai_asset_type: cloudfunctions.googleapis.com/CloudFunction + cai_asset_type_source: override + category: functions + aliases: [] + typed_collector: false + mandatory: false + gcp_functionsv2_function_policies: + cai_asset_type: cloudfunctions.googleapis.com/FunctionPolicy + cai_asset_type_source: heuristic + category: functionsv2 + aliases: [] + typed_collector: false + mandatory: false + gcp_functionsv2_functions: + cai_asset_type: cloudfunctions.googleapis.com/Function + cai_asset_type_source: override + category: functionsv2 + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_deny_policies: + cai_asset_type: iam.googleapis.com/DenyPolicy + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_organizational_roles: + cai_asset_type: iam.googleapis.com/OrganizationalRole + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_predefined_roles: + cai_asset_type: iam.googleapis.com/PredefinedRole + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_roles: + cai_asset_type: iam.googleapis.com/Role + cai_asset_type_source: override + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_service_account_keys: + cai_asset_type: iam.googleapis.com/ServiceAccountKey + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_service_account_policies: + cai_asset_type: iam.googleapis.com/ServiceAccountPolicy + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_service_accounts: + cai_asset_type: iam.googleapis.com/ServiceAccount + cai_asset_type_source: override + category: iam + aliases: [] + typed_collector: true + mandatory: false + gcp_iam_workload_identity_pool_providers: + cai_asset_type: iam.googleapis.com/WorkloadIdentityPoolProvider + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_iam_workload_identity_pools: + cai_asset_type: iam.googleapis.com/WorkloadIdentityPool + cai_asset_type_source: heuristic + category: iam + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_accounts: + cai_asset_type: identitytoolkit.googleapis.com/Account + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_default_supported_idps: + cai_asset_type: identitytoolkit.googleapis.com/DefaultSupportedIdp + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_inbound_saml_configs: + cai_asset_type: identitytoolkit.googleapis.com/InboundSamlConfig + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_oauth_idp_configs: + cai_asset_type: identitytoolkit.googleapis.com/OauthIdpConfig + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_project_configs: + cai_asset_type: identitytoolkit.googleapis.com/ProjectConfig + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_project_public_configs: + cai_asset_type: identitytoolkit.googleapis.com/ProjectPublicConfig + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_recaptcha_params: + cai_asset_type: identitytoolkit.googleapis.com/RecaptchaParam + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_identitytoolkit_tenants: + cai_asset_type: identitytoolkit.googleapis.com/Tenant + cai_asset_type_source: heuristic + category: identitytoolkit + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_crypto_key_versions: + cai_asset_type: cloudkms.googleapis.com/CryptoKeyVersion + cai_asset_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_crypto_keys: + cai_asset_type: cloudkms.googleapis.com/CryptoKey + cai_asset_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_ekm_connections: + cai_asset_type: cloudkms.googleapis.com/EkmConnection + cai_asset_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_import_jobs: + cai_asset_type: cloudkms.googleapis.com/ImportJob + cai_asset_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_keyrings: + cai_asset_type: cloudkms.googleapis.com/KeyRing + cai_asset_type_source: override + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_kms_locations: + cai_asset_type: cloudkms.googleapis.com/Location + cai_asset_type_source: heuristic + category: kms + aliases: [] + typed_collector: false + mandatory: false + gcp_livestream_channels: + cai_asset_type: livestream.googleapis.com/Channel + cai_asset_type_source: heuristic + category: livestream + aliases: [] + typed_collector: false + mandatory: false + gcp_livestream_inputs: + cai_asset_type: livestream.googleapis.com/Input + cai_asset_type_source: heuristic + category: livestream + aliases: [] + typed_collector: false + mandatory: false + gcp_logging_audit_logs: + cai_asset_type: logging.googleapis.com/AuditLog + cai_asset_type_source: heuristic + category: logging + aliases: [] + typed_collector: false + mandatory: false + gcp_logging_metrics: + cai_asset_type: logging.googleapis.com/Metric + cai_asset_type_source: heuristic + category: logging + aliases: [] + typed_collector: false + mandatory: false + gcp_logging_sinks: + cai_asset_type: logging.googleapis.com/Sink + cai_asset_type_source: heuristic + category: logging + aliases: [] + typed_collector: false + mandatory: false + gcp_looker_instance_operations: + cai_asset_type: looker.googleapis.com/InstanceOperation + cai_asset_type_source: heuristic + category: looker + aliases: [] + typed_collector: false + mandatory: false + gcp_looker_instances: + cai_asset_type: looker.googleapis.com/Instance + cai_asset_type_source: heuristic + category: looker + aliases: [] + typed_collector: false + mandatory: false + gcp_memorystore_instances: + cai_asset_type: memorystore.googleapis.com/Instance + cai_asset_type_source: heuristic + category: memorystore + aliases: [] + typed_collector: false + mandatory: false + gcp_memorystore_locations: + cai_asset_type: memorystore.googleapis.com/Location + cai_asset_type_source: heuristic + category: memorystore + aliases: [] + typed_collector: false + mandatory: false + gcp_memorystore_operations: + cai_asset_type: memorystore.googleapis.com/Operation + cai_asset_type_source: heuristic + category: memorystore + aliases: [] + typed_collector: false + mandatory: false + gcp_monitoring_alert_policies: + cai_asset_type: monitoring.googleapis.com/AlertPolicy + cai_asset_type_source: heuristic + category: monitoring + aliases: [] + typed_collector: false + mandatory: false + gcp_networkconnectivity_internal_ranges: + cai_asset_type: networkconnectivity.googleapis.com/InternalRange + cai_asset_type_source: heuristic + category: networkconnectivity + aliases: [] + typed_collector: false + mandatory: false + gcp_networkconnectivity_locations: + cai_asset_type: networkconnectivity.googleapis.com/Location + cai_asset_type_source: heuristic + category: networkconnectivity + aliases: [] + typed_collector: false + mandatory: false + gcp_networkintelligencecenter_connectivity_tests: + cai_asset_type: networkmanagement.googleapis.com/ConnectivityTest + cai_asset_type_source: heuristic + category: networkintelligencecenter + aliases: [] + typed_collector: false + mandatory: false + gcp_networkintelligencecenter_locations: + cai_asset_type: networkmanagement.googleapis.com/Location + cai_asset_type_source: heuristic + category: networkintelligencecenter + aliases: [] + typed_collector: false + mandatory: false + gcp_networkintelligencecenter_operations: + cai_asset_type: networkmanagement.googleapis.com/Operation + cai_asset_type_source: heuristic + category: networkintelligencecenter + aliases: [] + typed_collector: false + mandatory: false + gcp_networksecurity_address_groups: + cai_asset_type: networksecurity.googleapis.com/AddressGroup + cai_asset_type_source: heuristic + category: networksecurity + aliases: [] + typed_collector: false + mandatory: false + gcp_networksecurity_firewall_endpoints: + cai_asset_type: networksecurity.googleapis.com/FirewallEndpoint + cai_asset_type_source: heuristic + category: networksecurity + aliases: [] + typed_collector: false + mandatory: false + gcp_networksecurity_security_profile_groups: + cai_asset_type: networksecurity.googleapis.com/SecurityProfileGroup + cai_asset_type_source: heuristic + category: networksecurity + aliases: [] + typed_collector: false + mandatory: false + gcp_networksecurity_security_profiles: + cai_asset_type: networksecurity.googleapis.com/SecurityProfile + cai_asset_type_source: heuristic + category: networksecurity + aliases: [] + typed_collector: false + mandatory: false + gcp_networksecurity_tls_inspection_policies: + cai_asset_type: networksecurity.googleapis.com/TlsInspectionPolicy + cai_asset_type_source: heuristic + category: networksecurity + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_endpoint_policies: + cai_asset_type: networkservices.googleapis.com/EndpointPolicy + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_gateways: + cai_asset_type: networkservices.googleapis.com/Gateway + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_grpc_routes: + cai_asset_type: networkservices.googleapis.com/GrpcRoute + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_http_routes: + cai_asset_type: networkservices.googleapis.com/HttpRoute + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_meshes: + cai_asset_type: networkservices.googleapis.com/Mesh + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_service_bindings: + cai_asset_type: networkservices.googleapis.com/ServiceBinding + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_tcp_routes: + cai_asset_type: networkservices.googleapis.com/TcpRoute + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_networkservices_tls_routes: + cai_asset_type: networkservices.googleapis.com/TlsRoute + cai_asset_type_source: heuristic + category: networkservices + aliases: [] + typed_collector: false + mandatory: false + gcp_organization_folders_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/FoldersPolicy + cai_asset_type_source: heuristic + category: organization + aliases: [] + typed_collector: false + mandatory: false + gcp_organization_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/Policy + cai_asset_type_source: heuristic + category: organization + aliases: [] + typed_collector: false + mandatory: false + gcp_organization_projects_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectsPolicy + cai_asset_type_source: heuristic + category: organization + aliases: [] + typed_collector: false + mandatory: false + gcp_policyanalyzer_activities: + cai_asset_type: policyanalyzer.googleapis.com/Activity + cai_asset_type_source: heuristic + category: policyanalyzer + aliases: [] + typed_collector: false + mandatory: false + gcp_privateca_authorities: + cai_asset_type: privateca.googleapis.com/Authority + cai_asset_type_source: heuristic + category: privateca + aliases: [] + typed_collector: false + mandatory: false + gcp_privateca_certificates: + cai_asset_type: privateca.googleapis.com/Certificate + cai_asset_type_source: heuristic + category: privateca + aliases: [] + typed_collector: false + mandatory: false + gcp_privateca_pools: + cai_asset_type: privateca.googleapis.com/Pool + cai_asset_type_source: heuristic + category: privateca + aliases: [] + typed_collector: false + mandatory: false + gcp_projects: + cai_asset_type: cloudresourcemanager.googleapis.com/Project + cai_asset_type_source: override + category: projects + aliases: + - project + typed_collector: true + mandatory: true + gcp_pubsub_schema_revisions: + cai_asset_type: pubsub.googleapis.com/SchemaRevision + cai_asset_type_source: heuristic + category: pubsub + aliases: [] + typed_collector: false + mandatory: false + gcp_pubsub_schemas: + cai_asset_type: pubsub.googleapis.com/Schema + cai_asset_type_source: heuristic + category: pubsub + aliases: [] + typed_collector: false + mandatory: false + gcp_pubsub_snapshots: + cai_asset_type: pubsub.googleapis.com/Snapshot + cai_asset_type_source: override + category: pubsub + aliases: [] + typed_collector: false + mandatory: false + gcp_pubsub_subscriptions: + cai_asset_type: pubsub.googleapis.com/Subscription + cai_asset_type_source: override + category: pubsub + aliases: [] + typed_collector: true + mandatory: false + gcp_pubsub_topics: + cai_asset_type: pubsub.googleapis.com/Topic + cai_asset_type_source: override + category: pubsub + aliases: [] + typed_collector: true + mandatory: false + gcp_recommendations_folders: + cai_asset_type: recommendations.googleapis.com/Folder + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_folders_insights: + cai_asset_type: recommendations.googleapis.com/FoldersInsight + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_folders_locations: + cai_asset_type: recommendations.googleapis.com/FoldersLocation + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_organizations: + cai_asset_type: recommendations.googleapis.com/Organization + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_organizations_insights: + cai_asset_type: recommendations.googleapis.com/OrganizationsInsight + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_organizations_locations: + cai_asset_type: recommendations.googleapis.com/OrganizationsLocation + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_projects: + cai_asset_type: recommendations.googleapis.com/Project + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_projects_insights: + cai_asset_type: recommendations.googleapis.com/ProjectsInsight + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_recommendations_projects_locations: + cai_asset_type: recommendations.googleapis.com/ProjectsLocation + cai_asset_type_source: heuristic + category: recommendations + aliases: [] + typed_collector: false + mandatory: false + gcp_redis_instances: + cai_asset_type: redis.googleapis.com/Instance + cai_asset_type_source: override + category: redis + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_folder_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/FolderPolicy + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_folders: + cai_asset_type: cloudresourcemanager.googleapis.com/Folder + cai_asset_type_source: override + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_organization_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/OrganizationPolicy + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_organization_projects: + cai_asset_type: cloudresourcemanager.googleapis.com/OrganizationProject + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_organization_tag_keys: + cai_asset_type: cloudresourcemanager.googleapis.com/OrganizationTagKey + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_organization_tag_values: + cai_asset_type: cloudresourcemanager.googleapis.com/OrganizationTagValue + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_project_policies: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectPolicy + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_project_tag_bindings: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectTagBinding + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_project_tag_keys: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectTagKey + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_project_tag_values: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectTagValue + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_projects: + cai_asset_type: cloudresourcemanager.googleapis.com/Project + cai_asset_type_source: override + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_projects_search: + cai_asset_type: cloudresourcemanager.googleapis.com/ProjectsSearch + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_resourcemanager_subfolders: + cai_asset_type: cloudresourcemanager.googleapis.com/Subfolder + cai_asset_type_source: heuristic + category: resourcemanager + aliases: [] + typed_collector: false + mandatory: false + gcp_run_executions: + cai_asset_type: run.googleapis.com/Execution + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_job_policies: + cai_asset_type: run.googleapis.com/JobPolicy + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_jobs: + cai_asset_type: run.googleapis.com/Job + cai_asset_type_source: override + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_locations: + cai_asset_type: run.googleapis.com/Location + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_revisions: + cai_asset_type: run.googleapis.com/Revision + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_service_policies: + cai_asset_type: run.googleapis.com/ServicePolicy + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_services: + cai_asset_type: run.googleapis.com/Service + cai_asset_type_source: override + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_tasks: + cai_asset_type: run.googleapis.com/Task + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_worker_pool_policies: + cai_asset_type: run.googleapis.com/WorkerPoolPolicy + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_run_worker_pools: + cai_asset_type: run.googleapis.com/WorkerPool + cai_asset_type_source: heuristic + category: run + aliases: [] + typed_collector: false + mandatory: false + gcp_secretmanager_secrets: + cai_asset_type: secretmanager.googleapis.com/Secret + cai_asset_type_source: override + category: secretmanager + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_folder_event_threat_detection: + cai_asset_type: securitycenter.googleapis.com/FolderEventThreatDetection + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_folder_findings: + cai_asset_type: securitycenter.googleapis.com/FolderFinding + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_org_event_threat_detection_settings: + cai_asset_type: securitycenter.googleapis.com/OrgEventThreatDetectionSetting + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_organization_findings: + cai_asset_type: securitycenter.googleapis.com/OrganizationFinding + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_project_event_threat_detection: + cai_asset_type: securitycenter.googleapis.com/ProjectEventThreatDetection + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_securitycenter_project_findings: + cai_asset_type: securitycenter.googleapis.com/ProjectFinding + cai_asset_type_source: heuristic + category: securitycenter + aliases: [] + typed_collector: false + mandatory: false + gcp_servicehealth_events: + cai_asset_type: servicehealth.googleapis.com/Event + cai_asset_type_source: heuristic + category: servicehealth + aliases: [] + typed_collector: false + mandatory: false + gcp_servicehealth_locations: + cai_asset_type: servicehealth.googleapis.com/Location + cai_asset_type_source: heuristic + category: servicehealth + aliases: [] + typed_collector: false + mandatory: false + gcp_serviceusage_service_project_quota_metrics: + cai_asset_type: serviceusage.googleapis.com/ServiceProjectQuotaMetric + cai_asset_type_source: heuristic + category: serviceusage + aliases: [] + typed_collector: false + mandatory: false + gcp_serviceusage_services: + cai_asset_type: serviceusage.googleapis.com/Service + cai_asset_type_source: heuristic + category: serviceusage + aliases: [] + typed_collector: false + mandatory: false + gcp_sourcerepo_config: + cai_asset_type: sourcerepo.googleapis.com/Config + cai_asset_type_source: heuristic + category: sourcerepo + aliases: [] + typed_collector: false + mandatory: false + gcp_sourcerepo_repos: + cai_asset_type: sourcerepo.googleapis.com/Repo + cai_asset_type_source: heuristic + category: sourcerepo + aliases: [] + typed_collector: false + mandatory: false + gcp_spanner_databases: + cai_asset_type: spanner.googleapis.com/Database + cai_asset_type_source: override + category: spanner + aliases: [] + typed_collector: false + mandatory: false + gcp_spanner_instances: + cai_asset_type: spanner.googleapis.com/Instance + cai_asset_type_source: override + category: spanner + aliases: [] + typed_collector: false + mandatory: false + gcp_sql_backups: + cai_asset_type: sqladmin.googleapis.com/Backup + cai_asset_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + gcp_sql_databases: + cai_asset_type: sqladmin.googleapis.com/Databas + cai_asset_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + gcp_sql_instances: + cai_asset_type: sqladmin.googleapis.com/Instance + cai_asset_type_source: override + category: sql + aliases: [] + typed_collector: false + mandatory: false + gcp_sql_ssl_certs: + cai_asset_type: sqladmin.googleapis.com/SslCert + cai_asset_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + gcp_sql_users: + cai_asset_type: sqladmin.googleapis.com/User + cai_asset_type_source: heuristic + category: sql + aliases: [] + typed_collector: false + mandatory: false + gcp_storage_bucket_objects: + cai_asset_type: storage.googleapis.com/BucketObject + cai_asset_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + gcp_storage_bucket_policies: + cai_asset_type: storage.googleapis.com/BucketPolicy + cai_asset_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + gcp_storage_bucket_tag_bindings: + cai_asset_type: storage.googleapis.com/BucketTagBinding + cai_asset_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + gcp_storage_buckets: + cai_asset_type: storage.googleapis.com/Bucket + cai_asset_type_source: override + category: storage + aliases: [] + typed_collector: true + mandatory: false + gcp_storage_hmac_keys: + cai_asset_type: storage.googleapis.com/HmacKey + cai_asset_type_source: heuristic + category: storage + aliases: [] + typed_collector: false + mandatory: false + gcp_storagetransfer_agent_pools: + cai_asset_type: storagetransfer.googleapis.com/AgentPool + cai_asset_type_source: heuristic + category: storagetransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_storagetransfer_transfer_jobs: + cai_asset_type: storagetransfer.googleapis.com/TransferJob + cai_asset_type_source: heuristic + category: storagetransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_storagetransfer_transfer_operations: + cai_asset_type: storagetransfer.googleapis.com/TransferOperation + cai_asset_type_source: heuristic + category: storagetransfer + aliases: [] + typed_collector: false + mandatory: false + gcp_translate_glossaries: + cai_asset_type: translate.googleapis.com/Glossary + cai_asset_type_source: heuristic + category: translate + aliases: [] + typed_collector: false + mandatory: false + gcp_videotranscoder_job_templates: + cai_asset_type: transcoder.googleapis.com/JobTemplate + cai_asset_type_source: heuristic + category: videotranscoder + aliases: [] + typed_collector: false + mandatory: false + gcp_videotranscoder_jobs: + cai_asset_type: transcoder.googleapis.com/Job + cai_asset_type_source: heuristic + category: videotranscoder + aliases: [] + typed_collector: false + mandatory: false + gcp_vision_product_reference_images: + cai_asset_type: vision.googleapis.com/ProductReferenceImage + cai_asset_type_source: heuristic + category: vision + aliases: [] + typed_collector: false + mandatory: false + gcp_vision_products: + cai_asset_type: vision.googleapis.com/Product + cai_asset_type_source: heuristic + category: vision + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_groups: + cai_asset_type: vmmigration.googleapis.com/Group + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_source_datacenter_connectors: + cai_asset_type: vmmigration.googleapis.com/SourceDatacenterConnector + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_source_migrating_vm_clone_jobs: + cai_asset_type: vmmigration.googleapis.com/SourceMigratingVmCloneJob + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_source_migrating_vm_cutover_jobs: + cai_asset_type: vmmigration.googleapis.com/SourceMigratingVmCutoverJob + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_source_migrating_vms: + cai_asset_type: vmmigration.googleapis.com/SourceMigratingVm + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_source_utilization_reports: + cai_asset_type: vmmigration.googleapis.com/SourceUtilizationReport + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_sources: + cai_asset_type: vmmigration.googleapis.com/Source + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vmmigration_target_projects: + cai_asset_type: vmmigration.googleapis.com/TargetProject + cai_asset_type_source: heuristic + category: vmmigration + aliases: [] + typed_collector: false + mandatory: false + gcp_vpcaccess_connectors: + cai_asset_type: vpcaccess.googleapis.com/Connector + cai_asset_type_source: heuristic + category: vpcaccess + aliases: [] + typed_collector: false + mandatory: false + gcp_vpcaccess_locations: + cai_asset_type: vpcaccess.googleapis.com/Location + cai_asset_type_source: heuristic + category: vpcaccess + aliases: [] + typed_collector: false + mandatory: false + gcp_websecurityscanner_scan_config_scan_run_crawled_urls: + cai_asset_type: websecurityscanner.googleapis.com/ScanConfigScanRunCrawledUrl + cai_asset_type_source: heuristic + category: websecurityscanner + aliases: [] + typed_collector: false + mandatory: false + gcp_websecurityscanner_scan_config_scan_run_findings: + cai_asset_type: websecurityscanner.googleapis.com/ScanConfigScanRunFinding + cai_asset_type_source: heuristic + category: websecurityscanner + aliases: [] + typed_collector: false + mandatory: false + gcp_websecurityscanner_scan_config_scan_runs: + cai_asset_type: websecurityscanner.googleapis.com/ScanConfigScanRun + cai_asset_type_source: heuristic + category: websecurityscanner + aliases: [] + typed_collector: false + mandatory: false + gcp_websecurityscanner_scan_configs: + cai_asset_type: websecurityscanner.googleapis.com/ScanConfig + cai_asset_type_source: heuristic + category: websecurityscanner + aliases: [] + typed_collector: false + mandatory: false + gcp_workflows_workflows: + cai_asset_type: workflows.googleapis.com/Workflow + cai_asset_type_source: heuristic + category: workflows + aliases: [] + typed_collector: false + mandatory: false diff --git a/src/indexers/gcpapi.py b/src/indexers/gcpapi.py new file mode 100644 index 000000000..5dd5f4a0a --- /dev/null +++ b/src/indexers/gcpapi.py @@ -0,0 +1,533 @@ +""" +Native GCP SDK indexer. + +Replaces the CloudQuery-based GCP path with direct Cloud Asset Inventory (CAI) +and ``google-cloud-*`` calls while keeping the registry output compatible: each +resource is normalized into the same flat dict shape +``GCPPlatformHandler.parse_resource_data`` already accepts, and writes flow +through the :class:`ResourceWriter` seam. + +Discovery model (simpler than Azure - GCP level-of-detail is per-project, with +no resource-group dimension): + +* The configured ``projects`` are the discovery scope. Each project's LOD comes + from ``projectLevelOfDetails[]`` (falling back to the workspace default); + projects whose effective LOD is ``NONE`` are skipped entirely (selective + discovery), keeping the resource store focused on what gen rules need. +* The ``project`` resource is the mandatory anchor every other GCP resource + links to; it is synthesized directly from config and written first. +* The per-service typed SDK collectors are the supported FUNCTIONAL baseline: + a typed tier (compute instances/disks/snapshots/networks/subnetworks/ + firewalls/addresses, storage buckets, GKE clusters, Pub/Sub topics & + subscriptions, IAM service accounts) uses ``google-cloud-*`` SDKs for rich + payloads and needs only the relevant per-service viewer roles. These run + whether or not Cloud Asset Inventory is available. +* Cloud Asset Inventory is an OPTIONAL accelerator that broadens coverage to + resource types lacking a typed collector: a single ``list_assets`` call per + project (scoped to the CAI asset types referenced by generation rules) + returns full-payload assets, which we route by ``asset_type`` back to the + registry-mapped ``resource_type_name``. The typed asset types are excluded + from the CAI pass so a resource is never written twice. CAI is NOT required; + if it is not enabled / not permitted, discovery proceeds on the typed + collectors and the absence is logged informationally (never as an error). + +Coexists with the CloudQuery indexer behind the ``GCP_INDEXER_BACKEND`` setting: + +* ``"cloudquery"`` (default): this indexer is a no-op; CloudQuery handles GCP. +* ``"gcpapi"``: this indexer discovers GCP resources and the + CloudQuery indexer skips the GCP block. + +Component name: ``gcpapi``. Stage: ``INDEXER``. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +from component import Context, Setting, SettingDependency +from enrichers.generation_rule_types import ( + PLATFORM_HANDLERS_PROPERTY_NAME, + LevelOfDetail, + PlatformHandler, +) +from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY +from exceptions import WorkspaceBuilderException +from resources import ResourceTypeSpec + +from .common import CLOUD_CONFIG_SETTING +from .gcp_common import ( + gcp_get_credentials_and_projects, + gcp_has_discovery_config, + get_gcp_auth_type, + has_excluded_tags, + has_included_tags, +) +from .gcpapi_normalizers import ( + make_project_resource_data, + normalize_gcp_asset, + normalize_gcp_sdk_model, +) +from .gcpapi_resource_types import ( + PROJECTS_TABLE, + collect_assets_for_project, + find_spec, + find_spec_by_cai_type, +) +from .resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_PATH_SETTING, + get_resource_writer, +) + +logger = logging.getLogger(__name__) + +GCP_PLATFORM = "gcp" + +# Stable, grep-able marker emitted (at INFO) when the OPTIONAL Cloud Asset +# Inventory generic pass is not accessible (e.g. the API is not enabled or the +# service account lacks the CAI viewer role). It is purely informational: CAI is +# an accelerator that broadens coverage, NOT a requirement, so its absence must +# never fail discovery or CI. Operators can still grep for the token to confirm +# whether the CAI pass ran. +CAI_PERMISSION_DENIED_TOKEN = "GCP_CAI_PERMISSION_DENIED" + +DOCUMENTATION = "Index GCP resources using Cloud Asset Inventory and the google-cloud SDKs" + +# --------------------------------------------------------------------------- +# Settings +# --------------------------------------------------------------------------- + +GCP_INDEXER_BACKEND_SETTING = Setting( + "GCP_INDEXER_BACKEND", + "gcpIndexerBackend", + Setting.Type.STRING, + "Selects the backend used to discover GCP resources. " + "'cloudquery' (default) uses the legacy CloudQuery-based path; " + "'gcpapi' uses the native Cloud Asset Inventory + google-cloud-* indexer.", + "cloudquery", +) + +SETTINGS = ( + SettingDependency(CLOUD_CONFIG_SETTING, False), + SettingDependency(GCP_INDEXER_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_PATH_SETTING, False), +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _resolve_default_lod(context: Context, platform_cfg: dict[str, Any]) -> LevelOfDetail: + raw = ( + platform_cfg.get("defaultLOD") + or platform_cfg.get("defaultLevelOfDetail") + or context.get_setting("DEFAULT_LOD") + ) + if raw is None: + return LevelOfDetail.BASIC + try: + return LevelOfDetail.construct_from_config(raw) + except Exception: + return LevelOfDetail.BASIC + + +def _project_lod( + platform_cfg: dict[str, Any], + project_id: str, + default_lod: LevelOfDetail, +) -> LevelOfDetail: + """Effective LOD for ``project_id``. Mirrors the resolution chain in + ``GCPPlatformHandler.parse_resource_data`` (per-project override -> + workspace default).""" + cfg = platform_cfg.get("projectLevelOfDetails", {}) or {} + raw = cfg.get(project_id) + if raw is None: + return default_lod + try: + return LevelOfDetail.construct_from_config(raw) + except Exception: + return default_lod + + +def _accessed_gcp_type_names(context: Context) -> set[str]: + """Return the set of GCP resource-type names referenced by loaded + generation rules. Both the registry name and the CloudQuery table name are + accepted as valid spec values; the result mixes them.""" + all_specs: Optional[dict[str, dict[ResourceTypeSpec, Any]]] = context.get_property( + RESOURCE_TYPE_SPECS_PROPERTY + ) + if not all_specs: + return set() + gcp_specs = all_specs.get(GCP_PLATFORM, {}) + return {spec.resource_type_name for spec in gcp_specs.keys()} + + +def _resolve_platform_handler(context: Context) -> PlatformHandler: + handlers: Optional[dict[str, PlatformHandler]] = context.get_property( + PLATFORM_HANDLERS_PROPERTY_NAME + ) + if handlers and GCP_PLATFORM in handlers: + return handlers[GCP_PLATFORM] + from enrichers.gcp import GCPPlatformHandler + + return GCPPlatformHandler() + + +def _is_permission_denied(exc: Exception) -> bool: + """Best-effort detection of a GCP 403 / PermissionDenied without importing + the google SDK exception types at module scope (keeps this module importable + where the SDK is absent). Covers both the gRPC ``PermissionDenied`` and the + REST ``Forbidden`` shapes, plus a string fallback.""" + if type(exc).__name__ in ("PermissionDenied", "Forbidden"): + return True + code = getattr(exc, "code", None) + # google.api_core PermissionDenied exposes code==403; grpc status objects + # expose a callable .code(). Handle both without hard-depending on either. + try: + if callable(code): + code = code() + except Exception: # pragma: no cover - defensive + code = None + if code == 403 or getattr(code, "value", None) == 403: + return True + if str(getattr(code, "name", "")).upper() == "PERMISSION_DENIED": + return True + text = str(exc).lower() + return "403" in text and "permission" in text + + +def _note_cai_unavailable(context: Context, project_id: str, exc: Exception) -> None: + """Note, informationally, that the OPTIONAL Cloud Asset Inventory generic + pass was not accessible for this project. + + CAI is an accelerator that broadens coverage to resource types without a + typed collector; it is NOT required for native GCP discovery. The per-service + typed SDK collectors are the supported functional path and run independently + of CAI, so a 403 / disabled-API here is normal and non-fatal. This is logged + at INFO (no error, no banner, no warning) so CAI's absence never reads as a + failure or fails CI.""" + message = ( + f"{CAI_PERMISSION_DENIED_TOKEN}: Cloud Asset Inventory was not accessible " + f"for GCP project '{project_id}' ({exc}). This is informational, not an " + f"error: CAI is an OPTIONAL accelerator that broadens coverage to resource " + f"types lacking a typed collector. The per-service typed SDK collectors " + f"(compute, storage, GKE, Pub/Sub, IAM, ...) are the functional discovery " + f"path and continue to run normally. Enabling the Cloud Asset Inventory API " + f"with a CAI viewer role is optional and only increases coverage breadth; " + f"it is not required, so no action is needed." + ) + logger.info(message) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def index(context: Context) -> None: + backend = context.get_setting(GCP_INDEXER_BACKEND_SETTING) + if backend != "gcpapi": + logger.info( + f"GCP indexer backend: '{backend}' (gcpIndexerBackend in " + f"workspaceInfo.yaml). Native gcpapi indexer is a no-op; the " + f"CloudQuery indexer will handle GCP." + ) + return + + cloud_config = context.get_setting(CLOUD_CONFIG_SETTING) or {} + platform_cfg = cloud_config.get(GCP_PLATFORM) + if not platform_cfg: + logger.info( + "GCP indexer backend: 'gcpapi' selected, but no 'gcp' section in " + "cloudConfig; nothing to discover." + ) + return + + if not gcp_has_discovery_config(platform_cfg): + logger.info( + "GCP indexer backend: 'gcpapi' selected, but cloudConfig.gcp has no " + "projects to discover (set 'projects' or 'projectId'). Skipping." + ) + return + + logger.info( + "GCP indexer backend: 'gcpapi' (Cloud Asset Inventory + google-cloud-* " + "SDK). Starting GCP resource discovery." + ) + + creds_info = gcp_get_credentials_and_projects(platform_cfg) + credentials = creds_info["credentials"] + project_ids: list[str] = creds_info["project_ids"] + for key, value in (creds_info.get("env") or {}).items(): + os.environ[key] = value + + if not project_ids: + logger.warning("GCP indexer: no project IDs resolved; nothing to discover.") + return + + # Mirror cloudquery.py so enrichers.gcp (get_project_name etc.) and any other + # downstream code sees the same active project/auth we resolved here. + auth_type, auth_secret = get_gcp_auth_type(platform_cfg) + try: + from enrichers.gcp import set_gcp_credentials + + set_gcp_credentials( + project_id=creds_info.get("quota_project"), + service_account_key=platform_cfg.get("serviceAccountKey"), + auth_type=auth_type, + auth_secret=auth_secret, + ) + except Exception as e: + logger.warning(f"Could not update enrichers.gcp credentials: {e}") + + accessed_names = _accessed_gcp_type_names(context) + logger.info( + f"GCP resource types referenced by generation rules: " + f"{sorted(accessed_names) if accessed_names else '(none)'}" + ) + + # Resolve accessed gen-rule names -> specs. Split into the typed (SDK) + # tier and the CAI generic tier. Typed asset types are excluded from the + # CAI filter so each resource is written exactly once. + typed_specs_to_collect = [] + seen_typed: set[str] = set() + generic_cai_types: dict[str, Any] = {} # lower(cai_type) -> spec + typed_cai_types: set[str] = set() + + for accessed in accessed_names: + spec = find_spec(accessed) + if spec is None: + warning = ( + f'GCP indexer: gen-rule references unknown GCP resource type ' + f'"{accessed}". Verify the name or add it to ' + f'scripts/gcp/gcp_resource_type_overrides.yaml and rerun the ' + f'sync script.' + ) + logger.warning(warning) + context.add_warning(warning) + continue + if spec.cloudquery_table_name == PROJECTS_TABLE: + # The project anchor is always emitted below; no extra pass needed. + continue + if spec.collector is not None: + if spec.cloudquery_table_name not in seen_typed: + typed_specs_to_collect.append(spec) + seen_typed.add(spec.cloudquery_table_name) + if spec.cai_asset_type: + typed_cai_types.add(spec.cai_asset_type.lower()) + elif spec.cai_asset_type: + generic_cai_types[spec.cai_asset_type.lower()] = spec + + platform_handler = _resolve_platform_handler(context) + include_tags = platform_cfg.get("includeTags", {}) + exclude_tags = platform_cfg.get("excludeTags", {}) + writer = get_resource_writer(context) + default_lod = _resolve_default_lod(context, platform_cfg) + + # Determine which projects are in scope (LOD != NONE). + in_scope_projects: list[str] = [] + for pid in project_ids: + lod = _project_lod(platform_cfg, pid, default_lod) + if lod is LevelOfDetail.NONE: + logger.info(f"GCP project '{pid}': effective LOD is NONE; skipping.") + continue + in_scope_projects.append(pid) + logger.info(f"GCP project '{pid}': in scope (LOD={lod}).") + + stats = { + "discovered": 0, + "added": 0, + "added_projects": 0, + "added_typed": 0, + "added_generic": 0, + "generic_unmatched_cai_type": 0, + "skipped_tag_filter": 0, + "skipped_parse_error": 0, + "skipped_collector_error": 0, + "cai_permission_denied": 0, + } + + def _process(spec, project_id, resource_data, *, source: str) -> None: + stats["discovered"] += 1 + if exclude_tags and has_excluded_tags(resource_data, exclude_tags): + stats["skipped_tag_filter"] += 1 + return + if include_tags and not has_included_tags(resource_data, include_tags): + stats["skipped_tag_filter"] += 1 + return + try: + resource_name, qualified_name, resource_attributes = ( + platform_handler.parse_resource_data( + resource_data, spec.resource_type_name, platform_cfg, context + ) + ) + except WorkspaceBuilderException as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data rejected {spec.resource_type_name} in " + f"project {project_id}: {e}" + ) + return + except (KeyError, ValueError, TypeError, AttributeError) as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"parse_resource_data raised {type(e).__name__} for " + f"{spec.resource_type_name} in project {project_id}: {e}" + ) + return + + resource_attributes["resource"] = resource_data + resource_attributes["auth_type"] = auth_type + resource_attributes["auth_secret"] = auth_secret + + writer.add_resource( + GCP_PLATFORM, + spec.resource_type_name, + resource_name, + qualified_name, + resource_attributes, + ) + stats["added"] += 1 + if source == "project": + stats["added_projects"] += 1 + elif source == "typed": + stats["added_typed"] += 1 + else: + stats["added_generic"] += 1 + + # Phase 0: emit the project anchors first so child resources can link to + # their parent project at parse time (the handler does an immediate + # registry lookup). + project_spec = find_spec(PROJECTS_TABLE) + for pid in in_scope_projects: + _process( + project_spec, + pid, + make_project_resource_data(pid), + source="project", + ) + + if not accessed_names: + writer.finalize() + logger.info( + f"GCP indexing complete (anchors only): " + f"added_projects={stats['added_projects']}." + ) + return + + # Phase 1: typed (SDK) collectors per project. + for pid in in_scope_projects: + for spec in typed_specs_to_collect: + try: + models = list(spec.collector(credentials, pid)) + except Exception as e: + stats["skipped_collector_error"] += 1 + logger.error( + f"Failed to collect {spec.resource_type_name} in project " + f"{pid}: {e}" + ) + context.add_warning( + f"Failed to collect GCP {spec.resource_type_name} in " + f"project {pid}: {e}" + ) + continue + logger.info( + f"Collected {len(models)} {spec.resource_type_name} (typed) " + f"from project {pid}" + ) + for model in models: + try: + resource_data = normalize_gcp_sdk_model( + model, project_id=pid, resource_type_name=spec.resource_type_name + ) + except Exception as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"Failed to normalize GCP {spec.resource_type_name} " + f"model in project {pid}: {e}" + ) + continue + _process(spec, pid, resource_data, source="typed") + + # Phase 2: Cloud Asset Inventory generic pass per project, scoped to the + # CAI asset types referenced by gen rules (minus the typed ones). + if generic_cai_types: + cai_filter = sorted( + { + spec.cai_asset_type + for spec in generic_cai_types.values() + if spec.cai_asset_type + } + ) + for pid in in_scope_projects: + try: + assets = list(collect_assets_for_project(credentials, pid, cai_filter)) + except Exception as e: + if _is_permission_denied(e): + # Optional accelerator unavailable: informational, not an error. + stats["cai_permission_denied"] += 1 + _note_cai_unavailable(context, pid, e) + else: + stats["skipped_collector_error"] += 1 + logger.error( + f"Cloud Asset Inventory list_assets failed for project " + f"{pid}: {e}" + ) + context.add_warning( + f"Failed to list GCP assets in project {pid}: {e}" + ) + continue + logger.info( + f"Collected {len(assets)} assets (Cloud Asset Inventory) from " + f"project {pid}" + ) + for asset in assets: + asset_type = ( + getattr(asset, "asset_type", None) + or (asset.get("asset_type") if isinstance(asset, dict) else None) + ) + spec = find_spec_by_cai_type(asset_type) + if spec is None or ( + spec.cai_asset_type + and spec.cai_asset_type.lower() not in generic_cai_types + ): + stats["generic_unmatched_cai_type"] += 1 + continue + try: + resource_data = normalize_gcp_asset( + asset, project_id=pid, resource_type_name=spec.resource_type_name + ) + except Exception as e: + stats["skipped_parse_error"] += 1 + logger.warning( + f"Failed to normalize CAI asset (type={asset_type}) in " + f"project {pid}: {e}" + ) + continue + _process(spec, pid, resource_data, source="generic") + + writer.finalize() + + logger.info( + f"GCP indexing complete: " + f"discovered={stats['discovered']}, added={stats['added']} " + f"(projects={stats['added_projects']}, typed={stats['added_typed']}, " + f"generic={stats['added_generic']}), " + f"generic_unmatched_cai_type={stats['generic_unmatched_cai_type']}, " + f"skipped_tag_filter={stats['skipped_tag_filter']}, " + f"skipped_parse_error={stats['skipped_parse_error']}, " + f"skipped_collector_error={stats['skipped_collector_error']}, " + f"cai_permission_denied={stats['cai_permission_denied']}" + ) + + if stats["cai_permission_denied"]: + logger.info( + "GCP discovery completed via the typed SDK collectors; the OPTIONAL " + "Cloud Asset Inventory accelerator was not accessible this run (see %s " + "above). This is expected when CAI is not enabled and does not indicate " + "a failure -- the typed collectors are the functional discovery path.", + CAI_PERMISSION_DENIED_TOKEN, + ) diff --git a/src/indexers/gcpapi_normalizers.py b/src/indexers/gcpapi_normalizers.py new file mode 100644 index 000000000..dcc591d9a --- /dev/null +++ b/src/indexers/gcpapi_normalizers.py @@ -0,0 +1,290 @@ +""" +Normalize Cloud Asset Inventory (CAI) assets and ``google-cloud-*`` SDK models +into the flat dict shape that ``GCPPlatformHandler.parse_resource_data`` already +understands. + +Goal: produce a ``resource_data`` dict behaviourally equivalent to a row from +the legacy CloudQuery SQLite intermediate. ``parse_resource_data`` relies on a +small handful of fields: + +* ``project_id`` - the owning project; the parser links every resource + to its parent ``project`` resource via this. +* ``name`` - short resource name (last path segment). +* ``id`` - full resource path; used as a name fallback. +* ``zone`` / ``region`` / ``location`` - placement; the parser derives region + from zone when needed. +* ``tags`` - dict, defaults to ``{}``. GCP user ``labels`` are + copied here so cross-cloud include/exclude tag + matchers work unchanged. + +Everything else (the full API representation) is passed through at the top +level of the dict so generation-rule path matching keeps working, and the +original ``labels`` map is preserved for the ``gcp-tags.yaml`` template which +matches ``match_resource.labels``. + +This module must not import any ``google-*`` packages: it operates on plain +dicts and duck-typed objects so the test suite runs without the GCP SDK. +""" + +from __future__ import annotations + +import datetime +import enum +import logging +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +def _sanitize(value: Any) -> Any: + """Recursively convert SDK values into YAML-friendly primitives. + + Mirrors ``azureapi_normalizers._sanitize``: datetimes / enums become + strings so PyYAML's safe dumper (used by the resource store) is happy and + the output matches what CloudQuery serialized. + """ + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, datetime.date): + return value.isoformat() + if isinstance(value, enum.Enum): + return value.value if isinstance(value.value, (str, int, float, bool)) else str(value) + if isinstance(value, dict): + return {k: _sanitize(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_sanitize(v) for v in value] + return value + + +def _to_plain_dict(obj: Any) -> dict[str, Any]: + """Best-effort conversion of an SDK message / asset into a plain dict. + + Handles, in order: an already-plain dict; proto-plus messages + (``type(msg).to_dict(msg)``); objects exposing ``as_dict()`` / + ``to_dict()``; finally a shallow ``vars()`` fallback. + """ + if isinstance(obj, dict): + return dict(obj) + + # proto-plus message: the message *class* exposes a static to_dict. + to_dict_cls = getattr(type(obj), "to_dict", None) + if callable(to_dict_cls): + try: + return to_dict_cls(obj) + except Exception: # pragma: no cover - defensive + pass + + for attr in ("as_dict", "to_dict"): + fn = getattr(obj, attr, None) + if callable(fn): + try: + return fn() + except Exception: # pragma: no cover - defensive + continue + + if hasattr(obj, "__dict__"): + return {k: v for k, v in vars(obj).items() if not k.startswith("_")} + + raise TypeError(f"Cannot convert GCP object of type {type(obj).__name__} to dict") + + +def _short_name(full_name: Optional[str]) -> Optional[str]: + """Return the last path segment of a GCP resource name/self-link.""" + if not full_name: + return None + # CAI names look like //compute.googleapis.com/projects/p/zones/z/instances/i + return full_name.rstrip("/").split("/")[-1] or None + + +def _project_from_name(full_name: Optional[str]) -> Optional[str]: + """Extract the project id from a ``.../projects//...`` path.""" + if not full_name: + return None + parts = full_name.split("/") + for i, part in enumerate(parts): + if part == "projects" and i + 1 < len(parts): + return parts[i + 1] + return None + + +def _region_from_location(location: Optional[str]) -> tuple[Optional[str], Optional[str]]: + """Split a CAI ``location`` into ``(zone, region)`` heuristically. + + GCP locations are zones (``us-central1-a``), regions (``us-central1``), + multi-regions (``us``/``eu``), or ``global``. We treat a 3-segment value as + a zone and a 2-segment value as a region; everything else is left as a + bare location for the handler to place. + """ + if not location: + return None, None + segments = location.split("-") + if len(segments) >= 3: + return location, "-".join(segments[:-1]) + if len(segments) == 2: + return None, location + return None, None + + +def normalize_gcp_asset( + asset: Any, + *, + project_id: Optional[str] = None, + resource_type_name: str = "", +) -> dict[str, Any]: + """Convert a Cloud Asset Inventory asset into a ``resource_data`` dict. + + ``asset`` is a ``google.cloud.asset_v1.Asset`` (or an equivalent dict in + tests). With ``content_type=RESOURCE`` the asset carries the full API + representation under ``resource.data``; we hoist that to the top level so + generation-rule path matching sees the same fields CloudQuery exposed. + """ + raw = _to_plain_dict(asset) + raw = _sanitize(raw) + + asset_name = raw.get("name") # //service.googleapis.com/projects/.../ + asset_type = raw.get("asset_type") or raw.get("assetType") + + resource_blob = raw.get("resource") or {} + if isinstance(resource_blob, dict): + data = resource_blob.get("data") or {} + location = resource_blob.get("location") + self_link = resource_blob.get("discovery_name") and None # placeholder, ignored + self_link = resource_blob.get("self_link") or resource_blob.get("selfLink") + else: + data = {} + location = None + self_link = None + + if not isinstance(data, dict): + data = {} + + # Start from the full API payload so path-matching rules work, then stamp + # the well-known fields the handler reads. + out: dict[str, Any] = dict(data) + + out["asset_type"] = asset_type + if self_link or data.get("selfLink"): + out["self_link"] = self_link or data.get("selfLink") + + # name: prefer the API payload's name, else the last segment of the asset name. + name = data.get("name") or _short_name(asset_name) + # Some GCP API payloads put a full path in ``name``; collapse to the leaf. + if isinstance(name, str) and "/" in name: + name = _short_name(name) + if name: + out["name"] = name + + # id: the full asset path is the most stable identifier. + out["id"] = asset_name or data.get("id") or data.get("selfLink") + + # project_id: caller-supplied wins; else parse from the asset path. + pid = project_id or _project_from_name(asset_name) or data.get("project") + if pid: + out["project_id"] = pid + + # placement: zone/region from the payload, else derived from CAI location. + zone = data.get("zone") + region = data.get("region") + if isinstance(zone, str) and "/" in zone: + zone = _short_name(zone) + if isinstance(region, str) and "/" in region: + region = _short_name(region) + if not zone and not region: + zone, region = _region_from_location(location) + if zone: + out["zone"] = zone + if region: + out["region"] = region + elif location: + out["location"] = location + + # labels -> tags so cross-cloud include/exclude matchers work; keep labels + # too for the gcp-tags.yaml template. + labels = data.get("labels") + if not isinstance(labels, dict): + labels = {} + out["labels"] = labels + out["tags"] = dict(labels) + + if not out.get("id") and not out.get("name"): + logger.warning( + f"CAI asset for resource_type_name={resource_type_name!r} " + f"(asset_type={asset_type!r}) has neither id nor name; " + f"parse_resource_data will likely fail to link it." + ) + return out + + +def normalize_gcp_sdk_model( + model: Any, + *, + project_id: str, + resource_type_name: str = "", + location: Optional[str] = None, +) -> dict[str, Any]: + """Convert a typed ``google-cloud-*`` SDK model into a ``resource_data`` dict. + + Used by the typed-collector pass for rich-payload resources (compute + instances, GKE clusters, ...). The model is flattened to a dict and the + handler-read fields are stamped, keeping the output shape identical to the + CAI path so both passes are interchangeable. + """ + raw = _to_plain_dict(model) + raw = _sanitize(raw) + + out: dict[str, Any] = dict(raw) + out["project_id"] = project_id + + name = raw.get("name") or raw.get("display_name") or raw.get("displayName") + if isinstance(name, str) and "/" in name: + name = _short_name(name) + if name: + out["name"] = name + + if not out.get("id"): + out["id"] = raw.get("self_link") or raw.get("selfLink") or raw.get("id") or name + + zone = raw.get("zone") + region = raw.get("region") + loc = location or raw.get("location") + if isinstance(zone, str) and "/" in zone: + zone = _short_name(zone) + if isinstance(region, str) and "/" in region: + region = _short_name(region) + if not zone and not region: + zone, region = _region_from_location(loc) + if zone: + out["zone"] = zone + if region: + out["region"] = region + elif loc: + out["location"] = loc + + labels = raw.get("labels") + if not isinstance(labels, dict): + labels = {} + out["labels"] = labels + out["tags"] = dict(labels) + + return out + + +def make_project_resource_data(project_id: str, display_name: Optional[str] = None) -> dict[str, Any]: + """Build the ``resource_data`` dict for a synthesized ``project`` resource. + + ``gcp_projects`` is the mandatory anchor every other GCP resource links to. + We can materialize it from the configured project id alone (no API call): + ``parse_resource_data`` only requires ``project_id`` for the ``project`` + type and uses it as both the name and qualified name. + """ + data: dict[str, Any] = { + "project_id": project_id, + "name": project_id, + "id": f"//cloudresourcemanager.googleapis.com/projects/{project_id}", + "asset_type": "cloudresourcemanager.googleapis.com/Project", + "tags": {}, + "labels": {}, + } + if display_name: + data["display_name"] = display_name + return data diff --git a/src/indexers/gcpapi_resource_types.py b/src/indexers/gcpapi_resource_types.py new file mode 100644 index 000000000..f3a456d63 --- /dev/null +++ b/src/indexers/gcpapi_resource_types.py @@ -0,0 +1,333 @@ +""" +GCP resource-type specs for the native GCP SDK indexer. + +Unlike Azure - where the generic ``resources.list()`` pass returns only a +sparse envelope - GCP's Cloud Asset Inventory (CAI) ``list_assets`` / +``search_all_resources`` with ``content_type=RESOURCE`` returns the **full** +API representation of each asset. CAI is therefore the parity workhorse: a +single call per project covers every registry type that has a CAI asset type, +with rich payloads. Typed ``google-cloud-*`` collectors are a thin enrichment +layer for a handful of high-value resources (and the synthesized ``project`` +anchor). + +Each :class:`GcpResourceTypeSpec` describes one collectable type. Specs are +materialized from the registry in ``gcp_resource_type_registry.yaml`` (see +:mod:`gcp_resource_type_registry`). To enable a richer typed collector for a +type: + +* Make sure the table is in the registry (regenerate via + ``scripts/gcp/sync_gcp_resource_type_registry.py`` if needed, and flag it as + a ``typed_collector`` in the overrides YAML). +* Implement ``_collect_`` below and register it in + :data:`_TYPED_COLLECTORS` keyed by canonical CloudQuery table name. + +The public surface (``GCP_RESOURCE_TYPE_SPECS``, ``GcpResourceTypeSpec``, +``find_spec``, ``find_spec_by_cai_type``) is what ``indexers.gcpapi`` consumes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Optional + +from .gcp_resource_type_registry import GcpResourceTypeEntry, load_registry + +# A typed collector takes (credentials, project_id) and yields SDK models. +GcpCollector = Callable[[Any, str], Iterable[Any]] + +# The canonical table name for the synthesized project anchor. +PROJECTS_TABLE = "gcp_projects" + + +@dataclass(frozen=True) +class GcpResourceTypeSpec: + resource_type_name: str + cloudquery_table_name: str + cai_asset_type: Optional[str] + mandatory: bool + # ``True`` when a hand-written google-cloud-* collector ships for this type + # (rich/extra payload, or the synthesized project anchor). ``False`` means + # the type is materialized from the Cloud Asset Inventory generic pass. + typed: bool = False + # Hand-written collector callable; ``None`` for generic specs and for the + # synthesized ``project`` anchor (handled directly by the orchestrator). + collector: Optional[GcpCollector] = None + + +# --------------------------------------------------------------------------- +# Typed collectors (rich-payload enrichment tier) +# --------------------------------------------------------------------------- +# +# All google-cloud-* imports are lazy so this module stays importable - and the +# test suite stays runnable - without the GCP SDK installed. + +def _collect_compute_instances(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + client = compute_v1.InstancesClient(credentials=credentials) + # aggregated_list returns (zone, scoped_list) pairs across all zones. + for _zone, scoped in client.aggregated_list(project=project_id): + for instance in getattr(scoped, "instances", None) or []: + yield instance + + +def _collect_storage_buckets(credentials, project_id): + from google.cloud import storage # noqa: WPS433 + + client = storage.Client(project=project_id, credentials=credentials) + return client.list_buckets() + + +def _collect_container_clusters(credentials, project_id): + from google.cloud import container_v1 # noqa: WPS433 + + client = container_v1.ClusterManagerClient(credentials=credentials) + parent = f"projects/{project_id}/locations/-" + response = client.list_clusters(parent=parent) + return getattr(response, "clusters", None) or [] + + +# ---- Tier 1 compute fallbacks (no new dependency; google-cloud-compute) ----- +# +# These mirror ``_collect_compute_instances``: aggregated_list yields +# ``(scope, scoped_list)`` pairs across every zone/region, where each +# ``scoped_list`` carries either a per-scope resource list or a ``warning`` +# entry (for scopes with nothing / no permission). We defensively read the +# resource attribute and skip warning-only scopes. The plain ``list`` calls +# cover global resources. Each call returns a pager that transparently handles +# pagination, so materializing it (``list(...)`` in the orchestrator) walks +# every page. + +def _collect_compute_disks(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + client = compute_v1.DisksClient(credentials=credentials) + for _zone, scoped in client.aggregated_list(project=project_id): + for disk in getattr(scoped, "disks", None) or []: + yield disk + + +def _collect_compute_snapshots(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + # Snapshots are a global resource: a flat list, no aggregation. + client = compute_v1.SnapshotsClient(credentials=credentials) + return client.list(project=project_id) + + +def _collect_compute_networks(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + # Networks (VPCs) are global. + client = compute_v1.NetworksClient(credentials=credentials) + return client.list(project=project_id) + + +def _collect_compute_subnetworks(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + client = compute_v1.SubnetworksClient(credentials=credentials) + for _region, scoped in client.aggregated_list(project=project_id): + for subnet in getattr(scoped, "subnetworks", None) or []: + yield subnet + + +def _collect_compute_firewalls(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + # Firewall rules are global. + client = compute_v1.FirewallsClient(credentials=credentials) + return client.list(project=project_id) + + +def _collect_compute_addresses(credentials, project_id): + from google.cloud import compute_v1 # noqa: WPS433 + + # Regional addresses; aggregated_list spans all regions. (Global/static + # IPs are a separate table, gcp_compute_global_addresses.) + client = compute_v1.AddressesClient(credentials=credentials) + for _region, scoped in client.aggregated_list(project=project_id): + for address in getattr(scoped, "addresses", None) or []: + yield address + + +# ---- Tier 2 service fallbacks (each adds one idiomatic google-cloud-* dep) --- + +def _collect_pubsub_topics(credentials, project_id): + from google.cloud import pubsub_v1 # noqa: WPS433 + + client = pubsub_v1.PublisherClient(credentials=credentials) + # list_topics returns a pager yielding Topic messages whose ``name`` is the + # full path projects/

/topics/; the normalizer collapses it to the leaf. + return client.list_topics(request={"project": f"projects/{project_id}"}) + + +def _collect_pubsub_subscriptions(credentials, project_id): + from google.cloud import pubsub_v1 # noqa: WPS433 + + client = pubsub_v1.SubscriberClient(credentials=credentials) + return client.list_subscriptions( + request={"project": f"projects/{project_id}"} + ) + + +def _collect_iam_service_accounts(credentials, project_id): + from google.cloud import iam_admin_v1 # noqa: WPS433 + + client = iam_admin_v1.IAMClient(credentials=credentials) + # The pager is iterable, yielding ServiceAccount messages (name is the full + # path projects/

/serviceAccounts/); pagination is transparent. + request = iam_admin_v1.types.ListServiceAccountsRequest( + name=f"projects/{project_id}" + ) + return client.list_service_accounts(request=request) + + +# Maps canonical CQ table name -> typed collector callable. Adding an entry here +# automatically (a) flips the spec's ``typed`` flag and (b) excludes the type's +# CAI asset type from the Cloud Asset Inventory generic filter in +# ``gcpapi.index`` (write-once) - so each type below is discovered via its SDK +# collector whether or not CAI is available. +_TYPED_COLLECTORS: dict[str, GcpCollector] = { + # Pre-existing rich-payload tier. + "gcp_compute_instances": _collect_compute_instances, + "gcp_storage_buckets": _collect_storage_buckets, + "gcp_container_clusters": _collect_container_clusters, + # Tier 1 - high-value compute types (no new dependency). + "gcp_compute_disks": _collect_compute_disks, + "gcp_compute_snapshots": _collect_compute_snapshots, + "gcp_compute_networks": _collect_compute_networks, + "gcp_compute_subnetworks": _collect_compute_subnetworks, + "gcp_compute_firewalls": _collect_compute_firewalls, + "gcp_compute_addresses": _collect_compute_addresses, + # Tier 2 - idiomatic single-call service clients. + "gcp_pubsub_topics": _collect_pubsub_topics, + "gcp_pubsub_subscriptions": _collect_pubsub_subscriptions, + "gcp_iam_service_accounts": _collect_iam_service_accounts, +} + + +# --------------------------------------------------------------------------- +# Cloud Asset Inventory generic collector (the catch-all parity pass) +# --------------------------------------------------------------------------- + +def collect_assets_for_project( + credentials, + project_id: str, + asset_types: Optional[list[str]] = None, +): + """List Cloud Asset Inventory assets for one project. + + Returns ``google.cloud.asset_v1.Asset`` objects with ``content_type=RESOURCE`` + so each asset carries the full API representation under ``resource.data``. + ``asset_types`` (CAI type strings, e.g. ``compute.googleapis.com/Instance``) + scopes the call to exactly what generation rules referenced; passing + ``None`` lists everything CAI tracks for the project. + """ + from google.cloud import asset_v1 # noqa: WPS433 + + client = asset_v1.AssetServiceClient(credentials=credentials) + request = { + "parent": f"projects/{project_id}", + "content_type": asset_v1.ContentType.RESOURCE, + } + if asset_types: + request["asset_types"] = list(asset_types) + return client.list_assets(request=request) + + +# --------------------------------------------------------------------------- +# Spec materialization +# --------------------------------------------------------------------------- + +def _legacy_resource_type_name(entry: GcpResourceTypeEntry) -> str: + """Pick the ``resource_type_name`` surfaced to generation rules. + + Historically RWL gen rules referenced a few GCP types by short legacy + names (``project``, ``compute_instance``) and others by the CloudQuery + table name. The registry encodes both via ``cloudquery_table_name`` + + ``aliases``; prefer the first alias (legacy short name) when present. + """ + if entry.aliases: + return entry.aliases[0] + return entry.cloudquery_table_name + + +def _make_spec(entry: GcpResourceTypeEntry) -> GcpResourceTypeSpec: + collector = _TYPED_COLLECTORS.get(entry.cloudquery_table_name) + return GcpResourceTypeSpec( + resource_type_name=_legacy_resource_type_name(entry), + cloudquery_table_name=entry.cloudquery_table_name, + cai_asset_type=entry.cai_asset_type, + mandatory=entry.mandatory, + typed=bool(entry.typed_collector or collector is not None), + collector=collector, + ) + + +def _build_specs() -> tuple[GcpResourceTypeSpec, ...]: + """Materialize one ``GcpResourceTypeSpec`` per registry entry.""" + registry = load_registry() + specs: list[GcpResourceTypeSpec] = [] + + # Project anchor first (mandatory bootstrap every other resource links to). + projects_entry = registry.find(PROJECTS_TABLE) + if projects_entry is not None: + specs.append(_make_spec(projects_entry)) + + for entry in registry: + if entry.cloudquery_table_name == PROJECTS_TABLE: + continue + specs.append(_make_spec(entry)) + + return tuple(specs) + + +GCP_RESOURCE_TYPE_SPECS: tuple[GcpResourceTypeSpec, ...] = _build_specs() + +# Convenience subset for callers that only want the typed (rich/SDK) tier. +GCP_TYPED_RESOURCE_TYPE_SPECS: tuple[GcpResourceTypeSpec, ...] = tuple( + s for s in GCP_RESOURCE_TYPE_SPECS if s.collector is not None +) + + +# --------------------------------------------------------------------------- +# Lookup +# --------------------------------------------------------------------------- + +def find_spec(name_or_table: str) -> Optional[GcpResourceTypeSpec]: + """Look up a GCP resource type spec by registry name, alias, or CQ table. + + Returns a spec for any name the registry knows about. Returning ``None`` + means "this name is not a registered GCP resource type at all". + """ + if not name_or_table: + return None + for spec in GCP_RESOURCE_TYPE_SPECS: + if name_or_table in (spec.resource_type_name, spec.cloudquery_table_name): + return spec + entry = load_registry().find(name_or_table) + if entry is None: + return None + for spec in GCP_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None + + +def find_spec_by_cai_type(cai_asset_type: Optional[str]) -> Optional[GcpResourceTypeSpec]: + """Look up the spec whose ``cai_asset_type`` matches this string. + + Used by the Cloud Asset Inventory generic pass in ``gcpapi.index`` to route + each ``asset.asset_type`` back to the spec that owns its + ``resource_type_name``. Case-insensitive. + """ + if not cai_asset_type: + return None + entry = load_registry().find_by_cai_type(cai_asset_type) + if entry is None: + return None + for spec in GCP_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name == entry.cloudquery_table_name: + return spec + return None diff --git a/src/indexers/resource_writer.py b/src/indexers/resource_writer.py new file mode 100644 index 000000000..bf1506938 --- /dev/null +++ b/src/indexers/resource_writer.py @@ -0,0 +1,227 @@ +""" +ResourceWriter - the indexer/registry seam. + +Today, indexers call ``Registry.add_resource(...)`` directly, which mutates the +in-memory ``Registry`` carried on the workspace builder ``Context``. That works +for the existing single-request, in-memory pipeline, but it tightly couples +every indexer to the in-memory registry implementation. + +The future direction for runwhen-local is a small local resource DB queried by +a fast read-only REST API (extending the workspace-builder FastAPI service). To make that swap a +plug-in instead of a rewrite, all *new* indexers funnel writes through the +``ResourceWriter`` protocol declared here. The legacy ``cloudquery`` indexer is +still wired directly to the registry (intentional - we don't want to perturb +the existing path during the Azure SDK migration); it will migrate in a +follow-up once the AWS / GCP indexers are also native. + +Shape of the contract +--------------------- + +``add_resource(platform, resource_type, name, qualified_name, attributes)`` is +the canonical write. ``attributes`` is the dict produced by the platform +handler's ``parse_resource_data`` (e.g. ``AzurePlatformHandler``) plus the +indexer-supplied keys ``resource``, ``auth_type``, ``auth_secret``. This is +exactly what ``cloudquery.py`` passes to ``registry.add_resource(...)`` today, +so the contract is fixed by current behaviour. + +``finalize()`` runs once after all resources have been written. For the +in-memory implementation, that's where deferred RG resolution happens. Future +DB-backed implementations would commit their transaction here. + +Implementations +--------------- + +* :class:`InMemoryRegistryWriter` - the default writer; delegates to + ``Registry.add_resource`` and runs ``resolve_deferred_azure_relationships`` + on ``finalize()``. + +* :class:`indexers.sqlite_resource_writer.SqliteResourceWriter` - dual writer + that composes the in-memory writer **and** snapshots the registry into a + local SQLite database on ``finalize()``. Selected via the + ``resourceStoreBackend`` setting. The DB lands in the workspace output via + the active ``Outputter`` so it works for both filesystem and tar runs. + +* ``RestApiResourceWriter`` - **future**. POST resources to a fast REST + service so we can decouple workspace builder from the storage layer + entirely. Same protocol, different transport. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Optional, Protocol, runtime_checkable + +from component import Setting + +if TYPE_CHECKING: + from component import Context + from enrichers.generation_rule_types import PlatformHandler + from resources import Registry, Resource + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Settings +# +# Declared at module scope so individual indexer SETTINGS tuples can reference +# them (the workspace builder picks up settings via component.SETTINGS, so a +# setting must be reachable from at least one active component to appear in +# the schema). +# --------------------------------------------------------------------------- + +RESOURCE_STORE_BACKEND_MEMORY = "memory" +RESOURCE_STORE_BACKEND_SQLITE = "sqlite" + +RESOURCE_STORE_BACKEND_SETTING = Setting( + "RESOURCE_STORE_BACKEND", + "resourceStoreBackend", + Setting.Type.STRING, + "Selects the backend used by ResourceWriter for indexers that go through " + "the writer seam. 'memory' (default) keeps the in-memory Registry only; " + "'sqlite' additionally snapshots the discovered resource graph into a " + "local SQLite database (path configurable via 'resourceStorePath') so " + "the future read-only REST service has something to query.", + RESOURCE_STORE_BACKEND_MEMORY, +) + +RESOURCE_STORE_PATH_SETTING = Setting( + "RESOURCE_STORE_PATH", + "resourceStorePath", + Setting.Type.STRING, + "Output path (relative to the workspace output directory) for the " + "SQLite resource store. Only used when 'resourceStoreBackend' is " + "'sqlite'.", + "resources.sqlite", +) + + +@runtime_checkable +class ResourceWriter(Protocol): + """Protocol implemented by every indexer storage backend. + + Indexers should NOT depend on the concrete writer; they receive an + instance via ``get_resource_writer(context)`` (see below) and call + ``add_resource`` / ``finalize``. + """ + + def add_resource( + self, + platform: str, + resource_type: str, + name: str, + qualified_name: str, + attributes: dict[str, Any], + ) -> "Resource": + ... + + def finalize(self) -> None: + ... + + +class InMemoryRegistryWriter: + """Default writer - delegates to the in-memory ``Registry`` on the context. + + This is the only writer wired into the workspace builder today. + + On ``finalize()``, runs the Azure deferred-RG resolution pass so that + child resources discovered before their resource group still end up + correctly linked. The pass is a no-op if no Azure resources are present. + """ + + def __init__(self, context: "Context"): + from resources import REGISTRY_PROPERTY_NAME # local import to avoid cycles + + self._context = context + self._registry: "Registry" = context.get_property(REGISTRY_PROPERTY_NAME) + if self._registry is None: + raise RuntimeError( + "InMemoryRegistryWriter requires a Registry on the Context " + f"under property '{REGISTRY_PROPERTY_NAME}'" + ) + + @property + def registry(self) -> "Registry": + return self._registry + + def add_resource( + self, + platform: str, + resource_type: str, + name: str, + qualified_name: str, + attributes: dict[str, Any], + ) -> "Resource": + return self._registry.add_resource( + platform, resource_type, name, qualified_name, attributes + ) + + def finalize(self) -> None: + from enrichers.generation_rule_types import PLATFORM_HANDLERS_PROPERTY_NAME + from .azure_common import resolve_deferred_azure_relationships + + platform_handlers: Optional[dict[str, "PlatformHandler"]] = ( + self._context.get_property(PLATFORM_HANDLERS_PROPERTY_NAME) + ) + if platform_handlers is None: + logger.debug( + "InMemoryRegistryWriter.finalize: no platform handlers on context; " + "skipping deferred Azure RG resolution" + ) + return + resolve_deferred_azure_relationships(self._registry, platform_handlers) + + +def get_resource_writer(context: "Context") -> ResourceWriter: + """Return the active ``ResourceWriter`` for the given workspace-builder context. + + Centralised so indexers don't pick the implementation themselves. The + backend is selected via the ``resourceStoreBackend`` setting: + + * ``memory`` (default): :class:`InMemoryRegistryWriter`. + * ``sqlite``: :class:`indexers.sqlite_resource_writer.SqliteResourceWriter`, + which composes the in-memory writer (so reads via the ``Registry`` + keep working) and additionally snapshots the registry into a local + SQLite database on ``finalize()``. + + Unknown values fall back to the in-memory writer with a warning so a + typo in ``workspaceInfo.yaml`` doesn't silently break indexing. + """ + backend = (context.get_setting(RESOURCE_STORE_BACKEND_SETTING) or "").strip().lower() + + if backend == RESOURCE_STORE_BACKEND_SQLITE: + # Local import to avoid pulling sqlite3 / tempfile / json at module + # import time when only the in-memory writer is needed. + from .sqlite_resource_writer import SqliteResourceWriter + + db_path = context.get_setting(RESOURCE_STORE_PATH_SETTING) or "resources.sqlite" + logger.info( + "ResourceWriter: using SQLite backend (path=%s)", db_path + ) + return SqliteResourceWriter(context, db_path=db_path) + + if backend and backend != RESOURCE_STORE_BACKEND_MEMORY: + logger.warning( + "Unknown resourceStoreBackend %r; falling back to in-memory writer. " + "Valid values are %r and %r.", + backend, + RESOURCE_STORE_BACKEND_MEMORY, + RESOURCE_STORE_BACKEND_SQLITE, + ) + + return InMemoryRegistryWriter(context) + + +_RESOURCE_STORE_FINALIZED_PROPERTY = "RESOURCE_STORE_FINALIZED" + + +def finalize_resource_store(context: "Context") -> None: + """Persist resources and rendered workspace artifacts when backend is sqlite.""" + from .sqlite_resource_writer import persist_sqlite_store + + backend = (context.get_setting(RESOURCE_STORE_BACKEND_SETTING) or "").strip().lower() + if backend != RESOURCE_STORE_BACKEND_SQLITE: + return + if context.get_property(_RESOURCE_STORE_FINALIZED_PROPERTY): + return + persist_sqlite_store(context) diff --git a/src/indexers/sqlite_resource_writer.py b/src/indexers/sqlite_resource_writer.py new file mode 100644 index 000000000..65e2bb890 --- /dev/null +++ b/src/indexers/sqlite_resource_writer.py @@ -0,0 +1,689 @@ +""" +SQLite-backed :class:`ResourceWriter`. + +The long-term direction for ``runwhen-local`` is to replace the in-memory +``Registry`` + FastAPI REST shell with a small local resource DB queried by a +fast read-only REST API. :class:`SqliteResourceWriter` is the first concrete +step: it persists the indexer-discovered resource graph to a SQLite database +so the future read service has something to talk to. + +It is a *dual* writer today: + +1. It composes :class:`indexers.resource_writer.InMemoryRegistryWriter` and + forwards every ``add_resource`` / ``finalize`` call to it. This keeps the + in-memory ``Registry`` populated for enrichers / renderers, which still + read from it. +2. On ``finalize()`` (after deferred-RG resolution has run via the in-memory + writer), it walks the **full** ``Registry`` and snapshots it into the + SQLite database in one transaction. + +The snapshot-at-finalize approach intentionally captures the *whole* registry, +not just resources the writer itself recorded. That means the DB reflects +state contributed by every indexer that happened to run before whichever +indexer triggered ``finalize()`` (today only the ``azureapi`` path goes +through ``ResourceWriter``; ``kubeapi`` and ``cloudquery`` still mutate the +registry directly). The migration roadmap in +``docs/architecture/resource-writer.md`` covers how this becomes a complete +picture once all indexers funnel writes through the seam. + +Schema +------ + +Three tables form a normalised resource graph:: + + platforms (name PK) + resource_types (platform FK, name, custom_attributes JSON, PK platform+name) + resources (platform, resource_type, qualified_name) PK + + name, attributes_json, created_at, updated_at + +``attributes_json`` is the JSON-encoded payload produced by +:func:`encode_attributes`, which preserves rich types via a small set of +reserved markers (``$ref`` for inter-resource references, ``$lod`` for +``LevelOfDetail`` enums, ``$datetime`` / ``$date`` for date types). The full +encoding is deterministic and round-trippable so a future REST service can +reconstruct cross-resource references when needed. +""" + +from __future__ import annotations + +import datetime as _dt +import enum +import json +import logging +import os +import sqlite3 +import tempfile +from typing import TYPE_CHECKING, Any, Optional + +from .resource_writer import InMemoryRegistryWriter + +if TYPE_CHECKING: + from component import Context + from resources import Registry, Resource + +logger = logging.getLogger(__name__) + + +# Bumped when the on-disk schema changes in a backwards-incompatible way. +SCHEMA_VERSION = 2 + + +_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS platforms ( + name TEXT PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS resource_types ( + platform TEXT NOT NULL, + name TEXT NOT NULL, + custom_attributes TEXT NOT NULL, + PRIMARY KEY (platform, name), + FOREIGN KEY (platform) REFERENCES platforms(name) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS resources ( + platform TEXT NOT NULL, + resource_type TEXT NOT NULL, + qualified_name TEXT NOT NULL, + name TEXT NOT NULL, + attributes_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (platform, resource_type, qualified_name), + FOREIGN KEY (platform, resource_type) + REFERENCES resource_types(platform, name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_resources_name + ON resources (platform, resource_type, name); + +CREATE TABLE IF NOT EXISTS workspace_artifacts ( + workspace_name TEXT NOT NULL, + relative_path TEXT NOT NULL, + artifact_kind TEXT NOT NULL, + media_type TEXT NOT NULL, + slx_directory TEXT, + content TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_name, relative_path) +); + +CREATE INDEX IF NOT EXISTS idx_workspace_artifacts_kind + ON workspace_artifacts (workspace_name, artifact_kind); +CREATE INDEX IF NOT EXISTS idx_workspace_artifacts_slx_dir + ON workspace_artifacts (slx_directory); +""" + + +# --------------------------------------------------------------------------- +# Encoder / decoder +# --------------------------------------------------------------------------- + +# Marker keys used inside ``attributes_json``. Chosen with a leading ``$`` so +# they cannot clash with attribute names that originate from cloud APIs (those +# are snake_case). Centralised here so the decoder mirror knows about them too. +_REF_KEY = "$ref" +_LOD_KEY = "$lod" +_DATETIME_KEY = "$datetime" +_DATE_KEY = "$date" +_ENUM_KEY = "$enum" + + +def encode_attributes(attributes: dict[str, Any]) -> str: + """Serialise an attributes dict to a deterministic JSON string. + + See module docstring for the marker scheme. ``sort_keys=True`` keeps + output stable for diff-based testing. + """ + return json.dumps(_encode_value(attributes), sort_keys=True) + + +def decode_attributes(attributes_json: str) -> dict[str, Any]: + """Round-trip companion to :func:`encode_attributes`. + + Resource ``$ref`` markers are *not* resolved against a live registry here - + callers (e.g. a future REST service) decide whether to dereference. The + decoder leaves them as plain dicts so they remain visible to consumers + that want to follow them on demand. + """ + raw = json.loads(attributes_json) + return _decode_value(raw) + + +def _encode_value(value: Any) -> Any: + # Local imports to avoid pulling enrichers / resources at module import + # time (this writer is constructed late in the pipeline). + from resources import Resource + try: + from enrichers.generation_rule_types import LevelOfDetail + except Exception: + LevelOfDetail = None # type: ignore[assignment] + + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, dict): + return {str(k): _encode_value(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [_encode_value(v) for v in value] + if isinstance(value, Resource): + rt = getattr(value, "resource_type", None) + platform_name = None + type_name = None + if rt is not None: + type_name = getattr(rt, "name", None) + platform = getattr(rt, "platform", None) + if platform is not None: + platform_name = getattr(platform, "name", None) + return { + _REF_KEY: { + "platform": platform_name, + "resource_type": type_name, + "qualified_name": getattr(value, "qualified_name", None), + "name": getattr(value, "name", None), + } + } + if LevelOfDetail is not None and isinstance(value, LevelOfDetail): + return {_LOD_KEY: value.name} + if isinstance(value, _dt.datetime): + return {_DATETIME_KEY: value.isoformat()} + if isinstance(value, _dt.date): + return {_DATE_KEY: value.isoformat()} + if isinstance(value, enum.Enum): + return { + _ENUM_KEY: {"class": type(value).__name__, "name": value.name} + } + if isinstance(value, (bytes, bytearray)): + # Bytes are not expected in resource attributes; encode as string so + # the snapshot succeeds and surface a debug log so we notice if a new + # path produces them. + logger.debug( + "encode_attributes: coercing %s to string; bytes-typed attributes " + "are not supported by the SQLite store.", + type(value).__name__, + ) + try: + return bytes(value).decode("utf-8") + except UnicodeDecodeError: + return bytes(value).hex() + # Fallback: stringify. Logged so this is visible in CI if a new attribute + # type starts flowing through. + logger.debug( + "encode_attributes: unsupported type %s; coercing via str()", + type(value).__name__, + ) + return str(value) + + +def _decode_value(value: Any) -> Any: + if isinstance(value, dict): + if _DATETIME_KEY in value and len(value) == 1: + return _dt.datetime.fromisoformat(value[_DATETIME_KEY]) + if _DATE_KEY in value and len(value) == 1: + return _dt.date.fromisoformat(value[_DATE_KEY]) + if _LOD_KEY in value and len(value) == 1: + try: + from enrichers.generation_rule_types import LevelOfDetail + return LevelOfDetail[value[_LOD_KEY]] + except Exception: + return value + # Refs / enums are passed through as-is - callers resolve. + return {k: _decode_value(v) for k, v in value.items()} + if isinstance(value, list): + return [_decode_value(v) for v in value] + return value + + +# --------------------------------------------------------------------------- +# Writer +# --------------------------------------------------------------------------- + +class SqliteResourceWriter: + """Dual writer that persists resources to SQLite alongside the in-memory + registry. + + Construction is cheap (no DB I/O); the SQLite file is built only on + :meth:`finalize`. Until then the writer behaves like the in-memory one, + which lets the rest of the pipeline (parse_resource_data, generation + rules, renderers) run unchanged. + """ + + def __init__( + self, + context: "Context", + db_path: str = "resources.sqlite", + ): + self._context = context + self._memory = InMemoryRegistryWriter(context) + self._db_path = db_path + + @property + def registry(self) -> "Registry": + return self._memory.registry + + @property + def db_output_path(self) -> str: + """Path the SQLite file will be written to via the outputter on finalize.""" + return self._db_path + + def add_resource( + self, + platform: str, + resource_type: str, + name: str, + qualified_name: str, + attributes: dict[str, Any], + ) -> "Resource": + # Forward to the in-memory writer; SQLite snapshot happens in finalize. + return self._memory.add_resource( + platform, resource_type, name, qualified_name, attributes + ) + + def finalize(self) -> None: + # Registry relationship fixes (e.g. deferred Azure RG resolution). The + # on-disk SQLite snapshot is written later by :func:`persist_sqlite_store` + # once renderers have produced SLX/SLI/runbook artifacts. + self._memory.finalize() + + +# --------------------------------------------------------------------------- +# Schema / snapshot helpers (also used by tests) +# --------------------------------------------------------------------------- + +def _init_schema(conn: sqlite3.Connection) -> None: + conn.executescript(_SCHEMA_SQL) + conn.execute( + "INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)", + ("schema_version", str(SCHEMA_VERSION)), + ) + + +def _snapshot_workspace_artifacts( + conn: sqlite3.Connection, + workspace_name: str, + artifacts: list[dict[str, Any]], +) -> None: + now = _dt.datetime.now(_dt.timezone.utc).isoformat() + conn.execute("DELETE FROM workspace_artifacts WHERE workspace_name = ?", (workspace_name,)) + for artifact in artifacts: + conn.execute( + "INSERT INTO workspace_artifacts " + "(workspace_name, relative_path, artifact_kind, media_type, slx_directory, " + " content, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + workspace_name, + artifact["relative_path"], + artifact["artifact_kind"], + artifact["media_type"], + artifact.get("slx_directory"), + artifact["content"], + now, + now, + ), + ) + + +def persist_sqlite_store(context: "Context", db_path: str | None = None) -> None: + """Write the SQLite store with discovered resources and rendered workspace artifacts.""" + from component import WORKSPACE_NAME_SETTING + from resources import REGISTRY_PROPERTY_NAME + from indexers.resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_BACKEND_SQLITE, + RESOURCE_STORE_PATH_SETTING, + _RESOURCE_STORE_FINALIZED_PROPERTY, + ) + from renderers.rendered_artifacts import RENDERED_ARTIFACTS_PROPERTY + + backend = (context.get_setting(RESOURCE_STORE_BACKEND_SETTING) or "").strip().lower() + if backend != RESOURCE_STORE_BACKEND_SQLITE: + return + + if db_path is None: + db_path = context.get_setting(RESOURCE_STORE_PATH_SETTING) or "resources.sqlite" + + registry = context.get_property(REGISTRY_PROPERTY_NAME) + if registry is None: + logger.warning("persist_sqlite_store: no registry on context; skipping") + return + + workspace_name = context.get_setting(WORKSPACE_NAME_SETTING) or "workspace" + artifacts = context.get_property(RENDERED_ARTIFACTS_PROPERTY, []) + + with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp: + tmp_path = tmp.name + try: + conn = sqlite3.connect(tmp_path) + try: + conn.execute("PRAGMA foreign_keys = ON") + _init_schema(conn) + _snapshot_registry(conn, registry) + _snapshot_workspace_artifacts(conn, workspace_name, artifacts) + conn.commit() + finally: + conn.close() + with open(tmp_path, "rb") as fh: + payload = fh.read() + context.outputter.write_file(db_path, payload) + logger.info( + "persist_sqlite_store wrote %d byte DB to %s " + "(%d resources in registry snapshot, %d workspace artifacts)", + len(payload), + db_path, + sum( + len(rt.instances or {}) + for platform in (registry.platforms or {}).values() + for rt in (platform.resource_types or {}).values() + ), + len(artifacts), + ) + context.set_property(_RESOURCE_STORE_FINALIZED_PROPERTY, True) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _snapshot_registry(conn: sqlite3.Connection, registry: "Registry") -> None: + """Replace the contents of the SQLite DB with a fresh snapshot of ``registry``.""" + now = _dt.datetime.now(_dt.timezone.utc).isoformat() + + conn.execute("DELETE FROM resources") + conn.execute("DELETE FROM resource_types") + conn.execute("DELETE FROM platforms") + + for platform_name, platform in (registry.platforms or {}).items(): + conn.execute( + "INSERT INTO platforms (name) VALUES (?)", (platform_name,) + ) + for type_name, resource_type in (platform.resource_types or {}).items(): + custom_attrs = sorted(resource_type.custom_attributes or []) + conn.execute( + "INSERT INTO resource_types (platform, name, custom_attributes) " + "VALUES (?, ?, ?)", + (platform_name, type_name, json.dumps(custom_attrs)), + ) + for qualified_name, resource in (resource_type.instances or {}).items(): + attrs = _collect_resource_attributes(resource, resource_type) + conn.execute( + "INSERT INTO resources " + "(platform, resource_type, qualified_name, name, " + " attributes_json, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + platform_name, + type_name, + qualified_name, + getattr(resource, "name", ""), + encode_attributes(attrs), + now, + now, + ), + ) + + +def _collect_resource_attributes( + resource: "Resource", resource_type +) -> dict[str, Any]: + """Return the dict of resource attributes to persist. + + Walks ``vars(resource)`` rather than ``resource_type.custom_attributes`` + so attributes set via ``setattr`` (e.g. ``resource_group`` populated by + :func:`indexers.azure_common.resolve_deferred_azure_relationships`) are + captured even if they were never added to the type's + ``custom_attributes`` set. The structural fields ``name`` / + ``qualified_name`` / ``resource_type`` are stored as columns and so are + excluded here. + """ + structural = {"name", "qualified_name", "resource_type"} + return { + attr_name: value + for attr_name, value in vars(resource).items() + if attr_name not in structural + } + + +# --------------------------------------------------------------------------- +# Read API +# +# Thin convenience helpers for consumers that want to read out of the +# SQLite store today (smoke tests, debug scripts, an eventual REST service). +# Intentionally minimal - this is not the future query layer. +# --------------------------------------------------------------------------- + +def open_database(path: str) -> sqlite3.Connection: + conn = sqlite3.connect(path) + conn.execute("PRAGMA foreign_keys = ON") + return conn + + +def list_platforms(conn: sqlite3.Connection) -> list[str]: + return [ + row[0] + for row in conn.execute("SELECT name FROM platforms ORDER BY name") + ] + + +def list_resource_types( + conn: sqlite3.Connection, platform: Optional[str] = None +) -> list[dict[str, Any]]: + sql = ( + "SELECT platform, name, custom_attributes FROM resource_types" + + (" WHERE platform = ?" if platform else "") + + " ORDER BY platform, name" + ) + params: tuple[Any, ...] = (platform,) if platform else () + return [ + { + "platform": row[0], + "name": row[1], + "custom_attributes": json.loads(row[2]) if row[2] else [], + } + for row in conn.execute(sql, params) + ] + + +def list_resources( + conn: sqlite3.Connection, + platform: Optional[str] = None, + resource_type: Optional[str] = None, +) -> list[dict[str, Any]]: + where: list[str] = [] + params: list[Any] = [] + if platform: + where.append("platform = ?") + params.append(platform) + if resource_type: + where.append("resource_type = ?") + params.append(resource_type) + sql = ( + "SELECT platform, resource_type, qualified_name, name, " + "attributes_json, created_at, updated_at FROM resources" + ) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY platform, resource_type, qualified_name" + + return [ + { + "platform": row[0], + "resource_type": row[1], + "qualified_name": row[2], + "name": row[3], + "attributes": decode_attributes(row[4]), + "created_at": row[5], + "updated_at": row[6], + } + for row in conn.execute(sql, tuple(params)) + ] + + +def get_resource( + conn: sqlite3.Connection, + platform: str, + resource_type: str, + qualified_name: str, +) -> Optional[dict[str, Any]]: + row = conn.execute( + "SELECT name, attributes_json, created_at, updated_at FROM resources " + "WHERE platform = ? AND resource_type = ? AND qualified_name = ?", + (platform, resource_type, qualified_name), + ).fetchone() + if not row: + return None + return { + "platform": platform, + "resource_type": resource_type, + "qualified_name": qualified_name, + "name": row[0], + "attributes": decode_attributes(row[1]), + "created_at": row[2], + "updated_at": row[3], + } + + +def get_schema_version(conn: sqlite3.Connection) -> Optional[int]: + row = conn.execute( + "SELECT value FROM schema_meta WHERE key = 'schema_version'" + ).fetchone() + if not row: + return None + try: + return int(row[0]) + except (TypeError, ValueError): + return None + + +def count_workspace_artifacts( + conn: sqlite3.Connection, + workspace_name: Optional[str] = None, + artifact_kind: Optional[str] = None, + q: Optional[str] = None, +) -> int: + where: list[str] = [] + params: list[Any] = [] + if workspace_name: + where.append("workspace_name = ?") + params.append(workspace_name) + if artifact_kind: + where.append("artifact_kind = ?") + params.append(artifact_kind) + if q: + where.append("(relative_path LIKE ? OR content LIKE ?)") + pattern = f"%{q}%" + params.extend([pattern, pattern]) + sql = "SELECT COUNT(*) FROM workspace_artifacts" + if where: + sql += " WHERE " + " AND ".join(where) + return int(conn.execute(sql, tuple(params)).fetchone()[0]) + + +def list_workspace_artifact_kinds( + conn: sqlite3.Connection, workspace_name: Optional[str] = None +) -> list[dict[str, Any]]: + sql = ( + "SELECT workspace_name, artifact_kind, COUNT(*) " + "FROM workspace_artifacts" + ) + params: tuple[Any, ...] = () + if workspace_name: + sql += " WHERE workspace_name = ?" + params = (workspace_name,) + sql += " GROUP BY workspace_name, artifact_kind ORDER BY workspace_name, artifact_kind" + return [ + {"workspace_name": row[0], "artifact_kind": row[1], "count": row[2]} + for row in conn.execute(sql, params) + ] + + +def search_workspace_artifacts( + conn: sqlite3.Connection, + workspace_name: Optional[str] = None, + artifact_kind: Optional[str] = None, + q: Optional[str] = None, + limit: int = 100, + offset: int = 0, +) -> list[dict[str, Any]]: + where: list[str] = [] + params: list[Any] = [] + if workspace_name: + where.append("workspace_name = ?") + params.append(workspace_name) + if artifact_kind: + where.append("artifact_kind = ?") + params.append(artifact_kind) + if q: + where.append("(relative_path LIKE ? OR content LIKE ?)") + pattern = f"%{q}%" + params.extend([pattern, pattern]) + sql = ( + "SELECT workspace_name, relative_path, artifact_kind, media_type, " + "slx_directory, content, created_at, updated_at FROM workspace_artifacts" + ) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY workspace_name, slx_directory, relative_path LIMIT ? OFFSET ?" + params.extend([limit, offset]) + return [ + { + "workspace_name": row[0], + "relative_path": row[1], + "artifact_kind": row[2], + "media_type": row[3], + "slx_directory": row[4], + "content": row[5], + "created_at": row[6], + "updated_at": row[7], + } + for row in conn.execute(sql, tuple(params)) + ] + + +def get_workspace_artifact( + conn: sqlite3.Connection, + workspace_name: str, + relative_path: str, +) -> Optional[dict[str, Any]]: + row = conn.execute( + "SELECT artifact_kind, media_type, slx_directory, content, created_at, updated_at " + "FROM workspace_artifacts WHERE workspace_name = ? AND relative_path = ?", + (workspace_name, relative_path), + ).fetchone() + if not row: + return None + return { + "workspace_name": workspace_name, + "relative_path": relative_path, + "artifact_kind": row[0], + "media_type": row[1], + "slx_directory": row[2], + "content": row[3], + "created_at": row[4], + "updated_at": row[5], + } + + +__all__ = [ + "SCHEMA_VERSION", + "SqliteResourceWriter", + "encode_attributes", + "decode_attributes", + "open_database", + "list_platforms", + "list_resource_types", + "list_resources", + "get_resource", + "get_schema_version", + "count_workspace_artifacts", + "list_workspace_artifact_kinds", + "search_workspace_artifacts", + "get_workspace_artifact", + "persist_sqlite_store", +] diff --git a/src/indexers/test_aws_resource_type_registry.py b/src/indexers/test_aws_resource_type_registry.py new file mode 100644 index 000000000..084fb1e7c --- /dev/null +++ b/src/indexers/test_aws_resource_type_registry.py @@ -0,0 +1,322 @@ +""" +Unit tests for the AWS resource-type registry loader and the spec layer it +powers (``indexers.awsapi_resource_types``). + +These tests pin the contract that: + +1. The loader produces a fully-populated registry from the YAML on disk (1119 + entries today, with stable metadata fields and alias resolution). + +2. The hand-written typed-collector specs in ``awsapi_resource_types`` resolve + the legacy ``resource_type_name`` values generation rules expect + (``account``, ``ec2_instance``) and the canonical CQ table names. + +3. Aliases route to the canonical CQ table name; CloudFormation resource types + reverse-map back to the owning table. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import textwrap +from pathlib import Path +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.aws_resource_type_registry import ( # noqa: E402 + find_entry, + load_registry, + load_registry_from_path, + reset_cache, +) + + +class RegistryLoaderTests(TestCase): + def setUp(self) -> None: + reset_cache() + + def tearDown(self) -> None: + reset_cache() + + def test_default_registry_loads_with_expected_metadata(self) -> None: + registry = load_registry() + self.assertGreater(len(registry), 1000) # 1119 today, allow for growth + self.assertEqual(len(registry), registry.metadata.total_tables) + self.assertGreaterEqual(registry.metadata.typed_collectors, 1) + self.assertEqual( + registry.metadata.generator, + "scripts/aws/sync_aws_resource_type_registry.py", + ) + self.assertTrue( + registry.metadata.source.startswith("https://"), + f"metadata.source should be a URL, got {registry.metadata.source!r}", + ) + + def test_registry_lookup_by_canonical_name(self) -> None: + registry = load_registry() + accounts = registry.find("aws_iam_accounts") + self.assertIsNotNone(accounts) + self.assertEqual(accounts.cloudquery_table_name, "aws_iam_accounts") + # The synthesized account anchor has no Cloud Control type. + self.assertIsNone(accounts.cfn_type) + self.assertTrue(accounts.mandatory) + self.assertTrue(accounts.typed_collector) + + def test_registry_lookup_by_alias(self) -> None: + registry = load_registry() + legacy_account = registry.find("account") + self.assertIsNotNone(legacy_account) + self.assertEqual(legacy_account.cloudquery_table_name, "aws_iam_accounts") + + legacy_instance = registry.find("ec2_instance") + self.assertIsNotNone(legacy_instance) + self.assertEqual( + legacy_instance.cloudquery_table_name, "aws_ec2_instances" + ) + + def test_registry_find_by_cfn_type(self) -> None: + registry = load_registry() + entry = registry.find_by_cfn_type("AWS::EC2::Instance") + self.assertIsNotNone(entry) + self.assertEqual(entry.cloudquery_table_name, "aws_ec2_instances") + # Case-insensitive. + entry2 = registry.find_by_cfn_type("aws::ec2::instance") + self.assertIs(entry2, entry) + + def test_registry_unknown_name_returns_none(self) -> None: + self.assertIsNone(load_registry().find("not_a_real_table")) + self.assertIsNone(load_registry().find("")) + + def test_module_level_find_entry_uses_cache(self) -> None: + e1 = find_entry("account") + e2 = find_entry("aws_iam_accounts") + self.assertIs(e1, e2) + + def test_membership_protocol(self) -> None: + registry = load_registry() + self.assertIn("account", registry) + self.assertIn("aws_iam_accounts", registry) + self.assertNotIn("nope", registry) + self.assertNotIn(123, registry) + + def test_all_cfn_types_are_aws_prefixed(self) -> None: + registry = load_registry() + cfn_types = registry.all_cfn_types() + self.assertGreater(len(cfn_types), 0) + for value in cfn_types: + self.assertIsInstance(value, str) + self.assertTrue(value.startswith("AWS::"), value) + # CFN type names always have three segments: AWS::Service::Entity. + self.assertEqual(value.count("::"), 2, value) + + def test_typed_collector_tables_match_registry_flag(self) -> None: + registry = load_registry() + typed = set(registry.typed_collector_tables()) + for entry in registry: + if entry.typed_collector: + self.assertIn(entry.cloudquery_table_name, typed) + else: + self.assertNotIn(entry.cloudquery_table_name, typed) + + def test_mandatory_includes_accounts(self) -> None: + registry = load_registry() + self.assertIn("aws_iam_accounts", registry.mandatory_tables()) + + def test_regions_table_has_no_cfn_type(self) -> None: + # Tables with no Cloud Control equivalent are pinned to null in the + # overrides so generic discovery skips them. + registry = load_registry() + entry = registry.find("aws_regions") + self.assertIsNotNone(entry) + self.assertIsNone(entry.cfn_type) + + def test_every_table_resolves(self) -> None: + # Parity guard: every canonical name resolves to itself (the gen-rule + # contract that any CloudQuery table name is a valid resource_type). + registry = load_registry() + for name in registry.all_canonical_names(): + self.assertIsNotNone(registry.find(name), name) + + +class RegistryFromTempYamlTests(TestCase): + """Loader behavior tests that don't depend on the shipped registry.""" + + def _write(self, contents: str) -> Path: + tmp = tempfile.NamedTemporaryFile( + "w", suffix=".yaml", delete=False, encoding="utf-8" + ) + tmp.write(textwrap.dedent(contents)) + tmp.close() + path = Path(tmp.name) + self.addCleanup(path.unlink, missing_ok=True) + return path + + def test_minimal_registry_round_trips(self) -> None: + path = self._write( + """ + metadata: + source: https://example.test + snapshot_date: 2026-01-01 + total_tables: 1 + typed_collectors: 0 + cfn_types_assigned: 1 + generator: tests + notes: synthetic + types: + aws_test_widgets: + cfn_type: AWS::Test::Widget + cfn_type_source: heuristic + category: test + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertEqual(len(registry), 1) + entry = registry.find("aws_test_widgets") + self.assertIsNotNone(entry) + self.assertEqual(entry.cfn_type, "AWS::Test::Widget") + + def test_aliases_resolve_to_canonical(self) -> None: + path = self._write( + """ + metadata: {} + types: + aws_alpha_things: + cfn_type: AWS::Alpha::Thing + cfn_type_source: heuristic + category: a + aliases: [legacy_alpha, vintage_alpha] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertIs( + registry.find("legacy_alpha"), registry.find("aws_alpha_things") + ) + self.assertIs( + registry.find("vintage_alpha"), registry.find("aws_alpha_things") + ) + + def test_alias_collision_raises(self) -> None: + path = self._write( + """ + metadata: {} + types: + aws_alpha_things: + cfn_type: AWS::Alpha::Thing + aliases: [shared] + typed_collector: false + mandatory: false + aws_beta_things: + cfn_type: AWS::Beta::Thing + aliases: [shared] + typed_collector: false + mandatory: false + """ + ) + with self.assertRaises(ValueError): + load_registry_from_path(path) + + def test_missing_file_raises_filenotfound(self) -> None: + with self.assertRaises(FileNotFoundError): + load_registry_from_path(Path("/nonexistent/registry.yaml")) + + def test_null_cfn_type_is_allowed(self) -> None: + path = self._write( + """ + metadata: {} + types: + aws_no_type_things: + cfn_type: null + cfn_type_source: null + category: x + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + entry = registry.find("aws_no_type_things") + self.assertIsNotNone(entry) + self.assertIsNone(entry.cfn_type) + # A null cfn type must not be indexed for reverse lookup. + self.assertIsNone(registry.find_by_cfn_type(None)) + + +class AwsapiResourceTypesSpecTests(TestCase): + """The spec layer must surface the legacy ``resource_type_name`` values + generation rules know about and bind the SDK collectors.""" + + def setUp(self) -> None: + reset_cache() + import importlib + + import indexers.awsapi_resource_types as mod + + self.mod = importlib.reload(mod) + + def test_typed_sdk_collectors_present(self) -> None: + names = {s.cloudquery_table_name for s in self.mod.AWS_TYPED_RESOURCE_TYPE_SPECS} + self.assertEqual( + names, + { + "aws_ec2_instances", + "aws_s3_buckets", + }, + ) + + def test_account_spec_is_first_and_mandatory(self) -> None: + first = self.mod.AWS_RESOURCE_TYPE_SPECS[0] + self.assertEqual(first.resource_type_name, "account") + self.assertEqual(first.cloudquery_table_name, "aws_iam_accounts") + self.assertTrue(first.mandatory) + + def test_legacy_short_names_resolve_via_find_spec(self) -> None: + spec = self.mod.find_spec("account") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "aws_iam_accounts") + + inst = self.mod.find_spec("ec2_instance") + self.assertIsNotNone(inst) + self.assertEqual(inst.cloudquery_table_name, "aws_ec2_instances") + + def test_canonical_table_name_resolves_via_find_spec(self) -> None: + # The gen-rule contract: the CloudQuery table name is always a valid + # resource_type, regardless of backend. + spec = self.mod.find_spec("aws_ec2_instances") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "aws_ec2_instances") + + def test_find_spec_by_cfn_type(self) -> None: + spec = self.mod.find_spec_by_cfn_type("AWS::S3::Bucket") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "aws_s3_buckets") + + def test_unknown_name_returns_none(self) -> None: + self.assertIsNone(self.mod.find_spec("totally_made_up")) + self.assertIsNone(self.mod.find_spec("")) + + def test_typed_specs_have_callable_collectors(self) -> None: + for spec in self.mod.AWS_TYPED_RESOURCE_TYPE_SPECS: + self.assertTrue(callable(spec.collector), spec.cloudquery_table_name) + + def test_ec2_is_regional_s3_is_global(self) -> None: + self.assertTrue(self.mod.find_spec("aws_ec2_instances").regional) + self.assertFalse(self.mod.find_spec("aws_s3_buckets").regional) + + def test_generic_specs_have_no_collector(self) -> None: + # A type with a CFN mapping but no SDK collector (e.g. RDS instances) + # is materialized as a generic spec the Cloud Control pass owns. + spec = self.mod.find_spec("aws_rds_instances") + self.assertIsNotNone(spec) + self.assertIsNone(spec.collector) + self.assertEqual(spec.cfn_type, "AWS::RDS::DBInstance") diff --git a/src/indexers/test_awsapi_normalizers.py b/src/indexers/test_awsapi_normalizers.py new file mode 100644 index 000000000..6c803babd --- /dev/null +++ b/src/indexers/test_awsapi_normalizers.py @@ -0,0 +1,225 @@ +""" +Unit tests for ``indexers.awsapi_normalizers``. + +The native AWS indexer claims output compatible with the legacy CloudQuery +path. These tests pin the contract: + +1. ``normalize_cloudcontrol_resource`` flattens a Cloud Control + ``ResourceDescription`` (with its JSON ``Properties`` blob) into the flat + dict shape ``AWSPlatformHandler.parse_resource_data`` accepts (``arn``, + ``name``, ``account_id``, ``region``, ``tags``). + +2. ``normalize_aws_resource`` does the same for a typed boto3 payload, including + synthesizing an ARN when the payload carries none. + +3. Feeding those dicts into ``AWSPlatformHandler.parse_resource_data`` yields a + sensible ``(name, qualified_name, attributes)`` tuple. + +These tests run without boto3 installed because the normalizers operate on +plain dicts. +""" + +from __future__ import annotations + +import datetime +import json +import os +import sys +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.awsapi_normalizers import ( # noqa: E402 + _sanitize, + make_account_resource_data, + normalize_aws_resource, + normalize_cloudcontrol_resource, + normalize_tags, +) + +ACCOUNT_ID = "123456789012" +REGION = "us-east-1" + + +class SanitizeAndTagTests(TestCase): + def test_sanitize_handles_datetimes_enums_nested(self): + import enum + + class Color(enum.Enum): + RED = "red" + + payload = { + "ts": datetime.datetime(2024, 1, 2, 3, 4, 5), + "color": Color.RED, + "nested": {"more_ts": datetime.datetime(2025, 1, 1)}, + "list": [Color.RED, datetime.date(2025, 6, 1)], + } + out = _sanitize(payload) + self.assertEqual(out["ts"], "2024-01-02T03:04:05") + self.assertEqual(out["color"], "red") + self.assertEqual(out["nested"]["more_ts"], "2025-01-01T00:00:00") + self.assertEqual(out["list"][0], "red") + self.assertEqual(out["list"][1], "2025-06-01") + + def test_normalize_tags_from_aws_list(self): + raw = [{"Key": "env", "Value": "prod"}, {"Key": "team", "Value": "sre"}] + self.assertEqual(normalize_tags(raw), {"env": "prod", "team": "sre"}) + + def test_normalize_tags_from_dict(self): + self.assertEqual(normalize_tags({"env": "prod"}), {"env": "prod"}) + + def test_normalize_tags_empty(self): + self.assertEqual(normalize_tags(None), {}) + self.assertEqual(normalize_tags([]), {}) + + +class NormalizeCloudControlTests(TestCase): + def _bucket_description(self): + return { + "Identifier": "my-bucket", + "Properties": json.dumps( + { + "BucketName": "my-bucket", + "Arn": "arn:aws:s3:::my-bucket", + "Tags": [{"Key": "env", "Value": "prod"}], + } + ), + } + + def test_bucket_top_level_shape(self): + data = normalize_cloudcontrol_resource( + self._bucket_description(), + account_id=ACCOUNT_ID, + region=REGION, + cfn_type="AWS::S3::Bucket", + ) + self.assertEqual(data["name"], "my-bucket") + self.assertEqual(data["arn"], "arn:aws:s3:::my-bucket") + self.assertEqual(data["account_id"], ACCOUNT_ID) + self.assertEqual(data["region"], REGION) + self.assertEqual(data["tags"], {"env": "prod"}) + self.assertEqual(data["cfn_type"], "AWS::S3::Bucket") + + def test_synthesize_arn_when_absent(self): + desc = { + "Identifier": "q1", + "Properties": json.dumps({"QueueName": "q1"}), + } + data = normalize_cloudcontrol_resource( + desc, + account_id=ACCOUNT_ID, + region="us-west-2", + cfn_type="AWS::SQS::Queue", + ) + self.assertEqual(data["arn"], f"arn:aws:sqs:us-west-2:{ACCOUNT_ID}:queue/q1") + self.assertEqual(data["name"], "q1") + + def test_malformed_properties_string_is_tolerated(self): + desc = {"Identifier": "abc", "Properties": "{not valid json"} + data = normalize_cloudcontrol_resource( + desc, account_id=ACCOUNT_ID, region=REGION, cfn_type="AWS::EC2::Vpc" + ) + # Falls back to the identifier-based name + synthesized ARN. + self.assertEqual(data["name"], "abc") + self.assertTrue(data["arn"].startswith("arn:aws:ec2:")) + + +class NormalizeAwsResourceTests(TestCase): + def test_typed_payload_with_arn(self): + payload = { + "InstanceId": "i-0abc", + "Arn": f"arn:aws:ec2:{REGION}:{ACCOUNT_ID}:instance/i-0abc", + "Tags": [{"Key": "Name", "Value": "web-1"}], + "name": "web-1", + } + data = normalize_aws_resource( + payload, + account_id=ACCOUNT_ID, + region=REGION, + cfn_type="AWS::EC2::Instance", + identifier="i-0abc", + ) + self.assertEqual(data["name"], "web-1") + self.assertEqual(data["arn"], f"arn:aws:ec2:{REGION}:{ACCOUNT_ID}:instance/i-0abc") + self.assertEqual(data["tags"], {"Name": "web-1"}) + self.assertEqual(data["account_id"], ACCOUNT_ID) + + def test_global_payload_keeps_own_region_when_region_none(self): + payload = {"name": "b", "Arn": "arn:aws:s3:::b", "region": "eu-west-1"} + data = normalize_aws_resource( + payload, account_id=ACCOUNT_ID, region=None, cfn_type="AWS::S3::Bucket" + ) + self.assertEqual(data["region"], "eu-west-1") + + def test_make_account_resource_data(self): + data = make_account_resource_data(ACCOUNT_ID, account_name="Acme") + self.assertEqual(data["account_id"], ACCOUNT_ID) + self.assertEqual(data["name"], "Acme") + self.assertEqual(data["arn"], f"arn:aws:iam::{ACCOUNT_ID}:root") + self.assertEqual(data["region"], "global") + + +class NormalizerParserRoundTripTests(TestCase): + """End-to-end: normalizer dict -> AWSPlatformHandler.parse_resource_data.""" + + def setUp(self): + import component + from component import Context + from outputter import FileItemOutputter + from resources import REGISTRY_PROPERTY_NAME, Registry + from enrichers.generation_rule_types import LevelOfDetail + + # The AWS handler resolves LOD via ``Context.get_setting("DEFAULT_LOD")`` + # (string form), which needs the global settings registry populated. + component.init_components() + + self.context = Context( + setting_values={"DEFAULT_LOD": LevelOfDetail.BASIC}, + outputter=FileItemOutputter(), + ) + self.context.set_property(REGISTRY_PROPERTY_NAME, Registry()) + + def test_account_anchor_round_trip(self): + from enrichers.aws import AWSPlatformHandler + + platform_cfg = {"_account_names": {ACCOUNT_ID: "Acme"}} + data = make_account_resource_data(ACCOUNT_ID, account_name="Acme") + handler = AWSPlatformHandler() + name, qualified_name, attrs = handler.parse_resource_data( + data, "account", platform_cfg, self.context + ) + self.assertEqual(name, "Acme") + self.assertEqual(attrs["account_id"], ACCOUNT_ID) + self.assertEqual(attrs["region"], "global") + self.assertEqual(attrs["service"], "iam") + self.assertEqual(attrs["account_name"], "Acme") + self.assertEqual(qualified_name, f"{ACCOUNT_ID}:global:Acme") + self.assertIsNotNone(attrs.get("lod")) + + def test_generic_resource_round_trip(self): + from enrichers.aws import AWSPlatformHandler + + platform_cfg = {"_account_names": {ACCOUNT_ID: "Acme"}} + desc = { + "Identifier": "my-bucket", + "Properties": json.dumps( + {"BucketName": "my-bucket", "Arn": "arn:aws:s3:::my-bucket"} + ), + } + data = normalize_cloudcontrol_resource( + desc, account_id=ACCOUNT_ID, region=REGION, cfn_type="AWS::S3::Bucket" + ) + handler = AWSPlatformHandler() + name, qualified_name, attrs = handler.parse_resource_data( + data, "aws_s3_buckets", platform_cfg, self.context + ) + self.assertEqual(name, "my-bucket") + self.assertEqual(attrs["service"], "s3") + self.assertEqual(attrs["account_id"], ACCOUNT_ID) + # S3 bucket ARNs carry no region segment; handler falls back to the + # normalized region field. + self.assertEqual(attrs["region"], REGION) + self.assertEqual(attrs["account_name"], "Acme") diff --git a/src/indexers/test_awsapi_selective.py b/src/indexers/test_awsapi_selective.py new file mode 100644 index 000000000..7097d875a --- /dev/null +++ b/src/indexers/test_awsapi_selective.py @@ -0,0 +1,280 @@ +""" +Unit tests for the AWS indexer orchestrator (``indexers.awsapi``). + +Coverage: +* ``_account_lod`` resolves per-account LOD, falling back to the workspace + default and tolerating garbage values. +* ``index()`` honours account-level selective discovery: an account whose + effective LOD is NONE is skipped entirely (no anchor, no typed pass, no Cloud + Control pass). +* The account anchor is emitted (Phase 0) whenever the account is in scope. +* Typed (boto3) collectors run per region for regional types and once for + global types, for accessed typed types only. +* The Cloud Control pass is scoped to exactly the CFN types of accessed + *generic* types (typed types are excluded so nothing is written twice), once + per region. + +The dispatch tests drive ``index()`` with stubbed credentials / collectors / +writer / handler so they need no AWS SDK or network. +""" + +from __future__ import annotations + +import os +import sys +from unittest import TestCase, mock + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from enrichers.generation_rule_types import LevelOfDetail # noqa: E402 +from indexers import awsapi # noqa: E402 +from indexers.awsapi_resource_types import AwsResourceTypeSpec # noqa: E402 + +ACCOUNT_ID = "123456789012" + + +class AccountLodTests(TestCase): + def test_explicit_account_override(self): + cfg = {"accountLevelOfDetails": {ACCOUNT_ID: "detailed"}} + self.assertEqual( + awsapi._account_lod(cfg, ACCOUNT_ID, LevelOfDetail.BASIC), + LevelOfDetail.DETAILED, + ) + + def test_falls_back_to_default(self): + cfg = {"accountLevelOfDetails": {ACCOUNT_ID: "detailed"}} + self.assertEqual( + awsapi._account_lod(cfg, "999999999999", LevelOfDetail.BASIC), + LevelOfDetail.BASIC, + ) + + def test_none_override(self): + cfg = {"accountLevelOfDetails": {ACCOUNT_ID: "none"}} + self.assertEqual( + awsapi._account_lod(cfg, ACCOUNT_ID, LevelOfDetail.BASIC), + LevelOfDetail.NONE, + ) + + def test_garbage_falls_back_to_default(self): + cfg = {"accountLevelOfDetails": {ACCOUNT_ID: "garbage"}} + self.assertEqual( + awsapi._account_lod(cfg, ACCOUNT_ID, LevelOfDetail.DETAILED), + LevelOfDetail.DETAILED, + ) + + +class _FakeRuleSpec: + """Mimics a generation-rule ResourceTypeSpec (only resource_type_name read).""" + + def __init__(self, resource_type_name: str): + self.resource_type_name = resource_type_name + + def __hash__(self): + return hash(self.resource_type_name) + + def __eq__(self, other): + return getattr(other, "resource_type_name", None) == self.resource_type_name + + +class DispatchTests(TestCase): + def setUp(self): + self._calls = {"typed_ec2": [], "typed_s3": [], "cc": []} + + def _stub_ec2(session, account_id, region): + self._calls["typed_ec2"].append(region) + return [] + + def _stub_s3(session, account_id, region): + self._calls["typed_s3"].append(region) + return [] + + self.account_spec = AwsResourceTypeSpec( + resource_type_name="account", + cloudquery_table_name="aws_iam_accounts", + cfn_type=None, + mandatory=True, + typed=True, + collector=None, + ) + self.ec2_spec = AwsResourceTypeSpec( + resource_type_name="ec2_instance", + cloudquery_table_name="aws_ec2_instances", + cfn_type="AWS::EC2::Instance", + mandatory=False, + typed=True, + collector=_stub_ec2, + regional=True, + ) + self.s3_spec = AwsResourceTypeSpec( + resource_type_name="aws_s3_buckets", + cloudquery_table_name="aws_s3_buckets", + cfn_type="AWS::S3::Bucket", + mandatory=False, + typed=True, + collector=_stub_s3, + regional=False, + ) + self.rds_spec = AwsResourceTypeSpec( + resource_type_name="aws_rds_instances", + cloudquery_table_name="aws_rds_instances", + cfn_type="AWS::RDS::DBInstance", + mandatory=False, + typed=False, + collector=None, + ) + self._by_name = { + "aws_iam_accounts": self.account_spec, + "account": self.account_spec, + "ec2_instance": self.ec2_spec, + "aws_ec2_instances": self.ec2_spec, + "aws_s3_buckets": self.s3_spec, + "aws_rds_instances": self.rds_spec, + } + self._by_cfn = { + "AWS::EC2::Instance": self.ec2_spec, + "AWS::S3::Bucket": self.s3_spec, + "AWS::RDS::DBInstance": self.rds_spec, + } + + def _run(self, *, regions, account_lod, accessed): + from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY + from enrichers.generation_rule_types import PLATFORM_HANDLERS_PROPERTY_NAME + + platform_cfg = { + "regions": list(regions), + "accountLevelOfDetails": dict(account_lod), + } + + def _stub_cc(session, region, cfn_type): + self._calls["cc"].append((region, cfn_type)) + return [] + + handler = mock.MagicMock() + handler.parse_resource_data.return_value = ("nm", "q/nm", {}) + writer = mock.MagicMock() + + rule_specs = {"aws": {_FakeRuleSpec(name): {} for name in accessed}} + + class FakeContext: + def __init__(self): + self._cloud = {"aws": dict(platform_cfg)} + self._props = {RESOURCE_TYPE_SPECS_PROPERTY: rule_specs} + + def get_setting(self, setting): + name = getattr(setting, "name", setting) + return { + "AWS_INDEXER_BACKEND": "awsapi", + "CLOUD_CONFIG": self._cloud, + "RESOURCE_STORE_BACKEND": "memory", + "RESOURCE_STORE_PATH": None, + "DEFAULT_LOD": None, + }.get(name) + + def get_property(self, name): + if name == PLATFORM_HANDLERS_PROPERTY_NAME: + return None + return self._props.get(name) + + def add_warning(self, msg): + pass + + scope = { + "session": object(), + "account_id": ACCOUNT_ID, + "account_alias": "acme", + "account_name": "Acme", + "account_names": {ACCOUNT_ID: "Acme"}, + "regions": list(regions), + "auth_type": "aws_explicit", + "auth_secret": None, + "region": regions[0] if regions else None, + } + + with mock.patch.object( + awsapi, "aws_get_session_and_scope", return_value=scope + ), mock.patch.object(awsapi, "get_resource_writer", return_value=writer), \ + mock.patch.object(awsapi, "_resolve_platform_handler", return_value=handler), \ + mock.patch.object(awsapi, "collect_cloudcontrol_resources", _stub_cc), \ + mock.patch.object(awsapi, "find_spec", side_effect=lambda n: self._by_name.get(n)), \ + mock.patch.object( + awsapi, "find_spec_by_cfn_type", + side_effect=lambda t: self._by_cfn.get(t), + ), \ + mock.patch("enrichers.aws.set_aws_credentials"): + awsapi.index(FakeContext()) + + return writer + + def test_skips_none_lod_account(self): + writer = self._run( + regions=["us-east-1"], + account_lod={ACCOUNT_ID: "none"}, + accessed=["ec2_instance", "aws_rds_instances"], + ) + # Nothing written, no collectors invoked. + self.assertEqual(writer.add_resource.call_count, 0) + self.assertEqual(self._calls["typed_ec2"], []) + self.assertEqual(self._calls["cc"], []) + + def test_account_anchor_emitted_when_no_accessed_types(self): + writer = self._run(regions=["us-east-1"], account_lod={}, accessed=[]) + # Only the account anchor is written; neither pass runs. + self.assertEqual(writer.add_resource.call_count, 1) + self.assertEqual(writer.add_resource.call_args_list[0].args[1], "account") + self.assertEqual(self._calls["typed_ec2"], []) + self.assertEqual(self._calls["cc"], []) + + def test_typed_excluded_from_generic_pass(self): + # When only typed types are accessed, the Cloud Control pass never runs. + self._run( + regions=["us-east-1", "us-west-2"], + account_lod={}, + accessed=["ec2_instance", "aws_s3_buckets"], + ) + # EC2 is regional: runs once per region. + self.assertEqual(self._calls["typed_ec2"], ["us-east-1", "us-west-2"]) + # S3 is global: runs once, with the primary region. + self.assertEqual(self._calls["typed_s3"], ["us-east-1"]) + # No generic types -> no Cloud Control calls. + self.assertEqual(self._calls["cc"], []) + + def test_generic_only_runs_cloud_control_per_region(self): + self._run( + regions=["us-east-1", "us-west-2"], + account_lod={}, + accessed=["aws_rds_instances"], + ) + self.assertEqual(self._calls["typed_ec2"], []) + self.assertEqual(self._calls["typed_s3"], []) + self.assertEqual( + self._calls["cc"], + [ + ("us-east-1", "AWS::RDS::DBInstance"), + ("us-west-2", "AWS::RDS::DBInstance"), + ], + ) + + def test_no_op_when_backend_not_awsapi(self): + # When the backend is not 'awsapi', index() returns immediately. + from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY + from enrichers.generation_rule_types import PLATFORM_HANDLERS_PROPERTY_NAME + + writer = mock.MagicMock() + + class FakeContext: + def get_setting(self, setting): + name = getattr(setting, "name", setting) + return {"AWS_INDEXER_BACKEND": "cloudquery"}.get(name) + + def get_property(self, name): + return None + + def add_warning(self, msg): + pass + + with mock.patch.object(awsapi, "get_resource_writer", return_value=writer): + awsapi.index(FakeContext()) + self.assertEqual(writer.add_resource.call_count, 0) diff --git a/src/indexers/test_azure_resource_type_registry.py b/src/indexers/test_azure_resource_type_registry.py new file mode 100644 index 000000000..da2467a05 --- /dev/null +++ b/src/indexers/test_azure_resource_type_registry.py @@ -0,0 +1,348 @@ +""" +Unit tests for the Azure resource-type registry loader and the back-compat +shim it powers (``indexers.azureapi_resource_types``). + +These tests pin the contract that: + +1. The loader produces a fully-populated registry from the YAML on disk + (619 entries today, with stable metadata fields and alias resolution). + +2. The hand-written typed-collector specs in ``azureapi_resource_types`` + stay byte-compatible with what generation rules expect: + ``resource_group``, ``virtual_machine``, ``azure_keyvault_vaults``, + ``azure_storage_accounts`` etc. all still resolve via ``find_spec``. + +3. Aliases route to the canonical CQ table name in both directions: legacy + short names (``resource_group``) and renamed CQ tables + (``azure_keyvault_vaults`` -> ``azure_keyvault_keyvaults``). +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import textwrap +from pathlib import Path +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.azure_resource_type_registry import ( # noqa: E402 + AzureResourceTypeEntry, + AzureResourceTypeRegistry, + AzureRegistryMetadata, + find_entry, + load_registry, + load_registry_from_path, + reset_cache, +) + + +class RegistryLoaderTests(TestCase): + def setUp(self) -> None: + # Each test starts from a clean cache so module-level state doesn't + # leak between tests (e.g. a temp-file load shouldn't poison the + # default-path cache). + reset_cache() + + def tearDown(self) -> None: + reset_cache() + + def test_default_registry_loads_with_expected_metadata(self) -> None: + registry = load_registry() + self.assertGreater(len(registry), 100) # 619 today, allow for growth + self.assertEqual(len(registry), registry.metadata.total_tables) + self.assertGreaterEqual(registry.metadata.typed_collectors, 1) + self.assertEqual( + registry.metadata.generator, + "scripts/azure/sync_azure_resource_type_registry.py", + ) + self.assertTrue( + registry.metadata.source.startswith("https://"), + f"metadata.source should be a URL, got {registry.metadata.source!r}", + ) + + def test_registry_lookup_by_canonical_name(self) -> None: + registry = load_registry() + rg = registry.find("azure_resources_resource_groups") + self.assertIsNotNone(rg) + self.assertEqual(rg.cloudquery_table_name, "azure_resources_resource_groups") + self.assertEqual(rg.arm_type, "Microsoft.Resources/resourceGroups") + self.assertTrue(rg.mandatory) + self.assertTrue(rg.typed_collector) + + def test_registry_lookup_by_alias(self) -> None: + registry = load_registry() + legacy = registry.find("resource_group") + self.assertIsNotNone(legacy) + self.assertEqual(legacy.cloudquery_table_name, "azure_resources_resource_groups") + + legacy_vm = registry.find("virtual_machine") + self.assertIsNotNone(legacy_vm) + self.assertEqual(legacy_vm.cloudquery_table_name, "azure_compute_virtual_machines") + + def test_registry_keyvault_alias_routes_to_canonical_cq_name(self) -> None: + # The canonical CQ table is ``azure_keyvault_keyvaults``; + # ``azure_keyvault_vaults`` is the legacy alias RWL used to ship. + registry = load_registry() + canonical = registry.find("azure_keyvault_keyvaults") + self.assertIsNotNone(canonical) + alias = registry.find("azure_keyvault_vaults") + self.assertIs(alias, canonical) + self.assertEqual(canonical.arm_type, "Microsoft.KeyVault/vaults") + + def test_registry_unknown_name_returns_none(self) -> None: + self.assertIsNone(load_registry().find("not_a_real_table")) + self.assertIsNone(load_registry().find("")) + + def test_module_level_find_entry_uses_cache(self) -> None: + e1 = find_entry("resource_group") + e2 = find_entry("azure_resources_resource_groups") + self.assertIs(e1, e2) # alias and canonical return the same object + + def test_membership_protocol(self) -> None: + registry = load_registry() + self.assertIn("resource_group", registry) + self.assertIn("azure_resources_resource_groups", registry) + self.assertNotIn("nope", registry) + self.assertNotIn(123, registry) # non-string in -> False, never raises + + def test_all_arm_types_returns_unique_strings(self) -> None: + registry = load_registry() + arm_types = registry.all_arm_types() + self.assertGreater(len(arm_types), 0) + for value in arm_types: + self.assertIsInstance(value, str) + self.assertTrue(value.startswith("Microsoft."), value) + + def test_typed_collector_tables_match_registry_flag(self) -> None: + registry = load_registry() + typed = set(registry.typed_collector_tables()) + for entry in registry: + if entry.typed_collector: + self.assertIn(entry.cloudquery_table_name, typed) + else: + self.assertNotIn(entry.cloudquery_table_name, typed) + + def test_mandatory_includes_resource_groups(self) -> None: + registry = load_registry() + mandatory = registry.mandatory_tables() + self.assertIn("azure_resources_resource_groups", mandatory) + + +class RegistryFromTempYamlTests(TestCase): + """Loader behavior tests that don't depend on the shipped registry.""" + + def _write(self, contents: str) -> Path: + tmp = tempfile.NamedTemporaryFile( + "w", suffix=".yaml", delete=False, encoding="utf-8" + ) + tmp.write(textwrap.dedent(contents)) + tmp.close() + path = Path(tmp.name) + self.addCleanup(path.unlink, missing_ok=True) + return path + + def test_minimal_registry_round_trips(self) -> None: + path = self._write( + """ + metadata: + source: https://example.test + snapshot_date: 2026-01-01 + total_tables: 1 + typed_collectors: 0 + arm_types_assigned: 1 + generator: tests + notes: synthetic + types: + azure_test_widgets: + arm_type: Microsoft.Test/widgets + arm_type_source: heuristic + category: test + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertEqual(len(registry), 1) + entry = registry.find("azure_test_widgets") + self.assertIsNotNone(entry) + self.assertEqual(entry.arm_type, "Microsoft.Test/widgets") + + def test_aliases_resolve_to_canonical(self) -> None: + path = self._write( + """ + metadata: {} + types: + azure_alpha: + arm_type: Microsoft.A/things + arm_type_source: heuristic + category: a + aliases: [legacy_alpha, vintage_alpha] + typed_collector: false + mandatory: false + azure_beta: + arm_type: Microsoft.B/things + arm_type_source: heuristic + category: b + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertIs(registry.find("legacy_alpha"), registry.find("azure_alpha")) + self.assertIs(registry.find("vintage_alpha"), registry.find("azure_alpha")) + + def test_alias_collision_raises(self) -> None: + path = self._write( + """ + metadata: {} + types: + azure_alpha: + arm_type: Microsoft.A/things + arm_type_source: heuristic + category: a + aliases: [shared] + typed_collector: false + mandatory: false + azure_beta: + arm_type: Microsoft.B/things + arm_type_source: heuristic + category: b + aliases: [shared] + typed_collector: false + mandatory: false + """ + ) + with self.assertRaises(ValueError): + load_registry_from_path(path) + + def test_missing_file_raises_filenotfound(self) -> None: + with self.assertRaises(FileNotFoundError): + load_registry_from_path(Path("/nonexistent/registry.yaml")) + + def test_metadata_defaults_when_omitted(self) -> None: + path = self._write( + """ + types: + azure_solo: + arm_type: Microsoft.X/y + arm_type_source: heuristic + category: x + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertEqual(registry.metadata.total_tables, 0) + self.assertIsNone(registry.metadata.source) + + +class AzureapiResourceTypesShimTests(TestCase): + """The hand-written collector specs must surface the legacy + ``resource_type_name`` values generation rules know about.""" + + def setUp(self) -> None: + reset_cache() + # Reload the shim module so AZURE_RESOURCE_TYPE_SPECS is rebuilt + # against the fresh registry. + import importlib + + import indexers.azureapi_resource_types as mod + + self.mod = importlib.reload(mod) + + def test_typed_collectors_present(self) -> None: + # Pin the full set of CloudQuery tables that have a hand-written + # ``azure-mgmt-*`` collector. Whenever a new SDK collector is added + # in ``azureapi_resource_types`` (and registered via the overrides), + # this set should grow accordingly. + # + # ``AZURE_RESOURCE_TYPE_SPECS`` now contains one entry per registry + # row (typed *and* generic-catch-all specs), so we filter to the + # typed slice for this assertion. + names = { + s.cloudquery_table_name + for s in self.mod.AZURE_RESOURCE_TYPE_SPECS + if s.typed + } + self.assertEqual( + names, + { + # Bootstrap (no per-RG endpoint exists for resource groups). + "azure_resources_resource_groups", + # Compute. + "azure_compute_virtual_machines", + "azure_compute_disks", + "azure_compute_snapshots", + "azure_compute_virtual_machine_scale_sets", + # Storage / KeyVault. + "azure_storage_accounts", + "azure_keyvault_keyvaults", + # Network. + "azure_network_virtual_networks", + "azure_network_security_groups", + "azure_network_load_balancers", + "azure_network_application_gateways", + # Container service. + "azure_containerservice_managed_clusters", + "azure_containerregistry_registries", + # Subscription as a top-level resource. + "azure_subscription_subscriptions", + # App Service. + "azure_appservice_plans", + "azure_appservice_web_apps", + # RDBMS family. + "azure_mysql_servers", + "azure_mysqlflexibleservers_servers", + "azure_postgresql_databases", + # One-off SDKs. + "azure_redis_caches", + "azure_servicebus_namespaces", + "azure_datafactory_factories", + "azure_apimanagement_service", + "azure_cosmos_sql_databases", + "azure_azurearcdata_sql_server_instances", + }, + ) + + def test_resource_group_is_first_and_mandatory(self) -> None: + first = self.mod.AZURE_RESOURCE_TYPE_SPECS[0] + self.assertEqual(first.resource_type_name, "resource_group") + self.assertEqual(first.cloudquery_table_name, "azure_resources_resource_groups") + self.assertTrue(first.mandatory) + + def test_legacy_short_names_resolve_via_find_spec(self) -> None: + spec = self.mod.find_spec("resource_group") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "azure_resources_resource_groups") + + vm = self.mod.find_spec("virtual_machine") + self.assertIsNotNone(vm) + self.assertEqual(vm.cloudquery_table_name, "azure_compute_virtual_machines") + + def test_canonical_cq_name_resolves_via_find_spec(self) -> None: + # ``azure_keyvault_keyvaults`` is the canonical CQ name; the legacy + # RWL spec name was ``azure_keyvault_vaults``. + canonical = self.mod.find_spec("azure_keyvault_keyvaults") + self.assertIsNotNone(canonical) + self.assertEqual(canonical.cloudquery_table_name, "azure_keyvault_keyvaults") + + legacy = self.mod.find_spec("azure_keyvault_vaults") + self.assertIs(legacy, canonical) + self.assertEqual(legacy.resource_type_name, "azure_keyvault_vaults") + + def test_unknown_name_returns_none(self) -> None: + self.assertIsNone(self.mod.find_spec("totally_made_up")) + self.assertIsNone(self.mod.find_spec("")) + + def test_collector_callables_are_callable(self) -> None: + for spec in self.mod.AZURE_RESOURCE_TYPE_SPECS: + self.assertTrue(callable(spec.collector), spec.cloudquery_table_name) diff --git a/src/indexers/test_azureapi_normalizers.py b/src/indexers/test_azureapi_normalizers.py new file mode 100644 index 000000000..27ea7cc6f --- /dev/null +++ b/src/indexers/test_azureapi_normalizers.py @@ -0,0 +1,327 @@ +""" +Unit tests for ``indexers.azureapi_normalizers``. + +The fundamental claim of the new Azure SDK indexer is "byte-compatible output" +with the legacy CloudQuery path. These tests pin the contract: + +1. ``normalize_azure_resource`` produces a dict whose keys / nested shape match + what the CloudQuery sqlite intermediate produces (top-level snake_case for a + small set of well-known keys, ``properties``/``sku``/``identity`` preserved + verbatim, ``tags`` defaulted to ``{}``, ``subscription_id`` always set). + +2. Feeding that dict into ``AzurePlatformHandler.parse_resource_data`` yields + the same ``(name, qualified_name, attributes)`` tuple the existing + resource-dump fixture in ``.test/azure/multi-subscription-aks/output/`` + shows for the equivalent CloudQuery-discovered resource. + +These tests run without any Azure SDK package installed because +``normalize_azure_resource`` accepts plain dicts and SDK-shaped fakes via +``as_dict()``. +""" + +from __future__ import annotations + +import os +import sys +from unittest import TestCase + +# Tests live in the indexers package; ensure the project root is on sys.path +# when these are run directly (e.g. ``python -m unittest indexers.test_...``) +# so the local ``component``, ``resources``, ``enrichers`` modules import. +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.azureapi_normalizers import ( # noqa: E402 + _camel_to_snake, + _sanitize, + normalize_azure_resource, +) + + +SUBSCRIPTION_ID = "2a0cf760-baef-4446-b75c-75c4f8a6267f" + + +class _FakeAzureModel: + """Mimics an Azure SDK model object that exposes ``as_dict()``.""" + + def __init__(self, payload: dict): + self._payload = payload + + def as_dict(self, keep_readonly: bool = True): + return dict(self._payload) + + +def _resource_group_payload(): + return { + "id": f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/sandbox-contoso-rg", + "name": "sandbox-contoso-rg", + "type": "Microsoft.Resources/resourceGroups", + "location": "eastus", + "tags": None, + "managedBy": None, + "properties": {"provisioningState": "Succeeded"}, + } + + +def _storage_account_payload(): + return { + "id": ( + f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/" + f"azure-apps-func-hlth-functions-rg/providers/" + f"Microsoft.Storage/storageAccounts/rwapsfuncstorm1bxh" + ), + "name": "rwapsfuncstorm1bxh", + "type": "Microsoft.Storage/storageAccounts", + "kind": "StorageV2", + "location": "canadacentral", + "tags": { + "env": "test", + "lifecycle": "deleteme", + "product": "runwhen", + }, + "extendedLocation": None, + "identity": {"type": "None"}, + "sku": {"name": "Standard_LRS", "tier": "Standard"}, + "properties": { + "accessTier": "Hot", + "allowBlobPublicAccess": True, + "provisioningState": "Succeeded", + "primaryEndpoints": { + "blob": "https://rwapsfuncstorm1bxh.blob.core.windows.net/", + }, + }, + } + + +# --------------------------------------------------------------------------- +# normalize_azure_resource +# --------------------------------------------------------------------------- + +class NormalizeAzureResourceTests(TestCase): + + def test_camel_to_snake(self): + self.assertEqual("foo_bar", _camel_to_snake("fooBar")) + self.assertEqual("a_b_c", _camel_to_snake("aBC")) + self.assertEqual("simple", _camel_to_snake("simple")) + + def test_sanitize_handles_datetimes_enums_nested(self): + import datetime + import enum + + class Color(enum.Enum): + RED = "red" + + payload = { + "ts": datetime.datetime(2024, 1, 2, 3, 4, 5), + "color": Color.RED, + "nested": {"more_ts": datetime.datetime(2025, 1, 1)}, + "list": [Color.RED, datetime.date(2025, 6, 1)], + } + out = _sanitize(payload) + self.assertEqual(out["ts"], "2024-01-02T03:04:05") + self.assertEqual(out["color"], "red") + self.assertEqual(out["nested"]["more_ts"], "2025-01-01T00:00:00") + self.assertEqual(out["list"][0], "red") + self.assertEqual(out["list"][1], "2025-06-01") + + def test_normalize_resource_group_top_level_shape(self): + model = _FakeAzureModel(_resource_group_payload()) + data = normalize_azure_resource( + model, subscription_id=SUBSCRIPTION_ID, resource_type_name="resource_group" + ) + self.assertEqual(data["name"], "sandbox-contoso-rg") + self.assertEqual(data["type"], "Microsoft.Resources/resourceGroups") + self.assertEqual(data["location"], "eastus") + self.assertEqual(data["subscription_id"], SUBSCRIPTION_ID) + # tags must be a dict, never None - generation rules path-match into it. + self.assertEqual(data["tags"], {}) + # Azure SDK's camelCase 'managedBy' should be renamed to 'managed_by' so + # it matches the CQ row shape that existing rules may path-match. + self.assertIn("managed_by", data) + self.assertNotIn("managedBy", data) + # 'properties' is the Azure REST payload, kept camelCase as in CQ. + self.assertEqual(data["properties"], {"provisioningState": "Succeeded"}) + + def test_normalize_storage_account_preserves_nested_camel_case(self): + model = _FakeAzureModel(_storage_account_payload()) + data = normalize_azure_resource( + model, + subscription_id=SUBSCRIPTION_ID, + resource_type_name="azure_storage_accounts", + ) + # Top-level extendedLocation must be renamed; nested camelCase under + # 'properties' / 'sku' must be left exactly as Azure returns it. + self.assertIn("extended_location", data) + self.assertNotIn("extendedLocation", data) + self.assertEqual(data["sku"], {"name": "Standard_LRS", "tier": "Standard"}) + self.assertEqual(data["properties"]["accessTier"], "Hot") + self.assertEqual( + data["properties"]["primaryEndpoints"]["blob"], + "https://rwapsfuncstorm1bxh.blob.core.windows.net/", + ) + self.assertEqual(data["kind"], "StorageV2") + + def test_normalize_accepts_plain_dict(self): + # When tests / AWS / GCP code passes through plain dicts (e.g. from + # CloudQuery), the normalizer should still produce the canonical shape. + data = normalize_azure_resource( + _resource_group_payload(), + subscription_id=SUBSCRIPTION_ID, + resource_type_name="resource_group", + ) + self.assertEqual(data["subscription_id"], SUBSCRIPTION_ID) + self.assertEqual(data["tags"], {}) + + def test_subscription_id_overwritten_to_collector_value(self): + # The collector knows which subscription the resource was discovered + # in; that wins over whatever the SDK payload happens to contain. + payload = _resource_group_payload() + payload["subscriptionId"] = "garbage-value" + model = _FakeAzureModel(payload) + data = normalize_azure_resource( + model, subscription_id=SUBSCRIPTION_ID, resource_type_name="resource_group" + ) + self.assertEqual(data["subscription_id"], SUBSCRIPTION_ID) + + +# --------------------------------------------------------------------------- +# Round-trip: normalizer → AzurePlatformHandler.parse_resource_data +# --------------------------------------------------------------------------- + +class NormalizerParserRoundTripTests(TestCase): + """End-to-end check: the dict produced by the normalizer feeds cleanly + through ``AzurePlatformHandler.parse_resource_data`` and yields the same + ``(name, qualified_name, attributes)`` shape the legacy CloudQuery dump + shows for an equivalent resource. + """ + + def setUp(self): + # The platform handler uses ``Context.get_setting("DEFAULT_LOD")`` and + # the registry on the context; we provide the minimum surface here. + from component import Context # local import: needs component bootstrap + from outputter import FileItemOutputter + from resources import REGISTRY_PROPERTY_NAME, Registry + from enrichers.generation_rule_types import LevelOfDetail + + self.context = Context( + setting_values={"DEFAULT_LOD": LevelOfDetail.BASIC}, + outputter=FileItemOutputter(), + ) + self.context.set_property(REGISTRY_PROPERTY_NAME, Registry()) + + def test_resource_group_round_trip_matches_dump_shape(self): + from enrichers.azure import AzurePlatformHandler + + platform_cfg = { + "subscriptions": [{"subscriptionId": SUBSCRIPTION_ID}], + "subscriptionResourceGroupLevelOfDetails": { + SUBSCRIPTION_ID: {"sandbox-contoso-rg": "none"}, + }, + } + data = normalize_azure_resource( + _FakeAzureModel(_resource_group_payload()), + subscription_id=SUBSCRIPTION_ID, + resource_type_name="resource_group", + ) + handler = AzurePlatformHandler() + name, qualified_name, attrs = handler.parse_resource_data( + data, "resource_group", platform_cfg, self.context + ) + self.assertEqual(name, "sandbox-contoso-rg") + self.assertEqual(qualified_name, "sandbox-contoso-rg") + self.assertEqual(attrs["tags"], {}) + self.assertEqual(attrs["subscription_id"], SUBSCRIPTION_ID) + self.assertIsNotNone(attrs.get("lod")) + + def test_child_resource_round_trip_links_to_resource_group(self): + from enrichers.azure import AZURE_PLATFORM, AzurePlatformHandler + from resources import REGISTRY_PROPERTY_NAME + + platform_cfg = { + "subscriptions": [{"subscriptionId": SUBSCRIPTION_ID}], + "subscriptionResourceGroupLevelOfDetails": { + SUBSCRIPTION_ID: {"*": "detailed"}, + }, + } + # Pre-populate registry with the parent RG so the parser can link. + registry = self.context.get_property(REGISTRY_PROPERTY_NAME) + registry.add_resource( + AZURE_PLATFORM, + "resource_group", + "azure-apps-func-hlth-functions-rg", + "azure-apps-func-hlth-functions-rg", + {"subscription_id": SUBSCRIPTION_ID, "tags": {}}, + ) + + data = normalize_azure_resource( + _FakeAzureModel(_storage_account_payload()), + subscription_id=SUBSCRIPTION_ID, + resource_type_name="azure_storage_accounts", + ) + handler = AzurePlatformHandler() + name, qualified_name, attrs = handler.parse_resource_data( + data, "azure_storage_accounts", platform_cfg, self.context + ) + self.assertEqual(name, "rwapsfuncstorm1bxh") + # The CQ dump for this resource shows the qualified_name as + # "/"; the parser should reproduce it. + self.assertEqual( + qualified_name, + "azure-apps-func-hlth-functions-rg/rwapsfuncstorm1bxh", + ) + self.assertEqual(attrs["subscription_id"], SUBSCRIPTION_ID) + self.assertIn("resource_group", attrs) + self.assertEqual( + attrs["resource_group"].name, "azure-apps-func-hlth-functions-rg" + ) + + def test_child_resource_with_missing_rg_records_deferred_lookup(self): + # Mirror CQ behavior: when a child is indexed and the resource_group + # type exists in the registry but not the specific parent RG, the + # parser records a deferred lookup so + # ``resolve_deferred_azure_relationships`` can fix the link later. + from enrichers.azure import AZURE_PLATFORM, AzurePlatformHandler + from resources import REGISTRY_PROPERTY_NAME + + platform_cfg = { + "subscriptions": [{"subscriptionId": SUBSCRIPTION_ID}], + "subscriptionResourceGroupLevelOfDetails": { + SUBSCRIPTION_ID: {"*": "basic"}, + }, + } + + # Seed the registry with an unrelated RG so the ``resource_group`` + # type exists. This matches the real flow where RGs from one + # subscription have already been written before children from + # another are processed. + registry = self.context.get_property(REGISTRY_PROPERTY_NAME) + registry.add_resource( + AZURE_PLATFORM, + "resource_group", + "unrelated-rg", + "unrelated-rg", + {"subscription_id": "00000000-0000-0000-0000-000000000000", "tags": {}}, + ) + + data = normalize_azure_resource( + _FakeAzureModel(_storage_account_payload()), + subscription_id=SUBSCRIPTION_ID, + resource_type_name="azure_storage_accounts", + ) + handler = AzurePlatformHandler() + _, qualified_name, attrs = handler.parse_resource_data( + data, "azure_storage_accounts", platform_cfg, self.context + ) + # When the RG isn't in the registry yet, qualified_name is rg/name + # too (parser computes it from the ID), and the deferred-lookup info + # is stamped on the attributes for later resolution. + self.assertEqual( + qualified_name, + "azure-apps-func-hlth-functions-rg/rwapsfuncstorm1bxh", + ) + deferred = attrs.get("_deferred_rg_lookup") + self.assertIsNotNone(deferred) + self.assertEqual(deferred["rg_name"], "azure-apps-func-hlth-functions-rg") + self.assertEqual(deferred["subscription_id"], SUBSCRIPTION_ID) diff --git a/src/indexers/test_azureapi_selective.py b/src/indexers/test_azureapi_selective.py new file mode 100644 index 000000000..77bba2005 --- /dev/null +++ b/src/indexers/test_azureapi_selective.py @@ -0,0 +1,773 @@ +""" +Unit tests for the selective-indexing helpers in ``indexers.azureapi``. + +Coverage: +* ``_extract_rg_name_from_arm_id`` and ``_extract_subscription_id_from_arm_id`` + parse the ARM resource ID grammar correctly (case-insensitive markers, + preserves value casing, tolerates missing pieces). +* ``_compute_effective_lod`` mirrors the lookup chain documented in + ``AzurePlatformHandler.parse_resource_data``. +* ``_resource_is_in_scope`` returns False iff the effective LOD resolves to + ``LevelOfDetail.NONE``, dropping out-of-scope resources before they reach + the writer. + +The tests deliberately operate on the helper functions directly rather than +running the whole ``index()`` pipeline so they stay fast and don't require +any Azure SDK clients or stubs. +""" + +from __future__ import annotations + +import os +import sys +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from enrichers.generation_rule_types import LevelOfDetail # noqa: E402 + +from indexers.azureapi import ( # noqa: E402 + _build_subscription_rg_lod_map, + _compute_effective_lod, + _extract_rg_name_from_arm_id, + _extract_subscription_id_from_arm_id, + _resource_is_in_scope, + _rgs_in_scope_from_config, + _safe_lod, +) +from indexers.azureapi_resource_types import ( # noqa: E402 + AZURE_RESOURCE_TYPE_SPECS, + AzureResourceTypeSpec, + find_spec, + find_spec_by_arm_type, +) + + +_KEEP_ARM_ID = ( + "/subscriptions/abc-123/resourceGroups/keepRG" + "/providers/Microsoft.Storage/storageAccounts/keep" +) +_DROP_ARM_ID = ( + "/subscriptions/abc-123/resourceGroups/dropRG" + "/providers/Microsoft.Storage/storageAccounts/drop" +) + + +class ArmIdParserTests(TestCase): + + def test_extract_rg_basic(self) -> None: + self.assertEqual(_extract_rg_name_from_arm_id(_KEEP_ARM_ID), "keepRG") + + def test_extract_rg_case_insensitive_marker_preserves_value_case(self) -> None: + # Real-world Azure IDs occasionally use ``resourceGroups`` casing + # variations; the marker match must be case-insensitive but the + # extracted value must preserve casing for registry lookups. + arm = ( + "/SUBSCRIPTIONS/SUB/RESOURCEGROUPS/MixedCaseRG" + "/providers/Microsoft.Storage/storageAccounts/x" + ) + self.assertEqual(_extract_rg_name_from_arm_id(arm), "MixedCaseRG") + + def test_extract_rg_handles_trailing_rg_only(self) -> None: + arm = "/subscriptions/sub/resourceGroups/onlyRG" + self.assertEqual(_extract_rg_name_from_arm_id(arm), "onlyRG") + + def test_extract_rg_returns_none_for_missing_marker(self) -> None: + self.assertIsNone(_extract_rg_name_from_arm_id("")) + self.assertIsNone(_extract_rg_name_from_arm_id(None)) + self.assertIsNone(_extract_rg_name_from_arm_id("/subscriptions/sub/whatever")) + + def test_extract_subscription_basic(self) -> None: + self.assertEqual(_extract_subscription_id_from_arm_id(_KEEP_ARM_ID), "abc-123") + + def test_extract_subscription_case_insensitive_marker(self) -> None: + arm = "/SUBSCRIPTIONS/sub-xyz/resourceGroups/rg" + self.assertEqual(_extract_subscription_id_from_arm_id(arm), "sub-xyz") + + def test_extract_subscription_returns_none_for_garbage(self) -> None: + self.assertIsNone(_extract_subscription_id_from_arm_id("/no/markers/here")) + self.assertIsNone(_extract_subscription_id_from_arm_id(None)) + + +class LevelOfDetailLookupTests(TestCase): + + def _platform_cfg_with_subscriptions(self) -> dict: + # Mimic the workspaceInfo "subscriptions" block; the per-subscription + # ``defaultLOD: none`` is what makes "drop everything not explicitly + # whitelisted" work. + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": { + "keepRG": "detailed", + "dropRG": "none", + }, + } + ], + } + _build_subscription_rg_lod_map(cfg) + return cfg + + def test_keep_rg_resolves_detailed(self) -> None: + cfg = self._platform_cfg_with_subscriptions() + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "keepRG", LevelOfDetail.BASIC), + LevelOfDetail.DETAILED, + ) + + def test_drop_rg_resolves_none(self) -> None: + cfg = self._platform_cfg_with_subscriptions() + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "dropRG", LevelOfDetail.BASIC), + LevelOfDetail.NONE, + ) + + def test_unknown_rg_under_known_sub_uses_per_sub_default(self) -> None: + # The per-subscription ``defaultLOD: none`` is stuffed into ``sub_map["*"]`` + # so unknown RGs in that subscription inherit it. + cfg = self._platform_cfg_with_subscriptions() + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "noSuchRG", LevelOfDetail.BASIC), + LevelOfDetail.NONE, + ) + + def test_unknown_subscription_falls_through_to_default(self) -> None: + cfg = self._platform_cfg_with_subscriptions() + self.assertEqual( + _compute_effective_lod(cfg, "different-sub", "anyRG", LevelOfDetail.BASIC), + LevelOfDetail.BASIC, + ) + + def test_top_level_lod_map_with_explicit_overrides(self) -> None: + # Top-level (no subscriptions list): only ``resourceGroupLevelOfDetails`` + # is fed into the LOD map; the workspace ``defaultLOD`` falls through + # to the default arg. + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "detailed", + "resourceGroupLevelOfDetails": {"keepRG": "detailed"}, + } + _build_subscription_rg_lod_map(cfg) + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "keepRG", LevelOfDetail.BASIC), + LevelOfDetail.DETAILED, + ) + # Unknown RG falls through to the default arg passed in. + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "noSuchRG", LevelOfDetail.BASIC), + LevelOfDetail.BASIC, + ) + + def test_invalid_lod_string_falls_back_to_default(self) -> None: + cfg = { + "subscriptionResourceGroupLevelOfDetails": { + "abc-123": {"weirdRG": "garbage"} + } + } + self.assertEqual( + _compute_effective_lod(cfg, "abc-123", "weirdRG", LevelOfDetail.BASIC), + LevelOfDetail.BASIC, + ) + + +class ResourceIsInScopeTests(TestCase): + + def _cfg(self) -> dict: + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": { + "keepRG": "detailed", + "dropRG": "none", + }, + } + ], + } + _build_subscription_rg_lod_map(cfg) + return cfg + + def test_storage_under_keep_rg_in_scope(self) -> None: + in_scope, lod = _resource_is_in_scope( + self._cfg(), + {"id": _KEEP_ARM_ID, "name": "keep"}, + "azure_storage_accounts", + "abc-123", + LevelOfDetail.BASIC, + ) + self.assertTrue(in_scope) + self.assertEqual(lod, LevelOfDetail.DETAILED) + + def test_storage_under_drop_rg_out_of_scope(self) -> None: + in_scope, lod = _resource_is_in_scope( + self._cfg(), + {"id": _DROP_ARM_ID, "name": "drop"}, + "azure_storage_accounts", + "abc-123", + LevelOfDetail.BASIC, + ) + self.assertFalse(in_scope) + self.assertEqual(lod, LevelOfDetail.NONE) + + def test_resource_group_keep_in_scope(self) -> None: + in_scope, lod = _resource_is_in_scope( + self._cfg(), + { + "id": "/subscriptions/abc-123/resourceGroups/keepRG", + "name": "keepRG", + }, + "resource_group", + "abc-123", + LevelOfDetail.BASIC, + ) + self.assertTrue(in_scope) + self.assertEqual(lod, LevelOfDetail.DETAILED) + + def test_resource_group_drop_out_of_scope(self) -> None: + in_scope, lod = _resource_is_in_scope( + self._cfg(), + { + "id": "/subscriptions/abc-123/resourceGroups/dropRG", + "name": "dropRG", + }, + "resource_group", + "abc-123", + LevelOfDetail.BASIC, + ) + self.assertFalse(in_scope) + self.assertEqual(lod, LevelOfDetail.NONE) + + def test_subscription_resources_always_in_scope(self) -> None: + # Subscription resources are never dropped at the index layer because + # downstream code (rendering / RG attribution) needs them. + in_scope, lod = _resource_is_in_scope( + self._cfg(), + {"id": "/subscriptions/abc-123", "name": "abc-123"}, + "azure_subscription_subscriptions", + "abc-123", + LevelOfDetail.BASIC, + ) + self.assertTrue(in_scope) + self.assertEqual(lod, LevelOfDetail.BASIC) + + def test_unknown_subscription_uses_default(self) -> None: + # Resources whose subscription isn't in the LOD map at all fall + # through to the workspace default; if the default is BASIC they + # remain in scope. + in_scope, lod = _resource_is_in_scope( + self._cfg(), + { + "id": "/subscriptions/different-sub/resourceGroups/anyRG" + "/providers/Microsoft.Storage/storageAccounts/x", + "name": "x", + }, + "azure_storage_accounts", + "different-sub", + LevelOfDetail.BASIC, + ) + self.assertTrue(in_scope) + self.assertEqual(lod, LevelOfDetail.BASIC) + + +class DiscoveryScopeTests(TestCase): + """``_rgs_in_scope_from_config`` decides whether the indexer should + enumerate per-RG (finite list returned) or subscription-wide (None). + These tests pin the contract that selective discovery is triggered + only when *every* escape hatch (per-sub wildcard, global wildcard, + workspace default) resolves to NONE.""" + + def test_selective_mode_returns_finite_rg_list(self) -> None: + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": { + "keepRG": "detailed", + "alsoKeep": "basic", + "dropRG": "none", + }, + } + ], + } + _build_subscription_rg_lod_map(cfg) + rgs = _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + self.assertIsNotNone(rgs) + self.assertEqual(sorted(rgs), ["alsoKeep", "keepRG"]) + + def test_default_lod_basic_returns_none(self) -> None: + cfg = {"subscriptionId": "abc-123", "defaultLOD": "basic"} + _build_subscription_rg_lod_map(cfg) + self.assertIsNone( + _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.BASIC) + ) + + def test_default_lod_detailed_returns_none(self) -> None: + cfg = {"subscriptionId": "abc-123", "defaultLOD": "detailed"} + _build_subscription_rg_lod_map(cfg) + self.assertIsNone( + _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.DETAILED) + ) + + def test_per_sub_wildcard_non_none_returns_none(self) -> None: + # Top-level defaultLOD is none, but the per-subscription default is + # 'basic' so the subscription-wide wildcard kicks in. + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "abc-123", + "defaultLOD": "basic", + "resourceGroupLevelOfDetails": {"keepRG": "detailed"}, + } + ], + } + _build_subscription_rg_lod_map(cfg) + self.assertIsNone( + _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + ) + + def test_global_wildcard_non_none_returns_none(self) -> None: + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": {"*": "detailed", "keepRG": "detailed"}, + } + _build_subscription_rg_lod_map(cfg) + self.assertIsNone( + _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + ) + + def test_legacy_top_level_rg_overrides_promote_to_in_scope(self) -> None: + # No subscriptions[]; user uses the legacy top-level + # resourceGroupLevelOfDetails dict + defaultLOD: none. + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": {"oldStyleRG": "detailed"}, + } + _build_subscription_rg_lod_map(cfg) + rgs = _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + self.assertIsNotNone(rgs) + self.assertEqual(rgs, ["oldStyleRG"]) + + def test_explicit_none_override_excluded_from_finite_list(self) -> None: + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "abc-123", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": { + "keepRG": "detailed", + "dropRG": "none", + }, + } + ], + } + _build_subscription_rg_lod_map(cfg) + rgs = _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + self.assertEqual(rgs, ["keepRG"]) + self.assertNotIn("dropRG", rgs) + + def test_no_overrides_at_all_returns_empty_list(self) -> None: + # When defaultLOD is none and no per-RG overrides exist anywhere, + # selective discovery resolves to "discover nothing". This is + # behaviorally identical to "discover everything and drop everything" + # but burns zero SDK calls. + cfg = { + "subscriptionId": "abc-123", + "defaultLOD": "none", + } + _build_subscription_rg_lod_map(cfg) + rgs = _rgs_in_scope_from_config(cfg, "abc-123", LevelOfDetail.NONE) + self.assertEqual(rgs, []) + + def test_unknown_subscription_falls_through_to_default(self) -> None: + cfg = {"subscriptionId": "abc-123", "defaultLOD": "detailed"} + _build_subscription_rg_lod_map(cfg) + # An unknown subscription with workspace default 'detailed' is + # subscription-wide. + self.assertIsNone( + _rgs_in_scope_from_config(cfg, "no-such-sub", LevelOfDetail.DETAILED) + ) + # But with workspace default 'none', the unknown sub also hits the + # default-NONE branch and resolves to "selective with empty list". + self.assertEqual( + _rgs_in_scope_from_config(cfg, "no-such-sub", LevelOfDetail.NONE), + [], + ) + + def test_safe_lod_helpers(self) -> None: + self.assertIsNone(_safe_lod(None)) + self.assertIsNone(_safe_lod("garbage")) + self.assertEqual(_safe_lod("none"), LevelOfDetail.NONE) + self.assertEqual(_safe_lod("DETAILED"), LevelOfDetail.DETAILED) + # Already-an-enum input is fine. + self.assertEqual(_safe_lod(LevelOfDetail.BASIC), LevelOfDetail.BASIC) + + +class DiscoveryDispatchTests(TestCase): + """End-to-end ``index()`` invocations with stubbed collectors, asserting + that selective discovery actually calls per-RG SDK methods (not the + subscription-wide ones) when the workspaceInfo declares a finite scope. + + The test wires: + * a fake azureapi backend setting, + * a stub credentials provider, + * stubbed ``collector_all`` and ``collector_in_rg`` callables that + record their invocations. + + What we assert: + * In selective mode, ``collector_in_rg`` is called exactly for the + whitelisted RGs and ``collector_all`` is *not* called for the spec. + * In subscription-wide mode, the inverse holds. + * Resource-group enumeration always uses ``collector_all`` (RGs have + no per-RG endpoint). + """ + + def setUp(self) -> None: + self._calls = {"all": [], "in_rg": []} + + def _make_spec(self, *, supports_in_rg: bool = True): + """Build a minimal AzureResourceTypeSpec with stubbed collectors.""" + from indexers.azureapi_resource_types import AzureResourceTypeSpec + + def _fake_all(credential, subscription_id): + self._calls["all"].append((subscription_id,)) + return [] + + def _fake_in_rg(credential, subscription_id, rg_name): + self._calls["in_rg"].append((subscription_id, rg_name)) + return [] + + return AzureResourceTypeSpec( + resource_type_name="azure_storage_accounts", + cloudquery_table_name="azure_storage_accounts", + # Mandatory in the test so we don't have to wire up the + # generation-rule access list; production specs gate via + # ``RESOURCE_TYPE_SPECS_PROPERTY`` on the context. + mandatory=True, + collector_all=_fake_all, + collector_in_rg=_fake_in_rg if supports_in_rg else None, + ) + + def _make_rg_spec(self): + from indexers.azureapi_resource_types import AzureResourceTypeSpec + + def _fake_rg_list(credential, subscription_id): + self._calls["all"].append((subscription_id, "rg-list")) + return [] + + return AzureResourceTypeSpec( + resource_type_name="resource_group", + cloudquery_table_name="azure_resources_resource_groups", + mandatory=True, + collector_all=_fake_rg_list, + collector_in_rg=None, + ) + + def _run_index(self, *, platform_cfg, specs, subscription_ids): + """Drive ``index()`` with patched dependencies. Returns the indexer's + captured stats dict.""" + from unittest import mock + + from indexers import azureapi + + captured_stats = {} + + # The indexer reads several context settings + helpers; patch the + # exact pieces it touches and let everything else stay real. + with mock.patch.object(azureapi, "AZURE_RESOURCE_TYPE_SPECS", tuple(specs)), \ + mock.patch.object( + azureapi, "az_get_credentials_and_subscription_id", + return_value={ + "credential": object(), + "subscription_ids": list(subscription_ids), + "AZURE_TENANT_ID": "t", + "AZURE_CLIENT_ID": "c", + "AZURE_CLIENT_SECRET": "s", + }, + ), \ + mock.patch.object(azureapi, "_resolve_platform_handler") as resolve_handler, \ + mock.patch.object(azureapi, "get_resource_writer") as get_writer, \ + mock.patch.object(azureapi, "find_spec") as find_spec_mock, \ + mock.patch("enrichers.azure.set_azure_credentials"): + + # parse_resource_data is bypassed by our stub collectors returning + # empty lists (no models -> no parse calls). + resolve_handler.return_value = mock.MagicMock() + writer = mock.MagicMock() + get_writer.return_value = writer + # Pretend every spec the test cares about is registered. + find_spec_mock.side_effect = lambda name: next( + (s for s in specs + if s.resource_type_name == name or s.cloudquery_table_name == name), + None, + ) + + # Build a Context that returns our cloud_config + indexer backend. + class FakeContext: + def __init__(self, platform_cfg): + self._cloud_cfg = {"azure": dict(platform_cfg)} + self._properties = {} + + def get_setting(self, setting): + name = getattr(setting, "env_var", None) or getattr(setting, "name", "") + if name == "AZURE_INDEXER_BACKEND": + return "azureapi" + if name == "CLOUD_CONFIG": + return self._cloud_cfg + if name == "RESOURCE_STORE_BACKEND": + return "memory" + if name == "RESOURCE_STORE_PATH": + return None + if name == "DEFAULT_LOD": + return None + return None + + def get_property(self, name): + return self._properties.get(name) + + def add_warning(self, msg): + pass + + ctx = FakeContext(platform_cfg) + azureapi.index(ctx) + + captured_stats["writer_calls"] = writer.add_resource.call_count + + return captured_stats + + def test_selective_mode_uses_per_rg_collector(self) -> None: + rg_spec = self._make_rg_spec() + storage_spec = self._make_spec(supports_in_rg=True) + + platform_cfg = { + "subscriptionId": "sub-A", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "sub-A", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": { + "keepA": "detailed", + "keepB": "basic", + }, + } + ], + } + + self._run_index( + platform_cfg=platform_cfg, + specs=[rg_spec, storage_spec], + subscription_ids=["sub-A"], + ) + + # RG enumeration always uses collector_all. + self.assertIn(("sub-A", "rg-list"), self._calls["all"]) + # Storage discovery should be per-RG, exactly for keepA + keepB. + self.assertEqual( + sorted(self._calls["in_rg"]), + [("sub-A", "keepA"), ("sub-A", "keepB")], + ) + # And the subscription-wide storage list endpoint was NOT hit. + self.assertNotIn(("sub-A",), self._calls["all"]) + + def test_unbounded_mode_uses_subscription_wide_collector(self) -> None: + rg_spec = self._make_rg_spec() + storage_spec = self._make_spec(supports_in_rg=True) + + platform_cfg = { + "subscriptionId": "sub-A", + "defaultLOD": "detailed", + } + + self._run_index( + platform_cfg=platform_cfg, + specs=[rg_spec, storage_spec], + subscription_ids=["sub-A"], + ) + + self.assertIn(("sub-A", "rg-list"), self._calls["all"]) + self.assertIn(("sub-A",), self._calls["all"]) + self.assertEqual(self._calls["in_rg"], []) + + def test_selective_mode_falls_back_to_all_when_no_per_rg_collector(self) -> None: + """A spec that opted out of per-RG collection still works: the + indexer falls back to ``collector_all`` and emits a warning.""" + rg_spec = self._make_rg_spec() + opted_out_spec = self._make_spec(supports_in_rg=False) + + platform_cfg = { + "subscriptionId": "sub-A", + "defaultLOD": "none", + "subscriptions": [ + { + "subscriptionId": "sub-A", + "defaultLOD": "none", + "resourceGroupLevelOfDetails": {"keepA": "detailed"}, + } + ], + } + + self._run_index( + platform_cfg=platform_cfg, + specs=[rg_spec, opted_out_spec], + subscription_ids=["sub-A"], + ) + + # subscription-wide list_all was called for the opted-out spec. + self.assertIn(("sub-A",), self._calls["all"]) + # and per-RG was NOT called (no callable to call). + self.assertEqual(self._calls["in_rg"], []) + + def test_new_typed_collectors_are_registered(self) -> None: + """Lock in the comprehensive collector coverage added in the + Bucket A/B/C/D pass: every CQ table that the indexer is supposed + to have a typed collector for must show up in + ``AZURE_RESOURCE_TYPE_SPECS`` and resolve via ``find_spec``. + + Failing this test means the registry / overrides / dispatch dict + have drifted out of sync with one another. + """ + from indexers.azureapi_resource_types import ( + AZURE_RESOURCE_TYPE_SPECS, + find_spec, + ) + + expected_typed_tables = { + "azure_apimanagement_service", + "azure_appservice_plans", + "azure_appservice_web_apps", + "azure_azurearcdata_sql_server_instances", + "azure_compute_disks", + "azure_compute_snapshots", + "azure_compute_virtual_machine_scale_sets", + "azure_compute_virtual_machines", + "azure_containerregistry_registries", + "azure_containerservice_managed_clusters", + "azure_cosmos_sql_databases", + "azure_datafactory_factories", + "azure_keyvault_keyvaults", + "azure_mysql_servers", + "azure_mysqlflexibleservers_servers", + "azure_network_application_gateways", + "azure_network_load_balancers", + "azure_network_security_groups", + "azure_network_virtual_networks", + "azure_postgresql_databases", + "azure_redis_caches", + "azure_resources_resource_groups", + "azure_servicebus_namespaces", + "azure_storage_accounts", + "azure_subscription_subscriptions", + } + registered = {s.cloudquery_table_name for s in AZURE_RESOURCE_TYPE_SPECS} + missing = expected_typed_tables - registered + self.assertFalse(missing, f"Missing typed collectors: {sorted(missing)}") + + # find_spec should resolve every expected table by CQ name. + for table in expected_typed_tables: + self.assertIsNotNone( + find_spec(table), + f"find_spec returned None for registered table {table}", + ) + + # Subscriptions opt out of selective per-RG enumeration; everything + # else exposes both ``collector_all`` and ``collector_in_rg``. + for spec in AZURE_RESOURCE_TYPE_SPECS: + if spec.cloudquery_table_name in ( + "azure_resources_resource_groups", + "azure_subscription_subscriptions", + ): + self.assertFalse( + spec.supports_in_rg, + f"{spec.cloudquery_table_name} should not have a per-RG collector", + ) + else: + self.assertTrue( + spec.supports_in_rg, + f"{spec.cloudquery_table_name} is missing a per-RG collector", + ) + + def test_generic_specs_materialize_for_every_registry_entry(self) -> None: + """Every registry entry must resolve via ``find_spec`` - typed + when a hand-written collector exists, generic-catch-all otherwise. + Together this is what gives the indexer coverage parity with the + CloudQuery plugin's resource table. + """ + from indexers.azure_resource_type_registry import load_registry + + registry = load_registry() + # Every CQ table the registry knows about should have a spec. + for entry in registry: + spec = find_spec(entry.cloudquery_table_name) + self.assertIsNotNone( + spec, f"registry entry {entry.cloudquery_table_name} has no spec" + ) + # The set of typed specs is a strict subset. + typed = {s for s in AZURE_RESOURCE_TYPE_SPECS if s.typed} + all_specs = set(AZURE_RESOURCE_TYPE_SPECS) + self.assertLess(len(typed), len(all_specs)) + for s in typed: + self.assertIn(s, all_specs) + + def test_find_spec_by_arm_type_routes_generic_resources(self) -> None: + # An ARM type without a typed collector should round-trip through + # the registry to a generic spec. ``Microsoft.Logic/workflows`` is + # a representative non-typed ARM type today. + spec = find_spec_by_arm_type("Microsoft.Logic/workflows") + self.assertIsNotNone(spec) + self.assertFalse(spec.typed) + self.assertEqual(spec.cloudquery_table_name, "azure_logic_workflows") + # Case-insensitive on ARM type. + same = find_spec_by_arm_type("microsoft.logic/WORKFLOWS") + self.assertEqual(same, spec) + # And a typed ARM type still resolves to the typed spec. + kv = find_spec_by_arm_type("Microsoft.KeyVault/vaults") + self.assertIsNotNone(kv) + self.assertTrue(kv.typed) + self.assertEqual(kv.cloudquery_table_name, "azure_keyvault_keyvaults") + # Unknown ARM types return None. + self.assertIsNone(find_spec_by_arm_type("Microsoft.Nope/notathing")) + self.assertIsNone(find_spec_by_arm_type(None)) + self.assertIsNone(find_spec_by_arm_type("")) + + def test_no_in_scope_rgs_means_zero_sdk_calls_for_storage(self) -> None: + """With defaultLOD=none and no per-RG overrides, selective scope + resolves to an empty list. The RG enumeration still runs once + (we need the RG list to know what exists); storage enumeration + is skipped entirely.""" + rg_spec = self._make_rg_spec() + storage_spec = self._make_spec(supports_in_rg=True) + + platform_cfg = { + "subscriptionId": "sub-A", + "defaultLOD": "none", + } + + self._run_index( + platform_cfg=platform_cfg, + specs=[rg_spec, storage_spec], + subscription_ids=["sub-A"], + ) + + self.assertIn(("sub-A", "rg-list"), self._calls["all"]) + self.assertEqual(self._calls["in_rg"], []) + # No subscription-wide storage list either. + self.assertNotIn(("sub-A",), self._calls["all"]) diff --git a/src/indexers/test_gcp_resource_type_registry.py b/src/indexers/test_gcp_resource_type_registry.py new file mode 100644 index 000000000..52676a4f3 --- /dev/null +++ b/src/indexers/test_gcp_resource_type_registry.py @@ -0,0 +1,339 @@ +""" +Unit tests for the GCP resource-type registry loader and the spec layer it +powers (``indexers.gcpapi_resource_types``). + +These tests pin the contract that: + +1. The loader produces a fully-populated registry from the YAML on disk + (404 entries today, with stable metadata fields and alias resolution). + +2. The hand-written typed-collector specs in ``gcpapi_resource_types`` resolve + the legacy ``resource_type_name`` values generation rules expect + (``project``, ``compute_instance``) and the canonical CQ table names. + +3. Aliases route to the canonical CQ table name; Cloud Asset Inventory asset + types reverse-map back to the owning table. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import textwrap +from pathlib import Path +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.gcp_resource_type_registry import ( # noqa: E402 + find_entry, + load_registry, + load_registry_from_path, + reset_cache, +) + + +class RegistryLoaderTests(TestCase): + def setUp(self) -> None: + reset_cache() + + def tearDown(self) -> None: + reset_cache() + + def test_default_registry_loads_with_expected_metadata(self) -> None: + registry = load_registry() + self.assertGreater(len(registry), 100) # 404 today, allow for growth + self.assertEqual(len(registry), registry.metadata.total_tables) + self.assertGreaterEqual(registry.metadata.typed_collectors, 1) + self.assertEqual( + registry.metadata.generator, + "scripts/gcp/sync_gcp_resource_type_registry.py", + ) + self.assertTrue( + registry.metadata.source.startswith("https://"), + f"metadata.source should be a URL, got {registry.metadata.source!r}", + ) + + def test_registry_lookup_by_canonical_name(self) -> None: + registry = load_registry() + projects = registry.find("gcp_projects") + self.assertIsNotNone(projects) + self.assertEqual(projects.cloudquery_table_name, "gcp_projects") + self.assertEqual( + projects.cai_asset_type, "cloudresourcemanager.googleapis.com/Project" + ) + self.assertTrue(projects.mandatory) + self.assertTrue(projects.typed_collector) + + def test_registry_lookup_by_alias(self) -> None: + registry = load_registry() + legacy_project = registry.find("project") + self.assertIsNotNone(legacy_project) + self.assertEqual(legacy_project.cloudquery_table_name, "gcp_projects") + + legacy_instance = registry.find("compute_instance") + self.assertIsNotNone(legacy_instance) + self.assertEqual( + legacy_instance.cloudquery_table_name, "gcp_compute_instances" + ) + + def test_registry_find_by_cai_type(self) -> None: + registry = load_registry() + entry = registry.find_by_cai_type("compute.googleapis.com/Instance") + self.assertIsNotNone(entry) + self.assertEqual(entry.cloudquery_table_name, "gcp_compute_instances") + # Case-insensitive. + entry2 = registry.find_by_cai_type("COMPUTE.GOOGLEAPIS.COM/INSTANCE") + self.assertIs(entry2, entry) + + def test_registry_unknown_name_returns_none(self) -> None: + self.assertIsNone(load_registry().find("not_a_real_table")) + self.assertIsNone(load_registry().find("")) + + def test_module_level_find_entry_uses_cache(self) -> None: + e1 = find_entry("project") + e2 = find_entry("gcp_projects") + self.assertIs(e1, e2) + + def test_membership_protocol(self) -> None: + registry = load_registry() + self.assertIn("project", registry) + self.assertIn("gcp_projects", registry) + self.assertNotIn("nope", registry) + self.assertNotIn(123, registry) + + def test_all_cai_types_are_googleapis(self) -> None: + registry = load_registry() + cai_types = registry.all_cai_types() + self.assertGreater(len(cai_types), 0) + for value in cai_types: + self.assertIsInstance(value, str) + self.assertIn(".googleapis.com/", value, value) + + def test_typed_collector_tables_match_registry_flag(self) -> None: + registry = load_registry() + typed = set(registry.typed_collector_tables()) + for entry in registry: + if entry.typed_collector: + self.assertIn(entry.cloudquery_table_name, typed) + else: + self.assertNotIn(entry.cloudquery_table_name, typed) + + def test_mandatory_includes_projects(self) -> None: + registry = load_registry() + self.assertIn("gcp_projects", registry.mandatory_tables()) + + def test_billing_accounts_has_no_cai_type(self) -> None: + # Tables with no Cloud Asset Inventory equivalent are pinned to null + # in the overrides so generic discovery skips them. + registry = load_registry() + entry = registry.find("gcp_billing_billing_accounts") + self.assertIsNotNone(entry) + self.assertIsNone(entry.cai_asset_type) + + +class RegistryFromTempYamlTests(TestCase): + """Loader behavior tests that don't depend on the shipped registry.""" + + def _write(self, contents: str) -> Path: + tmp = tempfile.NamedTemporaryFile( + "w", suffix=".yaml", delete=False, encoding="utf-8" + ) + tmp.write(textwrap.dedent(contents)) + tmp.close() + path = Path(tmp.name) + self.addCleanup(path.unlink, missing_ok=True) + return path + + def test_minimal_registry_round_trips(self) -> None: + path = self._write( + """ + metadata: + source: https://example.test + snapshot_date: 2026-01-01 + total_tables: 1 + typed_collectors: 0 + cai_types_assigned: 1 + generator: tests + notes: synthetic + types: + gcp_test_widgets: + cai_asset_type: test.googleapis.com/Widget + cai_asset_type_source: heuristic + category: test + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertEqual(len(registry), 1) + entry = registry.find("gcp_test_widgets") + self.assertIsNotNone(entry) + self.assertEqual(entry.cai_asset_type, "test.googleapis.com/Widget") + + def test_aliases_resolve_to_canonical(self) -> None: + path = self._write( + """ + metadata: {} + types: + gcp_alpha: + cai_asset_type: a.googleapis.com/Thing + cai_asset_type_source: heuristic + category: a + aliases: [legacy_alpha, vintage_alpha] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + self.assertIs(registry.find("legacy_alpha"), registry.find("gcp_alpha")) + self.assertIs(registry.find("vintage_alpha"), registry.find("gcp_alpha")) + + def test_alias_collision_raises(self) -> None: + path = self._write( + """ + metadata: {} + types: + gcp_alpha: + cai_asset_type: a.googleapis.com/Thing + aliases: [shared] + typed_collector: false + mandatory: false + gcp_beta: + cai_asset_type: b.googleapis.com/Thing + aliases: [shared] + typed_collector: false + mandatory: false + """ + ) + with self.assertRaises(ValueError): + load_registry_from_path(path) + + def test_missing_file_raises_filenotfound(self) -> None: + with self.assertRaises(FileNotFoundError): + load_registry_from_path(Path("/nonexistent/registry.yaml")) + + def test_null_cai_type_is_allowed(self) -> None: + path = self._write( + """ + metadata: {} + types: + gcp_no_asset: + cai_asset_type: null + cai_asset_type_source: null + category: x + aliases: [] + typed_collector: false + mandatory: false + """ + ) + registry = load_registry_from_path(path) + entry = registry.find("gcp_no_asset") + self.assertIsNotNone(entry) + self.assertIsNone(entry.cai_asset_type) + # A null cai type must not be indexed for reverse lookup. + self.assertIsNone(registry.find_by_cai_type(None)) + + +class GcpapiResourceTypesSpecTests(TestCase): + """The spec layer must surface the legacy ``resource_type_name`` values + generation rules know about and bind the SDK collectors.""" + + def setUp(self) -> None: + reset_cache() + import importlib + + import indexers.gcpapi_resource_types as mod + + self.mod = importlib.reload(mod) + + def test_typed_sdk_collectors_present(self) -> None: + # The set of tables with a hand-written google-cloud-* collector. These + # are the high-value types that survive even when Cloud Asset Inventory + # is unavailable (CAI is an accelerator, not a hard dependency). + names = {s.cloudquery_table_name for s in self.mod.GCP_TYPED_RESOURCE_TYPE_SPECS} + self.assertEqual( + names, + { + # Original rich-payload tier. + "gcp_compute_instances", + "gcp_storage_buckets", + "gcp_container_clusters", + # Tier 1 compute fallbacks (google-cloud-compute). + "gcp_compute_disks", + "gcp_compute_snapshots", + "gcp_compute_networks", + "gcp_compute_subnetworks", + "gcp_compute_firewalls", + "gcp_compute_addresses", + # Tier 2 service-client fallbacks. + "gcp_pubsub_topics", + "gcp_pubsub_subscriptions", + "gcp_iam_service_accounts", + }, + ) + + def test_new_fallback_types_are_typed_and_excluded_from_cai_generic(self) -> None: + # Each newly-added fallback type must resolve via find_spec, carry a + # bound collector, and keep its CAI asset type (so the CAI pass can be + # used as an accelerator) - but because it now has a collector, the + # orchestrator's write-once logic excludes it from the CAI *generic* + # filter. We assert the spec-level invariants that drive that exclusion. + expected_cai = { + "gcp_compute_disks": "compute.googleapis.com/Disk", + "gcp_compute_snapshots": "compute.googleapis.com/Snapshot", + "gcp_compute_networks": "compute.googleapis.com/Network", + "gcp_compute_subnetworks": "compute.googleapis.com/Subnetwork", + "gcp_compute_firewalls": "compute.googleapis.com/Firewall", + "gcp_compute_addresses": "compute.googleapis.com/Address", + "gcp_pubsub_topics": "pubsub.googleapis.com/Topic", + "gcp_pubsub_subscriptions": "pubsub.googleapis.com/Subscription", + "gcp_iam_service_accounts": "iam.googleapis.com/ServiceAccount", + } + for table, cai_type in expected_cai.items(): + spec = self.mod.find_spec(table) + self.assertIsNotNone(spec, table) + self.assertTrue(spec.typed, table) + self.assertTrue(callable(spec.collector), table) + self.assertEqual(spec.cai_asset_type, cai_type, table) + + def test_project_spec_is_first_and_mandatory(self) -> None: + first = self.mod.GCP_RESOURCE_TYPE_SPECS[0] + self.assertEqual(first.resource_type_name, "project") + self.assertEqual(first.cloudquery_table_name, "gcp_projects") + self.assertTrue(first.mandatory) + + def test_legacy_short_names_resolve_via_find_spec(self) -> None: + spec = self.mod.find_spec("project") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "gcp_projects") + + inst = self.mod.find_spec("compute_instance") + self.assertIsNotNone(inst) + self.assertEqual(inst.cloudquery_table_name, "gcp_compute_instances") + + def test_find_spec_by_cai_type(self) -> None: + spec = self.mod.find_spec_by_cai_type("storage.googleapis.com/Bucket") + self.assertIsNotNone(spec) + self.assertEqual(spec.cloudquery_table_name, "gcp_storage_buckets") + + def test_unknown_name_returns_none(self) -> None: + self.assertIsNone(self.mod.find_spec("totally_made_up")) + self.assertIsNone(self.mod.find_spec("")) + + def test_typed_specs_have_callable_collectors(self) -> None: + for spec in self.mod.GCP_TYPED_RESOURCE_TYPE_SPECS: + self.assertTrue(callable(spec.collector), spec.cloudquery_table_name) + + def test_generic_specs_have_no_collector(self) -> None: + # A type with a CAI mapping but no SDK collector (e.g. sql instances) + # is materialized as a generic spec the CAI pass owns. + spec = self.mod.find_spec("gcp_sql_instances") + self.assertIsNotNone(spec) + self.assertIsNone(spec.collector) + self.assertEqual(spec.cai_asset_type, "sqladmin.googleapis.com/Instance") diff --git a/src/indexers/test_gcpapi_normalizers.py b/src/indexers/test_gcpapi_normalizers.py new file mode 100644 index 000000000..84b65c9c6 --- /dev/null +++ b/src/indexers/test_gcpapi_normalizers.py @@ -0,0 +1,375 @@ +""" +Unit tests for ``indexers.gcpapi_normalizers``. + +The native GCP indexer claims output compatible with the legacy CloudQuery +path. These tests pin the contract: + +1. ``normalize_gcp_asset`` flattens a Cloud Asset Inventory asset into the flat + dict shape ``GCPPlatformHandler.parse_resource_data`` accepts (``project_id``, + ``name``, ``id``, ``zone``/``region``/``location``, ``tags`` from labels). + +2. ``normalize_gcp_sdk_model`` does the same for a typed google-cloud-* model. + +3. Feeding those dicts into ``GCPPlatformHandler.parse_resource_data`` yields a + sensible ``(name, qualified_name, attributes)`` tuple, including linking a + child resource to its parent ``project``. + +These tests run without any google-* package installed because the normalizers +operate on plain dicts and duck-typed objects. +""" + +from __future__ import annotations + +import os +import sys +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.gcpapi_normalizers import ( # noqa: E402 + _region_from_location, + _sanitize, + make_project_resource_data, + normalize_gcp_asset, + normalize_gcp_sdk_model, +) + +PROJECT_ID = "my-project" + + +def _instance_asset(): + return { + "name": ( + "//compute.googleapis.com/projects/my-project/zones/" + "us-central1-a/instances/web-1" + ), + "asset_type": "compute.googleapis.com/Instance", + "resource": { + "location": "us-central1-a", + "data": { + "name": "web-1", + "labels": {"env": "prod", "team": "sre"}, + "zone": ( + "https://www.googleapis.com/compute/v1/projects/" + "my-project/zones/us-central1-a" + ), + "status": "RUNNING", + }, + }, + } + + +def _bucket_asset(): + return { + "name": "//storage.googleapis.com/projects/_/buckets/my-bucket", + "asset_type": "storage.googleapis.com/Bucket", + "resource": { + "location": "us-east1", + "data": {"name": "my-bucket", "labels": {}, "storageClass": "STANDARD"}, + }, + } + + +class SanitizeAndLocationTests(TestCase): + def test_sanitize_handles_datetimes_enums_nested(self): + import datetime + import enum + + class Color(enum.Enum): + RED = "red" + + payload = { + "ts": datetime.datetime(2024, 1, 2, 3, 4, 5), + "color": Color.RED, + "nested": {"more_ts": datetime.datetime(2025, 1, 1)}, + "list": [Color.RED, datetime.date(2025, 6, 1)], + } + out = _sanitize(payload) + self.assertEqual(out["ts"], "2024-01-02T03:04:05") + self.assertEqual(out["color"], "red") + self.assertEqual(out["nested"]["more_ts"], "2025-01-01T00:00:00") + self.assertEqual(out["list"][0], "red") + self.assertEqual(out["list"][1], "2025-06-01") + + def test_region_from_location(self): + self.assertEqual(_region_from_location("us-central1-a"), ("us-central1-a", "us-central1")) + self.assertEqual(_region_from_location("us-east1"), (None, "us-east1")) + self.assertEqual(_region_from_location("us"), (None, None)) + self.assertEqual(_region_from_location("global"), (None, None)) + self.assertEqual(_region_from_location(None), (None, None)) + + +class NormalizeGcpAssetTests(TestCase): + def test_instance_asset_top_level_shape(self): + data = normalize_gcp_asset(_instance_asset()) + self.assertEqual(data["name"], "web-1") + self.assertEqual(data["project_id"], PROJECT_ID) + self.assertEqual(data["asset_type"], "compute.googleapis.com/Instance") + # Zone full-URL collapsed to the leaf; region derivation left to handler. + self.assertEqual(data["zone"], "us-central1-a") + # labels copied into tags so cross-cloud filters work; labels preserved. + self.assertEqual(data["tags"], {"env": "prod", "team": "sre"}) + self.assertEqual(data["labels"], {"env": "prod", "team": "sre"}) + # Full payload fields are hoisted to the top level for path matching. + self.assertEqual(data["status"], "RUNNING") + self.assertTrue(data["id"].startswith("//compute.googleapis.com/")) + + def test_explicit_project_id_wins(self): + data = normalize_gcp_asset(_instance_asset(), project_id="override-proj") + self.assertEqual(data["project_id"], "override-proj") + + def test_bucket_region_from_location(self): + data = normalize_gcp_asset(_bucket_asset(), project_id=PROJECT_ID) + self.assertEqual(data["name"], "my-bucket") + self.assertEqual(data["project_id"], PROJECT_ID) + self.assertEqual(data.get("region"), "us-east1") + self.assertIsNone(data.get("zone")) + self.assertEqual(data["tags"], {}) + + def test_labels_default_to_empty_dict(self): + asset = { + "name": "//x.googleapis.com/projects/p/things/t", + "asset_type": "x.googleapis.com/Thing", + "resource": {"data": {"name": "t"}}, + } + data = normalize_gcp_asset(asset, project_id="p") + self.assertEqual(data["tags"], {}) + self.assertEqual(data["labels"], {}) + + +class NormalizeGcpSdkModelTests(TestCase): + def test_sdk_model_dict(self): + model = { + "name": "web-2", + "labels": {"env": "dev"}, + "zone": "https://www.googleapis.com/compute/v1/projects/p/zones/us-west1-b", + "selfLink": "https://.../instances/web-2", + } + data = normalize_gcp_sdk_model(model, project_id=PROJECT_ID) + self.assertEqual(data["name"], "web-2") + self.assertEqual(data["project_id"], PROJECT_ID) + self.assertEqual(data["zone"], "us-west1-b") + self.assertEqual(data["tags"], {"env": "dev"}) + + def test_make_project_resource_data(self): + data = make_project_resource_data(PROJECT_ID) + self.assertEqual(data["project_id"], PROJECT_ID) + self.assertEqual(data["name"], PROJECT_ID) + self.assertEqual(data["asset_type"], "cloudresourcemanager.googleapis.com/Project") + + +class TypedFallbackRoundTripTests(TestCase): + """Per-type round-trips for the new typed fallback collectors. + + Each fallback collector yields an SDK model that flows through + ``normalize_gcp_sdk_model`` and then ``GCPPlatformHandler.parse_resource_data``. + These tests pin that, for a representative SDK payload of each new type, the + normalized dict carries the handler-read fields (name, zone/region, tags) and + parses into the expected ``(name, qualified_name)`` under its CloudQuery + ``resource_type`` - identical to what the Cloud Asset Inventory path produced + for the same type, so generation rules are unaffected by the source. + """ + + def setUp(self): + from component import Context + from outputter import FileItemOutputter + from resources import REGISTRY_PROPERTY_NAME, Registry + from enrichers.generation_rule_types import LevelOfDetail + + self.context = Context( + setting_values={"DEFAULT_LOD": LevelOfDetail.BASIC}, + outputter=FileItemOutputter(), + ) + self.context.set_property(REGISTRY_PROPERTY_NAME, Registry()) + self.platform_cfg = {"projectLevelOfDetails": {PROJECT_ID: "detailed"}} + + from enrichers.gcp import GCP_PLATFORM, GCPPlatformHandler + from resources import REGISTRY_PROPERTY_NAME as _RP + + self.handler = GCPPlatformHandler() + # Write the project anchor so child resources link to it (Phase 0). + registry = self.context.get_property(_RP) + proj = make_project_resource_data(PROJECT_ID) + pname, pqual, pattrs = self.handler.parse_resource_data( + proj, "project", self.platform_cfg, self.context + ) + registry.add_resource(GCP_PLATFORM, "project", pname, pqual, pattrs) + + def _round_trip(self, model, resource_type_name, *, expected_name): + data = normalize_gcp_sdk_model( + model, project_id=PROJECT_ID, resource_type_name=resource_type_name + ) + name, qualified_name, attrs = self.handler.parse_resource_data( + data, resource_type_name, self.platform_cfg, self.context + ) + self.assertEqual(name, expected_name) + self.assertEqual(qualified_name, f"{PROJECT_ID}/{expected_name}") + self.assertEqual(attrs["project_id"], PROJECT_ID) + self.assertIn("project", attrs) + return data, attrs + + def test_compute_disk_zonal(self): + model = { + "name": "disk-1", + "labels": {"env": "prod"}, + "zone": ( + "https://www.googleapis.com/compute/v1/projects/" + "my-project/zones/us-central1-a" + ), + "sizeGb": "100", + } + data, attrs = self._round_trip( + model, "gcp_compute_disks", expected_name="disk-1" + ) + self.assertEqual(data["zone"], "us-central1-a") + self.assertEqual(data["tags"], {"env": "prod"}) + # Region derived from zone by the handler. + self.assertEqual(attrs["region"], "us-central1") + + def test_compute_snapshot_global(self): + model = {"name": "snap-1", "labels": {}, "diskSizeGb": "50"} + data, _ = self._round_trip( + model, "gcp_compute_snapshots", expected_name="snap-1" + ) + self.assertEqual(data["tags"], {}) + self.assertIsNone(data.get("zone")) + self.assertIsNone(data.get("region")) + + def test_compute_network_global_no_labels(self): + model = { + "name": "default", + "autoCreateSubnetworks": True, + "selfLink": "https://www.googleapis.com/compute/v1/projects/p/global/networks/default", + } + data, _ = self._round_trip( + model, "gcp_compute_networks", expected_name="default" + ) + self.assertEqual(data["tags"], {}) + + def test_compute_subnetwork_regional(self): + model = { + "name": "subnet-1", + "region": ( + "https://www.googleapis.com/compute/v1/projects/" + "my-project/regions/us-central1" + ), + "ipCidrRange": "10.0.0.0/24", + } + data, attrs = self._round_trip( + model, "gcp_compute_subnetworks", expected_name="subnet-1" + ) + self.assertEqual(data["region"], "us-central1") + self.assertEqual(attrs["region"], "us-central1") + + def test_compute_firewall_global(self): + model = {"name": "allow-ssh", "network": ".../networks/default"} + self._round_trip(model, "gcp_compute_firewalls", expected_name="allow-ssh") + + def test_compute_address_regional(self): + model = { + "name": "ip-1", + "address": "34.1.2.3", + "region": ( + "https://www.googleapis.com/compute/v1/projects/" + "my-project/regions/europe-west1" + ), + } + data, attrs = self._round_trip( + model, "gcp_compute_addresses", expected_name="ip-1" + ) + self.assertEqual(data["region"], "europe-west1") + + def test_pubsub_topic_full_path_name_and_labels(self): + # PublisherClient yields Topic messages whose name is the full path. + model = { + "name": f"projects/{PROJECT_ID}/topics/orders", + "labels": {"team": "payments"}, + } + data, _ = self._round_trip( + model, "gcp_pubsub_topics", expected_name="orders" + ) + self.assertEqual(data["tags"], {"team": "payments"}) + + def test_pubsub_subscription_full_path_name(self): + model = { + "name": f"projects/{PROJECT_ID}/subscriptions/orders-sub", + "topic": f"projects/{PROJECT_ID}/topics/orders", + "labels": {}, + } + self._round_trip( + model, "gcp_pubsub_subscriptions", expected_name="orders-sub" + ) + + def test_iam_service_account_name_collapses_to_email(self): + email = f"runner@{PROJECT_ID}.iam.gserviceaccount.com" + model = { + "name": f"projects/{PROJECT_ID}/serviceAccounts/{email}", + "email": email, + "display_name": "CI Runner", + "unique_id": "10293847", + } + data, _ = self._round_trip( + model, "gcp_iam_service_accounts", expected_name=email + ) + # Service accounts have no labels -> tags default to {}. + self.assertEqual(data["tags"], {}) + + +class NormalizerParserRoundTripTests(TestCase): + """End-to-end: normalizer dict -> GCPPlatformHandler.parse_resource_data.""" + + def setUp(self): + from component import Context + from outputter import FileItemOutputter + from resources import REGISTRY_PROPERTY_NAME, Registry + from enrichers.generation_rule_types import LevelOfDetail + + self.context = Context( + setting_values={"DEFAULT_LOD": LevelOfDetail.BASIC}, + outputter=FileItemOutputter(), + ) + self.context.set_property(REGISTRY_PROPERTY_NAME, Registry()) + + def test_project_round_trip(self): + from enrichers.gcp import GCPPlatformHandler + + platform_cfg = {"projectLevelOfDetails": {PROJECT_ID: "detailed"}} + data = make_project_resource_data(PROJECT_ID) + handler = GCPPlatformHandler() + name, qualified_name, attrs = handler.parse_resource_data( + data, "project", platform_cfg, self.context + ) + self.assertEqual(name, PROJECT_ID) + self.assertEqual(qualified_name, PROJECT_ID) + self.assertIsNotNone(attrs.get("lod")) + + def test_child_resource_links_to_project(self): + from enrichers.gcp import GCP_PLATFORM, GCPPlatformHandler + from resources import REGISTRY_PROPERTY_NAME + + platform_cfg = {"projectLevelOfDetails": {PROJECT_ID: "detailed"}} + handler = GCPPlatformHandler() + + # Write the project anchor first (the orchestrator does this in Phase 0). + registry = self.context.get_property(REGISTRY_PROPERTY_NAME) + proj_data = make_project_resource_data(PROJECT_ID) + pname, pqual, pattrs = handler.parse_resource_data( + proj_data, "project", platform_cfg, self.context + ) + registry.add_resource(GCP_PLATFORM, "project", pname, pqual, pattrs) + + data = normalize_gcp_asset(_instance_asset()) + name, qualified_name, attrs = handler.parse_resource_data( + data, "compute_instance", platform_cfg, self.context + ) + self.assertEqual(name, "web-1") + self.assertEqual(qualified_name, f"{PROJECT_ID}/web-1") + self.assertEqual(attrs["project_id"], PROJECT_ID) + self.assertIn("project", attrs) + self.assertEqual(attrs["project"].name, PROJECT_ID) + # zone present; region derived by the handler from the zone. + self.assertEqual(attrs["zone"], "us-central1-a") + self.assertEqual(attrs["region"], "us-central1") diff --git a/src/indexers/test_gcpapi_selective.py b/src/indexers/test_gcpapi_selective.py new file mode 100644 index 000000000..b2b79041a --- /dev/null +++ b/src/indexers/test_gcpapi_selective.py @@ -0,0 +1,423 @@ +""" +Unit tests for the GCP indexer orchestrator (``indexers.gcpapi``). + +Coverage: +* ``_project_lod`` resolves per-project LOD, falling back to the workspace + default and tolerating garbage values. +* ``index()`` honours per-project selective discovery: projects whose effective + LOD is NONE are skipped entirely (no anchor, no typed pass, no CAI pass). +* The project anchor is emitted for every in-scope project. +* Typed (SDK) collectors run per in-scope project for accessed typed types. +* The Cloud Asset Inventory pass is scoped to exactly the CAI asset types of + accessed *generic* types (typed types are excluded so nothing is written + twice). + +The dispatch tests drive ``index()`` with stubbed credentials / collectors / +writer / handler so they need no GCP SDK or network. +""" + +from __future__ import annotations + +import os +import sys +from unittest import TestCase, mock + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from enrichers.generation_rule_types import LevelOfDetail # noqa: E402 +from indexers import gcpapi # noqa: E402 +from indexers.gcpapi_resource_types import GcpResourceTypeSpec # noqa: E402 + + +class ProjectLodTests(TestCase): + def test_explicit_project_override(self): + cfg = {"projectLevelOfDetails": {"proj-a": "detailed"}} + self.assertEqual( + gcpapi._project_lod(cfg, "proj-a", LevelOfDetail.BASIC), + LevelOfDetail.DETAILED, + ) + + def test_falls_back_to_default(self): + cfg = {"projectLevelOfDetails": {"proj-a": "detailed"}} + self.assertEqual( + gcpapi._project_lod(cfg, "proj-b", LevelOfDetail.BASIC), + LevelOfDetail.BASIC, + ) + + def test_none_override(self): + cfg = {"projectLevelOfDetails": {"proj-a": "none"}} + self.assertEqual( + gcpapi._project_lod(cfg, "proj-a", LevelOfDetail.BASIC), + LevelOfDetail.NONE, + ) + + def test_garbage_falls_back_to_default(self): + cfg = {"projectLevelOfDetails": {"proj-a": "garbage"}} + self.assertEqual( + gcpapi._project_lod(cfg, "proj-a", LevelOfDetail.DETAILED), + LevelOfDetail.DETAILED, + ) + + +class _FakeRuleSpec: + """Mimics a generation-rule ResourceTypeSpec (only resource_type_name read).""" + + def __init__(self, resource_type_name: str): + self.resource_type_name = resource_type_name + + def __hash__(self): + return hash(self.resource_type_name) + + def __eq__(self, other): + return getattr(other, "resource_type_name", None) == self.resource_type_name + + +class DispatchTests(TestCase): + def setUp(self): + self._calls = {"typed": [], "cai": []} + + def _stub_compute(credentials, project_id): + self._calls["typed"].append(project_id) + return [] + + self.project_spec = GcpResourceTypeSpec( + resource_type_name="project", + cloudquery_table_name="gcp_projects", + cai_asset_type="cloudresourcemanager.googleapis.com/Project", + mandatory=True, + typed=True, + collector=None, + ) + self.compute_spec = GcpResourceTypeSpec( + resource_type_name="compute_instance", + cloudquery_table_name="gcp_compute_instances", + cai_asset_type="compute.googleapis.com/Instance", + mandatory=False, + typed=True, + collector=_stub_compute, + ) + self.sql_spec = GcpResourceTypeSpec( + resource_type_name="gcp_sql_instances", + cloudquery_table_name="gcp_sql_instances", + cai_asset_type="sqladmin.googleapis.com/Instance", + mandatory=False, + typed=False, + collector=None, + ) + + def _stub_disks(credentials, project_id): + self._calls["typed"].append(project_id) + return [] + + # A representative new Tier-1 typed fallback: having a collector must + # exclude its CAI asset type from the generic pass (write-once), so the + # type is discovered via the SDK collector whether or not CAI is up. + self.disks_spec = GcpResourceTypeSpec( + resource_type_name="gcp_compute_disks", + cloudquery_table_name="gcp_compute_disks", + cai_asset_type="compute.googleapis.com/Disk", + mandatory=False, + typed=True, + collector=_stub_disks, + ) + self._by_name = { + "gcp_projects": self.project_spec, + "compute_instance": self.compute_spec, + "gcp_compute_instances": self.compute_spec, + "gcp_sql_instances": self.sql_spec, + "gcp_compute_disks": self.disks_spec, + } + self._by_cai = { + "sqladmin.googleapis.com/Instance": self.sql_spec, + "compute.googleapis.com/Instance": self.compute_spec, + "compute.googleapis.com/Disk": self.disks_spec, + } + + def _run(self, *, projects, project_lod, accessed): + from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY + from enrichers.generation_rule_types import PLATFORM_HANDLERS_PROPERTY_NAME + + platform_cfg = { + "projects": list(projects), + "projectLevelOfDetails": dict(project_lod), + } + + def _stub_cai(credentials, project_id, asset_types=None): + self._calls["cai"].append((project_id, tuple(sorted(asset_types or [])))) + return [] + + handler = mock.MagicMock() + handler.parse_resource_data.return_value = ("nm", "q/nm", {}) + writer = mock.MagicMock() + + rule_specs = { + "gcp": {_FakeRuleSpec(name): {} for name in accessed} + } + + class FakeContext: + def __init__(self): + self._cloud = {"gcp": dict(platform_cfg)} + self._props = {RESOURCE_TYPE_SPECS_PROPERTY: rule_specs} + + def get_setting(self, setting): + name = getattr(setting, "name", setting) + return { + "GCP_INDEXER_BACKEND": "gcpapi", + "CLOUD_CONFIG": self._cloud, + "RESOURCE_STORE_BACKEND": "memory", + "RESOURCE_STORE_PATH": None, + "DEFAULT_LOD": None, + }.get(name) + + def get_property(self, name): + if name == PLATFORM_HANDLERS_PROPERTY_NAME: + return None + return self._props.get(name) + + def add_warning(self, msg): + pass + + with mock.patch.object( + gcpapi, "gcp_get_credentials_and_projects", + return_value={ + "credentials": object(), + "project_ids": list(projects), + "quota_project": projects[0] if projects else None, + "env": {}, + }, + ), mock.patch.object(gcpapi, "get_resource_writer", return_value=writer), \ + mock.patch.object(gcpapi, "_resolve_platform_handler", return_value=handler), \ + mock.patch.object(gcpapi, "collect_assets_for_project", _stub_cai), \ + mock.patch.object(gcpapi, "find_spec", side_effect=lambda n: self._by_name.get(n)), \ + mock.patch.object( + gcpapi, "find_spec_by_cai_type", + side_effect=lambda t: self._by_cai.get(t), + ), \ + mock.patch("enrichers.gcp.set_gcp_credentials"): + gcpapi.index(FakeContext()) + + return writer + + def test_skips_none_lod_project(self): + writer = self._run( + projects=["proj-keep", "proj-skip"], + project_lod={"proj-skip": "none"}, + accessed=["compute_instance", "gcp_sql_instances"], + ) + # Typed compute collector ran only for the in-scope project. + self.assertEqual(self._calls["typed"], ["proj-keep"]) + # CAI pass ran only for the in-scope project, scoped to the SQL type + # (compute is owned by the typed pass and excluded from the filter). + self.assertEqual( + self._calls["cai"], + [("proj-keep", ("sqladmin.googleapis.com/Instance",))], + ) + + def test_project_anchor_emitted_per_in_scope_project(self): + writer = self._run( + projects=["proj-a", "proj-b"], + project_lod={}, + accessed=[], + ) + # With no accessed types, only the project anchors are written (one per + # project) and neither the typed nor CAI passes run. + self.assertEqual(writer.add_resource.call_count, 2) + self.assertEqual(self._calls["typed"], []) + self.assertEqual(self._calls["cai"], []) + for call in writer.add_resource.call_args_list: + self.assertEqual(call.args[1], "project") # resource_type arg + + def test_typed_excluded_from_cai_filter(self): + # When only a typed type is accessed, the CAI pass should not run at all + # (no generic types to fetch). + self._run( + projects=["proj-a"], + project_lod={}, + accessed=["compute_instance"], + ) + self.assertEqual(self._calls["typed"], ["proj-a"]) + self.assertEqual(self._calls["cai"], []) + + def test_generic_only_runs_cai(self): + self._run( + projects=["proj-a"], + project_lod={}, + accessed=["gcp_sql_instances"], + ) + self.assertEqual(self._calls["typed"], []) + self.assertEqual( + self._calls["cai"], + [("proj-a", ("sqladmin.googleapis.com/Instance",))], + ) + + def test_new_typed_fallback_excluded_from_cai_filter(self): + # gcp_compute_disks now has a typed collector, so when accessed + # alongside a generic type, the disk's CAI asset type is dropped from + # the CAI generic filter (write-once); the SDK collector runs instead. + self._run( + projects=["proj-a"], + project_lod={}, + accessed=["gcp_compute_disks", "gcp_sql_instances"], + ) + # The disks collector ran... + self.assertEqual(self._calls["typed"], ["proj-a"]) + # ...and the CAI pass is scoped to ONLY the still-generic SQL type, + # never compute.googleapis.com/Disk. + self.assertEqual( + self._calls["cai"], + [("proj-a", ("sqladmin.googleapis.com/Instance",))], + ) + + def test_only_new_typed_fallback_skips_cai_entirely(self): + # When the only accessed type is a typed fallback, the CAI pass does not + # run at all - proving the type survives with CAI fully unavailable. + self._run( + projects=["proj-a"], + project_lod={}, + accessed=["gcp_compute_disks"], + ) + self.assertEqual(self._calls["typed"], ["proj-a"]) + self.assertEqual(self._calls["cai"], []) + + +class _FakePermissionDenied(Exception): + """Mimics google.api_core.exceptions.PermissionDenied (code == 403).""" + + code = 403 + + +class CaiPermissionDeniedTests(TestCase): + """The CAI generic pass is the primary GCP discovery workhorse; a 403 must + be surfaced loudly (not silently swallowed) and tagged with the stable + token CI keys off of, so a degraded discovery cannot pass as a false green. + """ + + def test_is_permission_denied_detects_common_shapes(self): + self.assertTrue(gcpapi._is_permission_denied(_FakePermissionDenied())) + + class Forbidden(Exception): + pass + + self.assertTrue(gcpapi._is_permission_denied(Forbidden())) + self.assertTrue( + gcpapi._is_permission_denied( + Exception("403 The caller does not have permission") + ) + ) + self.assertFalse(gcpapi._is_permission_denied(Exception("404 not found"))) + self.assertFalse(gcpapi._is_permission_denied(ValueError("boom"))) + + def test_cai_403_is_informational_not_fatal(self): + from enrichers.generation_rules import RESOURCE_TYPE_SPECS_PROPERTY + from enrichers.generation_rule_types import PLATFORM_HANDLERS_PROPERTY_NAME + + sql_spec = GcpResourceTypeSpec( + resource_type_name="gcp_sql_instances", + cloudquery_table_name="gcp_sql_instances", + cai_asset_type="sqladmin.googleapis.com/Instance", + mandatory=False, + typed=False, + collector=None, + ) + project_spec = GcpResourceTypeSpec( + resource_type_name="project", + cloudquery_table_name="gcp_projects", + cai_asset_type="cloudresourcemanager.googleapis.com/Project", + mandatory=True, + typed=True, + collector=None, + ) + by_name = {"gcp_projects": project_spec, "gcp_sql_instances": sql_spec} + + warnings: list[str] = [] + + def _boom_cai(credentials, project_id, asset_types=None): + raise _FakePermissionDenied("403 caller lacks cloudasset.assets.listResource") + + class FakeContext: + def __init__(self): + self._cloud = { + "gcp": {"projects": ["proj-a"], "projectLevelOfDetails": {}} + } + self._props = { + RESOURCE_TYPE_SPECS_PROPERTY: { + "gcp": {_FakeRuleSpec("gcp_sql_instances"): {}} + } + } + + def get_setting(self, setting): + name = getattr(setting, "name", setting) + return { + "GCP_INDEXER_BACKEND": "gcpapi", + "CLOUD_CONFIG": self._cloud, + "RESOURCE_STORE_BACKEND": "memory", + "RESOURCE_STORE_PATH": None, + "DEFAULT_LOD": None, + }.get(name) + + def get_property(self, name): + if name == PLATFORM_HANDLERS_PROPERTY_NAME: + return None + return self._props.get(name) + + def add_warning(self, msg): + warnings.append(msg) + + handler = mock.MagicMock() + handler.parse_resource_data.return_value = ("nm", "q/nm", {}) + writer = mock.MagicMock() + + with mock.patch.object( + gcpapi, "gcp_get_credentials_and_projects", + return_value={ + "credentials": object(), + "project_ids": ["proj-a"], + "quota_project": "proj-a", + "env": {}, + }, + ), mock.patch.object(gcpapi, "get_resource_writer", return_value=writer), \ + mock.patch.object(gcpapi, "_resolve_platform_handler", return_value=handler), \ + mock.patch.object(gcpapi, "collect_assets_for_project", _boom_cai), \ + mock.patch.object(gcpapi, "find_spec", side_effect=lambda n: by_name.get(n)), \ + mock.patch.object(gcpapi, "find_spec_by_cai_type", side_effect=lambda t: None), \ + mock.patch("enrichers.gcp.set_gcp_credentials"): + # CAI is an OPTIONAL accelerator: a 403 must be logged informationally + # (INFO), never raised, and never elevated to a warning/error. It must + # not abort discovery (the project anchor is still written). + with self.assertLogs(gcpapi.logger, level="INFO") as captured: + gcpapi.index(FakeContext()) + + joined_logs = "\n".join(captured.output) + + # The stable, grep-able token is emitted at INFO (not ERROR). + self.assertTrue( + any( + gcpapi.CAI_PERMISSION_DENIED_TOKEN in rec.getMessage() + and rec.levelname == "INFO" + for rec in captured.records + ), + f"expected an INFO {gcpapi.CAI_PERMISSION_DENIED_TOKEN} log; got {captured.output}", + ) + # No ERROR severity for the CAI-denied path, and no "DEGRADED" framing. + self.assertFalse( + any(rec.levelno >= 40 for rec in captured.records), # >= ERROR + f"CAI-denied must not log at ERROR; got {captured.output}", + ) + self.assertNotIn("DEGRADED", joined_logs) + # The message frames CAI as optional and reassures it is not an error. + self.assertIn("optional", joined_logs.lower()) + + # CAI's absence must NOT surface a user-facing warning (it is normal). + self.assertFalse( + any(gcpapi.CAI_PERMISSION_DENIED_TOKEN in w for w in warnings), + f"CAI-denied must not add a warning; got {warnings}", + ) + + # Discovery still completed: the project anchor was written. + self.assertTrue( + writer.add_resource.called, + "discovery must continue (project anchor written) when CAI is denied", + ) diff --git a/src/indexers/test_sqlite_resource_writer.py b/src/indexers/test_sqlite_resource_writer.py new file mode 100644 index 000000000..34059ba89 --- /dev/null +++ b/src/indexers/test_sqlite_resource_writer.py @@ -0,0 +1,580 @@ +""" +Unit tests for ``indexers.sqlite_resource_writer``. + +These tests exercise the SQLite-backed :class:`ResourceWriter` end to end: + +1. The encoder round-trips primitives, datetimes, ``LevelOfDetail`` enums, + nested dicts/lists, and ``Resource`` cross-references via the documented + marker scheme. +2. ``SqliteResourceWriter.finalize`` produces a SQLite database via the + ``FileSystemOutputter`` containing every resource currently in the + ``Registry`` (including resources written by indexers that bypass the + writer). +3. Deferred Azure resource-group resolution runs *before* the snapshot, so + the persisted attributes reflect the resolved ``resource_group`` ref. +4. The selector ``get_resource_writer`` honours the ``resourceStoreBackend`` + setting. + +Tests use only the standard library + the local workspace-builder modules; +no Azure SDK or sqlalchemy required. +""" + +from __future__ import annotations + +import datetime as _dt +import os +import sqlite3 +import sys +import tempfile +from unittest import TestCase + +# Allow ``python -m unittest indexers.test_sqlite_resource_writer`` from any cwd. +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + +from indexers.sqlite_resource_writer import ( # noqa: E402 + SCHEMA_VERSION, + SqliteResourceWriter, + _snapshot_registry, + count_workspace_artifacts, + decode_attributes, + encode_attributes, + get_resource, + get_schema_version, + list_platforms, + list_resource_types, + list_resources, + open_database, + persist_sqlite_store, + search_workspace_artifacts, +) + + +def _make_context(tmpdir: str, sqlite: bool = False): + """Build a minimal workspace-builder Context backed by a real filesystem + outputter under ``tmpdir``.""" + from component import Context + from outputter import FileSystemOutputter + from resources import REGISTRY_PROPERTY_NAME, Registry + + setting_values: dict = {} + if sqlite: + from indexers.resource_writer import RESOURCE_STORE_BACKEND_SETTING + + setting_values[RESOURCE_STORE_BACKEND_SETTING.name] = "sqlite" + + ctx = Context( + setting_values=setting_values, + outputter=FileSystemOutputter(tmpdir), + ) + ctx.set_property(REGISTRY_PROPERTY_NAME, Registry()) + return ctx + + +class EncoderTests(TestCase): + def test_primitive_round_trip(self): + attrs = { + "name": "foo", + "count": 7, + "enabled": True, + "ratio": 1.5, + "missing": None, + } + encoded = encode_attributes(attrs) + self.assertIsInstance(encoded, str) + self.assertEqual(decode_attributes(encoded), attrs) + + def test_datetime_round_trip(self): + ts = _dt.datetime(2024, 1, 2, 3, 4, 5, tzinfo=_dt.timezone.utc) + encoded = encode_attributes({"created_at": ts}) + decoded = decode_attributes(encoded) + self.assertEqual(decoded["created_at"], ts) + + def test_date_round_trip(self): + d = _dt.date(2024, 5, 25) + encoded = encode_attributes({"on": d}) + decoded = decode_attributes(encoded) + self.assertEqual(decoded["on"], d) + + def test_level_of_detail_round_trip(self): + from enrichers.generation_rule_types import LevelOfDetail + + encoded = encode_attributes({"lod": LevelOfDetail.DETAILED}) + decoded = decode_attributes(encoded) + self.assertEqual(decoded["lod"], LevelOfDetail.DETAILED) + + def test_nested_dict_and_list_preserved(self): + attrs = { + "tags": {"env": "prod", "team": "sre"}, + "nics": [ + {"name": "n1", "ips": ["10.0.0.1", "10.0.0.2"]}, + {"name": "n2", "ips": []}, + ], + } + encoded = encode_attributes(attrs) + self.assertEqual(decode_attributes(encoded), attrs) + + def test_resource_reference_encoded_as_marker(self): + from resources import Registry + + registry = Registry() + rg = registry.add_resource( + "azure", "resource_group", "rg1", "rg1", {"subscription_id": "s1"} + ) + encoded = encode_attributes({"resource_group": rg}) + # The marker dict is stored verbatim (decoder leaves it as-is so a + # downstream caller can decide how to resolve refs). + decoded = decode_attributes(encoded) + ref = decoded["resource_group"]["$ref"] + self.assertEqual(ref["platform"], "azure") + self.assertEqual(ref["resource_type"], "resource_group") + self.assertEqual(ref["qualified_name"], "rg1") + self.assertEqual(ref["name"], "rg1") + + def test_set_serialised_as_list(self): + attrs = {"members": {"a", "b", "c"}} + encoded = encode_attributes(attrs) + decoded = decode_attributes(encoded) + self.assertCountEqual(decoded["members"], ["a", "b", "c"]) + + def test_unsupported_type_falls_back_to_string(self): + class Quirky: + def __repr__(self): + return "" + + encoded = encode_attributes({"weird": Quirky()}) + self.assertEqual(decode_attributes(encoded), {"weird": ""}) + + +class SchemaTests(TestCase): + def test_schema_initialised_with_version(self): + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + from indexers.resource_writer import RESOURCE_STORE_BACKEND_SETTING + + ctx.setting_values[RESOURCE_STORE_BACKEND_SETTING.name] = "sqlite" + persist_sqlite_store(ctx, db_path="resources.sqlite") + + db_file = os.path.join(tmpdir, "resources.sqlite") + self.assertTrue(os.path.exists(db_file)) + conn = open_database(db_file) + try: + self.assertEqual(get_schema_version(conn), SCHEMA_VERSION) + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + } + self.assertTrue( + { + "platforms", + "resource_types", + "resources", + "workspace_artifacts", + "schema_meta", + }.issubset(tables) + ) + finally: + conn.close() + + +class WriterTests(TestCase): + def test_dual_write_persists_full_registry(self): + from enrichers.generation_rule_types import LevelOfDetail + from resources import REGISTRY_PROPERTY_NAME + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + writer = SqliteResourceWriter(ctx, db_path="store/resources.sqlite") + + # 1) Write through the writer (azureapi-style call). + writer.add_resource( + "azure", + "resource_group", + "rg1", + "rg1", + { + "subscription_id": "sub-1", + "tags": {"env": "prod"}, + "lod": LevelOfDetail.BASIC, + }, + ) + + # 2) Bypass the writer to mimic kubeapi / cloudquery indexers + # that still mutate the registry directly. Snapshot must + # pick these up too. + registry = ctx.get_property(REGISTRY_PROPERTY_NAME) + registry.add_resource( + "kubernetes", + "cluster", + "demo", + "demo", + {"server": "https://demo.example.com"}, + ) + + writer.finalize() + persist_sqlite_store(ctx, db_path="store/resources.sqlite") + + db_file = os.path.join(tmpdir, "store", "resources.sqlite") + conn = open_database(db_file) + try: + self.assertEqual( + sorted(list_platforms(conn)), ["azure", "kubernetes"] + ) + + rg_rows = list_resources(conn, platform="azure", resource_type="resource_group") + self.assertEqual(len(rg_rows), 1) + rg = rg_rows[0] + self.assertEqual(rg["name"], "rg1") + self.assertEqual(rg["qualified_name"], "rg1") + self.assertEqual(rg["attributes"]["subscription_id"], "sub-1") + self.assertEqual(rg["attributes"]["tags"], {"env": "prod"}) + self.assertEqual(rg["attributes"]["lod"], LevelOfDetail.BASIC) + + kube_rows = list_resources(conn, platform="kubernetes") + self.assertEqual(len(kube_rows), 1) + self.assertEqual(kube_rows[0]["qualified_name"], "demo") + finally: + conn.close() + + def test_resource_reference_attributes_persist_as_ref_marker(self): + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + writer = SqliteResourceWriter(ctx, db_path="resources.sqlite") + + rg = writer.add_resource( + "azure", + "resource_group", + "rg1", + "rg1", + {"subscription_id": "sub-1"}, + ) + writer.add_resource( + "azure", + "azure_storage_accounts", + "store1", + "rg1/store1", + {"subscription_id": "sub-1", "resource_group": rg}, + ) + writer.finalize() + persist_sqlite_store(ctx, db_path="resources.sqlite") + + db_file = os.path.join(tmpdir, "resources.sqlite") + conn = open_database(db_file) + try: + stored = get_resource( + conn, + "azure", + "azure_storage_accounts", + "rg1/store1", + ) + self.assertIsNotNone(stored) + ref = stored["attributes"]["resource_group"]["$ref"] + self.assertEqual(ref["platform"], "azure") + self.assertEqual(ref["resource_type"], "resource_group") + self.assertEqual(ref["qualified_name"], "rg1") + finally: + conn.close() + + def test_finalize_runs_deferred_rg_resolution_before_snapshot(self): + """The in-memory writer's ``finalize`` resolves + ``_deferred_rg_lookup`` markers and re-keys child resources by + ``rg/name``. The SQLite snapshot must reflect the *resolved* state. + """ + from enrichers.azure import AzurePlatformHandler + from enrichers.generation_rule_types import ( + PLATFORM_HANDLERS_PROPERTY_NAME, + ) + from indexers.azureapi_normalizers import normalize_azure_resource + from resources import REGISTRY_PROPERTY_NAME + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + ctx.setting_values["DEFAULT_LOD"] = "basic" + ctx.set_property( + PLATFORM_HANDLERS_PROPERTY_NAME, + {"azure": AzurePlatformHandler()}, + ) + + writer = SqliteResourceWriter(ctx, db_path="resources.sqlite") + registry = ctx.get_property(REGISTRY_PROPERTY_NAME) + + sub_id = "11111111-1111-1111-1111-111111111111" + platform_cfg = { + "subscriptions": [{"subscriptionId": sub_id}], + "subscriptionResourceGroupLevelOfDetails": { + sub_id: {"*": "detailed"}, + }, + } + + # Simulate the order azureapi typically processes things in: a + # child resource referencing an RG that hasn't landed yet. + child_payload = { + "id": ( + f"/subscriptions/{sub_id}/resourceGroups/" + f"app-rg/providers/Microsoft.Storage/storageAccounts/store1" + ), + "name": "store1", + "type": "Microsoft.Storage/storageAccounts", + "location": "eastus", + "tags": {}, + "properties": {}, + } + # Seed registry with a different RG so the ``resource_group`` + # type table exists and the deferred-lookup branch is taken. + registry.add_resource( + "azure", + "resource_group", + "other-rg", + "other-rg", + {"subscription_id": sub_id, "tags": {}}, + ) + + class _Bag: + def __init__(self, d): + self._d = d + + def as_dict(self, keep_readonly=True): + return dict(self._d) + + data = normalize_azure_resource( + _Bag(child_payload), + subscription_id=sub_id, + resource_type_name="azure_storage_accounts", + ) + handler = AzurePlatformHandler() + name, qualified, attrs = handler.parse_resource_data( + data, "azure_storage_accounts", platform_cfg, ctx + ) + attrs["resource"] = data + attrs["auth_type"] = "managed_identity" + attrs["auth_secret"] = None + self.assertIn("_deferred_rg_lookup", attrs) + + writer.add_resource( + "azure", "azure_storage_accounts", name, qualified, attrs + ) + + # Now write the actual parent RG (this is the order azureapi + # uses but only because it sorts spec-major; the deferred path + # exists for cases where it doesn't). + registry.add_resource( + "azure", + "resource_group", + "app-rg", + "app-rg", + {"subscription_id": sub_id, "tags": {}}, + ) + + writer.finalize() + persist_sqlite_store(ctx, db_path="resources.sqlite") + + db_file = os.path.join(tmpdir, "resources.sqlite") + conn = open_database(db_file) + try: + rows = list_resources( + conn, + platform="azure", + resource_type="azure_storage_accounts", + ) + self.assertEqual(len(rows), 1) + row = rows[0] + # After deferred resolution, the child should be linked to + # the real RG and the deferred marker should be gone. + self.assertEqual(row["qualified_name"], "app-rg/store1") + self.assertNotIn("_deferred_rg_lookup", row["attributes"]) + ref = row["attributes"]["resource_group"]["$ref"] + self.assertEqual(ref["qualified_name"], "app-rg") + finally: + conn.close() + + +class SelectorTests(TestCase): + def test_default_selector_returns_in_memory_writer(self): + from indexers.resource_writer import ( + InMemoryRegistryWriter, + get_resource_writer, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir) + writer = get_resource_writer(ctx) + self.assertIsInstance(writer, InMemoryRegistryWriter) + + def test_sqlite_backend_selected_by_setting(self): + from indexers.resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_PATH_SETTING, + get_resource_writer, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + ctx.setting_values[RESOURCE_STORE_BACKEND_SETTING.name] = "sqlite" + ctx.setting_values[RESOURCE_STORE_PATH_SETTING.name] = "custom/path.sqlite" + writer = get_resource_writer(ctx) + self.assertIsInstance(writer, SqliteResourceWriter) + self.assertEqual(writer.db_output_path, "custom/path.sqlite") + + def test_unknown_backend_falls_back_to_in_memory(self): + from indexers.resource_writer import ( + InMemoryRegistryWriter, + RESOURCE_STORE_BACKEND_SETTING, + get_resource_writer, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir) + ctx.setting_values[RESOURCE_STORE_BACKEND_SETTING.name] = "postgres" + writer = get_resource_writer(ctx) + self.assertIsInstance(writer, InMemoryRegistryWriter) + + def test_finalize_resource_store_writes_sqlite_from_registry(self): + from indexers.resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + finalize_resource_store, + ) + from resources import REGISTRY_PROPERTY_NAME + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + ctx.setting_values[RESOURCE_STORE_BACKEND_SETTING.name] = "sqlite" + ctx.get_property(REGISTRY_PROPERTY_NAME).add_resource( + "kubernetes", + "Namespace", + "default", + "default", + {"lod": "basic"}, + ) + finalize_resource_store(ctx) + db_path = os.path.join(tmpdir, "resources.sqlite") + self.assertTrue(os.path.exists(db_path)) + conn = sqlite3.connect(db_path) + try: + rows = conn.execute("SELECT COUNT(*) FROM resources").fetchone()[0] + self.assertEqual(rows, 1) + finally: + conn.close() + + def test_persist_workspace_artifacts(self): + from renderers.rendered_artifacts import record_rendered_artifact + + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + ctx.setting_values["WORKSPACE_NAME"] = "demo-ws" + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/my-app/slx.yaml", + "kind: ServiceLevelX\nmetadata:\n name: my-app\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/my-app/sli.yaml", + "kind: ServiceLevelIndicator\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/my-app/runbook.yaml", + "kind: Runbook\n", + ) + persist_sqlite_store(ctx, db_path="resources.sqlite") + + conn = open_database(os.path.join(tmpdir, "resources.sqlite")) + try: + self.assertEqual(count_workspace_artifacts(conn, workspace_name="demo-ws"), 3) + rows = search_workspace_artifacts( + conn, workspace_name="demo-ws", artifact_kind="slx" + ) + self.assertEqual(len(rows), 1) + self.assertIn("ServiceLevelX", rows[0]["content"]) + finally: + conn.close() + + +class ReadApiTests(TestCase): + def test_list_resource_types_includes_custom_attributes(self): + with tempfile.TemporaryDirectory() as tmpdir: + ctx = _make_context(tmpdir, sqlite=True) + writer = SqliteResourceWriter(ctx, db_path="resources.sqlite") + writer.add_resource( + "azure", + "resource_group", + "rg1", + "rg1", + {"subscription_id": "sub-1", "tags": {}}, + ) + writer.finalize() + persist_sqlite_store(ctx, db_path="resources.sqlite") + + conn = open_database(os.path.join(tmpdir, "resources.sqlite")) + try: + types = list_resource_types(conn, platform="azure") + self.assertEqual(len(types), 1) + self.assertEqual(types[0]["name"], "resource_group") + self.assertIn("subscription_id", types[0]["custom_attributes"]) + self.assertIn("tags", types[0]["custom_attributes"]) + finally: + conn.close() + + def test_attributes_json_is_deterministic(self): + # Stable serialisation matters for diff-based tests downstream; + # encode the same dict twice in different insertion orders and + # confirm the JSON string matches. + first = encode_attributes({"b": 2, "a": 1, "c": [3, 2, 1]}) + second = encode_attributes({"a": 1, "b": 2, "c": [3, 2, 1]}) + self.assertEqual(first, second) + + +class SnapshotHelperTests(TestCase): + """Cover the lower-level ``_snapshot_registry`` directly so debug scripts + can re-use it without instantiating a writer.""" + + def test_snapshot_replaces_existing_rows(self): + from resources import Registry + + with tempfile.TemporaryDirectory() as tmpdir: + db_file = os.path.join(tmpdir, "snapshot.sqlite") + conn = sqlite3.connect(db_file) + try: + from indexers.sqlite_resource_writer import _init_schema + + _init_schema(conn) + + # First snapshot: one resource. + reg = Registry() + reg.add_resource( + "azure", + "resource_group", + "rg1", + "rg1", + {"subscription_id": "sub-1"}, + ) + _snapshot_registry(conn, reg) + conn.commit() + self.assertEqual( + conn.execute("SELECT COUNT(*) FROM resources").fetchone()[0], 1 + ) + + # Second snapshot with a *different* resource: old rows should + # be wiped, not appended to. + reg2 = Registry() + reg2.add_resource( + "azure", + "resource_group", + "rg2", + "rg2", + {"subscription_id": "sub-2"}, + ) + _snapshot_registry(conn, reg2) + conn.commit() + + names = [ + row[0] + for row in conn.execute("SELECT qualified_name FROM resources") + ] + self.assertEqual(names, ["rg2"]) + finally: + conn.close() diff --git a/src/indexers/test_startup_import_guard.py b/src/indexers/test_startup_import_guard.py new file mode 100644 index 000000000..f5a85e08c --- /dev/null +++ b/src/indexers/test_startup_import_guard.py @@ -0,0 +1,171 @@ +""" +Startup import regression guard. + +The workspace-builder REST service bootstraps by importing +``workspace_builder.api`` -> ``startup.bootstrap()`` -> ``component.init_components()``, +which imports *every* registered component module (including the native +``gcpapi`` / ``awsapi`` / ``azure_devops`` indexers) at service-start time. If +any of those modules grows a top-level import with a side effect that fails in a +clean / locked-down image, uvicorn crashes on startup and the whole REST service +never comes up -- which takes down every CI job that uses the image, not just +the cloud whose indexer regressed. + +The motivating regression (this file's reason to grow): ``indexers.azure_devops`` +imported the Azure DevOps SDK at module scope (``from azure.devops.connection +import Connection`` etc). Importing ``azure.devops`` has a filesystem side effect +-- ``azure/devops/_file_cache.get_cache_dir()`` runs ``os.makedirs($HOME/.azure-devops/...)`` +at import time. In the GCP/AWS CI containers ``$HOME`` resolves to the read-only +``/shared`` mount, so the makedirs raised ``PermissionError`` -> +``init_components()`` crashed -> uvicorn never started -> every CI job that uses +the image failed with "Total SLXs: 0" / "Error executing script". + +These tests fail fast and cheaply (no container, no network). Cloud SDK clients +MUST stay lazily imported inside functions, and importing an indexer module MUST +NOT touch the filesystem. +""" + +from __future__ import annotations + +import importlib +import os +import re +import site +import subprocess +import sys +import tempfile +from unittest import TestCase + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SRC_DIR = os.path.dirname(_THIS_DIR) +if _SRC_DIR not in sys.path: + sys.path.insert(0, _SRC_DIR) + + +def _registered_indexer_module_names() -> list[str]: + """Read the INDEXER-stage component list straight from component.py so this + guard automatically covers every indexer that init_components() imports. + + The list lives in the ``component_stages_init`` tuple as a local inside + ``init_components()``, so we parse the source rather than import a private. + """ + component_src_path = os.path.join(_SRC_DIR, "component.py") + with open(component_src_path, "r", encoding="utf-8") as f: + source = f.read() + + match = re.search(r"\(\s*Stage\.INDEXER\s*,\s*\[([^\]]*)\]", source) + if not match: + raise AssertionError( + "Could not locate the (Stage.INDEXER, [...]) component list in " + "component.py; update this guard if the registration moved." + ) + names = re.findall(r"""['"]([^'"]+)['"]""", match.group(1)) + if not names: + raise AssertionError("Parsed an empty INDEXER component list from component.py") + return [f"indexers.{n}" for n in names] + + +def _import_module_in_isolated_home(module_name: str) -> tuple[int, str, list[str]]: + """Import ``module_name`` in a fresh subprocess whose ``$HOME`` is an empty + temp dir, then report (returncode, stderr, files_created_under_home). + + This reproduces the exact failure class: a module whose import touches the + filesystem under ``$HOME`` (like the Azure DevOps SDK's + ``os.makedirs($HOME/.azure-devops)``). A side-effect-free import leaves the + temp HOME empty. + """ + with tempfile.TemporaryDirectory() as home: + env = dict(os.environ) + # Preserve the real interpreter's import path. Python derives the + # per-user site-packages dir from $HOME, so pointing HOME at a temp dir + # would hide deps installed under ~/.local (e.g. in local dev). Pin the + # real search path on PYTHONPATH so imports still resolve, while HOME + # itself is the empty temp dir we watch for write side effects. + extra_paths = [p for p in sys.path if p] + try: + extra_paths.append(site.getusersitepackages()) + except Exception: + pass + existing_pp = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + [p for p in extra_paths if p] + ([existing_pp] if existing_pp else []) + ) + env["HOME"] = home + # Make sure the dedicated cache-dir override can't redirect the SDK's + # makedirs away from HOME -- we want to observe writes under HOME. + env.pop("AZURE_DEVOPS_CACHE_DIR", None) + script = ( + "import importlib\n" + f"importlib.import_module({module_name!r})\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=_SRC_DIR, + capture_output=True, + text=True, + env=env, + ) + created = sorted(os.listdir(home)) + return result.returncode, result.stderr, created + + +class TopLevelImportGuardTests(TestCase): + def test_native_indexers_import_at_module_scope(self): + # A bare import must not require any cloud SDK to be installed. + importlib.import_module("indexers.gcpapi") + importlib.import_module("indexers.awsapi") + + def test_every_registered_indexer_imports_without_filesystem_side_effects(self): + """Importing ANY registered indexer module must succeed and must NOT + write to the filesystem (under $HOME). This is the generalized guard for + the whole INDEXER stage and reproduces the azure_devops regression: + pre-fix, importing ``indexers.azure_devops`` imported ``azure.devops``, + which ran ``os.makedirs($HOME/.azure-devops)`` -> a ``.azure-devops`` + entry would appear under the isolated HOME and fail this test (and in a + read-only HOME it crashed the import outright).""" + indexer_modules = _registered_indexer_module_names() + # azure_devops MUST be in the list we guard. + self.assertIn("indexers.azure_devops", indexer_modules) + + for module_name in indexer_modules: + returncode, stderr, created = _import_module_in_isolated_home(module_name) + self.assertEqual( + returncode, + 0, + f"Importing {module_name} crashed (this is exactly what takes " + f"down uvicorn on startup):\n{stderr}", + ) + self.assertEqual( + created, + [], + f"Importing {module_name} wrote {created} under $HOME at import " + f"time. Module imports must be side-effect-free: move any SDK " + f"import (and the filesystem work it triggers) inside the " + f"function(s) that use it. A read-only $HOME (the CI /shared " + f"mount) turns this write into a startup crash.", + ) + + def test_importing_azure_devops_does_not_import_sdk(self): + """Importing indexers.azure_devops must not pull ``azure.devops`` into + sys.modules. The dir-creation side effect only fires when ``azure.devops`` + is imported, so its absence proves the side effect could not have fired.""" + importlib.import_module("indexers.azure_devops") + self.assertNotIn( + "azure.devops", + sys.modules, + "indexers.azure_devops eagerly imported azure.devops, which runs " + "os.makedirs($HOME/.azure-devops) at import time and crashes the REST " + "service when $HOME is read-only. Keep the import lazy.", + ) + + def test_init_components_imports_every_registered_component(self): + import component + + # Must not raise: this is exactly what startup.bootstrap() runs and what + # would crash uvicorn (and thus the REST service) on a clean image. + component.init_components() + + names = set(component.all_components.keys()) + # The native cloud indexers must be registered (they are imported + # unconditionally for every run). + for required in ("gcpapi", "awsapi", "azureapi", "azure_devops"): + self.assertIn(required, names) diff --git a/src/manage.py b/src/manage.py deleted file mode 100755 index 8e7ac79b9..000000000 --- a/src/manage.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -"""Django's command-line utility for administrative tasks.""" -import os -import sys - - -def main(): - """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') - try: - from django.core.management import execute_from_command_line - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc - execute_from_command_line(sys.argv) - - -if __name__ == '__main__': - main() diff --git a/src/poetry.lock b/src/poetry.lock new file mode 100644 index 000000000..d1c73ada0 --- /dev/null +++ b/src/poetry.lock @@ -0,0 +1,2861 @@ +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.13.0" +description = "High-level concurrency and networking framework on top of asyncio or Trio" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, + {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, +] + +[package.dependencies] +idna = ">=2.8" + +[package.extras] +trio = ["trio (>=0.32.0)"] + +[[package]] +name = "attrs" +version = "26.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, + {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, +] + +[[package]] +name = "azure-common" +version = "1.1.28" +description = "Microsoft Azure Client Library for Python (Common)" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, + {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, +] + +[[package]] +name = "azure-core" +version = "1.41.0" +description = "Microsoft Azure Core Library for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d"}, + {file = "azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a"}, +] + +[package.dependencies] +requests = ">=2.21.0" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["aiohttp (>=3.0)"] +tracing = ["opentelemetry-api (>=1.26,<2.0)"] + +[[package]] +name = "azure-devops" +version = "7.1.0b4" +description = "Python wrapper around the Azure DevOps 7.x APIs" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "azure-devops-7.1.0b4.tar.gz", hash = "sha256:f04ba939112579f3d530cfecc044a74ef9e9339ba23c9ee1ece248241f07ff85"}, + {file = "azure_devops-7.1.0b4-py3-none-any.whl", hash = "sha256:f827e9fbc7c77bc6f2aaee46e5717514e9fe7d676c87624eccd0ca640b54f122"}, +] + +[package.dependencies] +msrest = ">=0.7.1,<0.8.0" + +[[package]] +name = "azure-identity" +version = "1.25.3" +description = "Microsoft Azure Identity Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c"}, + {file = "azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6"}, +] + +[package.dependencies] +azure-core = ">=1.31.0" +cryptography = ">=2.5" +msal = ">=1.35.1" +msal-extensions = ">=1.2.0" +typing-extensions = ">=4.0.0" + +[[package]] +name = "azure-mgmt-apimanagement" +version = "5.0.0" +description = "Microsoft Azure API Management Client Library for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "azure_mgmt_apimanagement-5.0.0-py3-none-any.whl", hash = "sha256:b88c42a392333b60722fb86f15d092dfc19a8d67510dccd15c217381dff4e6ec"}, + {file = "azure_mgmt_apimanagement-5.0.0.tar.gz", hash = "sha256:0ab7fe17e70fe3154cd840ff47d19d7a4610217003eaa7c21acf3511a6e57999"}, +] + +[package.dependencies] +azure-common = ">=1.1" +azure-mgmt-core = ">=1.3.2" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-azurearcdata" +version = "1.0.1" +description = "Microsoft Azure Azurearcdata Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_azurearcdata-1.0.1-py3-none-any.whl", hash = "sha256:b09883bda9c72e35e27540bdc58bd1d7477f798171349fa08ef5d84be4a6cccc"}, + {file = "azure_mgmt_azurearcdata-1.0.1.tar.gz", hash = "sha256:bc248b509e75bbb4b1104650f29aa09efdddc78c0a2482db1195be4d2644c51c"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-compute" +version = "38.0.0" +description = "Microsoft Azure Compute Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_compute-38.0.0-py3-none-any.whl", hash = "sha256:6307edc4d1dabb540dadf11204be7663565d3108e16bcd253494457300cb3609"}, + {file = "azure_mgmt_compute-38.0.0.tar.gz", hash = "sha256:46cb0864f943b88463ed50ad006738023120cf7e53f50f7e4a740ccd5337abaf"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-containerregistry" +version = "15.0.0" +description = "Microsoft Azure Containerregistry Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_containerregistry-15.0.0-py3-none-any.whl", hash = "sha256:e7d6302e8b993b53c3167a255dd8fae75fc2f1dc0e39ae9fa97fb4b7ed5d25d1"}, + {file = "azure_mgmt_containerregistry-15.0.0.tar.gz", hash = "sha256:a31a70b7b6811d343c26804a146f1e4c1b12f6660c6e381d2d5017b27bcab573"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-containerservice" +version = "40.2.0" +description = "Microsoft Azure Containerservice Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_containerservice-40.2.0-py3-none-any.whl", hash = "sha256:a73d9720dd0ae29257dd6dcf39942b9274f15b588ffe25f4564aa77798eb4703"}, + {file = "azure_mgmt_containerservice-40.2.0.tar.gz", hash = "sha256:acd55cae95b768efeb0377d83dea07d610c434eec0c089e02935ff31f0e3e07d"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +msrest = ">=0.7.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-core" +version = "1.6.0" +description = "Microsoft Azure Management Core Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_core-1.6.0-py3-none-any.whl", hash = "sha256:0460d11e85c408b71c727ee1981f74432bc641bb25dfcf1bb4e90a49e776dbc4"}, + {file = "azure_mgmt_core-1.6.0.tar.gz", hash = "sha256:b26232af857b021e61d813d9f4ae530465255cb10b3dde945ad3743f7a58e79c"}, +] + +[package.dependencies] +azure-core = ">=1.32.0" + +[[package]] +name = "azure-mgmt-cosmosdb" +version = "9.9.0" +description = "Microsoft Azure Cosmosdb Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_cosmosdb-9.9.0-py3-none-any.whl", hash = "sha256:31322770c61fdca6bcd1444e9dad501a5a225879c152ec1fd57ab5c68901a1fa"}, + {file = "azure_mgmt_cosmosdb-9.9.0.tar.gz", hash = "sha256:4678bf042bdc208aa24fca71767ac29b6f2a2722ac7872608371a5922f3b6c37"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +msrest = ">=0.7.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-datafactory" +version = "9.3.0" +description = "Microsoft Azure Datafactory Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_datafactory-9.3.0-py3-none-any.whl", hash = "sha256:fddb855a27e3f7b78328f184df146d71e433e1dfb9cc4923ea503c53813f0504"}, + {file = "azure_mgmt_datafactory-9.3.0.tar.gz", hash = "sha256:f5fdd5cd416f0ed71dfedf05dc7677b8f0e52f3428fd5b17b04c9200dd8d36b3"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-keyvault" +version = "14.0.1" +description = "Microsoft Azure Keyvault Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_keyvault-14.0.1-py3-none-any.whl", hash = "sha256:7710873c5b667e19d86109caf2898dddb902e5ed21013e01d7d85ebb496928d7"}, + {file = "azure_mgmt_keyvault-14.0.1.tar.gz", hash = "sha256:d141a8084ae4c7c5bd1cafeca49a8f3fbebc58dc5bc5290f322ea73d8b307ef7"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-network" +version = "30.2.0" +description = "Microsoft Azure Network Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_network-30.2.0-py3-none-any.whl", hash = "sha256:a87edffd7c38aa9d3494daa8d42914213b0863a9072cf8c7c7e48018b12b6532"}, + {file = "azure_mgmt_network-30.2.0.tar.gz", hash = "sha256:9b17c259e6344808aaa80a34bbc4b13f16bc01185dd9db137eaa0ae26664861a"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-rdbms" +version = "10.1.1" +description = "Microsoft Azure Rdbms Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_rdbms-10.1.1-py3-none-any.whl", hash = "sha256:cb0d85c016e616727f7be89132b288b26a1edcac2729bc769ec93eede61fcbfa"}, + {file = "azure_mgmt_rdbms-10.1.1.tar.gz", hash = "sha256:963455ace0d6566d299826713e54ed8fa87a88fec6467e8fd3932387ca0a2e8c"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +msrest = ">=0.7.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-redis" +version = "14.5.0" +description = "Microsoft Azure Redis Cache Management Client Library for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "azure_mgmt_redis-14.5.0-py3-none-any.whl", hash = "sha256:d98fe771a4478920bc520687ae158eb3c0e68faa073e5a27c2b6c65898572028"}, + {file = "azure_mgmt_redis-14.5.0.tar.gz", hash = "sha256:5c3434c82492688e25b93aaf5113ecff0b92b7ad6da2a4fd4695530f82b152fa"}, +] + +[package.dependencies] +azure-common = ">=1.1" +azure-mgmt-core = ">=1.3.2" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-resource" +version = "24.0.0" +description = "Microsoft Azure Resource Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_resource-24.0.0-py3-none-any.whl", hash = "sha256:27b32cd223e2784269f5a0db3c282042886ee4072d79cedc638438ece7cd0df4"}, + {file = "azure_mgmt_resource-24.0.0.tar.gz", hash = "sha256:cf6b8995fcdd407ac9ff1dd474087129429a1d90dbb1ac77f97c19b96237b265"}, +] + +[package.dependencies] +azure-common = ">=1.1" +azure-mgmt-core = ">=1.5.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-servicebus" +version = "9.0.0" +description = "Microsoft Azure Service Bus Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_servicebus-9.0.0-py3-none-any.whl", hash = "sha256:45c6832c68c197dcaa38dab9d58e1ca9f0f0b36a94d9d95e62520fb3eea7f003"}, + {file = "azure_mgmt_servicebus-9.0.0.tar.gz", hash = "sha256:1613d7e416304d6b5f4908f9622355c402614ac27ef49e56438c5c6f50f200e1"}, +] + +[package.dependencies] +azure-common = ">=1.1" +azure-mgmt-core = ">=1.5.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-storage" +version = "25.0.0" +description = "Microsoft Azure Storage Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_storage-25.0.0-py3-none-any.whl", hash = "sha256:7824840e8251aa6b2c27abf29cbd0ab86fc6685aa69e44f61c2f53168e71ff2d"}, + {file = "azure_mgmt_storage-25.0.0.tar.gz", hash = "sha256:52c4bb1fb395fcfa7a2e8fb024c1dabc5a67bd6fa23c0d2d5a7fb29314f172d2"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "azure-mgmt-web" +version = "11.0.0" +description = "Microsoft Azure Web Management Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_mgmt_web-11.0.0-py3-none-any.whl", hash = "sha256:ccaeeaa757de94a6deb62e7692c0274190fe9e252e67984baf43d7fa3913df7b"}, + {file = "azure_mgmt_web-11.0.0.tar.gz", hash = "sha256:1f98b29283ecb9c36ede7309c0da8d26db0455d77ae37e1cb6cdcd244044d6de"}, +] + +[package.dependencies] +azure-mgmt-core = ">=1.6.0" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[[package]] +name = "boto3" +version = "1.43.16" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "boto3-1.43.16-py3-none-any.whl", hash = "sha256:dffc8a3cd3edbc0ad95b9c6b983e873b76ede46d3aa0709f94db253f2ff2388f"}, + {file = "boto3-1.43.16.tar.gz", hash = "sha256:6c337bbe608aacc7d335c79e671f0c893870293b74d652f7a7af22ccd0dfef16"}, +] + +[package.dependencies] +botocore = ">=1.43.16,<1.44.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.17.0,<0.18.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.43.16" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "botocore-1.43.16-py3-none-any.whl", hash = "sha256:8ab05b1346d26a3c6d69c7338051f07bd4739a090f414d2cff43c0dbc1e18ca7"}, + {file = "botocore-1.43.16.tar.gz", hash = "sha256:813dae233d8b365c19aaf7865b32070e34d7e793654881bf86ecbbef3f4ad5c6"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" + +[package.extras] +crt = ["awscrt (==0.32.2)"] + +[[package]] +name = "certifi" +version = "2026.5.20" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897"}, + {file = "certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d"}, +] + +[[package]] +name = "cffi" +version = "2.0.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\"" +files = [ + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_armv7l.whl", hash = "sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win32.whl", hash = "sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win_amd64.whl", hash = "sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win32.whl", hash = "sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_amd64.whl", hash = "sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_arm64.whl", hash = "sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c"}, + {file = "charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d"}, + {file = "charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5"}, +] + +[[package]] +name = "click" +version = "8.4.1" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2"}, + {file = "click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "48.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +groups = ["main"] +files = [ + {file = "cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5"}, + {file = "cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321"}, + {file = "cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74"}, + {file = "cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4"}, + {file = "cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7"}, + {file = "cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057"}, + {file = "cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae"}, + {file = "cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c"}, + {file = "cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f"}, + {file = "cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12"}, + {file = "cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239"}, + {file = "cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c"}, + {file = "cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4"}, + {file = "cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd"}, + {file = "cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a"}, + {file = "cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + +[[package]] +name = "durationpy" +version = "0.10" +description = "Module for converting between datetime.timedelta and Go's Duration strings." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286"}, + {file = "durationpy-0.10.tar.gz", hash = "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba"}, +] + +[[package]] +name = "fastapi" +version = "0.115.14" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca"}, + {file = "fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.40.0,<0.47.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "gitdb" +version = "4.0.12" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.50" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, + {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] + +[[package]] +name = "google-api-core" +version = "2.30.3" +description = "Google API client core library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8"}, + {file = "google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.63.2,<2.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""} +grpcio-status = {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" +requests = ">=2.20.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] + +[[package]] +name = "google-auth" +version = "2.53.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68"}, + {file = "google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c"}, +] + +[package.dependencies] +cryptography = ">=38.0.3" +pyasn1-modules = ">=0.2.1" + +[package.extras] +aiohttp = ["aiohttp (>=3.8.0,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] +cryptography = ["cryptography (>=38.0.3)"] +enterprise-cert = ["pyopenssl"] +pyjwt = ["pyjwt (>=2.0)"] +pyopenssl = ["pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0)"] +rsa = ["rsa (>=3.1.4,<5)"] +testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.8.0,<4.0.0)", "aioresponses", "flask", "freezegun", "grpcio", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] +urllib3 = ["packaging", "urllib3"] + +[[package]] +name = "google-cloud-access-context-manager" +version = "0.5.0" +description = "Google Cloud Access Context Manager Protobufs" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_access_context_manager-0.5.0-py3-none-any.whl", hash = "sha256:825362c2aab06ae77bb80fef5db18c27c4b1715928e0fcbc13eb11484180d7cc"}, + {file = "google_cloud_access_context_manager-0.5.0.tar.gz", hash = "sha256:950ce6597308581e3ffd378ffd385ea0e16c099cdd566932857cc10c5e774752"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-asset" +version = "3.30.1" +description = "Google Cloud Asset API client library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_cloud_asset-3.30.1-py3-none-any.whl", hash = "sha256:a2692d0aa250cfd6697ec1114cd0e36cab0d60fccc36f6d79b600d50522f6c25"}, + {file = "google_cloud_asset-3.30.1.tar.gz", hash = "sha256:a0f0249bfcbc44ef7f8980b621424de7cfe29588d2dac90cb58cccc810d053cc"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +google-cloud-access-context-manager = ">=0.1.2,<1.0.0" +google-cloud-org-policy = ">=0.1.2,<2.0.0" +google-cloud-os-config = ">=1.0.0,<2.0.0" +grpc-google-iam-v1 = ">=0.14.0,<1.0.0" +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" + +[[package]] +name = "google-cloud-compute" +version = "1.47.0" +description = "Google Cloud Compute API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_compute-1.47.0-py3-none-any.whl", hash = "sha256:7e0329d1b226ec948cd6064aa88ba6f16d4556585a13b1ec2494f751783749d3"}, + {file = "google_cloud_compute-1.47.0.tar.gz", hash = "sha256:f2c7909299f230428b0b12e52e031efe76c39be5d28cae9998fe1130a223fc3a"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-container" +version = "2.64.0" +description = "Google Cloud Container API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_container-2.64.0-py3-none-any.whl", hash = "sha256:667c2488f61f1ebcd1e7ce9e894799db86cd58c62cad33e4cef239f70ad99b27"}, + {file = "google_cloud_container-2.64.0.tar.gz", hash = "sha256:b41593e189f25d4c5a5b5f796669c1e4384ba0a25f6b298f64b556915b1e374d"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-core" +version = "2.6.0" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e"}, + {file = "google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83"}, +] + +[package.dependencies] +google-api-core = ">=2.11.0,<3.0.0" +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" + +[package.extras] +grpc = ["grpcio (>=1.47.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.47.0,<2.0.0)"] + +[[package]] +name = "google-cloud-iam" +version = "2.23.0" +description = "Google Cloud Iam API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_iam-2.23.0-py3-none-any.whl", hash = "sha256:a123ac45080a5c1735218a6b3db4c6e6ea12a1cdc86feec1c30ad1ede6c91fc6"}, + {file = "google_cloud_iam-2.23.0.tar.gz", hash = "sha256:49246f6221026d381cff4f8d804daf1bb6416153f2504bf5ef54d4af2450b828"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-org-policy" +version = "1.17.0" +description = "Google Cloud Org Policy API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_org_policy-1.17.0-py3-none-any.whl", hash = "sha256:ee903d598958f58e69b0c38b558ff75d370d3d8879bbadd1325f02973b97fcac"}, + {file = "google_cloud_org_policy-1.17.0.tar.gz", hash = "sha256:f77189ee7f65a28f3755f855f710b8b67b9862e2a6b6a93b075e6f9da93bff20"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-os-config" +version = "1.24.0" +description = "Google Cloud Os Config API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_os_config-1.24.0-py3-none-any.whl", hash = "sha256:9abae5b6d25eda6047a1da81e3fdef4871f100fe8a0b4bc0b043c52673dd2392"}, + {file = "google_cloud_os_config-1.24.0.tar.gz", hash = "sha256:ddf64eac47c0da9483d573b7de9abed48a5a6cbe27931ae94396899f598e0192"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "google-cloud-pubsub" +version = "2.38.0" +description = "Google Cloud Pub/Sub API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_pubsub-2.38.0-py3-none-any.whl", hash = "sha256:fed2c40cfb77d58f6dced563a8146a8c34319c7dfbbb4d045b6c9c101e043db9"}, + {file = "google_cloud_pubsub-2.38.0.tar.gz", hash = "sha256:9212309f8d6cfaefb577bca52492b13464b56e584505408685d63e69346c56cf"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpc-google-iam-v1 = ">=0.14.0,<1.0.0" +grpcio-status = ">=1.33.2" +opentelemetry-api = ">=1.27.0" +opentelemetry-sdk = ">=1.27.0" +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +libcst = ["libcst (>=0.3.10)"] + +[[package]] +name = "google-cloud-storage" +version = "2.19.0" +description = "Google Cloud Storage API client library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_cloud_storage-2.19.0-py2.py3-none-any.whl", hash = "sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba"}, + {file = "google_cloud_storage-2.19.0.tar.gz", hash = "sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2"}, +] + +[package.dependencies] +google-api-core = ">=2.15.0,<3.0.0.dev0" +google-auth = ">=2.26.1,<3.0.dev0" +google-cloud-core = ">=2.3.0,<3.0.dev0" +google-crc32c = ">=1.0,<2.0.dev0" +google-resumable-media = ">=2.7.2" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +protobuf = ["protobuf (<6.0.0.dev0)"] +tracing = ["opentelemetry-api (>=1.1.0)"] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_crc32c-1.8.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff"}, + {file = "google_crc32c-1.8.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288"}, + {file = "google_crc32c-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d"}, + {file = "google_crc32c-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092"}, + {file = "google_crc32c-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733"}, + {file = "google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8"}, + {file = "google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7"}, + {file = "google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15"}, + {file = "google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a"}, + {file = "google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2"}, + {file = "google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113"}, + {file = "google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb"}, + {file = "google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411"}, + {file = "google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454"}, + {file = "google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962"}, + {file = "google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b"}, + {file = "google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27"}, + {file = "google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa"}, + {file = "google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8"}, + {file = "google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f"}, + {file = "google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697"}, + {file = "google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651"}, + {file = "google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2"}, + {file = "google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21"}, + {file = "google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2"}, + {file = "google_crc32c-1.8.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc"}, + {file = "google_crc32c-1.8.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2"}, + {file = "google_crc32c-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215"}, + {file = "google_crc32c-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a"}, + {file = "google_crc32c-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f"}, + {file = "google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93"}, + {file = "google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c"}, + {file = "google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79"}, +] + +[[package]] +name = "google-resumable-media" +version = "2.9.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "google_resumable_media-2.9.0-py3-none-any.whl", hash = "sha256:c8901e88e389af8bed64d9696c74d8bad961865eb2236e13e0bfca9bb0a65ca3"}, + {file = "google_resumable_media-2.9.0.tar.gz", hash = "sha256:f7cfb224846a9dd444d125115dfbe8ef02a2b893e78f087762fe716a255a734b"}, +] + +[package.dependencies] +google-crc32c = ">=1.0.0,<2.0.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "google-auth (>=1.22.0,<2.0.0)"] +requests = ["requests (>=2.18.0,<3.0.0)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed"}, + {file = "googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd"}, +] + +[package.dependencies] +grpcio = {version = ">=1.44.0,<2.0.0", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0)"] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.14.4" +description = "IAM API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpc_google_iam_v1-0.14.4-py3-none-any.whl", hash = "sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964"}, + {file = "grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038"}, +] + +[package.dependencies] +googleapis-common-protos = {version = ">=1.63.2,<2.0.0", extras = ["grpc"]} +grpcio = ">=1.44.0,<2.0.0" +protobuf = ">=4.25.8,<8.0.0" + +[[package]] +name = "grpcio" +version = "1.80.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, + {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81"}, + {file = "grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069"}, + {file = "grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58"}, + {file = "grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a"}, + {file = "grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6"}, + {file = "grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440"}, + {file = "grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9"}, + {file = "grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0"}, + {file = "grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f"}, + {file = "grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193"}, + {file = "grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff"}, + {file = "grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad"}, + {file = "grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7"}, + {file = "grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294"}, + {file = "grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50"}, + {file = "grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e"}, + {file = "grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e"}, + {file = "grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae"}, + {file = "grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f"}, + {file = "grpcio-1.80.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4"}, + {file = "grpcio-1.80.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614"}, + {file = "grpcio-1.80.0-cp39-cp39-win32.whl", hash = "sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141"}, + {file = "grpcio-1.80.0-cp39-cp39-win_amd64.whl", hash = "sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de"}, + {file = "grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257"}, +] + +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + +[package.extras] +protobuf = ["grpcio-tools (>=1.80.0)"] + +[[package]] +name = "grpcio-status" +version = "1.80.0" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe"}, + {file = "grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.80.0" +protobuf = ">=6.31.1,<7.0.0" + +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httptools" +version = "0.8.0" +description = "A collection of framework independent HTTP protocol utils." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "httptools-0.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bf3b6f807c8541503cecfbb8a8dffb385640d0d96102f3d112aa8740f9b7c826"}, + {file = "httptools-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da684f2e1aa2ee9bdcb083f3f3a68c5956750b375bc5df864d3a5f0c42a40b77"}, + {file = "httptools-0.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6f21e2a3b0067bbe7f67e34cfd16276af556e5e52f4c7503be0cb5f90e905e4"}, + {file = "httptools-0.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb"}, + {file = "httptools-0.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0d726cc107fceb7d45f978483b4b70dd8caa836f5914d3434bb18628eb73813"}, + {file = "httptools-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9878eb2785ba5eb70631ad269b37976f73d647955e26c91d490eb8a4edfda4ba"}, + {file = "httptools-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b205e5f5523fa039679da0dfe5a10132b2a4abeae6a86fdd1ddc035f7f836557"}, + {file = "httptools-0.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed377e64805bdba4943c82717333f8f8603a13b09aff9cead2717c6c817fb168"}, + {file = "httptools-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9518c406d7b310f05adb1a37f80acabac40504a575d7c0da6d3e365c695ac20d"}, + {file = "httptools-0.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376"}, + {file = "httptools-0.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d"}, + {file = "httptools-0.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085"}, + {file = "httptools-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124"}, + {file = "httptools-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1b4c8e7a489a0d750d91894e9a8cdc295838f1924c0ca903ae993456fddec07"}, + {file = "httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d"}, + {file = "httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5"}, + {file = "httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2"}, + {file = "httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09"}, + {file = "httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a"}, + {file = "httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745"}, + {file = "httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150"}, + {file = "httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8"}, + {file = "httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c"}, + {file = "httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7"}, + {file = "httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d"}, + {file = "httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681"}, + {file = "httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683"}, + {file = "httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1"}, + {file = "httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6"}, + {file = "httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b"}, + {file = "httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0"}, + {file = "httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e"}, + {file = "httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b"}, + {file = "httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0"}, + {file = "httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527"}, + {file = "httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568"}, + {file = "httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b"}, + {file = "httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca"}, + {file = "httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f"}, + {file = "httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d"}, + {file = "httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081"}, + {file = "httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77"}, + {file = "httptools-0.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:df31ef5494f406ab6cf827b7e64a22841c6e2d654100e6a116ea15b46d02d5e8"}, + {file = "httptools-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5eb911c515b96ee44bbd861e42cbefc488681d450545b1d02127f6136e3a86f5"}, + {file = "httptools-0.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c08ffe3e79756e0963cbc8fe410139f38a5884874b6f2e17761bef6563fdcd9b"}, + {file = "httptools-0.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe2a4c95aeba2209434e7b31172da572846cae8ca0bf1e7013e61b99fbbf5e72"}, + {file = "httptools-0.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7b71e7d7031928c650e1006e6c03e911bf967f7c69c011d37d541c3e7bf55005"}, + {file = "httptools-0.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9fc1644f415372cec4f8a5be3a64183737398f10dbb1263602a036427fe75247"}, + {file = "httptools-0.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5d7fa4ba7292c1139c0526f0b5aad507c6263c948206ea1b1cbca015c8af1b62"}, + {file = "httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999"}, +] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +description = "Consume Server-Sent Event (SSE) messages with HTTPX." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, + {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, +] + +[[package]] +name = "idna" +version = "3.16" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5"}, + {file = "idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d"}, +] + +[package.extras] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "isodate" +version = "0.7.2" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15"}, + {file = "isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6"}, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, + {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.3.6" +referencing = ">=0.28.4" +rpds-py = ">=0.25.0" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, + {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + +[[package]] +name = "kubernetes" +version = "34.1.0" +description = "Kubernetes python client" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "kubernetes-34.1.0-py2.py3-none-any.whl", hash = "sha256:bffba2272534e224e6a7a74d582deb0b545b7c9879d2cd9e4aae9481d1f2cc2a"}, + {file = "kubernetes-34.1.0.tar.gz", hash = "sha256:8fe8edb0b5d290a2f3ac06596b23f87c658977d46b5f8df9d0f4ea83d0003912"}, +] + +[package.dependencies] +certifi = ">=14.5.14" +durationpy = ">=0.7" +google-auth = ">=1.0.1" +python-dateutil = ">=2.5.3" +pyyaml = ">=5.4.1" +requests = "*" +requests-oauthlib = "*" +six = ">=1.9.0" +urllib3 = ">=1.24.2,<2.4.0" +websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" + +[package.extras] +adal = ["adal (>=1.0.2)"] + +[[package]] +name = "markupsafe" +version = "3.0.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, + {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, + {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, + {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, + {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, + {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, + {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, + {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, + {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, + {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, + {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, + {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, + {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, + {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, + {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, + {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, + {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, + {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, + {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, + {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, + {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, + {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, + {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, + {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, + {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, + {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, + {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, + {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, + {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, + {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, + {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, + {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, + {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, + {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, + {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, +] + +[[package]] +name = "mcp" +version = "1.27.1" +description = "Model Context Protocol SDK" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f"}, + {file = "mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924"}, +] + +[package.dependencies] +anyio = ">=4.5" +httpx = ">=0.27.1,<1.0.0" +httpx-sse = ">=0.4" +jsonschema = ">=4.20.0" +pydantic = ">=2.11.0,<3.0.0" +pydantic-settings = ">=2.5.2" +pyjwt = {version = ">=2.10.1", extras = ["crypto"]} +python-multipart = ">=0.0.9" +pywin32 = {version = ">=310", markers = "sys_platform == \"win32\""} +sse-starlette = ">=1.6.1" +starlette = ">=0.27" +typing-extensions = ">=4.9.0" +typing-inspection = ">=0.4.1" +uvicorn = {version = ">=0.31.1", markers = "sys_platform != \"emscripten\""} + +[package.extras] +cli = ["python-dotenv (>=1.0.0)", "typer (>=0.16.0)"] +rich = ["rich (>=13.9.4)"] +ws = ["websockets (>=15.0.1)"] + +[[package]] +name = "msal" +version = "1.36.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4"}, + {file = "msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b"}, +] + +[package.dependencies] +cryptography = ">=2.5,<49" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.14,<0.21) ; python_version >= \"3.8\" and platform_system == \"Windows\"", "pymsalruntime (>=0.17,<0.21) ; python_version >= \"3.8\" and platform_system == \"Darwin\"", "pymsalruntime (>=0.18,<0.21) ; python_version >= \"3.8\" and platform_system == \"Linux\""] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca"}, + {file = "msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4"}, +] + +[package.dependencies] +msal = ">=1.29,<2" + +[package.extras] +portalocker = ["portalocker (>=1.4,<4)"] + +[[package]] +name = "msrest" +version = "0.7.1" +description = "AutoRest swagger generator Python client runtime." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, + {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, +] + +[package.dependencies] +azure-core = ">=1.24.0" +certifi = ">=2017.4.17" +isodate = ">=0.6.0" +requests = ">=2.16,<3.0" +requests-oauthlib = ">=0.5.0" + +[package.extras] +async = ["aiodns ; python_version >= \"3.5\"", "aiohttp (>=3.0) ; python_version >= \"3.5\""] + +[[package]] +name = "oauthlib" +version = "3.3.1" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1"}, + {file = "oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "opentelemetry-api" +version = "1.42.1" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714"}, + {file = "opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716"}, +] + +[package.dependencies] +typing-extensions = ">=4.5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.42.1" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_sdk-1.42.1-py3-none-any.whl", hash = "sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d"}, + {file = "opentelemetry_sdk-1.42.1.tar.gz", hash = "sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7"}, +] + +[package.dependencies] +opentelemetry-api = "1.42.1" +opentelemetry-semantic-conventions = "0.63b1" +typing-extensions = ">=4.5.0" + +[package.extras] +file-configuration = ["jsonschema (>=4.0)", "pyyaml (>=6.0)"] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.63b1" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682"}, + {file = "opentelemetry_semantic_conventions-0.63b1.tar.gz", hash = "sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9"}, +] + +[package.dependencies] +opentelemetry-api = "1.42.1" +typing-extensions = ">=4.5.0" + +[[package]] +name = "proto-plus" +version = "1.28.0" +description = "Beautiful, Pythonic protocol buffers" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8"}, + {file = "proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9"}, +] + +[package.dependencies] +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "6.33.6" +description = "" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3"}, + {file = "protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326"}, + {file = "protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593"}, + {file = "protobuf-6.33.6-cp39-cp39-win32.whl", hash = "sha256:bd56799fb262994b2c2faa1799693c95cc2e22c62f56fb43af311cae45d26f0e"}, + {file = "protobuf-6.33.6-cp39-cp39-win_amd64.whl", hash = "sha256:f443a394af5ed23672bc6c486be138628fbe5c651ccbc536873d7da23d1868cf"}, + {file = "protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901"}, + {file = "protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135"}, +] + +[[package]] +name = "pyasn1" +version = "0.6.3" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, +] + +[package.dependencies] +pyasn1 = ">=0.6.1,<0.7.0" + +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba"}, + {file = "pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.46.4" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4"}, + {file = "pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39"}, + {file = "pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d"}, + {file = "pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf"}, + {file = "pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594"}, + {file = "pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d"}, + {file = "pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2"}, + {file = "pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a"}, + {file = "pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008"}, + {file = "pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d"}, + {file = "pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb"}, + {file = "pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596"}, + {file = "pydantic_core-2.46.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae"}, + {file = "pydantic_core-2.46.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9"}, + {file = "pydantic_core-2.46.4-cp39-cp39-win32.whl", hash = "sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1"}, + {file = "pydantic_core-2.46.4-cp39-cp39-win_amd64.whl", hash = "sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983"}, + {file = "pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1"}, +] + +[package.dependencies] +typing-extensions = ">=4.14.1" + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de"}, + {file = "pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" +typing-inspection = ">=0.4.0" + +[package.extras] +aws-secrets-manager = ["boto3 (>=1.35.0)", "types-boto3[secretsmanager]"] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] +gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "pyjwt" +version = "2.13.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728"}, + {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.2.2" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a"}, + {file = "python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-multipart" +version = "0.0.29" +description = "A streaming multipart parser for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69"}, + {file = "python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904"}, +] + +[[package]] +name = "pywin32" +version = "311" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, + {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, + {file = "pywin32-311-cp310-cp310-win_arm64.whl", hash = "sha256:0502d1facf1fed4839a9a51ccbcc63d952cf318f78ffc00a7e78528ac27d7a2b"}, + {file = "pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151"}, + {file = "pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503"}, + {file = "pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2"}, + {file = "pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31"}, + {file = "pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067"}, + {file = "pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852"}, + {file = "pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d"}, + {file = "pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d"}, + {file = "pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a"}, + {file = "pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee"}, + {file = "pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87"}, + {file = "pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42"}, + {file = "pywin32-311-cp38-cp38-win32.whl", hash = "sha256:6c6f2969607b5023b0d9ce2541f8d2cbb01c4f46bc87456017cf63b73f1e2d8c"}, + {file = "pywin32-311-cp38-cp38-win_amd64.whl", hash = "sha256:c8015b09fb9a5e188f83b7b04de91ddca4658cee2ae6f3bc483f0b21a77ef6cd"}, + {file = "pywin32-311-cp39-cp39-win32.whl", hash = "sha256:aba8f82d551a942cb20d4a83413ccbac30790b50efb89a75e4f586ac0bb8056b"}, + {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, + {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, + {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, + {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, + {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, + {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, + {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, + {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, + {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, + {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, + {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, +] + +[[package]] +name = "referencing" +version = "0.37.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, + {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "requests" +version = "2.34.2" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, + {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, +] + +[package.dependencies] +certifi = ">=2023.5.7" +charset_normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.26,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] + +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=3.4" +groups = ["main"] +files = [ + {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, + {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "robotframework" +version = "7.3.2" +description = "Generic automation framework for acceptance testing and robotic process automation (RPA)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "robotframework-7.3.2-py3-none-any.whl", hash = "sha256:14ef2afa905285cc073df6ce06d0cd3af4a113df6f815532718079e00c98cca4"}, + {file = "robotframework-7.3.2.tar.gz", hash = "sha256:3bb3e299831ecb1664f3d5082f6ff9f08ba82d61a745bef2227328ef3049e93a"}, +] + +[[package]] +name = "rpds-py" +version = "2026.5.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3397a5ed7174dc2786bb214030232fc36fe8e5584fec43a9952cc542b1a12036"}, + {file = "rpds_py-2026.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:99ab6ba7bfa2cb0f96a04e3652355bf04e3f51aceb1e943b8541dab7ba4828cc"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0efbe45632665e53e3db8fe1e5692db58fc5cb9bab4459d570b83efefe11164"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01d17b29c0c23d82b1f4751147ec49cf451f1fc2554eb9ef5f957e55d2656ead"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7559f72b94ae52659086c595dfa017cde03155f7832071d30959049052cb3ece"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e25b7088f9ccbfc0dfcaa52bf969300ca229e10ecf758974ebcbb080a4b37bb"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613fc4ee9eaef26dc5840666214dd6fbcebcf32f46e76f4abc473059f4e13dda"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:85264a90ff4c05c1568dd65f5921c837614b67c60358fb4c17df3b7f2e90690a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe71bca7d547acb17027c7fd1624ff8aae623499c498d3e7011182c4de5c25e0"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05fa4f41f37ec97c9c260441a940450a192f78d774d2b097eee1379f1e1246a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df1d2a1996755b24b9ecee92cb4d36c28f86f464a6a173349c26bab41e94b8c2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8895840ac4809e5f60c88fd07617cd71326e73d6e5a8aa783c5c0f7c24985de2"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win32.whl", hash = "sha256:3684a59b158a7683aaeb8e25352e9a9dd2122cec78f2d8530266e4f91b4c7b3f"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:7bd530e6a530bb3ea892f194fafa455f3516ac25ecf7143fd33c09be62b0470a"}, + {file = "rpds_py-2026.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:0a5ae4dbe43c1076983b72616496919872ae7bbe7a1e21cc48336bc3154d130b"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d"}, + {file = "rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24"}, + {file = "rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870"}, + {file = "rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d"}, + {file = "rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b"}, + {file = "rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f"}, + {file = "rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf"}, + {file = "rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc"}, + {file = "rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9"}, + {file = "rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec"}, + {file = "rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02"}, + {file = "rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838"}, + {file = "rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a"}, + {file = "rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd"}, + {file = "rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01"}, + {file = "rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325"}, + {file = "rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df"}, + {file = "rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:edf2765d84e42447f112ad877af8fe1db0089aaec5b28e88d6eab45e7fe99cea"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad3773236e95f7f33991eb125224b7da66f206504d032a253a02da7e134519fb"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a04df86b3f0fade39ec8fd0e0aab089b1da9fbd2b48df778a57ef96f5e7d38df"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6142dbd80c4df62a5d899f0d616d417f84e0bc8d32526c8e5589019d75d028a7"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b35217adefe87f2fe4db7e9766cabe84744bfe9616d9667be18988928c7f2dc"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b95d5e11fc712b752081183a55a244c03cd00570489edd7014d8899f8ceb8162"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141c9498daf2ace9eda35d2b0e376f9ea8b058d84f2aef4f96fccfd449a2f251"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:6f249f8b860a200ad35193af961183ebe9132710484e6f6ce0cf89fd83c63a9a"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4abbf391a70be864920858bf360f4fb380577c9a0f732438a1996726e2c195b"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c74005a7bb87752acf351c93897ec63ad77a07a0da7ecad9c050e32e7286ba34"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:8213afbe8a3a906fb9acb2014423fe3359ee783d0bf90995f70623a3217bfa6c"}, + {file = "rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049"}, + {file = "rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256"}, +] + +[[package]] +name = "s3transfer" +version = "0.17.1" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "s3transfer-0.17.1-py3-none-any.whl", hash = "sha256:5b9827d1044159bbb01b86ef8902760ea39281927f5de31de75e1d657177bf4c"}, + {file = "s3transfer-0.17.1.tar.gz", hash = "sha256:042dd5e3b1b512355e35a23f0223e426b7042e80b97830ea2680ddce327fc45e"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "smmap" +version = "5.0.3" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f"}, + {file = "smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c"}, +] + +[[package]] +name = "sse-starlette" +version = "3.0.3" +description = "SSE plugin for Starlette" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sse_starlette-3.0.3-py3-none-any.whl", hash = "sha256:af5bf5a6f3933df1d9c7f8539633dc8444ca6a97ab2e2a7cd3b6e431ac03a431"}, + {file = "sse_starlette-3.0.3.tar.gz", hash = "sha256:88cfb08747e16200ea990c8ca876b03910a23b547ab3bd764c0d8eb81019b971"}, +] + +[package.dependencies] +anyio = ">=4.7.0" + +[package.extras] +daphne = ["daphne (>=4.2.0)"] +examples = ["aiosqlite (>=0.21.0)", "fastapi (>=0.115.12)", "sqlalchemy[asyncio] (>=2.0.41)", "starlette (>=0.49.1)", "uvicorn (>=0.34.0)"] +granian = ["granian (>=2.3.1)"] +uvicorn = ["uvicorn (>=0.34.0)"] + +[[package]] +name = "starlette" +version = "0.46.2" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35"}, + {file = "starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5"}, +] + +[package.dependencies] +anyio = ">=3.6.2,<5" + +[package.extras] +full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, + {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[[package]] +name = "urllib3" +version = "2.3.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "uvicorn" +version = "0.34.3" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "uvicorn-0.34.3-py3-none-any.whl", hash = "sha256:16246631db62bdfbf069b0645177d6e8a77ba950cfedbfd093acef9444e4d885"}, + {file = "uvicorn-0.34.3.tar.gz", hash = "sha256:35919a9a979d7a59334b6b10e05d77c1d0d574c50e0fc98b8b1a0f165708b55a"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} +h11 = ">=0.8" +httptools = {version = ">=0.6.3", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} +uvloop = {version = ">=0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} + +[package.extras] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "uvloop" +version = "0.22.1" +description = "Fast implementation of asyncio event loop on top of libuv" +optional = false +python-versions = ">=3.8.1" +groups = ["main"] +markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"" +files = [ + {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c"}, + {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792"}, + {file = "uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86"}, + {file = "uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd"}, + {file = "uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2"}, + {file = "uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec"}, + {file = "uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9"}, + {file = "uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77"}, + {file = "uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21"}, + {file = "uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702"}, + {file = "uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733"}, + {file = "uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473"}, + {file = "uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42"}, + {file = "uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6"}, + {file = "uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370"}, + {file = "uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4"}, + {file = "uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2"}, + {file = "uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0"}, + {file = "uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705"}, + {file = "uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8"}, + {file = "uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d"}, + {file = "uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e"}, + {file = "uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e"}, + {file = "uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad"}, + {file = "uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142"}, + {file = "uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74"}, + {file = "uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35"}, + {file = "uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25"}, + {file = "uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6"}, + {file = "uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079"}, + {file = "uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289"}, + {file = "uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3"}, + {file = "uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c"}, + {file = "uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21"}, + {file = "uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88"}, + {file = "uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e"}, + {file = "uvloop-0.22.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:80eee091fe128e425177fbd82f8635769e2f32ec9daf6468286ec57ec0313efa"}, + {file = "uvloop-0.22.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:017bd46f9e7b78e81606329d07141d3da446f8798c6baeec124260e22c262772"}, + {file = "uvloop-0.22.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3e5c6727a57cb6558592a95019e504f605d1c54eb86463ee9f7a2dbd411c820"}, + {file = "uvloop-0.22.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:57df59d8b48feb0e613d9b1f5e57b7532e97cbaf0d61f7aa9aa32221e84bc4b6"}, + {file = "uvloop-0.22.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:55502bc2c653ed2e9692e8c55cb95b397d33f9f2911e929dc97c4d6b26d04242"}, + {file = "uvloop-0.22.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4a968a72422a097b09042d5fa2c5c590251ad484acf910a651b4b620acd7f193"}, + {file = "uvloop-0.22.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b45649628d816c030dba3c80f8e2689bab1c89518ed10d426036cdc47874dfc4"}, + {file = "uvloop-0.22.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ea721dd3203b809039fcc2983f14608dae82b212288b346e0bfe46ec2fab0b7c"}, + {file = "uvloop-0.22.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ae676de143db2b2f60a9696d7eca5bb9d0dd6cc3ac3dad59a8ae7e95f9e1b54"}, + {file = "uvloop-0.22.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17d4e97258b0172dfa107b89aa1eeba3016f4b1974ce85ca3ef6a66b35cbf659"}, + {file = "uvloop-0.22.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:05e4b5f86e621cf3927631789999e697e58f0d2d32675b67d9ca9eb0bca55743"}, + {file = "uvloop-0.22.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:286322a90bea1f9422a470d5d2ad82d38080be0a29c4dd9b3e6384320a4d11e7"}, + {file = "uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f"}, +] + +[package.extras] +dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"] + +[[package]] +name = "watchfiles" +version = "1.2.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "watchfiles-1.2.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bb68bf4df85abebe5efddc53cf2075520f243a59868d9b3973278b23e76962a9"}, + {file = "watchfiles-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c16cb06dd17d43b9d185094268459eac92c9538356f050e55b54e82cf700e1d4"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a0feab9af4c021c581f695258c642b3d10c5fd4c676e33a0d8606425d82631"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a16ffe19bf5cf9f5edaa1ad1dd830c5a816e8feec430c522302ab55483a4b994"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204f299afcbd65918ab78dbc52626b0ae45e9d8cef403fdbf33ecf9e40eac66e"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11743adfa510bfffebe97659fb280182b5c9b238708f667e866f308c3430dc19"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb72919d93e3a16fc451d3aa3d4b1698423daca1b382d3d959c9ac51297c12a8"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62f042afde2dde21ec1d2c1a74361e804673df86f51e418a999c9acfe671b07"}, + {file = "watchfiles-1.2.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:027ae72bfdfd254862065d8b3e2a815c6ab9b1853ce41e6648ece84afd34a551"}, + {file = "watchfiles-1.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1cfd51e97e13ff3bd047c140764d277fc9b95b7cb5da59e46a47d167adab310"}, + {file = "watchfiles-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:24b2405c0a46738dd9e1cf7135aa5dbdb9d42d024628651b3b13d5117e99f8df"}, + {file = "watchfiles-1.2.0-cp310-cp310-win32.whl", hash = "sha256:8c520725602756229f045b032a1ff33d7ef0f7404189d62f6c2438cb6d8ef6a1"}, + {file = "watchfiles-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:03b14855c6f35539e2d95c442ae9530a75762f1e26567152b9ed05f96534a74d"}, + {file = "watchfiles-1.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:704fd259e332e01f9b9c178f4bce9e49027e5587cc2600eeeaf8e76e1c846201"}, + {file = "watchfiles-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6543cf55d170003296d185c0af981f3e1311564907e1f4e08671fc7693a890a5"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:772b80df316480d894a0e3165fdd19cf77f5d17f9a787f94029465ad0e3529d1"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d158cd89df6053823533e06fb1d73c549133bff5f0396170c0e53d9559340717"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d516b3283a758e087841aedb8031549fb41ced08f3db10aa6d2bf32dc042525b"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53b2290c92e0506d102cd448fbc610d87079553f86caa39d67440856a8b8bba5"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e"}, + {file = "watchfiles-1.2.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:e2ca07fa7d89195ec0865d3d285666286740bfa83d83e5cee204043a31ecc165"}, + {file = "watchfiles-1.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0618518f282c4ebff60f5e5b1247b6d91bb8b9f4476947563a1e74acc66f3c6"}, + {file = "watchfiles-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d191c054d0715c3c95c99df9b8dbf6fd096d8c1e021e8f212e1bd8bc444ccb5"}, + {file = "watchfiles-1.2.0-cp311-cp311-win32.whl", hash = "sha256:9342472aff9b093c5acd4f6d8f70ae0937964ab56542502bcf5579782da69ae8"}, + {file = "watchfiles-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:dbd6c97045dad81227c8d040173da044c1de08de64a5ea8b555da4aee1d5fa22"}, + {file = "watchfiles-1.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:57a2d9fa4fb4c2ecae57b13dfff2c7ab53e21a2ba674fe9f05506680fcdcc0d7"}, + {file = "watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26"}, + {file = "watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5"}, + {file = "watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d"}, + {file = "watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c"}, + {file = "watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906"}, + {file = "watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898"}, + {file = "watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379"}, + {file = "watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5"}, + {file = "watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98"}, + {file = "watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71"}, + {file = "watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3"}, + {file = "watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0"}, + {file = "watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427"}, + {file = "watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799"}, + {file = "watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9"}, + {file = "watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077"}, + {file = "watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08"}, + {file = "watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9"}, + {file = "watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa"}, + {file = "watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44"}, + {file = "watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72"}, + {file = "watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4"}, + {file = "watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7"}, + {file = "watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e"}, + {file = "watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06"}, + {file = "watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba"}, + {file = "watchfiles-1.2.0-cp314-cp314-win32.whl", hash = "sha256:c0db965c5f79aa49fe672d297cf1febc5ad149b658594944f49a54a2b96270a7"}, + {file = "watchfiles-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:71283b39fd17e5408eb123bd37aeecfd9d54c81fc184421943208aadb879d103"}, + {file = "watchfiles-1.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:c5c19526f4e54a00f2666a6c0e9e40d582c09e865055ea7378bf0009aab857b3"}, + {file = "watchfiles-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d73a585accffa5ae39c17264c36ec3166d2fad7000c780f5ef83b2722afb9dd2"}, + {file = "watchfiles-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae99b14c5f21e026e0e9d96f40e07d8570ebee6cafd9d8fc318354606daa7a28"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925"}, + {file = "watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b"}, + {file = "watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30"}, + {file = "watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5"}, + {file = "watchfiles-1.2.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:37a6721cdf3f65dbb13aa9503510ccb4451603ac837e44d265d7992a597e1374"}, + {file = "watchfiles-1.2.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2b37d10b5a63bd4d87e18472d80fa525bd670586fae62e5dd580452764879b65"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4"}, + {file = "watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488"}, + {file = "watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb"}, + {file = "watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377"}, + {file = "watchfiles-1.2.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4674d49eb94706dfe666c069fc0a1b646ffcf920473492e209f6d5f60d3f0cc2"}, + {file = "watchfiles-1.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:094b9b70103d4e963499bdea001ee3c2697b144cd9ae6218a62c0f89ec9e31db"}, + {file = "watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ef001f8c25ad0fa9529f914c1600647ecd0f542d11c19b7894768c67b6acb7"}, + {file = "watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a88fc94e647bc4eec523f1caa540258eb71d14278b9daf72fa1e2658a98df0f0"}, + {file = "watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + +[[package]] +name = "websocket-client" +version = "1.9.0" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef"}, + {file = "websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx_rtd_theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["pytest", "websockets"] + +[[package]] +name = "websockets" +version = "16.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a"}, + {file = "websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0"}, + {file = "websockets-16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957"}, + {file = "websockets-16.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72"}, + {file = "websockets-16.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde"}, + {file = "websockets-16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3"}, + {file = "websockets-16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3"}, + {file = "websockets-16.0-cp310-cp310-win32.whl", hash = "sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9"}, + {file = "websockets-16.0-cp310-cp310-win_amd64.whl", hash = "sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35"}, + {file = "websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8"}, + {file = "websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad"}, + {file = "websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d"}, + {file = "websockets-16.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe"}, + {file = "websockets-16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b"}, + {file = "websockets-16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5"}, + {file = "websockets-16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64"}, + {file = "websockets-16.0-cp311-cp311-win32.whl", hash = "sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6"}, + {file = "websockets-16.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac"}, + {file = "websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00"}, + {file = "websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79"}, + {file = "websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39"}, + {file = "websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c"}, + {file = "websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f"}, + {file = "websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1"}, + {file = "websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2"}, + {file = "websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89"}, + {file = "websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea"}, + {file = "websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9"}, + {file = "websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230"}, + {file = "websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c"}, + {file = "websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5"}, + {file = "websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82"}, + {file = "websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8"}, + {file = "websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f"}, + {file = "websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a"}, + {file = "websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156"}, + {file = "websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0"}, + {file = "websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904"}, + {file = "websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4"}, + {file = "websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e"}, + {file = "websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4"}, + {file = "websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1"}, + {file = "websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3"}, + {file = "websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8"}, + {file = "websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d"}, + {file = "websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244"}, + {file = "websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e"}, + {file = "websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641"}, + {file = "websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8"}, + {file = "websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e"}, + {file = "websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944"}, + {file = "websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206"}, + {file = "websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6"}, + {file = "websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd"}, + {file = "websockets-16.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d"}, + {file = "websockets-16.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03"}, + {file = "websockets-16.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da"}, + {file = "websockets-16.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c"}, + {file = "websockets-16.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767"}, + {file = "websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec"}, + {file = "websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5"}, +] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.14,<3.15" +content-hash = "caca939decd8d8f31599f9dd1dc3885db75b4889413c8eed46c03f1a1eb84af1" diff --git a/src/pyproject.toml b/src/pyproject.toml index f52cf7ac1..279a98d9b 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -9,9 +9,9 @@ package-mode = false [tool.poetry.dependencies] python = ">=3.14,<3.15" -Django = "^5.1" -django-extensions = "^3.1.3" -djangorestframework = "^3.15.2" +fastapi = "^0.115.0" +uvicorn = {version = "^0.34.0", extras = ["standard"]} +httpx = "^0.28.0" kubernetes = "^34.0.0" robotframework = "==7.3.2" GitPython = "^3.1.43" @@ -21,7 +21,28 @@ boto3 = "^1.34" azure-identity = "^1.19" azure-mgmt-resource = "^24.0.0" azure-mgmt-containerservice = "^40.0.0" +azure-mgmt-compute = "^38.0.0" +azure-mgmt-storage = "^25.0.0" +azure-mgmt-network = "^30.2.0" +azure-mgmt-keyvault = "^14.0.1" +azure-mgmt-web = "^11.0.0" +azure-mgmt-rdbms = "^10.1.1" +azure-mgmt-redis = "^14.5.0" +azure-mgmt-servicebus = "^9.0.0" +azure-mgmt-datafactory = "^9.3.0" +azure-mgmt-containerregistry = "^15.0.0" +azure-mgmt-apimanagement = "^5.0.0" +azure-mgmt-cosmosdb = "^9.9.0" +azure-mgmt-azurearcdata = "^1.0.1" azure-devops = "^7.1.0b1" +google-auth = "^2.38.0" +google-cloud-asset = "^3.27.0" +google-cloud-compute = "^1.23.0" +google-cloud-storage = "^2.19.0" +google-cloud-container = "^2.56.0" +google-cloud-pubsub = "^2.21.0" +google-cloud-iam = "^2.15.0" +mcp = "^1.27.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/src/renderers/dump_resources.py b/src/renderers/dump_resources.py index 02c5da299..a679559d2 100644 --- a/src/renderers/dump_resources.py +++ b/src/renderers/dump_resources.py @@ -1,6 +1,11 @@ from component import Context, Setting, SettingDependency from resources import Registry, REGISTRY_PROPERTY_NAME from enrichers.generation_rules import SLXS_PROPERTY # for access to aggregated SLX info +from indexers.resource_writer import ( + RESOURCE_STORE_BACKEND_SETTING, + RESOURCE_STORE_PATH_SETTING, + finalize_resource_store, +) import yaml DOCUMENTATION = "Render/dump resource state to a YAML file" @@ -14,9 +19,13 @@ SETTINGS = ( SettingDependency(RESOURCE_DUMP_PATH_SETTING, False), + SettingDependency(RESOURCE_STORE_BACKEND_SETTING, False), + SettingDependency(RESOURCE_STORE_PATH_SETTING, False), ) def render(context: Context): + finalize_resource_store(context) + resource_dump_path = context.get_setting(RESOURCE_DUMP_PATH_SETTING) if not resource_dump_path: return diff --git a/src/renderers/render_output_items.py b/src/renderers/render_output_items.py index fb72b0c2b..b683c3e1f 100644 --- a/src/renderers/render_output_items.py +++ b/src/renderers/render_output_items.py @@ -8,6 +8,7 @@ LOCATION_ID_SETTING, WORKSPACE_OUTPUT_PATH_SETTING from template import render_template_file from exceptions import WorkspaceBuilderException +from renderers.rendered_artifacts import init_rendered_artifacts, record_rendered_artifact logger = logging.getLogger(__name__) @@ -34,21 +35,28 @@ class OutputItem: template_name: str template_loader_func: Optional[Callable[[str], str]] template_variables: dict[str, Any] + raw_content: Optional[str] def __init__(self, path: str, template_name: str, template_variables: dict[str, Any], - template_loader_func: Optional[Callable[[str], str]] = None): + template_loader_func: Optional[Callable[[str], str]] = None, + raw_content: Optional[str] = None): self.path = path self.template_name = template_name self.template_loader_func = template_loader_func self.template_variables = template_variables + # When ``raw_content`` is set, the renderer writes it verbatim to ``path`` + # without invoking Jinja or post-processing. Used for codebundle overlay + # files such as ``Skill.md`` that ship as opaque text alongside an SLX. + self.raw_content = raw_content def load(context: Context): # Initialize the output items dict so that other components don't need to # check for None and initialize it themselves. context.set_property(OUTPUT_ITEMS_PROPERTY, dict()) + init_rendered_artifacts(context) def compute_resource_path_from_hierarchy(data: dict) -> None: @@ -320,6 +328,13 @@ def render(context: Context): logger.debug(f"Template variables for {output_item.path}: {output_item.template_variables}") try: + if output_item.raw_content is not None: + output_text = output_item.raw_content + context.write_file(output_item.path, output_text) + record_rendered_artifact(context, output_item.path, output_text) + render_stats['successfully_rendered'] += 1 + continue + # Render the template output_text = render_template_file(output_item.template_name, output_item.template_variables, @@ -333,6 +348,7 @@ def render(context: Context): # Write the deduplicated output to the file context.write_file(output_item.path, deduplicated_output) + record_rendered_artifact(context, output_item.path, deduplicated_output) render_stats['successfully_rendered'] += 1 except WorkspaceBuilderException as e: logger.warning(f"Skipping template {output_item.template_name} for {output_item.path}: {str(e)}") diff --git a/src/renderers/rendered_artifacts.py b/src/renderers/rendered_artifacts.py new file mode 100644 index 000000000..e657ded05 --- /dev/null +++ b/src/renderers/rendered_artifacts.py @@ -0,0 +1,70 @@ +"""Capture rendered workspace files for SQLite persistence.""" + +from __future__ import annotations + +import os +from typing import Any + +from component import Context + +RENDERED_ARTIFACTS_PROPERTY = "rendered_artifacts" + + +def init_rendered_artifacts(context: Context) -> None: + context.set_property(RENDERED_ARTIFACTS_PROPERTY, []) + + +def classify_workspace_artifact(relative_path: str) -> str: + base = os.path.basename(relative_path).lower() + if base == "slx.yaml": + return "slx" + if base == "sli.yaml" or base.endswith("-sli.yaml"): + return "sli" + if base == "runbook.yaml" or "runbook" in base: + return "runbook" + if base == "workspace.yaml": + return "workspace" + # Codebundle Skill overlay copied alongside each rendered SLX. The filename + # is matched case-insensitively because upstream codebundles publish it as + # ``SKILL.md`` or ``Skill.md`` (or, less commonly, ``skill.md``). + if base == "skill.md": + return "skill" + normalized = relative_path.replace("\\", "/") + if "/slxs/" in normalized: + return "slx_bundle" + return "other" + + +def classify_media_type(relative_path: str) -> str: + _, ext = os.path.splitext(relative_path.lower()) + if ext in (".yaml", ".yml"): + return "yaml" + if ext == ".md": + return "markdown" + if ext == ".json": + return "json" + return "text" + + +def slx_directory_for_path(relative_path: str) -> str | None: + normalized = relative_path.replace("\\", "/") + if "/slxs/" not in normalized: + return None + directory = os.path.dirname(normalized) + if os.path.basename(directory) and os.path.basename(directory) != "slxs": + return directory + return None + + +def record_rendered_artifact(context: Context, relative_path: str, content: str) -> None: + artifacts: list[dict[str, Any]] = context.get_property(RENDERED_ARTIFACTS_PROPERTY, []) + artifacts.append( + { + "relative_path": relative_path, + "artifact_kind": classify_workspace_artifact(relative_path), + "media_type": classify_media_type(relative_path), + "slx_directory": slx_directory_for_path(relative_path), + "content": content if isinstance(content, str) else content.decode("utf-8"), + } + ) + context.set_property(RENDERED_ARTIFACTS_PROPERTY, artifacts) diff --git a/src/renderers/test_skill_overlay.py b/src/renderers/test_skill_overlay.py new file mode 100644 index 000000000..86e38a679 --- /dev/null +++ b/src/renderers/test_skill_overlay.py @@ -0,0 +1,285 @@ +"""Tests for the Skill.md overlay end of the rendering pipeline.""" + +from __future__ import annotations + +import os +import tempfile +import unittest + +from component import Context +from outputter import FileSystemOutputter +from renderers.rendered_artifacts import ( + RENDERED_ARTIFACTS_PROPERTY, + classify_workspace_artifact, + init_rendered_artifacts, + record_rendered_artifact, + slx_directory_for_path, +) +from renderers.render_output_items import ( + OUTPUT_ITEMS_PROPERTY, + OutputItem, + render, +) + + +class ClassifyWorkspaceArtifactTests(unittest.TestCase): + def test_skill_md_is_classified_as_skill_regardless_of_case(self): + for variant in ("Skill.md", "SKILL.md", "skill.md", "sKiLL.Md"): + with self.subTest(filename=variant): + self.assertEqual( + "skill", + classify_workspace_artifact(f"workspaces/ws/slxs/foo/{variant}"), + ) + + def test_existing_kinds_still_classified(self): + self.assertEqual("slx", classify_workspace_artifact("workspaces/ws/slxs/foo/slx.yaml")) + self.assertEqual("sli", classify_workspace_artifact("workspaces/ws/slxs/foo/sli.yaml")) + self.assertEqual("runbook", classify_workspace_artifact("workspaces/ws/slxs/foo/runbook.yaml")) + self.assertEqual("workspace", classify_workspace_artifact("workspaces/ws/workspace.yaml")) + + def test_unrelated_md_files_under_slxs_are_slx_bundle(self): + # README and other markdown files under an SLX directory should not + # accidentally be classified as the canonical Skill overlay. + self.assertEqual( + "slx_bundle", + classify_workspace_artifact("workspaces/ws/slxs/foo/README.md"), + ) + + def test_slx_directory_picks_up_skill_overlay_path(self): + self.assertEqual( + "workspaces/ws/slxs/foo", + slx_directory_for_path("workspaces/ws/slxs/foo/SKILL.md"), + ) + + +class RawContentRenderingTests(unittest.TestCase): + def _make_context(self, output_dir: str) -> Context: + ctx = Context( + setting_values={ + "WORKSPACE_NAME": "demo-ws", + "LOCATION_ID": "location-01", + "WORKSPACE_OUTPUT_PATH": "workspaces", + }, + outputter=FileSystemOutputter(output_dir), + ) + ctx.set_property(OUTPUT_ITEMS_PROPERTY, {}) + init_rendered_artifacts(ctx) + return ctx + + def test_raw_content_is_written_verbatim_and_recorded(self): + with tempfile.TemporaryDirectory() as out_dir: + ctx = self._make_context(out_dir) + skill_text = "# Skill: namespace healthcheck\n\nDoes a thing.\n" + overlay_path = "workspaces/demo-ws/slxs/k8s-namespace/Skill.md" + + ctx.get_property(OUTPUT_ITEMS_PROPERTY)[overlay_path] = OutputItem( + path=overlay_path, + template_name="Skill.md", + template_variables={}, + template_loader_func=None, + raw_content=skill_text, + ) + render(ctx) + + disk_path = os.path.join(out_dir, overlay_path) + self.assertTrue(os.path.isfile(disk_path), f"expected overlay file at {disk_path}") + with open(disk_path, encoding="utf-8") as fh: + self.assertEqual(skill_text, fh.read()) + + artifacts = ctx.get_property(RENDERED_ARTIFACTS_PROPERTY, []) + self.assertEqual(1, len(artifacts)) + self.assertEqual("skill", artifacts[0]["artifact_kind"]) + self.assertEqual("markdown", artifacts[0]["media_type"]) + self.assertEqual("workspaces/demo-ws/slxs/k8s-namespace", artifacts[0]["slx_directory"]) + self.assertEqual(skill_text, artifacts[0]["content"]) + + def test_record_rendered_artifact_with_skill_md(self): + with tempfile.TemporaryDirectory() as out_dir: + ctx = self._make_context(out_dir) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/foo/Skill.md", + "# Hello\n", + ) + artifacts = ctx.get_property(RENDERED_ARTIFACTS_PROPERTY) + self.assertEqual(1, len(artifacts)) + self.assertEqual("skill", artifacts[0]["artifact_kind"]) + self.assertEqual("markdown", artifacts[0]["media_type"]) + + +class _StubCodeCollection: + """Minimal CodeCollection stand-in for `_emit_skill_overlay` tests. + + The overlay only touches ``repo_url`` (for cache keying) and + ``find_code_bundle_file``, so we don't need GitPython here. ``file_lookup`` + is keyed by ``(ref_name, code_bundle_name)`` and maps to a dict of + ``{filename_as_published: content}`` so each bundle can publish the Skill + overlay under any casing variant (``SKILL.md``, ``Skill.md``, etc.). + """ + + def __init__(self, repo_url: str, file_lookup: dict): + self.repo_url = repo_url + # Normalize legacy {(ref, bundle, name): text} shape for backwards compat. + normalized: dict[tuple, dict[str, str]] = {} + for key, value in file_lookup.items(): + if isinstance(key, tuple) and len(key) == 3 and isinstance(value, str): + ref, bundle, name = key + normalized.setdefault((ref, bundle), {})[name] = value + elif isinstance(key, tuple) and len(key) == 2 and isinstance(value, dict): + normalized[key] = dict(value) + else: + raise TypeError(f"Unexpected file_lookup entry: {key!r} -> {value!r}") + self._lookup = normalized + self.calls: list[tuple] = [] + + def find_code_bundle_file(self, ref_name, code_bundle_name, relative_path, case_insensitive=False): + self.calls.append((ref_name, code_bundle_name, relative_path, case_insensitive)) + bundle_files = self._lookup.get((ref_name, code_bundle_name)) + if not bundle_files: + return None + if case_insensitive and "/" not in relative_path and "\\" not in relative_path: + target = relative_path.lower() + for actual_name, text in bundle_files.items(): + if actual_name.lower() == target: + return actual_name, text + return None + text = bundle_files.get(relative_path) + return (relative_path, text) if text is not None else None + + def get_code_bundle_file_text(self, ref_name, code_bundle_name, relative_path, case_insensitive=False): + resolved = self.find_code_bundle_file( + ref_name, code_bundle_name, relative_path, case_insensitive=case_insensitive + ) + return None if resolved is None else resolved[1] + + +class _StubFileSpec: + def __init__(self, ref_name: str, code_bundle_name: str): + self.ref_name = ref_name + self.code_bundle_name = code_bundle_name + + +class _StubGenerationRuleInfo: + def __init__(self, code_collection, ref_name: str, code_bundle_name: str): + self.code_collection = code_collection + self.generation_rule_file_spec = _StubFileSpec(ref_name, code_bundle_name) + + +class EmitSkillOverlayTests(unittest.TestCase): + """Exercise the overlay wiring in generation_rules without GitPython.""" + + def _make_context(self) -> Context: + ctx = Context( + setting_values={"WORKSPACE_NAME": "demo-ws"}, + outputter=FileSystemOutputter("."), + ) + return ctx + + def test_overlay_emits_raw_content_renderer_item(self): + from enrichers.generation_rules import _emit_skill_overlay + + repo_url = "https://example.com/foo.git" + ref = "feat/skill-overlay" + bundle = "k8s-namespace-healthcheck" + skill_text = "# Skill: namespace healthcheck\n\nDoes a thing.\n" + cc = _StubCodeCollection(repo_url, {(ref, bundle, "Skill.md"): skill_text}) + gr_info = _StubGenerationRuleInfo(cc, ref, bundle) + + renderer_output_items: dict = {} + ctx = self._make_context() + + _emit_skill_overlay( + generation_rule_info=gr_info, + slx_directory_path="workspaces/demo-ws/slxs/cvb-ns-health", + slx_base_template_variables={"slx_name": "cvb-ns-health"}, + renderer_output_items=renderer_output_items, + context=ctx, + ) + + overlay_path = "workspaces/demo-ws/slxs/cvb-ns-health/Skill.md" + self.assertIn(overlay_path, renderer_output_items) + item = renderer_output_items[overlay_path] + self.assertEqual(skill_text, item.raw_content) + self.assertEqual("Skill.md", item.template_name) + self.assertIsNone(item.template_loader_func) + + def test_overlay_preserves_upstream_filename_casing(self): + from enrichers.generation_rules import _emit_skill_overlay + + # Upstream codebundle publishes the overlay as SKILL.md (uppercase). + # The overlay should write SKILL.md into the SLX directory verbatim, + # not normalize it to Skill.md. + repo_url = "https://example.com/foo.git" + ref = "feat/skill-overlay" + bundle = "k8s-namespace-healthcheck" + skill_text = "# SKILL: namespace healthcheck\n" + cc = _StubCodeCollection(repo_url, {(ref, bundle, "SKILL.md"): skill_text}) + gr_info = _StubGenerationRuleInfo(cc, ref, bundle) + + renderer_output_items: dict = {} + ctx = self._make_context() + + _emit_skill_overlay( + generation_rule_info=gr_info, + slx_directory_path="workspaces/demo-ws/slxs/foo", + slx_base_template_variables={}, + renderer_output_items=renderer_output_items, + context=ctx, + ) + + overlay_path = "workspaces/demo-ws/slxs/foo/SKILL.md" + self.assertIn(overlay_path, renderer_output_items) + # Lookup must have asked case-insensitively at least once. + self.assertTrue(any(call[3] is True for call in cc.calls)) + + def test_overlay_skipped_when_codebundle_has_no_skill_md(self): + from enrichers.generation_rules import _emit_skill_overlay + + cc = _StubCodeCollection("https://example.com/foo.git", file_lookup={}) + gr_info = _StubGenerationRuleInfo(cc, "main", "k8s-namespace-healthcheck") + + renderer_output_items: dict = {} + ctx = self._make_context() + + _emit_skill_overlay( + generation_rule_info=gr_info, + slx_directory_path="workspaces/demo-ws/slxs/foo", + slx_base_template_variables={}, + renderer_output_items=renderer_output_items, + context=ctx, + ) + self.assertEqual({}, renderer_output_items) + + def test_overlay_lookup_is_cached_across_slxs_for_same_bundle(self): + from enrichers.generation_rules import _emit_skill_overlay + + repo_url = "https://example.com/foo.git" + ref = "feat/skill-overlay" + bundle = "k8s-namespace-healthcheck" + skill_text = "# Skill\n" + cc = _StubCodeCollection(repo_url, {(ref, bundle, "Skill.md"): skill_text}) + gr_info = _StubGenerationRuleInfo(cc, ref, bundle) + + renderer_output_items: dict = {} + ctx = self._make_context() + + for slx_dir in ( + "workspaces/demo-ws/slxs/foo", + "workspaces/demo-ws/slxs/bar", + "workspaces/demo-ws/slxs/baz", + ): + _emit_skill_overlay( + generation_rule_info=gr_info, + slx_directory_path=slx_dir, + slx_base_template_variables={}, + renderer_output_items=renderer_output_items, + context=ctx, + ) + + self.assertEqual(3, len(renderer_output_items)) + # Cache hit means we hit the git tree exactly once per bundle/ref. + self.assertEqual(1, len(cc.calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/run.py b/src/run.py index 049e01443..9e829381c 100755 --- a/src/run.py +++ b/src/run.py @@ -22,15 +22,6 @@ from aws_utils import generate_kubeconfig_for_eks -debug_suppress_cheat_sheet = os.getenv("WB_DEBUG_SUPPRESS_CHEAT_SHEET", "true") -cheat_sheet_enabled = (debug_suppress_cheat_sheet.lower() == 'false' or - debug_suppress_cheat_sheet == '0') -if cheat_sheet_enabled: - import cheatsheet - -# FIXME: Since we currently also do the cheat sheet generation in this tool -# should the service name be something more generic, e.g. RunWhen Local or -# something like that? SERVICE_NAME = "Workspace Builder" REST_SERVICE_HOST_DEFAULT = "localhost" REST_SERVICE_PORT_DEFAULT = 8000 @@ -42,11 +33,6 @@ CUSTOMIZATION_RULES_DEFAULT = "map-customization-rules" -tmpdir_value = os.getenv("TMPDIR", "/tmp") # fallback to /tmp if TMPDIR not set -print("TMPDIR:", os.environ.get("TMPDIR", "not set")) -with tempfile.NamedTemporaryFile(delete=True) as f: - print("Actual temp file location:", f.name) - def read_file(file_path: bytes, mode="r") -> Union[str, bytes]: with open(file_path, mode) as f: @@ -63,12 +49,24 @@ def check_rest_service_error(response: requests.Response, command: str, verbose: # FIXME: Should probably also check for other 2xx status code, but currently # for the workspace builder service a successful execution always returns 200. if response.status_code != HTTPStatus.OK: - response_data = response.json() + # The service normally returns a JSON error body, but an unhandled + # exception (or a crashing/exiting server) can yield a non-JSON body or + # an empty response. Don't let that turn a clean server-side error into + # an opaque client-side JSONDecodeError -- surface the status + body. + try: + response_data = response.json() + except ValueError: + body = (response.text or "").strip() + fatal( + f'Error {response.status_code} from {SERVICE_NAME} service for ' + f'command "{command}": non-JSON response body:\n{body[:4000]}' + ) + return if verbose: print("Exception stack trace:") - print(response_data["stackTrace"]) + print(response_data.get("stackTrace")) print("Request data:") - print(response_data["originalRequestData"]) + print(response_data.get("originalRequestData")) fatal(f'Error {response.status_code} from {SERVICE_NAME} service for command "{command}": ' f'{response_data.get("message")}') @@ -433,6 +431,26 @@ def main(): code_collections = workspace_info.get("codeCollections") overrides = workspace_info.get("overrides", {}) task_tag_exclusions = workspace_info.get("taskTagExclusions") + resource_store_backend = coalesce( + workspace_info.get("resourceStoreBackend"), + os.getenv("WB_RESOURCE_STORE_BACKEND"), + ) + resource_store_path = coalesce( + workspace_info.get("resourceStorePath"), + os.getenv("WB_RESOURCE_STORE_PATH"), + ) + azure_indexer_backend = coalesce( + workspace_info.get("azureIndexerBackend"), + os.getenv("WB_AZURE_INDEXER_BACKEND"), + ) + gcp_indexer_backend = coalesce( + workspace_info.get("gcpIndexerBackend"), + os.getenv("WB_GCP_INDEXER_BACKEND"), + ) + aws_indexer_backend = coalesce( + workspace_info.get("awsIndexerBackend"), + os.getenv("WB_AWS_INDEXER_BACKEND"), + ) # ------------------------------------------------------------------ 4. validation guards missing = [] @@ -693,6 +711,16 @@ def main(): resource_load_data = read_file(resource_load_path, "rb") encoded_resource_load_data = base64.b64encode(resource_load_data).decode('utf-8') request_data['resourceLoadFile'] = encoded_resource_load_data + if resource_store_backend: + request_data['resourceStoreBackend'] = resource_store_backend + if resource_store_path: + request_data['resourceStorePath'] = resource_store_path + if azure_indexer_backend: + request_data['azureIndexerBackend'] = azure_indexer_backend + if gcp_indexer_backend: + request_data['gcpIndexerBackend'] = gcp_indexer_backend + if aws_indexer_backend: + request_data['awsIndexerBackend'] = aws_indexer_backend # Invoke the workspace builder /run REST endpoint run_url = f"http://{rest_service_host}:{rest_service_port}/run/" @@ -839,16 +867,6 @@ def main(): print("Workspace builder data uploaded successfully.") - # Add cheat-sheet integration, which points at the output items and - # generates the list of local commands that exist in the TaskSet. - # FIXME: I think it would probably be cleaner and more decoupled to move the - # command assist stuff to a separate command line tool and then have the - # run.sh script handle calling it after invoking this tool. - if cheat_sheet_enabled: - # Update cheat sheet status by copying index - mkdocs_dir=f"{tmpdir_value}/mkdocs-temp" - cheatsheet.cheat_sheet(output_path, mkdocs_dir) - if __name__ == "__main__": main() diff --git a/src/run.sh b/src/run.sh index 1571bab53..0a69111e6 100755 --- a/src/run.sh +++ b/src/run.sh @@ -54,9 +54,14 @@ fi touch "$LOCK_FILE" # Construct components string based on whether --disable-cloudquery is set -COMPONENTS="load_resources,kubeapi,azure_devops,generation_rules,render_output_items,dump_resources" +# `azureapi` / `gcpapi` / `awsapi` are the native Azure / GCP / AWS SDK +# indexers; each is a no-op unless its backend is selected in +# workspaceInfo.yaml (azureIndexerBackend=azureapi, gcpIndexerBackend=gcpapi, +# awsIndexerBackend=awsapi). Including them here unconditionally lets users opt +# in via config without changing the CLI. +COMPONENTS="load_resources,kubeapi,azureapi,gcpapi,awsapi,azure_devops,generation_rules,render_output_items,dump_resources" if [ $DISABLE_CLOUDQUERY -eq 0 ]; then - COMPONENTS="load_resources,kubeapi,cloudquery,azure_devops,generation_rules,render_output_items,dump_resources" + COMPONENTS="load_resources,kubeapi,azureapi,gcpapi,awsapi,cloudquery,azure_devops,generation_rules,render_output_items,dump_resources" fi # Run the Python script with your specified arguments diff --git a/src/workspace_builder/admin.py b/src/workspace_builder/admin.py deleted file mode 100644 index 8c38f3f3d..000000000 --- a/src/workspace_builder/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/src/workspace_builder/api.py b/src/workspace_builder/api.py new file mode 100644 index 000000000..819a4b071 --- /dev/null +++ b/src/workspace_builder/api.py @@ -0,0 +1,130 @@ +"""FastAPI application for the workspace-builder REST service.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from component import Stage, get_all_settings +from utils import get_version_info + +from . import startup +from .exceptions import build_error_response +from .explorer import router as explorer_router +from .mcp.server import build_mcp_lifespan, build_streamable_http_app, is_mcp_enabled +from .models import InfoResult +from .run_handler import execute_run +from .serialization import serialize_info, serialize_run_result + +startup.bootstrap() + +logger = logging.getLogger("workspace_builder") + +# The MCP server's Streamable HTTP transport runs a session manager whose +# lifecycle is exposed as an ASGI lifespan. FastAPI does not propagate +# nested-mount lifespans automatically, so we wire the MCP session +# manager into the parent app's lifespan explicitly. Without this, every +# JSON-RPC call into the mounted ``/mcp`` endpoint would fail with +# "Task group is not initialized". ``is_mcp_enabled()`` lets operators +# opt out via ``RW_MCP_DISABLED=true`` while keeping the rest of the +# server running. +_mcp_lifespan = build_mcp_lifespan() if is_mcp_enabled() else None + +app = FastAPI(title="RunWhen Local Workspace Builder", lifespan=_mcp_lifespan) +app.include_router(explorer_router) + +if _mcp_lifespan is not None: + # JSON-RPC endpoint reachable at ``http://:8000/mcp``. Inside + # the FastMCP server we configured ``streamable_http_path="/"``, so + # the mount path is the only thing clients need to point at. + app.mount("/mcp", build_streamable_http_app()) + logger.info("MCP server mounted at /mcp (set RW_MCP_DISABLED=true to disable)") +else: + logger.info("MCP server disabled via RW_MCP_DISABLED") + + +@app.get("/info/") +def info() -> dict[str, Any]: + settings = get_all_settings() + version_info = get_version_info() + result = InfoResult( + version_info["version"], + version_info["name"], + Stage.INDEXER.components, + Stage.ENRICHER.components, + Stage.RENDERER.components, + settings, + ) + return serialize_info(result) + + +@app.post("/run/") +async def run(request: Request) -> dict[str, Any]: + request_data = await request.json() + result = execute_run(request_data) + return serialize_run_result(result) + + +@app.get("/health/") +def health() -> dict[str, Any]: + try: + from .health import get_health_tracker + + health_tracker = get_health_tracker() + health_info = health_tracker.get_health_info() + is_healthy = health_tracker.is_healthy() + is_ready = health_tracker.is_ready() + + response_data: dict[str, Any] = { + "status": health_info.service_status, + "service_start_time": health_info.service_start_time, + "uptime_seconds": health_info.uptime_seconds, + "is_healthy": is_healthy, + "is_ready": is_ready, + } + + if health_info.last_run: + last_run = health_info.last_run + response_data["last_run"] = { + "start_time": last_run.start_time, + "end_time": last_run.end_time, + "status": last_run.status, + "error_message": last_run.error_message, + "stacktrace": last_run.stacktrace, + "warnings_count": last_run.warnings_count, + "parsing_errors_count": last_run.parsing_errors_count, + "components_run": last_run.components_run, + "current_stage": last_run.current_stage, + "current_component": last_run.current_component, + "slx_count": last_run.slx_count, + "duration_seconds": last_run.duration_seconds, + } + + return response_data + except Exception as exc: + print(f"Warning: Health tracker failed: {exc}") + import traceback + + traceback.print_exc() + return { + "status": "healthy", + "timestamp": datetime.now(timezone.utc).isoformat(), + "is_healthy": True, + "is_ready": True, + "error": str(exc), + } + + +@app.exception_handler(Exception) +async def workspace_builder_exception_handler(request: Request, exc: Exception) -> JSONResponse: + body: dict[str, Any] | None = None + try: + body = await request.json() + except Exception: + pass + payload, status_code = build_error_response(exc, request.url.path, body) + return JSONResponse(status_code=status_code, content=payload) diff --git a/src/workspace_builder/apps.py b/src/workspace_builder/apps.py deleted file mode 100644 index 0cd0e96e6..000000000 --- a/src/workspace_builder/apps.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.apps import AppConfig -from component import init_components -from enrichers.code_collection import init_code_collections - -class WorkspaceBuilderConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'workspace_builder' - - def ready(self): - init_components() - init_code_collections() diff --git a/src/workspace_builder/exceptions.py b/src/workspace_builder/exceptions.py index 6eae738e9..4b783967a 100644 --- a/src/workspace_builder/exceptions.py +++ b/src/workspace_builder/exceptions.py @@ -1,65 +1,53 @@ +"""Error response helpers for the workspace-builder REST service.""" + +from __future__ import annotations + import json import logging import traceback +from typing import Any -from rest_framework import views, status -from rest_framework.response import Response +from exceptions import WorkspaceBuilderUserException logger = logging.getLogger(__name__) -def handle(e, context): - """ - Return a response that has as much detail as practical to help debugging. - - Args: - e (BaseException): The exception that caused the issue - context (dict): A rich set of data (e.g. the view) given to us by the DRF framework for handling - """ - next_exc = e - stack_traces = [] +def _collect_stack_traces(exc: BaseException) -> str: + stack_traces: list[str] = [] + next_exc: BaseException | None = exc while next_exc: - next_stack_trace = "\n".join(traceback.format_tb(next_exc.__traceback__)) - stack_traces.append(next_stack_trace) + stack_traces.append("\n".join(traceback.format_tb(next_exc.__traceback__))) next_exc = next_exc.__cause__ - stack_trace = "\nCaused by:\n\n".join(stack_traces) + return "\nCaused by:\n\n".join(stack_traces) + + +def build_error_response( + exc: Exception, + path: str, + request_data: dict[str, Any] | None, +) -> tuple[dict[str, Any], int]: + """Return JSON payload and HTTP status compatible with run.py error handling.""" + stack_trace = _collect_stack_traces(exc) + status_code = 400 if isinstance(exc, WorkspaceBuilderUserException) else 500 + + redacted_request = dict(request_data) if request_data else {} + password = redacted_request.get("password") + if password: + redacted_request["password"] = "*******" + + payload: dict[str, Any] = { + "message": f"{exc}", + "exceptionType": str(type(exc)), + "stackTrace": stack_trace, + "originalRequestData": redacted_request, + "originalRequestPath": path, + } - try: - # Call the DRF default exception handler first, and decorate the results. - # The code below is DRF standard boilerplate - response = views.exception_handler(e, context) - if not response: - response = Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) - ed = dict() - ed["drf"] = str(response) - ed["message"] = f"{e}" - ed["exceptionType"] = str(type(e)) - ed["stackTrace"] = stack_trace - ed["context"] = str(context) - # ed["originalRequestHeaders"] = context.get("request").headers - ed["originalRequestData"] = context.get("request").data - pw = ed.get("originalRequestData", {}).get("password") - if pw: - ed["originalRequestData"]["password"] = '*******' - ed["originalRequestPath"] = context.get("request").path_info - # ed["originalRequestBody"] = context.get("request").text - # ed["originalRequestSession"] = context.get("request").session - if response.data: - if isinstance(response.data, dict): - ed.update(response.data) - else: - ed["pd"] = response.data - response.data = ed - logger.debug( - f"Error handling successful: type {type(e)}, " - f"returning response {response} with data {json.dumps(response.data, indent=2)}, " - f"stack trace {stack_trace}" - ) - return response - except Exception as exc: - logger.warning( - f"Error handling unsuccessful: type {type(e)}, " - f"e: {str(e)} and stacktrace {stack_trace} with context {str(context)}" - ) - logger.exception(exc) - raise exc + logger.debug( + "Error handling successful: type %s, returning status %s with data %s, stack trace %s", + type(exc), + status_code, + json.dumps(payload, indent=2), + stack_trace, + ) + return payload, status_code diff --git a/src/workspace_builder/explorer.html b/src/workspace_builder/explorer.html new file mode 100644 index 000000000..3372f1aa5 --- /dev/null +++ b/src/workspace_builder/explorer.html @@ -0,0 +1,787 @@ + + + + + + RunWhen Local Workspace Explorer + + + + + + +

+
+ RW + RunWhen Local · Workspace Explorer +
+
+
+ + +
+
+ +
+
+

Workspace Explorer

+

Browse the discovered cloud and Kubernetes resources, plus the SLXs, SLIs, runbooks and Skills that RunWhen Local rendered for them. Each SLX is an instance of a Skill defined by a CodeBundle, and every artifact below comes straight from the local SQLite resource store.

+ +
+
+
+ +
+
+
+ + + +
+ +
+ +
+
+
+ +
+
+
+
+ +
+ +
+ + + diff --git a/src/workspace_builder/explorer.py b/src/workspace_builder/explorer.py new file mode 100644 index 000000000..e93096f1b --- /dev/null +++ b/src/workspace_builder/explorer.py @@ -0,0 +1,203 @@ +"""Basic HTML/JSON explorer for the SQLite resource store.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Optional + +from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import HTMLResponse + +from .resource_store_reader import ( + count_resources, + get_store_summary, + json_safe, + list_slx_bundles, + resource_db_connection, + resolve_resource_db_path, + search_resources, +) +from indexers.sqlite_resource_writer import ( + count_workspace_artifacts, + get_resource, + get_workspace_artifact, + list_resource_types, + search_workspace_artifacts, +) + +router = APIRouter(prefix="/explorer", tags=["resource-explorer"]) + +_UI_PATH = Path(__file__).with_name("explorer.html") + + +def _build_tree(conn) -> list[dict[str, Any]]: + tree: list[dict[str, Any]] = [] + for platform in get_store_summary(conn)["platforms"]: + types = list_resource_types(conn, platform=platform) + platform_count = count_resources(conn, platform=platform) + type_nodes = [] + for resource_type in types: + type_count = count_resources( + conn, platform=platform, resource_type=resource_type["name"] + ) + type_nodes.append( + { + "name": resource_type["name"], + "count": type_count, + "custom_attributes": resource_type["custom_attributes"], + } + ) + tree.append( + { + "name": platform, + "count": platform_count, + "resource_types": type_nodes, + } + ) + return tree + + +@router.get("/", response_class=HTMLResponse) +def explorer_page() -> HTMLResponse: + return HTMLResponse(_UI_PATH.read_text(encoding="utf-8")) + + +@router.get("/api/summary") +def explorer_summary() -> dict[str, Any]: + try: + with resource_db_connection() as conn: + summary = get_store_summary(conn) + return { + **summary, + "db_path": str(resolve_resource_db_path()), + "tree": _build_tree(conn), + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/api/resources") +def explorer_resources( + platform: Optional[str] = None, + resource_type: Optional[str] = None, + q: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), +) -> dict[str, Any]: + try: + with resource_db_connection() as conn: + total = count_resources(conn, platform=platform, resource_type=resource_type, q=q) + items = search_resources( + conn, + platform=platform, + resource_type=resource_type, + q=q, + limit=limit, + offset=offset, + ) + return { + "total": total, + "limit": limit, + "offset": offset, + "items": json_safe(items), + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/api/resource") +def explorer_resource( + platform: str, + resource_type: str, + qualified_name: str, +) -> dict[str, Any]: + try: + with resource_db_connection() as conn: + resource = get_resource(conn, platform, resource_type, qualified_name) + if resource is None: + raise HTTPException(status_code=404, detail="Resource not found") + return json_safe(resource) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/api/artifacts") +def explorer_artifacts( + workspace_name: Optional[str] = None, + artifact_kind: Optional[str] = None, + q: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), +) -> dict[str, Any]: + try: + with resource_db_connection() as conn: + total = count_workspace_artifacts( + conn, + workspace_name=workspace_name, + artifact_kind=artifact_kind, + q=q, + ) + items = search_workspace_artifacts( + conn, + workspace_name=workspace_name, + artifact_kind=artifact_kind, + q=q, + limit=limit, + offset=offset, + ) + preview = [] + for item in items: + row = dict(item) + content = row.pop("content", "") + row["content_preview"] = content[:240] + ("..." if len(content) > 240 else "") + preview.append(row) + return { + "total": total, + "limit": limit, + "offset": offset, + "items": json_safe(preview), + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/api/artifact") +def explorer_artifact( + workspace_name: str, + relative_path: str, +) -> dict[str, Any]: + try: + with resource_db_connection() as conn: + artifact = get_workspace_artifact(conn, workspace_name, relative_path) + if artifact is None: + raise HTTPException(status_code=404, detail="Artifact not found") + return json_safe(artifact) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/api/slx-bundles") +def explorer_slx_bundles( + workspace_name: Optional[str] = None, + q: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), +) -> dict[str, Any]: + """Group rendered artifacts by SLX directory. + + Each bundle includes the SLX, SLI and runbook files that share an + ``slx_directory``. Useful for browsing the rendered workspace by SLX + rather than by individual file. + """ + try: + with resource_db_connection() as conn: + data = list_slx_bundles( + conn, + workspace_name=workspace_name, + q=q, + limit=limit, + offset=offset, + ) + return json_safe(data) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc diff --git a/src/workspace_builder/mcp/__init__.py b/src/workspace_builder/mcp/__init__.py new file mode 100644 index 000000000..c443f63fc --- /dev/null +++ b/src/workspace_builder/mcp/__init__.py @@ -0,0 +1,14 @@ +"""MCP (Model Context Protocol) server for RunWhen Local. + +This package exposes a tiny, read-only MCP server that lets agentic clients +(Claude Code, Cursor, Claude Desktop, custom agents...) discover and reason +about the Skills and resources that the workspace-builder has indexed. + +It is intentionally **read-only** in v1: search/list/get over the SQLite +resource store and rendered SLX bundles. A future iteration will layer +execution on top via a sandboxed micro-runtime. + +The server is mounted into the existing FastAPI app as a Streamable HTTP +ASGI sub-app at ``/mcp``. See ``server.py`` for wiring details and +``tools.py`` for the tool surface itself. +""" diff --git a/src/workspace_builder/mcp/search.py b/src/workspace_builder/mcp/search.py new file mode 100644 index 000000000..3d06ff0ea --- /dev/null +++ b/src/workspace_builder/mcp/search.py @@ -0,0 +1,128 @@ +"""Lightweight token-overlap ranking for SLX bundle search. + +We deliberately keep this dependency-free for v1: SQL ``LIKE`` filters in +:mod:`workspace_builder.resource_store_reader` get us to a candidate set, +and this module re-ranks the candidates by tokenised overlap with the +query so an agent's natural-language phrasing ("failing pods", "key vault +rotation") surfaces the most relevant Skill bundle first. + +A future iteration can swap this out for embeddings/semantic search +without changing the tool contract that clients see, because the only +thing this returns is a stable ``score`` + ``snippet`` pair attached to +each candidate dict. +""" + +from __future__ import annotations + +import re +from typing import Iterable + +# Field weights for token-overlap scoring. Heuristic, not learned: matches +# in the SLX name should dominate matches in the long-form runbook body. +_NAME_WEIGHT = 4.0 +_PATH_WEIGHT = 2.0 +_KIND_WEIGHT = 1.5 +_CONTENT_WEIGHT = 1.0 + +_TOKEN_RE = re.compile(r"[A-Za-z0-9]+") +_STOPWORDS = frozenset( + { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "but", + "by", + "for", + "from", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "this", + "to", + "was", + "with", + } +) + +# Chars to scan around the first match when building a snippet. Roughly two +# lines of context, enough for an agent to verify relevance before fetching +# the full bundle. +_SNIPPET_RADIUS = 120 + + +def tokenize(text: str) -> list[str]: + """Return a lowercased, stopword-filtered token list for ``text``.""" + if not text: + return [] + return [ + tok.lower() + for tok in _TOKEN_RE.findall(text) + if tok.lower() not in _STOPWORDS + ] + + +def score_candidate( + query_tokens: Iterable[str], + *, + name: str | None, + path: str | None, + kinds: Iterable[str] | None, + content: str | None, +) -> float: + """Score a single bundle against ``query_tokens``. + + Score is the weighted count of distinct query tokens that appear in + each field. We count distinct tokens (not occurrences) so a long + runbook can't drown out a tight name match. + """ + query_set = set(query_tokens) + if not query_set: + return 0.0 + + score = 0.0 + if name: + name_tokens = set(tokenize(name)) + score += _NAME_WEIGHT * len(query_set & name_tokens) + if path: + path_tokens = set(tokenize(path)) + score += _PATH_WEIGHT * len(query_set & path_tokens) + if kinds: + kind_tokens = set(tokenize(" ".join(kinds))) + score += _KIND_WEIGHT * len(query_set & kind_tokens) + if content: + content_tokens = set(tokenize(content)) + score += _CONTENT_WEIGHT * len(query_set & content_tokens) + return score + + +def make_snippet(content: str, query: str) -> str: + """Return a short snippet around the first query-token match in ``content``. + + Falls back to the head of ``content`` when no token matches (so callers + always have something to display in tool output). + """ + if not content: + return "" + haystack = content.lower() + for tok in tokenize(query): + idx = haystack.find(tok) + if idx != -1: + start = max(0, idx - _SNIPPET_RADIUS) + end = min(len(content), idx + len(tok) + _SNIPPET_RADIUS) + prefix = "..." if start > 0 else "" + suffix = "..." if end < len(content) else "" + return prefix + content[start:end].replace("\n", " ").strip() + suffix + head = content[: 2 * _SNIPPET_RADIUS].replace("\n", " ").strip() + return head + ("..." if len(content) > 2 * _SNIPPET_RADIUS else "") + + +__all__ = ["tokenize", "score_candidate", "make_snippet"] diff --git a/src/workspace_builder/mcp/server.py b/src/workspace_builder/mcp/server.py new file mode 100644 index 000000000..38f67389b --- /dev/null +++ b/src/workspace_builder/mcp/server.py @@ -0,0 +1,486 @@ +"""MCP server registration for RunWhen Local. + +Builds a :class:`mcp.server.fastmcp.FastMCP` instance with the read-only +tool surface defined in :mod:`workspace_builder.mcp.tools` and exposes a +Streamable HTTP ASGI app that the FastAPI parent mounts at ``/mcp``. + +The server is intentionally tiny: every tool delegates straight to the +same :mod:`workspace_builder.resource_store_reader` accessors that power +the explorer UI, so the MCP view is consistent with the human-facing +explorer at all times. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +from mcp.server.fastmcp import FastMCP + +from . import tools as _tools + +logger = logging.getLogger(__name__) + +# Singleton FastMCP instance. We instantiate at import time so the ASGI +# app's lifespan is available to FastAPI before any request is routed. +_MCP_SERVER_NAME = "runwhen-local" +_MCP_SERVER_INSTRUCTIONS = ( + "RunWhen Local exposes resources and Skills indexed from the user's " + "Kubernetes / Azure / AWS / GCP environment. Use these tools to " + "search the resource graph, browse generated agentic Skills (each " + "is a small SLX bundle of YAML + a SKILL.md), and pull the full " + "bundle when the agent decides to reason about or describe a " + "specific Skill. v1 is read-only; execution is a future iteration." +) + + +def _build_mcp_server() -> FastMCP: + """Construct the FastMCP server and register tools. + + The MCP HTTP path is configurable so the Streamable HTTP path under + the ``/mcp`` mount can be ``/mcp`` (the conventional default), giving + final URLs of ``/mcp/mcp`` for the JSON-RPC endpoint. Setting it to + ``/`` here means the JSON-RPC endpoint is served at exactly ``/mcp``, + which is what most clients expect. + """ + mcp = FastMCP( + name=_MCP_SERVER_NAME, + instructions=_MCP_SERVER_INSTRUCTIONS, + # Serve the JSON-RPC endpoint at the mount root. The parent + # FastAPI app mounts this under ``/mcp``, so clients reach it at + # ``http://:8000/mcp``. + streamable_http_path="/", + ) + + # ---- Workspace summary ------------------------------------------------- + + @mcp.tool( + name="get_workspace_summary", + description=( + "Return top-line stats about the indexed RunWhen Local " + "workspace: counts of resources by platform/type, number of " + "generated Skills, and whether discovery has been run. Call " + "this first to ground the agent in what the user actually " + "has indexed before searching for Skills or resources." + ), + ) + def get_workspace_summary() -> dict[str, Any]: + return _tools.get_workspace_summary() + + # ---- Skill (SLX bundle) tools ----------------------------------------- + + @mcp.tool( + name="search_skills", + description=( + "Search the generated agentic Skills (SLX bundles) by a " + "natural-language query. Each Skill bundles together an SLX " + "definition, an SLI, a runbook, and a SKILL.md describing " + "what it does. Returns ranked matches with a short snippet " + "around the first match - use get_skill to retrieve the " + "full bundle once the agent has chosen one." + ), + ) + def search_skills( + query: str, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, + ) -> dict[str, Any]: + return _tools.search_skills( + query=query, + platform=platform, + resource_type=resource_type, + limit=limit, + ) + + @mcp.tool( + name="list_skills", + description=( + "Browse all generated Skill bundles with pagination. Optional " + "platform / resource_type filters are applied as keyword " + "matches against the SLX path and contents. Use this when " + "the agent wants to enumerate what is available rather than " + "search for something specific." + ), + ) + def list_skills( + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, + offset: int = 0, + ) -> dict[str, Any]: + return _tools.list_skills( + platform=platform, + resource_type=resource_type, + limit=limit, + offset=offset, + ) + + @mcp.tool( + name="get_skill", + description=( + "Retrieve the full content of one Skill bundle by SLX name " + "(as returned by list_skills / search_skills). Includes the " + "SLX yaml, SLI yaml, runbook yaml, and SKILL.md markdown so " + "the agent can reason about what the Skill does and how to " + "describe or invoke it." + ), + ) + def get_skill(slx_name: str) -> dict[str, Any]: + return _tools.get_skill(slx_name) + + # ---- Resource tools --------------------------------------------------- + + @mcp.tool( + name="search_resources", + description=( + "Search the indexed resource graph (Kubernetes, Azure, AWS, " + "GCP) discovered by the workspace-builder. Filter by " + "platform, resource_type, or a free-text query against " + "resource name / qualified name. Use this to ground answers " + "about what infrastructure the user actually has." + ), + ) + def search_resources( + query: Optional[str] = None, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, + offset: int = 0, + ) -> dict[str, Any]: + return _tools.search_resources_tool( + query=query, + platform=platform, + resource_type=resource_type, + limit=limit, + offset=offset, + ) + + @mcp.tool( + name="get_resource", + description=( + "Retrieve one indexed resource by its (platform, " + "resource_type, qualified_name) triple, with the full set " + "of attributes the indexer captured. Use after " + "search_resources to drill into a specific item." + ), + ) + def get_resource( + platform: str, + resource_type: str, + qualified_name: str, + ) -> dict[str, Any]: + return _tools.get_resource_tool( + platform=platform, + resource_type=resource_type, + qualified_name=qualified_name, + ) + + # ---- Resource <-> Skill join ----------------------------------------- + + @mcp.tool( + name="get_skills_for_resource", + description=( + "Return Skill bundles bound to a specific resource - the " + "agent's natural \"I'm looking at this Pod / Key Vault / " + "Deployment, what runbooks apply?\" entry point. Matches " + "are best-effort against SLX directory names and rendered " + "content; future iterations will use an explicit binding " + "stored at render time." + ), + ) + def get_skills_for_resource( + platform: str, + resource_type: str, + qualified_name: str, + limit: Optional[int] = None, + ) -> dict[str, Any]: + return _tools.get_skills_for_resource( + platform=platform, + resource_type=resource_type, + qualified_name=qualified_name, + limit=limit, + ) + + # ---- Workspace introspection ----------------------------------------- + + @mcp.tool( + name="get_workspace_health", + description=( + "Return the discovery-run health state: whether the service " + "is healthy, when it last ran, how long the run took, how " + "many warnings or parsing errors it produced, and which " + "components ran. Use this when an agent needs to answer " + "\"did the last discovery succeed?\" or \"why is the " + "workspace empty?\"." + ), + ) + def get_workspace_health() -> dict[str, Any]: + return _tools.get_workspace_health() + + @mcp.tool( + name="list_codebundles", + description=( + "List the CodeCollections currently loaded by the workspace " + "builder, plus where each was cloned from. Lets the agent " + "explain *which repo* a Skill came from and helps the user " + "trust the provenance of the runbooks it's seeing." + ), + ) + def list_codebundles() -> dict[str, Any]: + return _tools.list_codebundles() + + # ---- Resource graph navigation --------------------------------------- + + @mcp.tool( + name="get_resource_neighbors", + description=( + "Walk one hop in the indexed resource graph. Returns " + "forward references (resources this one points at, e.g. a " + "Deployment -> its Service / Pods / ReplicaSet) and reverse " + "references (resources that point at it, e.g. a Namespace's " + "members). Useful for grounding a multi-resource " + "investigation." + ), + ) + def get_resource_neighbors( + platform: str, + resource_type: str, + qualified_name: str, + limit: Optional[int] = None, + ) -> dict[str, Any]: + return _tools.get_resource_neighbors( + platform=platform, + resource_type=resource_type, + qualified_name=qualified_name, + limit=limit, + ) + + # ---- Smarter Skill recommendation ------------------------------------ + + @mcp.tool( + name="recommend_skills", + description=( + "Recommend Skills given free-text context (a user message, " + "an error trace, a log line). Token-overlap scoring runs " + "against every Skill bundle, not just LIKE-prefiltered " + "matches, so longer/looser context still surfaces the " + "right runbook. Use when search_skills's curated query " + "form feels too narrow." + ), + ) + def recommend_skills( + context_text: str, + max_results: int = 5, + ) -> dict[str, Any]: + return _tools.recommend_skills( + context_text=context_text, + max_results=max_results, + ) + + # ---- Skill invocation preview (read-only) ---------------------------- + + @mcp.tool( + name="preview_skill_invocation", + description=( + "Return what *would* run for a given Skill - SLX directory, " + "runbook content, and an illustrative runwhen-cli command - " + "without executing anything. v1 of the MCP server is " + "read-only; the agent should describe or hand the command " + "to the user. The future micro-runtime tool will replace " + "this with sandboxed execution." + ), + ) + def preview_skill_invocation(slx_name: str) -> dict[str, Any]: + return _tools.preview_skill_invocation(slx_name) + + # ---- Prompts (canned investigation flows) ---------------------------- + # + # Prompts are pre-built starter templates that show up in slash-menus + # of MCP-aware clients (Cursor, Claude Code, ...). They let an OSS + # user pick "/triage-namespace payments-prod" and get a curated + # investigation flow without having to learn how to phrase the + # request to the agent. Each prompt orchestrates calls to the + # existing tools above. + + @mcp.prompt( + name="kickoff_investigation", + description=( + "Get oriented in a freshly-indexed RunWhen Local workspace. " + "Tells the agent to call get_workspace_summary, list a few " + "Skills and resources, and propose 2-3 next investigation " + "directions. Good first prompt after wiring runwhen-local " + "up to your client." + ), + ) + def kickoff_investigation_prompt() -> str: + return ( + "I've connected to a RunWhen Local instance via MCP. Please " + "help me get oriented:\n\n" + "1. Call `get_workspace_summary` to see what platforms and " + "resources are indexed.\n" + "2. Call `get_workspace_health` to confirm the last " + "discovery run succeeded and surface any warnings.\n" + "3. Call `list_codebundles` so I can see which Skill " + "libraries are loaded.\n" + "4. Call `list_skills` (limit ~10) to sample what Skills " + "are available.\n\n" + "Then summarise (a) what kind of environment is indexed, " + "(b) whether discovery looks healthy, and (c) 2-3 concrete " + "investigation directions I could pursue next, each as a " + "single-sentence pitch." + ) + + @mcp.prompt( + name="triage_kubernetes_namespace", + description=( + "Triage a Kubernetes namespace: enumerate its resources, " + "find Skills bound to those resources, and propose an " + "investigation order. Best when something feels off in a " + "namespace and you want a starter checklist." + ), + ) + def triage_kubernetes_namespace_prompt(namespace: str) -> str: + return ( + f"I want to triage the Kubernetes namespace `{namespace}`. " + "Please:\n\n" + f"1. Call `search_resources(platform='kubernetes', " + f"query='{namespace}')` to enumerate the resources in this " + "namespace.\n" + "2. For each Deployment / StatefulSet / DaemonSet you find, " + "call `get_skills_for_resource(platform='kubernetes', " + "resource_type=, qualified_name=)` to find " + "matching Skills.\n" + "3. If any of those Skills look directly relevant, call " + "`get_skill(slx_name=...)` to read the runbook.\n\n" + "Output: a concise triage plan with the top 3-5 things to " + "investigate, each linked to a specific Skill (by name) " + "where one exists. Flag any resources that have no matching " + "Skill - those are gaps worth raising." + ) + + @mcp.prompt( + name="diagnose_failing_deployment", + description=( + "Diagnose a specific failing Kubernetes Deployment. The " + "agent will pull the Deployment's resource attributes, " + "find Skills that match it, walk one hop of the resource " + "graph (Pods, Services, ReplicaSets), and propose the " + "single most-likely-relevant Skill to run." + ), + ) + def diagnose_failing_deployment_prompt( + namespace: str, + deployment: str, + ) -> str: + qn = f"{namespace}/{deployment}" + return ( + f"The Kubernetes Deployment `{deployment}` in namespace " + f"`{namespace}` (qualified_name `{qn}`) is misbehaving. " + "Please:\n\n" + f"1. Call `get_resource(platform='kubernetes', " + f"resource_type='Deployment', qualified_name='{qn}')` and " + "summarise its replicas, image, and key attributes.\n" + f"2. Call `get_resource_neighbors(platform='kubernetes', " + f"resource_type='Deployment', qualified_name='{qn}')` to " + "list its Pods / ReplicaSet / Services.\n" + f"3. Call `get_skills_for_resource(platform='kubernetes', " + f"resource_type='Deployment', qualified_name='{qn}')` to " + "find matching Skills.\n" + "4. Pick the single Skill you think is most likely to " + "diagnose the issue and call `preview_skill_invocation` on " + "it so I can see the exact command.\n\n" + "Be explicit about uncertainty: if the matching Skills " + "look generic, call `recommend_skills` with the " + "Deployment's image + a one-line description as context " + "to broaden the suggestion." + ) + + @mcp.prompt( + name="audit_azure_keyvaults", + description=( + "Audit indexed Azure Key Vaults for rotation, expiry, and " + "access-policy concerns. Returns a per-vault checklist " + "linked to the Skills that can investigate each concern." + ), + ) + def audit_azure_keyvaults_prompt() -> str: + return ( + "I want a security audit of the Azure Key Vaults RunWhen " + "Local has indexed. Please:\n\n" + "1. Call `search_resources(platform='azure', " + "resource_type='azure_keyvault_vaults')` to list every " + "indexed vault.\n" + "2. Call `search_skills(query='key vault rotation expiry " + "access policy')` to surface relevant Skills.\n" + "3. For each vault, call `get_skills_for_resource` so we " + "know which Skills are bound to which vault.\n" + "4. Produce a per-vault report: vault name, location, " + "the Skills that apply, and any obvious concerns from " + "the resource attributes (network ACLs, soft-delete, " + "purge protection).\n\n" + "Where Skills are missing for a vault, call this out as " + "a gap rather than fabricating an investigation." + ) + + return mcp + + +_mcp_server: Optional[FastMCP] = None + + +def get_mcp_server() -> FastMCP: + """Lazy singleton accessor. + + Used by tests so they can build a fresh server instance per case + without paying the registration cost at import time. + """ + global _mcp_server + if _mcp_server is None: + _mcp_server = _build_mcp_server() + return _mcp_server + + +def is_mcp_enabled() -> bool: + """Return whether the MCP HTTP route should be mounted. + + Defaults to enabled. Set ``RW_MCP_DISABLED=true`` to opt out (e.g. + in environments where the operator wants to keep the server + process tightly scoped to discovery only). + """ + flag = os.getenv("RW_MCP_DISABLED", "").strip().lower() + return flag not in ("1", "true", "yes", "on") + + +def build_streamable_http_app(): + """Return the Streamable HTTP ASGI app for FastAPI to mount. + + The FastMCP session manager has a lifespan that the parent ASGI app + *must* run, otherwise tool calls fail with "Task group is not + initialized". The caller is responsible for wiring this into the + parent's lifespan via :func:`build_mcp_lifespan`. + """ + server = get_mcp_server() + return server.streamable_http_app() + + +def build_mcp_lifespan(): + """Return an async context manager FastAPI can use as ``lifespan=``. + + Internally this drives the FastMCP server's session manager, which + is what initialises the streamable-HTTP task group. Without it, + every JSON-RPC call to the mounted ``/mcp`` endpoint fails with + "FastMCP's StreamableHTTPSessionManager task group was not + initialized." + """ + from contextlib import asynccontextmanager + + server = get_mcp_server() + + @asynccontextmanager + async def lifespan(app): # noqa: ANN001 - FastAPI passes the app instance + async with server.session_manager.run(): + yield + + return lifespan diff --git a/src/workspace_builder/mcp/tools.py b/src/workspace_builder/mcp/tools.py new file mode 100644 index 000000000..88b2c45b5 --- /dev/null +++ b/src/workspace_builder/mcp/tools.py @@ -0,0 +1,885 @@ +"""Tool implementations for the RunWhen Local MCP server. + +Each public function here corresponds 1:1 with a tool registered on the +MCP server in :mod:`workspace_builder.mcp.server`. The split lets us: + +* unit-test the behaviour without needing the MCP transport, and +* keep the tool registration file thin and focused on schemas/docstrings. + +All functions are read-only and side-effect-free; they query the SQLite +resource store via the same accessors as the ``/explorer/*`` REST API, +so anything an agent learns through MCP is consistent with what a human +sees in the explorer UI. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from indexers.sqlite_resource_writer import ( + get_resource as _get_resource, + get_workspace_artifact, + list_resource_types, + search_workspace_artifacts, +) +from utils import get_version_info + +from ..resource_store_reader import ( + count_resources, + get_store_summary, + json_safe, + list_slx_bundles, + resource_db_connection, + resolve_resource_db_path, + search_resources, +) +from .search import make_snippet, score_candidate, tokenize + +DEFAULT_LIMIT = 20 +MAX_LIMIT = 100 + +# Maximum content size we hand back per skill artifact (slx / sli / runbook / +# skill markdown). Agents have hard context limits; runbooks are rarely +# above this and capping prevents one giant SLX from blowing up a request. +MAX_CONTENT_CHARS = 32_000 + +# Pool of bundle candidates we re-rank inside ``search_skills``. Big enough +# that LIKE matches anywhere in a runbook can still surface a precise hit, +# small enough that ranking stays O(few-hundred) per query. +_SEARCH_CANDIDATE_POOL = 100 + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class StoreUnavailable(RuntimeError): + """Raised when the SQLite resource store has not been built yet. + + The MCP layer surfaces this as an explicit, actionable error so the + calling agent can tell the user "run discovery first" rather than + silently returning empty results. + """ + + +def _clip_limit(limit: Optional[int], default: int = DEFAULT_LIMIT) -> int: + if limit is None: + return default + if limit <= 0: + return default + return min(limit, MAX_LIMIT) + + +def _clip_content(content: str | None) -> str: + if not content: + return "" + if len(content) <= MAX_CONTENT_CHARS: + return content + return content[:MAX_CONTENT_CHARS] + "\n\n[... truncated; fetch via the explorer API for full content ...]" + + +# --------------------------------------------------------------------------- +# Workspace summary +# --------------------------------------------------------------------------- + + +def get_workspace_summary() -> dict[str, Any]: + """Top-line stats about the indexed workspace. + + Useful as a first call from an agent to ground itself: how many + resources are indexed, across which platforms, and how many Skills + have been generated. Returns an empty-but-valid summary when no + discovery run has happened yet (so agents can still call this before + the user has done anything). + """ + version = get_version_info().get("version") + db_path = str(resolve_resource_db_path()) + + if not os.path.isfile(db_path): + return { + "version": version, + "db_path": db_path, + "discovery_complete": False, + "message": ( + "No discovery has been run yet. Ask the user to run the " + "workspace-builder against their environment, then call this " + "tool again." + ), + } + + with resource_db_connection() as conn: + summary = get_store_summary(conn) + bundles = list_slx_bundles(conn, limit=1, offset=0) + + type_breakdown: list[dict[str, Any]] = [] + for platform in summary["platforms"]: + for resource_type in list_resource_types(conn, platform=platform): + type_breakdown.append( + { + "platform": platform, + "resource_type": resource_type["name"], + "count": count_resources( + conn, + platform=platform, + resource_type=resource_type["name"], + ), + } + ) + + return { + "version": version, + "db_path": db_path, + "discovery_complete": True, + "schema_version": summary["schema_version"], + "resource_count": summary["resource_count"], + "resource_type_count": summary["resource_type_count"], + "platform_count": summary["platform_count"], + "platforms": summary["platforms"], + "skill_bundle_count": bundles["total"], + "artifact_kinds": summary["artifact_kinds"], + "resources_by_type": sorted( + type_breakdown, + key=lambda r: r["count"], + reverse=True, + ), + } + + +# --------------------------------------------------------------------------- +# Skill (SLX bundle) tools +# --------------------------------------------------------------------------- + + +def list_skills( + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, + offset: int = 0, +) -> dict[str, Any]: + """List rendered SLX bundles, optionally narrowed by ``platform`` / + ``resource_type`` keyword. + + The resource_type filter is applied as a substring match against + the SLX directory + name + the contents of skill/runbook artifacts; + SLX bundles aren't directly tagged with a resource type, so this + keeps the tool useful without a schema migration. + """ + limit = _clip_limit(limit) + offset = max(0, int(offset)) + + query_terms = [t for t in (platform, resource_type) if t] + q = " ".join(query_terms) if query_terms else None + + with resource_db_connection() as conn: + data = list_slx_bundles(conn, q=q, limit=limit, offset=offset) + items = [ + _summarize_bundle(bundle) + for bundle in data["items"] + ] + return { + "total": data["total"], + "limit": limit, + "offset": offset, + "items": items, + } + + +def search_skills( + query: str, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, +) -> dict[str, Any]: + """Rank-search rendered Skill bundles by ``query``. + + Combines an SQL ``LIKE`` filter against the SLX directory / file path + / content (cheap, narrows to a candidate pool) with token-overlap + re-ranking against the SLX name + skill markdown body. Returns at + most ``limit`` bundles ordered by score, each with a short snippet + around the first query match. + """ + limit = _clip_limit(limit) + if not query or not query.strip(): + return {"total": 0, "limit": limit, "items": []} + + # Only the explicit platform / resource_type filters get used as a LIKE + # narrowing of the candidate pool. Using the free-text query as a LIKE + # forces all words to appear contiguously, which is wrong for natural- + # language queries ("rotate key vault secrets" rarely appears verbatim). + # Token-overlap ranking below does the real relevance work; the pool + # cap (_SEARCH_CANDIDATE_POOL) keeps this O(few-hundred) per query. + extra_terms = [t for t in (platform, resource_type) if t] + pool_query = " ".join(extra_terms) if extra_terms else None + + with resource_db_connection() as conn: + pool = list_slx_bundles( + conn, + q=pool_query, + limit=_SEARCH_CANDIDATE_POOL, + offset=0, + ) + query_tokens = tokenize(query) + ranked: list[tuple[float, dict[str, Any]]] = [] + for bundle in pool["items"]: + content = _bundle_content_for_ranking(conn, bundle) + score = score_candidate( + query_tokens, + name=bundle.get("slx_name"), + path=bundle.get("slx_directory"), + kinds=bundle.get("kinds"), + content=content, + ) + if score <= 0.0: + continue + summary = _summarize_bundle(bundle) + summary["score"] = score + summary["snippet"] = make_snippet(content, query) + ranked.append((score, summary)) + + ranked.sort(key=lambda pair: pair[0], reverse=True) + items = [item for _, item in ranked[:limit]] + + return { + "total": len(ranked), + "candidate_pool": pool["total"], + "limit": limit, + "items": items, + } + + +def get_skill(slx_name: str) -> dict[str, Any]: + """Return the full contents of a Skill bundle by SLX name. + + The name is matched against the basename of ``slx_directory`` (i.e. + what ``list_skills`` reports as ``slx_name``). Content is clipped + per artifact at :data:`MAX_CONTENT_CHARS` to stay within agent + context budgets; the explorer REST API is the escape hatch for full + content. + """ + if not slx_name: + raise ValueError("slx_name is required") + + with resource_db_connection() as conn: + candidates = list_slx_bundles(conn, q=slx_name, limit=50, offset=0) + match = None + for bundle in candidates["items"]: + if bundle.get("slx_name") == slx_name: + match = bundle + break + if match is None: + for bundle in candidates["items"]: + if (bundle.get("slx_directory") or "").endswith("/" + slx_name): + match = bundle + break + if match is None: + raise LookupError(f"No Skill bundle found with name: {slx_name}") + + files: list[dict[str, Any]] = [] + for file_meta in match["files"]: + full = get_workspace_artifact( + conn, + match["workspace_name"], + file_meta["relative_path"], + ) + if full is None: + continue + files.append( + { + "relative_path": file_meta["relative_path"], + "artifact_kind": full["artifact_kind"], + "media_type": full["media_type"], + "content": _clip_content(full.get("content", "")), + } + ) + + return { + "workspace_name": match["workspace_name"], + "slx_name": match["slx_name"], + "slx_directory": match["slx_directory"], + "kinds": match["kinds"], + "has_skill": match["has_skill"], + "has_sli": match["has_sli"], + "has_runbook": match["has_runbook"], + "files": files, + } + + +# --------------------------------------------------------------------------- +# Resource tools +# --------------------------------------------------------------------------- + + +def search_resources_tool( + query: Optional[str] = None, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + limit: Optional[int] = None, + offset: int = 0, +) -> dict[str, Any]: + """Search the indexed resource graph. + + Mirrors :func:`workspace_builder.resource_store_reader.search_resources` + but returns a JSON-safe payload suitable for direct return from an + MCP tool. + """ + limit = _clip_limit(limit) + offset = max(0, int(offset)) + + with resource_db_connection() as conn: + total = count_resources( + conn, + platform=platform, + resource_type=resource_type, + q=query, + ) + items = search_resources( + conn, + platform=platform, + resource_type=resource_type, + q=query, + limit=limit, + offset=offset, + ) + return { + "total": total, + "limit": limit, + "offset": offset, + "items": json_safe(items), + } + + +def get_resource_tool( + platform: str, + resource_type: str, + qualified_name: str, +) -> dict[str, Any]: + """Return one indexed resource with its full attribute payload.""" + if not platform or not resource_type or not qualified_name: + raise ValueError("platform, resource_type and qualified_name are required") + + with resource_db_connection() as conn: + resource = _get_resource(conn, platform, resource_type, qualified_name) + if resource is None: + raise LookupError( + f"Resource not found: platform={platform!r} " + f"resource_type={resource_type!r} qualified_name={qualified_name!r}" + ) + return json_safe(resource) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _summarize_bundle(bundle: dict[str, Any]) -> dict[str, Any]: + return { + "workspace_name": bundle.get("workspace_name"), + "slx_name": bundle.get("slx_name"), + "slx_directory": bundle.get("slx_directory"), + "file_count": bundle.get("file_count"), + "kinds": bundle.get("kinds", []), + "has_skill": bundle.get("has_skill", False), + "has_sli": bundle.get("has_sli", False), + "has_runbook": bundle.get("has_runbook", False), + } + + +def _bundle_content_for_ranking(conn, bundle: dict[str, Any]) -> str: + """Concatenate skill + runbook + sli content for a bundle. + + We score against the union so a query that matches a runbook + snippet still surfaces the bundle even when the SLX name itself is + generic. Skill markdown is weighted highest by ordering it first + (only matters if we ever truncate; harmless otherwise). + """ + workspace_name = bundle.get("workspace_name") + if not workspace_name: + return "" + slx_dir = bundle.get("slx_directory") or "" + parts: list[str] = [] + for kind in ("skill", "runbook", "sli", "slx"): + rows = search_workspace_artifacts( + conn, + workspace_name=workspace_name, + artifact_kind=kind, + limit=10, + offset=0, + ) + for row in rows: + if not row.get("relative_path", "").startswith(slx_dir): + continue + content = row.get("content") + if content: + parts.append(content) + return "\n\n".join(parts) + + +# Resource-availability helper for the server module so it can decide +# whether to register tools eagerly even when the DB doesn't exist yet +# (it does; tools surface a friendly StoreUnavailable / empty payload at +# call time). + + +def store_is_ready() -> bool: + """Return ``True`` when the SQLite resource store is on disk.""" + try: + return resolve_resource_db_path().is_file() + except Exception: + return False + + +# --------------------------------------------------------------------------- +# Join: Skills <-> Resources +# --------------------------------------------------------------------------- + + +def get_skills_for_resource( + platform: str, + resource_type: str, + qualified_name: str, + limit: Optional[int] = None, +) -> dict[str, Any]: + """Return Skill bundles that reference a specific resource. + + The workspace-builder doesn't store an explicit "this SLX is bound + to this resource" link in the SQLite store today; instead, the + binding is encoded into the rendered artifacts (``slx_directory`` + name, runbook ``configProvided`` values, SKILL.md prose). We do a + best-effort match by scanning artifact content + paths for the + resource's ``qualified_name`` and ``name``. + + This is the agent's natural "I'm looking at this Pod, what runbooks + apply?" entry point. Future iterations can swap the content scan + for an explicit binding once the renderer records one. + """ + if not platform or not resource_type or not qualified_name: + raise ValueError("platform, resource_type and qualified_name are required") + + limit = _clip_limit(limit) + + with resource_db_connection() as conn: + # Resolve the resource so we know its short name (used in slx_directory + # for many gen rule templates) and any reference attributes that might + # also identify it. + resource = _get_resource(conn, platform, resource_type, qualified_name) + if resource is None: + raise LookupError( + f"Resource not found: platform={platform!r} " + f"resource_type={resource_type!r} qualified_name={qualified_name!r}" + ) + + short_name = resource.get("name") or qualified_name.rsplit("/", 1)[-1] + match_terms = {qualified_name, short_name} + # SLX directories are usually slugified; basename of qualified_name + # often appears in directory names verbatim or with separators + # changed. Add the basename as an additional candidate. + for term in (qualified_name, short_name): + slug = term.replace("/", "-").replace(" ", "-").lower() + if slug: + match_terms.add(slug) + + seen_dirs: dict[str, dict[str, Any]] = {} + for term in match_terms: + if not term: + continue + data = list_slx_bundles( + conn, + q=term, + limit=_SEARCH_CANDIDATE_POOL, + offset=0, + ) + for bundle in data["items"]: + key = f"{bundle.get('workspace_name')}|{bundle.get('slx_directory')}" + if key in seen_dirs: + seen_dirs[key]["matched_terms"].add(term) + continue + summary = _summarize_bundle(bundle) + summary["matched_terms"] = {term} + seen_dirs[key] = summary + + items: list[dict[str, Any]] = [] + for entry in seen_dirs.values(): + entry["matched_terms"] = sorted(entry["matched_terms"]) + items.append(entry) + # Bundles that match more than one term are likely better-bound; sort + # most-matched first. + items.sort(key=lambda b: (-len(b["matched_terms"]), b.get("slx_name") or "")) + + return { + "resource": { + "platform": platform, + "resource_type": resource_type, + "qualified_name": qualified_name, + "name": short_name, + }, + "total": len(items), + "items": items[:limit], + } + + +# --------------------------------------------------------------------------- +# Workspace health +# --------------------------------------------------------------------------- + + +def get_workspace_health() -> dict[str, Any]: + """Surface the discovery-run health state so agents can answer + "did discovery succeed?" / "when did it last run?". + + Mirrors the shape of the existing ``GET /health/`` endpoint, with a + hard fallback so an agent always gets a usable response even if the + health tracker hasn't been initialised yet. + """ + try: + from ..health import get_health_tracker + + tracker = get_health_tracker() + info = tracker.get_health_info() + payload: dict[str, Any] = { + "status": info.service_status, + "service_start_time": info.service_start_time, + "uptime_seconds": info.uptime_seconds, + "is_healthy": tracker.is_healthy(), + "is_ready": tracker.is_ready(), + } + if info.last_run: + last = info.last_run + payload["last_run"] = { + "start_time": last.start_time, + "end_time": last.end_time, + "status": last.status, + "error_message": last.error_message, + "warnings_count": last.warnings_count, + "parsing_errors_count": last.parsing_errors_count, + "components_run": list(last.components_run or []), + "current_stage": last.current_stage, + "current_component": last.current_component, + "slx_count": last.slx_count, + "duration_seconds": last.duration_seconds, + } + else: + payload["last_run"] = None + return payload + except Exception as exc: + return { + "status": "unknown", + "is_healthy": False, + "is_ready": False, + "error": str(exc), + "last_run": None, + } + + +# --------------------------------------------------------------------------- +# CodeCollection introspection +# --------------------------------------------------------------------------- + + +def list_codebundles() -> dict[str, Any]: + """List the CodeCollections that have been loaded into the workspace + builder, plus a count of Skills each contributes. + + This lets an agent explain *where* a Skill came from when answering + a user ("this Skill ships in the rw-cli-codecollection repo, + branch main"). The Skill-count side is best-effort: we can't + definitively map an SLX to its source CodeCollection from the + SQLite store alone, so we surface the loaded collections and let + the explorer / authoring docs handle deeper provenance. + """ + try: + from enrichers import code_collection as _cc + + cache = getattr(_cc, "code_collection_cache", None) + if not cache: + return {"total": 0, "items": []} + + items: list[dict[str, Any]] = [] + for repo_url, collection in cache.items(): + items.append( + { + "repo_url": repo_url, + "repo_directory": getattr(collection, "repo_directory_path", None), + "loaded": getattr(collection, "repo", None) is not None, + } + ) + return {"total": len(items), "items": items} + except Exception as exc: + return {"total": 0, "items": [], "error": str(exc)} + + +# --------------------------------------------------------------------------- +# Resource neighbors (graph walk) +# --------------------------------------------------------------------------- + + +def get_resource_neighbors( + platform: str, + resource_type: str, + qualified_name: str, + limit: Optional[int] = None, +) -> dict[str, Any]: + """Return resources directly related to the given resource. + + Two passes: + + * **Forward refs**: walk the resource's attributes for ``$ref`` markers + (the encoder in :mod:`indexers.sqlite_resource_writer` records every + Resource cross-reference as a stable JSON marker) and resolve each + to a real row in the resource store. + * **Reverse refs**: SQL ``LIKE`` over ``attributes_json`` for the + resource's qualified name to find anything that points back at it. + + Both directions are bounded by ``limit`` so an agent can't run away + on a hub resource (e.g. a Resource Group with hundreds of + descendants). + """ + if not platform or not resource_type or not qualified_name: + raise ValueError("platform, resource_type and qualified_name are required") + + limit = _clip_limit(limit) + + forward: list[dict[str, Any]] = [] + reverse: list[dict[str, Any]] = [] + + with resource_db_connection() as conn: + resource = _get_resource(conn, platform, resource_type, qualified_name) + if resource is None: + raise LookupError( + f"Resource not found: platform={platform!r} " + f"resource_type={resource_type!r} qualified_name={qualified_name!r}" + ) + + # --- Forward refs: walk attributes for $ref markers -------------- + seen_forward: set[tuple[str, str, str]] = set() + for ref in _walk_refs(resource.get("attributes")): + key = ( + ref.get("platform") or "", + ref.get("resource_type") or "", + ref.get("qualified_name") or "", + ) + if not all(key) or key in seen_forward: + continue + seen_forward.add(key) + target = _get_resource(conn, key[0], key[1], key[2]) + if target is None: + # Reference to something not in the store (e.g. cross- + # subscription Azure RG or a referenced K8s resource we + # didn't index). Surface the marker so the agent can see + # the link even though we can't resolve it. + forward.append( + { + "platform": key[0], + "resource_type": key[1], + "qualified_name": key[2], + "name": ref.get("name"), + "resolved": False, + } + ) + else: + forward.append( + { + "platform": target["platform"], + "resource_type": target["resource_type"], + "qualified_name": target["qualified_name"], + "name": target.get("name"), + "resolved": True, + } + ) + if len(forward) >= limit: + break + + # --- Reverse refs: who references this resource? ----------------- + # We escape SQL LIKE wildcards so qualified_names containing + # underscores ("default/api_v2") don't match as wildcards. + like_pattern = "%" + qualified_name.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + rows = conn.execute( + "SELECT platform, resource_type, qualified_name, name " + "FROM resources " + "WHERE attributes_json LIKE ? ESCAPE '\\' " + " AND NOT (platform = ? AND resource_type = ? AND qualified_name = ?) " + "LIMIT ?", + (like_pattern, platform, resource_type, qualified_name, limit), + ).fetchall() + for row in rows: + reverse.append( + { + "platform": row[0], + "resource_type": row[1], + "qualified_name": row[2], + "name": row[3], + "resolved": True, + } + ) + + return { + "resource": { + "platform": platform, + "resource_type": resource_type, + "qualified_name": qualified_name, + "name": resource.get("name"), + }, + "forward_refs": forward, + "reverse_refs": reverse, + } + + +def _walk_refs(value: Any): + """Yield ``$ref`` marker dicts found anywhere inside a decoded + attribute tree. + + The encoder represents Resource cross-references as + ``{"$ref": {"platform": ..., "resource_type": ..., "qualified_name": ...}}``; + we just descend recursively and yield each one we find. + """ + if isinstance(value, dict): + ref = value.get("$ref") + if isinstance(ref, dict): + yield ref + for v in value.values(): + if isinstance(v, (dict, list)): + yield from _walk_refs(v) + elif isinstance(value, list): + for v in value: + if isinstance(v, (dict, list)): + yield from _walk_refs(v) + + +# --------------------------------------------------------------------------- +# Skill recommendation (smarter than search_skills) +# --------------------------------------------------------------------------- + + +def recommend_skills( + context_text: str, + max_results: int = 5, +) -> dict[str, Any]: + """Recommend Skills given a free-text context (an error trace, log + line, the user's last message, etc.). + + Differences from :func:`search_skills`: + + * Accepts longer free-text input, not a curated query string. + * Token-overlap scoring is run against every bundle (not gated by + the SQL ``LIKE`` pre-filter), so a long stack trace can still + surface a Skill whose relevance lives only in the runbook body. + * The default ``max_results`` is small (5) because this is meant + to feed an agent's "consider these next" list, not a browse. + + Implementation reuses the same ranking primitives so the contract + stays drop-in compatible if we later replace the keyword scorer + with embeddings. + """ + if not context_text or not context_text.strip(): + return {"total": 0, "items": []} + + max_results = max(1, min(int(max_results or 5), MAX_LIMIT)) + + with resource_db_connection() as conn: + pool = list_slx_bundles( + conn, + q=None, + limit=_SEARCH_CANDIDATE_POOL, + offset=0, + ) + query_tokens = tokenize(context_text) + ranked: list[tuple[float, dict[str, Any]]] = [] + for bundle in pool["items"]: + content = _bundle_content_for_ranking(conn, bundle) + score = score_candidate( + query_tokens, + name=bundle.get("slx_name"), + path=bundle.get("slx_directory"), + kinds=bundle.get("kinds"), + content=content, + ) + if score <= 0.0: + continue + summary = _summarize_bundle(bundle) + summary["score"] = score + summary["snippet"] = make_snippet(content, context_text) + ranked.append((score, summary)) + ranked.sort(key=lambda pair: pair[0], reverse=True) + items = [item for _, item in ranked[:max_results]] + + return { + "total": len(ranked), + "candidate_pool": pool["total"], + "items": items, + } + + +# --------------------------------------------------------------------------- +# Skill invocation preview (read-only "what would I run?") +# --------------------------------------------------------------------------- + + +def preview_skill_invocation(slx_name: str) -> dict[str, Any]: + """Return a human-readable preview of how the agent (or a user) would + invoke a Skill, without executing anything. + + v1 returns: + + * the SLX directory and workspace name (the path the + runwhen-cli expects), + * the runbook YAML inline (clipped), + * a templated invocation example that the user can copy. + + The actual execution surface is intentionally not part of v1; this + tool is the "agent describes what would happen" handoff while we + build the sandboxed micro-runtime. The exact CLI string is + illustrative - the runwhen-cli spelling may differ in the user's + install. The agent is expected to confirm with the user before + running anything. + """ + bundle = get_skill(slx_name) + + runbook_files = [f for f in bundle["files"] if f["artifact_kind"] == "runbook"] + runbook = runbook_files[0] if runbook_files else None + + invocation = ( + f"# Run this Skill via runwhen-cli (illustrative)\n" + f"runwhen-cli run \\\n" + f" --workspace {bundle['workspace_name']} \\\n" + f" --slx {bundle['slx_name']}" + ) + + return { + "slx_name": bundle["slx_name"], + "slx_directory": bundle["slx_directory"], + "workspace_name": bundle["workspace_name"], + "kinds": bundle["kinds"], + "runbook_path": runbook["relative_path"] if runbook else None, + "runbook_content": runbook["content"] if runbook else None, + "invocation_example": invocation, + "notes": ( + "v1 of the MCP server is read-only. This tool returns what " + "*would* run; the agent should confirm with the user and " + "either copy-paste the command or wait for the future " + "micro-runtime tool. The exact CLI invocation depends on " + "the user's runwhen-cli install." + ), + } + + +__all__ = [ + "DEFAULT_LIMIT", + "MAX_LIMIT", + "MAX_CONTENT_CHARS", + "StoreUnavailable", + "get_workspace_summary", + "list_skills", + "search_skills", + "get_skill", + "search_resources_tool", + "get_resource_tool", + "store_is_ready", + "get_skills_for_resource", + "get_workspace_health", + "list_codebundles", + "get_resource_neighbors", + "recommend_skills", + "preview_skill_invocation", +] diff --git a/src/workspace_builder/resource_store_reader.py b/src/workspace_builder/resource_store_reader.py new file mode 100644 index 000000000..f7e99ac7c --- /dev/null +++ b/src/workspace_builder/resource_store_reader.py @@ -0,0 +1,239 @@ +"""Read-only access to the persisted SQLite resource store.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Optional + +from indexers.sqlite_resource_writer import ( + count_workspace_artifacts, + get_resource, + get_schema_version, + list_platforms, + list_resource_types, + list_resources, + list_workspace_artifact_kinds, + open_database, +) + +DEFAULT_DB_FILENAME = "resources.sqlite" + + +def resolve_resource_db_path() -> Path: + """Return the filesystem path to the resource SQLite database.""" + explicit = os.getenv("RW_RESOURCE_STORE_PATH", "").strip() + if explicit: + path = Path(explicit) + if path.is_absolute(): + return path + output_dir = _output_directory() + return output_dir / explicit + + output_dir = _output_directory() + filename = os.getenv("RW_RESOURCE_STORE_FILENAME", DEFAULT_DB_FILENAME).strip() + return output_dir / filename + + +def _output_directory() -> Path: + output_dir = os.getenv("RW_OUTPUT_DIR", "").strip() + if output_dir: + return Path(output_dir) + shared = os.getenv("RUNWHEN_SHARED", "/shared").strip() or "/shared" + return Path(shared) / "output" + + +@contextmanager +def resource_db_connection() -> Iterator[sqlite3.Connection]: + db_path = resolve_resource_db_path() + if not db_path.is_file(): + raise FileNotFoundError( + f"Resource store database not found at {db_path}. " + "Run discovery with resourceStoreBackend: sqlite first." + ) + conn = open_database(str(db_path)) + try: + yield conn + finally: + conn.close() + + +def get_store_summary(conn: sqlite3.Connection) -> dict[str, Any]: + platforms = list_platforms(conn) + resource_types = list_resource_types(conn) + resource_count = conn.execute("SELECT COUNT(*) FROM resources").fetchone()[0] + artifact_count = count_workspace_artifacts(conn) + artifact_kinds = list_workspace_artifact_kinds(conn) + return { + "schema_version": get_schema_version(conn), + "platform_count": len(platforms), + "resource_type_count": len(resource_types), + "resource_count": resource_count, + "artifact_count": artifact_count, + "artifact_kinds": artifact_kinds, + "platforms": platforms, + } + + +def count_resources( + conn: sqlite3.Connection, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + q: Optional[str] = None, +) -> int: + where: list[str] = [] + params: list[Any] = [] + if platform: + where.append("platform = ?") + params.append(platform) + if resource_type: + where.append("resource_type = ?") + params.append(resource_type) + if q: + where.append("(name LIKE ? OR qualified_name LIKE ?)") + pattern = f"%{q}%" + params.extend([pattern, pattern]) + sql = "SELECT COUNT(*) FROM resources" + if where: + sql += " WHERE " + " AND ".join(where) + return int(conn.execute(sql, tuple(params)).fetchone()[0]) + + +def search_resources( + conn: sqlite3.Connection, + platform: Optional[str] = None, + resource_type: Optional[str] = None, + q: Optional[str] = None, + limit: int = 100, + offset: int = 0, +) -> list[dict[str, Any]]: + where: list[str] = [] + params: list[Any] = [] + if platform: + where.append("platform = ?") + params.append(platform) + if resource_type: + where.append("resource_type = ?") + params.append(resource_type) + if q: + where.append("(name LIKE ? OR qualified_name LIKE ?)") + pattern = f"%{q}%" + params.extend([pattern, pattern]) + + sql = ( + "SELECT platform, resource_type, qualified_name, name, " + "attributes_json, created_at, updated_at FROM resources" + ) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY platform, resource_type, qualified_name LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + from indexers.sqlite_resource_writer import decode_attributes + + return [ + { + "platform": row[0], + "resource_type": row[1], + "qualified_name": row[2], + "name": row[3], + "attributes": decode_attributes(row[4]), + "created_at": row[5], + "updated_at": row[6], + } + for row in conn.execute(sql, tuple(params)) + ] + + +def json_safe(value: Any) -> Any: + """Convert decoded attribute trees into JSON-serializable values.""" + if isinstance(value, dict): + return {k: json_safe(v) for k, v in value.items()} + if isinstance(value, list): + return [json_safe(v) for v in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def list_slx_bundles( + conn: sqlite3.Connection, + workspace_name: Optional[str] = None, + q: Optional[str] = None, + limit: int = 200, + offset: int = 0, +) -> dict[str, Any]: + """Group rendered artifacts by ``slx_directory`` into SLX bundles. + + Returns a dict with ``items`` (each bundle has ``slx_directory``, ``slx_name``, + ``workspace_name``, file paths grouped by kind) and a ``total`` count of distinct + bundles matching the filter. + """ + where: list[str] = ["slx_directory IS NOT NULL"] + params: list[Any] = [] + if workspace_name: + where.append("workspace_name = ?") + params.append(workspace_name) + if q: + where.append("(slx_directory LIKE ? OR relative_path LIKE ? OR content LIKE ?)") + pattern = f"%{q}%" + params.extend([pattern, pattern, pattern]) + where_sql = " WHERE " + " AND ".join(where) + + total = int( + conn.execute( + f"SELECT COUNT(DISTINCT workspace_name || '|' || slx_directory) " + f"FROM workspace_artifacts{where_sql}", + tuple(params), + ).fetchone()[0] + ) + + bundle_sql = ( + f"SELECT workspace_name, slx_directory, COUNT(*) AS file_count " + f"FROM workspace_artifacts{where_sql} " + f"GROUP BY workspace_name, slx_directory " + f"ORDER BY workspace_name, slx_directory " + f"LIMIT ? OFFSET ?" + ) + bundle_params = list(params) + [limit, offset] + bundle_rows = list(conn.execute(bundle_sql, tuple(bundle_params))) + + items: list[dict[str, Any]] = [] + for ws, slx_dir, file_count in bundle_rows: + files_sql = ( + "SELECT relative_path, artifact_kind, media_type, " + "length(content) AS size_bytes, updated_at " + "FROM workspace_artifacts " + "WHERE workspace_name = ? AND slx_directory = ? " + "ORDER BY artifact_kind, relative_path" + ) + files = [ + { + "relative_path": row[0], + "artifact_kind": row[1], + "media_type": row[2], + "size_bytes": row[3], + "updated_at": row[4], + } + for row in conn.execute(files_sql, (ws, slx_dir)) + ] + kinds = sorted({f["artifact_kind"] for f in files}) + items.append( + { + "workspace_name": ws, + "slx_directory": slx_dir, + "slx_name": os.path.basename(slx_dir) if slx_dir else None, + "file_count": file_count, + "kinds": kinds, + "has_slx": "slx" in kinds, + "has_sli": "sli" in kinds, + "has_runbook": "runbook" in kinds, + "has_skill": "skill" in kinds, + "files": files, + } + ) + + return {"total": total, "items": items, "limit": limit, "offset": offset} diff --git a/src/workspace_builder/run_handler.py b/src/workspace_builder/run_handler.py new file mode 100644 index 000000000..58c539598 --- /dev/null +++ b/src/workspace_builder/run_handler.py @@ -0,0 +1,145 @@ +"""Core /run endpoint logic shared by the REST server.""" + +from __future__ import annotations + +import io +import os +import tarfile +import tempfile +import traceback +from typing import Any + +from component import ( + Component, + Context, + Setting, + apply_component_dependencies, + get_active_settings, + get_component, + run_components, +) +from exceptions import WorkspaceBuilderUserException +from outputter import TarFileOutputter +from resources import REGISTRY_PROPERTY_NAME, Registry + +from .models import ArchiveRunResult + +_TMPDIR = os.getenv("TMPDIR", "/tmp") + + +def _parse_component_names(components_data: Any) -> list[str]: + if isinstance(components_data, list): + return [str(name).strip() for name in components_data if str(name).strip()] + if isinstance(components_data, str): + return [name.strip() for name in components_data.split(",") if name.strip()] + return [] + + +def execute_run(request_data: dict[str, Any]) -> ArchiveRunResult: + """Run the workspace-builder pipeline for a JSON request body.""" + try: + from .health import get_health_tracker + + health_tracker = get_health_tracker() + except Exception as exc: + print(f"Warning: Could not initialize health_tracker: {exc}") + health_tracker = None + + input_component_names = _parse_component_names(request_data.get("components", "")) + input_components = [get_component(name) for name in input_component_names] + components: list[Component] = apply_component_dependencies(input_components) + + if health_tracker: + try: + health_tracker.start_run(components) + except Exception as health_error: + print(f"Warning: Health tracker start_run failed: {health_error}") + + setting_temp_files: list[tempfile._TemporaryFileWrapper] = [] + setting_temp_dirs: list[tempfile.TemporaryDirectory] = [] + + try: + active_settings = get_active_settings(components) + setting_values: dict[str, Any] = {} + + for setting_dependency in active_settings.values(): + setting = setting_dependency.setting + value_string = request_data.get(setting.json_name) + using_default_value = False + + if value_string is not None: + value = setting.convert_value(value_string) + elif setting.default_value: + value = setting.default_value + using_default_value = True + elif setting_dependency.required: + raise WorkspaceBuilderUserException( + f"Required setting {setting.json_name} must be specified." + ) + else: + value = None + + if value is not None: + if setting.type == Setting.Type.FILE and not using_default_value: + try: + tar_stream = io.BytesIO(value) + archive = tarfile.open(fileobj=tar_stream, mode="r") + setting_temp_directory = tempfile.TemporaryDirectory(dir=_TMPDIR) + setting_temp_dirs.append(setting_temp_directory) + archive.extractall(setting_temp_directory.name) + value = setting_temp_directory.name + except Exception: + setting_temp_file = tempfile.NamedTemporaryFile(mode="wb+", delete=True) + setting_temp_files.append(setting_temp_file) + setting_temp_file.write(value) + setting_temp_file.flush() + value = setting_temp_file.name + setting_values[setting.name] = value + + outputter = TarFileOutputter() + context = Context(setting_values, outputter) + context.set_property(REGISTRY_PROPERTY_NAME, Registry()) + + overrides = request_data.get("overrides", {}) + if overrides: + context.set_property("overrides", overrides) + + run_components(context, components) + outputter.close() + archive_bytes = outputter.get_bytes() + + slx_count = None + try: + slxs = context.get_property("SLXS") + if slxs: + slx_count = len(slxs) + except Exception: + pass + + if health_tracker: + try: + health_tracker.complete_run(warnings=context.warnings, slx_count=slx_count) + except Exception as health_error: + print(f"Warning: Health tracker complete_run failed: {health_error}") + + return ArchiveRunResult( + "Workspace builder completed successfully.", + context.warnings, + archive_bytes, + ) + + except Exception as exc: + full_stacktrace = traceback.format_exc() + if health_tracker: + try: + health_tracker.fail_run(exc, full_stacktrace) + except Exception as health_error: + print(f"Warning: Health tracker fail_run failed: {health_error}") + print(full_stacktrace) + raise + + finally: + for setting_temp_file in setting_temp_files: + setting_temp_file.close() + for setting_temp_dir in setting_temp_dirs: + setting_temp_dir.cleanup() diff --git a/src/workspace_builder/serialization.py b/src/workspace_builder/serialization.py new file mode 100644 index 000000000..168c5ead7 --- /dev/null +++ b/src/workspace_builder/serialization.py @@ -0,0 +1,88 @@ +"""JSON helpers for workspace-builder REST responses (formerly DRF serializers).""" + +from __future__ import annotations + +from base64 import b64encode +from typing import Any + +from component import Component, Setting +from outputter import DirectoryItem, FileItem, OutputItem + +from .models import ArchiveRunResult, InfoResult + + +def _omit_empty(mapping: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in mapping.items() if v} + + +def _serialize_component(component: Component) -> dict[str, Any]: + return _omit_empty( + { + "name": component.name, + "documentation": component.documentation, + } + ) + + +def _serialize_setting(setting: Setting) -> dict[str, Any]: + default_value = setting.default_value + if default_value is not None and not isinstance(default_value, (str, int, float, bool)): + default_value = str(default_value) + return _omit_empty( + { + "name": setting.name, + "type": setting.type.value if hasattr(setting.type, "value") else str(setting.type), + "defaultValue": default_value, + "documentation": setting.documentation, + } + ) + + +def serialize_info(info: InfoResult) -> dict[str, Any]: + return _omit_empty( + { + "version": info.version, + "description": info.description, + "indexers": [_serialize_component(c) for c in info.indexers], + "enrichers": [_serialize_component(c) for c in info.enrichers], + "renderers": [_serialize_component(c) for c in info.renderers], + "settings": [_serialize_setting(s) for s in info.settings], + } + ) + + +def _serialize_file_item(item: FileItem) -> dict[str, Any]: + return _omit_empty( + { + "type": item.type.value, + "data": b64encode(item.data).decode("utf-8") if isinstance(item.data, bytes) else item.data, + } + ) + + +def _serialize_directory_item(item: DirectoryItem) -> dict[str, Any]: + children = { + name: ( + _serialize_file_item(child) + if child.type == OutputItem.Type.FILE + else _serialize_directory_item(child) + ) + for name, child in (item.children or {}).items() + } + return _omit_empty( + { + "type": item.type.value, + "children": children, + } + ) + + +def serialize_run_result(result: ArchiveRunResult) -> dict[str, Any]: + return _omit_empty( + { + "message": result.message, + "warnings": result.warnings, + "outputType": result.output_type, + "output": b64encode(result.output).decode("utf-8"), + } + ) diff --git a/src/workspace_builder/serializers.py b/src/workspace_builder/serializers.py deleted file mode 100644 index 3f5c8040f..000000000 --- a/src/workspace_builder/serializers.py +++ /dev/null @@ -1,111 +0,0 @@ -from base64 import b64encode - -from rest_framework import serializers -from outputter import OutputItem, FileItem, DirectoryItem - - -class OmitEmptyFieldsSerializer(serializers.Serializer): - """ - Serializer that omits empty fields (either None or empty lists or - dictionaries) from the serialization. - FIXME: I think this will currently prune all scalar values that - evaluate to False, e.g. also boolean and numerical values, which - may not always be desirable. It's not an issue in the current - context, because there are no values like that, but could be - an issue if this is copied/reused in other contexts. Could - potentially accept some arguments in __init__ that allow - more fine-grained control over which fields are pruned. - """ - def to_representation(self, instance): - result = super().to_representation(instance) - return {k: v for (k, v) in result.items() if v} - - -class ComponentSerializer(OmitEmptyFieldsSerializer): - name = serializers.CharField() - documentation = serializers.CharField() - - -class SettingSerializer(OmitEmptyFieldsSerializer): - name = serializers.CharField() - type = serializers.CharField() - defaultValue = serializers.CharField(source="default_value") - documentation = serializers.CharField() - - -class InfoResultSerializer(serializers.Serializer): - version = serializers.CharField() - description = serializers.CharField() - indexers = ComponentSerializer(many=True) - enrichers = ComponentSerializer(many=True) - renderers = ComponentSerializer(many=True) - settings = SettingSerializer(many=True) - - -class BytesField(serializers.Serializer): - """ - Serializer that converts binary data to its base64 representation, - so we can return it as a string value in the JSON response. - """ - def to_representation(self, value: bytes) -> str: - return b64encode(value).decode('utf-8') - - -class DirectoryChildrenField(serializers.Serializer): - @staticmethod - def serialize_child(value: [FileItem, DirectoryItem]): - if value.type == OutputItem.Type.FILE: - serializer = FileItemSerializer - elif value.type == OutputItem.Type.DIRECTORY: - serializer = DirectoryItemSerializer - else: - raise Exception(f"Invalid result type: {value.type}") - return serializer(value).data - - def to_representation(self, children: dict[str, [FileItem, DirectoryItem]]): - converted_children = {k: self.serialize_child(v) for (k, v) in children.items()} - return converted_children - - -class CommonItemSerializer(OmitEmptyFieldsSerializer): - type = serializers.CharField() - hexsha = serializers.CharField() - - -class FileItemSerializer(CommonItemSerializer): - data = BytesField() - - -class DirectoryItemSerializer(CommonItemSerializer): - children = DirectoryChildrenField() - - -class CommonRunResultSerializer(serializers.Serializer): - message = serializers.CharField() - warnings = serializers.ListField(child=serializers.CharField()) - outputType = serializers.CharField(source="output_type") - - -class ArchiveRunResultSerializer(CommonRunResultSerializer): - output = BytesField() - - -class FileItemsRunResultSerializer(CommonRunResultSerializer): - output = DirectoryItemSerializer() - - -class HealthResultSerializer(serializers.Serializer): - status = serializers.CharField() - service_start_time = serializers.CharField() - uptime_seconds = serializers.FloatField() - last_run_start = serializers.CharField(required=False, allow_null=True) - last_run_end = serializers.CharField(required=False, allow_null=True) - last_run_status = serializers.CharField(required=False, allow_null=True) - last_run_error = serializers.CharField(required=False, allow_null=True) - last_run_warnings_count = serializers.IntegerField(required=False, allow_null=True) - last_run_components = serializers.ListField(child=serializers.CharField(), required=False, allow_null=True) - last_run_parsing_errors_count = serializers.IntegerField(required=False, allow_null=True) - current_stage = serializers.CharField(required=False, allow_null=True) - current_component = serializers.CharField(required=False, allow_null=True) - is_healthy = serializers.BooleanField() - is_ready = serializers.BooleanField() diff --git a/src/workspace_builder/startup.py b/src/workspace_builder/startup.py new file mode 100644 index 000000000..723431ddf --- /dev/null +++ b/src/workspace_builder/startup.py @@ -0,0 +1,61 @@ +"""Application bootstrap (replaces Django AppConfig.ready).""" + +from __future__ import annotations + +import logging +import logging.config +import os +import sys + +from component import init_components +from enrichers.code_collection import init_code_collections + +_BOOTSTRAPPED = False + + +def configure_logging() -> None: + debug_logging = os.getenv("DEBUG_LOGGING", "false").lower() in ("true", "1") + root_log_level = "DEBUG" if debug_logging else "INFO" + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": { + "format": "[%(levelname)s] %(name)s: %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "simple", + }, + }, + "root": { + "handlers": ["console"], + "level": root_log_level, + }, + "loggers": { + "workspace_builder": { + "handlers": ["console"], + "level": root_log_level, + "propagate": False, + }, + }, + } + ) + + +def bootstrap() -> None: + global _BOOTSTRAPPED + if _BOOTSTRAPPED: + return + configure_logging() + init_components() + init_code_collections() + _BOOTSTRAPPED = True + + +# Import-time bootstrap so module workers and tests see initialized components. +bootstrap() diff --git a/src/workspace_builder/test_explorer.py b/src/workspace_builder/test_explorer.py new file mode 100644 index 000000000..2fa48fb3c --- /dev/null +++ b/src/workspace_builder/test_explorer.py @@ -0,0 +1,144 @@ +"""Tests for the resource explorer API.""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from indexers.sqlite_resource_writer import persist_sqlite_store +from outputter import FileSystemOutputter +from component import Context +from resources import REGISTRY_PROPERTY_NAME, Registry +from workspace_builder.api import app + + +def _seed_database(db_path: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + ctx = Context( + setting_values={"RESOURCE_STORE_BACKEND": "sqlite", "RESOURCE_STORE_PATH": "resources.sqlite"}, + outputter=FileSystemOutputter(tmpdir), + ) + ctx.set_property(REGISTRY_PROPERTY_NAME, Registry()) + ctx.get_property(REGISTRY_PROPERTY_NAME).add_resource( + "kubernetes", + "Namespace", + "default", + "default", + {"lod": "basic"}, + ) + ctx.get_property(REGISTRY_PROPERTY_NAME).add_resource( + "kubernetes", + "Deployment", + "api", + "default/api", + {"lod": "detailed"}, + ) + ctx = Context( + setting_values={ + "RESOURCE_STORE_BACKEND": "sqlite", + "RESOURCE_STORE_PATH": "resources.sqlite", + "WORKSPACE_NAME": "demo-ws", + }, + outputter=FileSystemOutputter(tmpdir), + ) + ctx.set_property(REGISTRY_PROPERTY_NAME, Registry()) + ctx.get_property(REGISTRY_PROPERTY_NAME).add_resource( + "kubernetes", + "Namespace", + "default", + "default", + {"lod": "basic"}, + ) + ctx.get_property(REGISTRY_PROPERTY_NAME).add_resource( + "kubernetes", + "Deployment", + "api", + "default/api", + {"lod": "detailed"}, + ) + from renderers.rendered_artifacts import record_rendered_artifact + + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/my-app/slx.yaml", + "kind: ServiceLevelX\n", + ) + persist_sqlite_store(ctx, db_path="resources.sqlite") + with open(os.path.join(tmpdir, "resources.sqlite"), "rb") as src, open(db_path, "wb") as dst: + dst.write(src.read()) + + +class ExplorerApiTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = TestClient(app) + + def test_explorer_summary_and_resources(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + summary = self.client.get("/explorer/api/summary") + self.assertEqual(200, summary.status_code) + data = summary.json() + self.assertEqual(2, data["resource_count"]) + self.assertGreaterEqual(data.get("artifact_count", 0), 1) + self.assertIn("kubernetes", data["platforms"]) + + resources = self.client.get("/explorer/api/resources?platform=kubernetes") + self.assertEqual(200, resources.status_code) + payload = resources.json() + self.assertEqual(2, payload["total"]) + self.assertEqual(2, len(payload["items"])) + + detail = self.client.get( + "/explorer/api/resource", + params={ + "platform": "kubernetes", + "resource_type": "Namespace", + "qualified_name": "default", + }, + ) + self.assertEqual(200, detail.status_code) + self.assertEqual("default", detail.json()["name"]) + + def test_explorer_page_loads(self): + response = self.client.get("/explorer/") + self.assertEqual(200, response.status_code) + self.assertIn("Workspace Explorer", response.text) + + def test_explorer_slx_bundles(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + response = self.client.get("/explorer/api/slx-bundles") + self.assertEqual(200, response.status_code) + payload = response.json() + self.assertGreaterEqual(payload["total"], 1) + bundle = payload["items"][0] + self.assertEqual("demo-ws", bundle["workspace_name"]) + self.assertIn("slx", bundle["kinds"]) + self.assertTrue(bundle["has_slx"]) + self.assertEqual(1, bundle["file_count"]) + + def test_missing_database_returns_404(self): + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": "/tmp/does-not-exist/resources.sqlite"}, + clear=False, + ): + response = self.client.get("/explorer/api/summary") + self.assertEqual(404, response.status_code) diff --git a/src/workspace_builder/test_mcp.py b/src/workspace_builder/test_mcp.py new file mode 100644 index 000000000..651f423c7 --- /dev/null +++ b/src/workspace_builder/test_mcp.py @@ -0,0 +1,665 @@ +"""Tests for the MCP server tool surface and ranking.""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from unittest.mock import patch + +from indexers.sqlite_resource_writer import persist_sqlite_store +from outputter import FileSystemOutputter +from component import Context +from resources import REGISTRY_PROPERTY_NAME, Registry + +from workspace_builder.mcp import search as mcp_search +from workspace_builder.mcp import tools as mcp_tools +from workspace_builder.mcp.server import ( + build_streamable_http_app, + get_mcp_server, + is_mcp_enabled, +) + + +# --------------------------------------------------------------------------- +# Database seeding helper +# --------------------------------------------------------------------------- + + +def _seed_database(db_path: str, with_skills: bool = True) -> None: + """Seed a SQLite resource store with two resources and (optionally) two + full SLX bundles for the MCP tools to exercise.""" + with tempfile.TemporaryDirectory() as tmpdir: + ctx = Context( + setting_values={ + "RESOURCE_STORE_BACKEND": "sqlite", + "RESOURCE_STORE_PATH": "resources.sqlite", + "WORKSPACE_NAME": "demo-ws", + }, + outputter=FileSystemOutputter(tmpdir), + ) + registry = Registry() + ctx.set_property(REGISTRY_PROPERTY_NAME, registry) + ns = registry.add_resource( + "kubernetes", + "Namespace", + "default", + "default", + {"lod": "basic"}, + ) + # Deployment carries a $ref-encodable attribute pointing at its + # Namespace so get_resource_neighbors has a forward edge to walk. + registry.add_resource( + "kubernetes", + "Deployment", + "api", + "default/api", + {"lod": "detailed", "replicas": 3, "namespace": ns}, + ) + registry.add_resource( + "azure", + "azure_keyvault_vaults", + "rwl-kv-demo", + "/subscriptions/sub-1/resourceGroups/keep/providers/Microsoft.KeyVault/vaults/rwl-kv-demo", + {"location": "eastus"}, + ) + + if with_skills: + from renderers.rendered_artifacts import record_rendered_artifact + + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-health/slx.yaml", + "kind: ServiceLevelX\nname: k8s-deployment-health\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-health/runbook.yaml", + "commands:\n - description: Check failing pods in the deployment\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-health/sli.yaml", + "kind: ServiceLevelIndicator\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-health/SKILL.md", + "# Deployment health\n\nDiagnoses failing pods, " + "crashloops, and image pull errors in a Kubernetes " + "Deployment.\n", + ) + + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/azure-keyvault-rotation/slx.yaml", + "kind: ServiceLevelX\nname: azure-keyvault-rotation\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/azure-keyvault-rotation/runbook.yaml", + "commands:\n - description: Rotate Azure Key Vault secrets and report stale entries\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/azure-keyvault-rotation/SKILL.md", + "# Azure Key Vault rotation\n\nFinds Key Vault secrets approaching " + "expiry and produces a rotation plan.\n", + ) + + # A *resource-bound* Skill: in real workspaces, generation rules + # render an SLX per matched resource and embed the resource's + # name in both the slx_directory and runbook configProvided. + # This is what get_skills_for_resource keys on, so we mirror it + # in the fixture so the join tool has something credible to find. + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-default-api-health/slx.yaml", + "kind: ServiceLevelX\nname: k8s-deployment-default-api-health\n" + "spec:\n configProvided:\n - name: NAMESPACE\n value: default\n" + " - name: DEPLOYMENT\n value: api\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-default-api-health/runbook.yaml", + "metadata:\n bound_to: default/api\n" + "commands:\n - description: Diagnose default/api Deployment\n", + ) + record_rendered_artifact( + ctx, + "workspaces/demo-ws/slxs/k8s-deployment-default-api-health/SKILL.md", + "# Health check for default/api\n\nPair-bound Skill targeting " + "the Deployment named `api` in the `default` namespace.\n", + ) + + persist_sqlite_store(ctx, db_path="resources.sqlite") + with open(os.path.join(tmpdir, "resources.sqlite"), "rb") as src, open(db_path, "wb") as dst: + dst.write(src.read()) + + +# --------------------------------------------------------------------------- +# Search ranking unit tests (no DB required) +# --------------------------------------------------------------------------- + + +class SearchRankingTests(unittest.TestCase): + def test_tokenize_drops_stopwords_and_lowercases(self): + self.assertEqual( + mcp_search.tokenize("The Failing Pod and the API"), + ["failing", "pod", "api"], + ) + + def test_tokenize_handles_empty_input(self): + self.assertEqual(mcp_search.tokenize(""), []) + self.assertEqual(mcp_search.tokenize(None), []) + + def test_score_weights_name_above_content(self): + """A name match should score higher than a content match for the same token.""" + query_tokens = ["foo"] + name_score = mcp_search.score_candidate( + query_tokens, + name="foo-bar", + path=None, + kinds=None, + content=None, + ) + content_score = mcp_search.score_candidate( + query_tokens, + name=None, + path=None, + kinds=None, + content="foo lives here", + ) + self.assertGreater(name_score, content_score) + + def test_score_zero_when_no_overlap(self): + score = mcp_search.score_candidate( + ["zzz"], + name="alpha", + path="alpha/beta", + kinds=["slx"], + content="this body has no overlap", + ) + self.assertEqual(score, 0.0) + + def test_make_snippet_centers_on_first_match(self): + body = "lorem ipsum dolor sit amet, consectetur adipiscing elit. The pod is failing." + snippet = mcp_search.make_snippet(body, "failing pod") + self.assertIn("failing", snippet) + + def test_make_snippet_falls_back_to_head(self): + body = "completely unrelated content body about other things" + snippet = mcp_search.make_snippet(body, "asdfqwerty") + self.assertTrue(snippet.startswith("completely unrelated")) + + +# --------------------------------------------------------------------------- +# Tool implementation tests (require DB) +# --------------------------------------------------------------------------- + + +class WorkspaceSummaryTests(unittest.TestCase): + def test_summary_when_db_missing(self): + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": "/tmp/__rwl-mcp-missing/db.sqlite"}, + clear=False, + ): + summary = mcp_tools.get_workspace_summary() + self.assertFalse(summary["discovery_complete"]) + self.assertIn("No discovery", summary["message"]) + + def test_summary_with_seeded_db(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + summary = mcp_tools.get_workspace_summary() + self.assertTrue(summary["discovery_complete"]) + self.assertEqual(3, summary["resource_count"]) + self.assertIn("kubernetes", summary["platforms"]) + self.assertIn("azure", summary["platforms"]) + self.assertGreaterEqual(summary["skill_bundle_count"], 2) + # Type breakdown should be sorted by count desc and include all three types. + names = {entry["resource_type"] for entry in summary["resources_by_type"]} + self.assertEqual( + names, + {"Namespace", "Deployment", "azure_keyvault_vaults"}, + ) + + +class SkillToolTests(unittest.TestCase): + def test_list_skills_returns_bundles(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + listing = mcp_tools.list_skills() + self.assertGreaterEqual(listing["total"], 2) + names = {bundle["slx_name"] for bundle in listing["items"]} + self.assertIn("k8s-deployment-health", names) + self.assertIn("azure-keyvault-rotation", names) + + def test_search_skills_ranks_keyvault_above_deployment(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + results = mcp_tools.search_skills(query="rotate key vault secrets") + self.assertGreaterEqual(results["total"], 1) + top = results["items"][0] + self.assertEqual("azure-keyvault-rotation", top["slx_name"]) + self.assertIn("snippet", top) + self.assertGreater(top["score"], 0) + + def test_search_skills_ranks_deployment_for_pod_query(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + results = mcp_tools.search_skills(query="failing pods") + self.assertGreaterEqual(results["total"], 1) + self.assertEqual("k8s-deployment-health", results["items"][0]["slx_name"]) + + def test_search_skills_returns_empty_for_blank_query(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + results = mcp_tools.search_skills(query=" ") + self.assertEqual(results["total"], 0) + + def test_get_skill_returns_full_bundle(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + bundle = mcp_tools.get_skill("k8s-deployment-health") + self.assertEqual("k8s-deployment-health", bundle["slx_name"]) + kinds = {f["artifact_kind"] for f in bundle["files"]} + self.assertIn("slx", kinds) + self.assertIn("runbook", kinds) + self.assertIn("sli", kinds) + self.assertIn("skill", kinds) + skill_md = next( + f for f in bundle["files"] if f["artifact_kind"] == "skill" + ) + self.assertIn("Deployment health", skill_md["content"]) + + def test_get_skill_raises_for_unknown_name(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + with self.assertRaises(LookupError): + mcp_tools.get_skill("nonexistent-skill") + + +class ResourceToolTests(unittest.TestCase): + def test_search_resources_filters_by_platform(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + k8s = mcp_tools.search_resources_tool(platform="kubernetes") + az = mcp_tools.search_resources_tool(platform="azure") + self.assertEqual(2, k8s["total"]) + self.assertEqual(1, az["total"]) + self.assertTrue( + all(item["platform"] == "kubernetes" for item in k8s["items"]) + ) + + def test_get_resource_returns_attributes(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + r = mcp_tools.get_resource_tool( + platform="kubernetes", + resource_type="Deployment", + qualified_name="default/api", + ) + self.assertEqual("api", r["name"]) + self.assertEqual(3, r["attributes"]["replicas"]) + + def test_get_resource_raises_for_unknown(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + with self.assertRaises(LookupError): + mcp_tools.get_resource_tool( + platform="kubernetes", + resource_type="Deployment", + qualified_name="missing", + ) + + +# --------------------------------------------------------------------------- +# Server registration smoke test +# --------------------------------------------------------------------------- + + +class GetSkillsForResourceTests(unittest.TestCase): + def test_finds_resource_bound_skill_for_deployment(self): + """The fixture seeds a Skill whose slx_directory and content + embed the Deployment's name (mirroring how a real generation + rule renders one SLX per matched resource). The join tool + should surface that Skill when asked for the Deployment.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + result = mcp_tools.get_skills_for_resource( + platform="kubernetes", + resource_type="Deployment", + qualified_name="default/api", + ) + self.assertEqual("api", result["resource"]["name"]) + names = {b["slx_name"] for b in result["items"]} + self.assertIn("k8s-deployment-default-api-health", names) + # That bundle should have hit on multiple match terms (qn + short + # name + slugified qn), proving the fan-out works. + bound = next( + b for b in result["items"] + if b["slx_name"] == "k8s-deployment-default-api-health" + ) + self.assertGreaterEqual(len(bound["matched_terms"]), 1) + + def test_raises_for_unknown_resource(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + with self.assertRaises(LookupError): + mcp_tools.get_skills_for_resource( + platform="kubernetes", + resource_type="Deployment", + qualified_name="missing/ns/dep", + ) + + def test_raises_for_unknown_resource(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + with self.assertRaises(LookupError): + mcp_tools.get_skills_for_resource( + platform="kubernetes", + resource_type="Deployment", + qualified_name="missing/ns/dep", + ) + + +class GetWorkspaceHealthTests(unittest.TestCase): + def test_returns_health_payload(self): + # The health tracker reads/writes a file at /tmp/health_status.json; + # a clean test environment will produce a "healthy" payload with no + # last_run, which is exactly what we want to assert. + result = mcp_tools.get_workspace_health() + self.assertIn("status", result) + self.assertIn("is_healthy", result) + self.assertIn("is_ready", result) + # last_run is allowed to be None on a fresh service; just make sure + # the key is there for stable agent consumption. + self.assertIn("last_run", result) + + +class ListCodebundlesTests(unittest.TestCase): + def test_returns_loaded_collections(self): + # init_code_collections runs at startup; in the test env at least + # the default-code-collections.yaml ones are loaded. The shape + # contract is what matters here. + result = mcp_tools.list_codebundles() + self.assertIn("total", result) + self.assertIn("items", result) + self.assertIsInstance(result["items"], list) + for item in result["items"]: + self.assertIn("repo_url", item) + self.assertIn("loaded", item) + + +class GetResourceNeighborsTests(unittest.TestCase): + def test_forward_ref_resolves_namespace(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + # The Deployment was seeded with namespace=ns, so the encoder + # records a $ref pointing at the Namespace. + result = mcp_tools.get_resource_neighbors( + platform="kubernetes", + resource_type="Deployment", + qualified_name="default/api", + ) + types = {r["resource_type"] for r in result["forward_refs"]} + self.assertIn("Namespace", types) + ns_refs = [ + r + for r in result["forward_refs"] + if r["resource_type"] == "Namespace" + ] + self.assertTrue(ns_refs[0]["resolved"]) + self.assertEqual("default", ns_refs[0]["qualified_name"]) + + def test_reverse_ref_finds_dependent(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + # The Deployment references the Namespace -> reverse lookup + # from the Namespace should surface the Deployment. + result = mcp_tools.get_resource_neighbors( + platform="kubernetes", + resource_type="Namespace", + qualified_name="default", + ) + rev = {r["resource_type"] for r in result["reverse_refs"]} + self.assertIn("Deployment", rev) + + def test_raises_for_unknown_resource(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + with self.assertRaises(LookupError): + mcp_tools.get_resource_neighbors( + platform="kubernetes", + resource_type="Deployment", + qualified_name="missing", + ) + + +class RecommendSkillsTests(unittest.TestCase): + def test_long_context_finds_relevant_skill(self): + """A long, naturalistic context should surface a Deployment- + related Skill at the top - either the generic + ``k8s-deployment-health`` or the resource-bound + ``k8s-deployment-default-api-health``. Both are correct + answers; what's important is that the unrelated key-vault + Skill is *not* the top recommendation.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + ctx = ( + "User reports: 'My deployment in default has 3 replicas " + "but pods keep restarting with CrashLoopBackOff and " + "ImagePullBackOff errors'. What runbooks should we run?" + ) + result = mcp_tools.recommend_skills(ctx, max_results=3) + self.assertGreaterEqual(result["total"], 1) + top_name = result["items"][0]["slx_name"] + self.assertIn( + top_name, + {"k8s-deployment-health", "k8s-deployment-default-api-health"}, + f"Expected a deployment-related Skill at the top; got {top_name!r}", + ) + + def test_blank_context_returns_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + result = mcp_tools.recommend_skills("", max_results=5) + self.assertEqual(0, result["total"]) + + +class PreviewSkillInvocationTests(unittest.TestCase): + def test_preview_includes_runbook_and_invocation(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "resources.sqlite") + _seed_database(db_path) + with patch.dict( + os.environ, + {"RW_RESOURCE_STORE_PATH": db_path}, + clear=False, + ): + preview = mcp_tools.preview_skill_invocation( + "k8s-deployment-health" + ) + self.assertEqual("k8s-deployment-health", preview["slx_name"]) + self.assertIn("runwhen-cli", preview["invocation_example"]) + self.assertIn("runbook.yaml", preview["runbook_path"] or "") + self.assertIn( + "Check failing pods", preview["runbook_content"] or "" + ) + + +class McpServerSmokeTests(unittest.TestCase): + def test_server_registers_all_tools(self): + server = get_mcp_server() + # FastMCP exposes registered tools via ``list_tools()``; v1.1 adds + # six new tools on top of v1's six. + import asyncio + + tools = asyncio.run(server.list_tools()) + names = {t.name for t in tools} + self.assertEqual( + names, + { + "get_workspace_summary", + "search_skills", + "list_skills", + "get_skill", + "search_resources", + "get_resource", + "get_skills_for_resource", + "get_workspace_health", + "list_codebundles", + "get_resource_neighbors", + "recommend_skills", + "preview_skill_invocation", + }, + ) + + def test_server_registers_prompts(self): + server = get_mcp_server() + import asyncio + + prompts = asyncio.run(server.list_prompts()) + names = {p.name for p in prompts} + self.assertEqual( + names, + { + "kickoff_investigation", + "triage_kubernetes_namespace", + "diagnose_failing_deployment", + "audit_azure_keyvaults", + }, + ) + + def test_streamable_http_app_is_asgi_callable(self): + app = build_streamable_http_app() + self.assertTrue(callable(app)) + # Should expose a lifespan attribute (Starlette router) for FastAPI to wire. + self.assertTrue(hasattr(app, "lifespan") or hasattr(app, "router")) + + def test_is_mcp_enabled_default_true(self): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("RW_MCP_DISABLED", None) + self.assertTrue(is_mcp_enabled()) + + def test_is_mcp_enabled_respects_env_var(self): + with patch.dict( + os.environ, + {"RW_MCP_DISABLED": "true"}, + clear=False, + ): + self.assertFalse(is_mcp_enabled()) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/workspace_builder/tests.py b/src/workspace_builder/tests.py index 85df9a8da..bdcb13430 100644 --- a/src/workspace_builder/tests.py +++ b/src/workspace_builder/tests.py @@ -4,34 +4,38 @@ import shutil import tarfile import time +import unittest from base64 import b64decode from base64 import b64encode from http import HTTPStatus from typing import Any -from utils import read_file -import yaml -from django.test import TestCase +import yaml +from fastapi.testclient import TestClient from exceptions import WorkspaceBuilderException -from utils import transform_client_cloud_config +from utils import read_file, transform_client_cloud_config +from workspace_builder.api import app git_repo_root = os.getenv("WB_GIT_REPO_ROOT") if not git_repo_root: - print("The WB_GIT_REPO_ROOT environment variable must be set to directory where the " - "code collection repos are cloned.") + print( + "The WB_GIT_REPO_ROOT environment variable must be set to directory where the " + "code collection repos are cloned." + ) workspace_info_name_default = os.getenv("WB_WORKSPACE_INFO_NAME", "workspaceInfo.yaml") upload_info_name_default = os.getenv("WB_UPLOAD_INFO_NAME", "uploadInfo.yaml") base_directory_default = os.getenv("WB_BASE_DIRECTORY", "shared") -def build_rest_request_data(base_directory, - workspace_info_data: dict[str, Any], - upload_info_data: dict[str, Any], - **kwargs) -> str: +def build_rest_request_data( + base_directory, + workspace_info_data: dict[str, Any], + upload_info_data: dict[str, Any], + **kwargs, +) -> dict[str, Any]: request_data = workspace_info_data.copy() - # FIXME: Hacky code that hard-codes the fields in upload info that we care about for key in ("workspaceName", "defaultLocation", "defaultLocationName"): value = upload_info_data.get(key) if value is not None: @@ -39,27 +43,17 @@ def build_rest_request_data(base_directory, for key, value in kwargs.items(): request_data[key] = value for key in request_data.keys(): - # FIXME: Shouldn't hard-code "kubeconfig" setting here - # FIXME: Should check that the all of the keys are valid - # FIXME: Should dynamically determine which settings are file-based settings if key == "kubeconfig": data = read_file(request_data[key], "rb") - request_data[key] = b64encode(data).decode('utf-8') + request_data[key] = b64encode(data).decode("utf-8") elif key == "resourceLoadFile": data = read_file(request_data[key], "rb") - request_data[key] = b64encode(data).decode('utf-8') + request_data[key] = b64encode(data).decode("utf-8") elif key == "cloudConfig": cloud_config_data = request_data[key] transform_client_cloud_config(base_directory, cloud_config_data) - elif key == 'mapCustomizationRules': + elif key == "mapCustomizationRules": map_customization_rules_path = request_data[key] - # FIXME: This code is (mostly) copied from run.py. - # Should figure out a way to factor this to eliminate the code duplication - # FIXME: Seems like the path should be resolved relative to the base directory? - # Otherwise, I don't see how you could load non-default map customizations from - # the shared directory? - # The logic in run.py is sort of weird and I can't remember why it's written that way. - # Should take another look at this to refresh my memory and clean it up. if not os.path.exists(map_customization_rules_path): raise WorkspaceBuilderException("Map customization rules path does not exist") if os.path.isdir(map_customization_rules_path): @@ -67,8 +61,8 @@ def build_rest_request_data(base_directory, map_customization_rules_tar = tarfile.open(mode="x:gz", fileobj=tar_bytes) children = os.listdir(map_customization_rules_path) for child in children: - child_map_customization_rule_path = os.path.join(map_customization_rules_path, child) - data_bytes = read_file(child_map_customization_rule_path, "rb") + child_map_customization_rules_path = os.path.join(map_customization_rules_path, child) + data_bytes = read_file(child_map_customization_rules_path, "rb") data_stream: io.BytesIO = io.BytesIO(data_bytes) info = tarfile.TarInfo(child) info.size = len(data_bytes) @@ -78,39 +72,35 @@ def build_rest_request_data(base_directory, map_customization_rules_data = tar_bytes.getvalue() else: map_customization_rules_data = read_file(map_customization_rules_path, "rb") - encoded_map_customization_rules = b64encode(map_customization_rules_data).decode('utf-8') + encoded_map_customization_rules = b64encode(map_customization_rules_data).decode("utf-8") request_data[key] = encoded_map_customization_rules - return json.dumps(request_data) + return request_data -class ProductionComponentTestCase(TestCase): +class ProductionComponentTestCase(unittest.TestCase): """ Defines some runs of component pipelines for standard use cases. Note that these tests can't be run successfully without doing some manual setup: - - neo4j server must be running with a user/credentials that matches what's set in NEO4J_AUTH - - the NEO4J_AUTH environment variable must be set with valid credentials - for Kubernetes-dependent testing a kubeconfig file must be copied to the workspace builder project root directory and named "kubeconfig" - for the gcpmetrics-based tests an appropriate shhh module containing the GCP credentials must be copied to the project root directory So these tests are more for manual execution during development, not as an automated build validation step. - - It would be nice if these were improved to be able to run in a more automated way, - but the secrets/credential dependencies make that tricky. Also, the dynamic nature of a - Kubernetes/GCP environment would make it tricky to get reproducible results for any - tests that are scanning those things. - - So for now these will rely on some degree of manual setup and testing for validation. - Probably should have some process/check-list in place for any of these manual testing - requirements that should gate release readiness. """ - def run_common(self, components: str, - base_directory=None, - workspace_info_name=None, - upload_info_name=None): + @classmethod + def setUpClass(cls): + cls.client = TestClient(app) + + def run_common( + self, + components: str, + base_directory=None, + workspace_info_name=None, + upload_info_name=None, + ): if base_directory is None: base_directory = base_directory_default if workspace_info_name is None: @@ -126,13 +116,15 @@ def run_common(self, components: str, upload_info_data = yaml.safe_load(upload_info_text) else: upload_info_data = dict() - request_data = build_rest_request_data(base_directory, - workspace_info_data, - upload_info_data, - components=components) - response = self.client.post("/run/", data=request_data, content_type="application/json") + request_data = build_rest_request_data( + base_directory, + workspace_info_data, + upload_info_data, + components=components, + ) + response = self.client.post("/run/", json=request_data) self.assertEqual(HTTPStatus.OK, response.status_code) - response_data = json.loads(response.content) + response_data = response.json() archive_bytes = b64decode(response_data["output"]) archive_file_obj = io.BytesIO(archive_bytes) archive = tarfile.open(fileobj=archive_file_obj, mode="r") @@ -149,11 +141,6 @@ def run_common(self, components: str, def test_generation_rules_workspace_gen(self): self.run_common("load_resources,kubeapi,cloudquery,generation_rules,render_output_items,dump_resources") - # Need to implement some sort of dump/load feature to the resource registry - # for this to make sense now that we're not using neo4j anymore. - # def test_generation_rules_workspace_gen_no_indexing(self): - # self.run_common("generation_rules,render_output_items") - def test_info(self): response = self.client.get("/info/") self.assertEqual(HTTPStatus.OK, response.status_code) diff --git a/src/workspace_builder/urls.py b/src/workspace_builder/urls.py deleted file mode 100644 index c6941ea28..000000000 --- a/src/workspace_builder/urls.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.urls import path - -from . import views - -urlpatterns = [ - path('info/', views.InfoView.as_view()), - path('run/', views.RunView.as_view()), - path('health/', views.HealthView.as_view()), -] diff --git a/src/workspace_builder/views.py b/src/workspace_builder/views.py deleted file mode 100644 index d9457d51d..000000000 --- a/src/workspace_builder/views.py +++ /dev/null @@ -1,237 +0,0 @@ -import io -import os -import tarfile -import tempfile -import traceback -from typing import Any - -from rest_framework.request import Request -from rest_framework.response import Response -from rest_framework.views import APIView - -from component import Component, Stage, Setting, Context, \ - get_active_settings, get_all_settings, get_component, apply_component_dependencies, run_components -from exceptions import WorkspaceBuilderUserException -from outputter import TarFileOutputter -from resources import Registry, REGISTRY_PROPERTY_NAME -from utils import get_version_info -from .models import InfoResult, ArchiveRunResult -from .serializers import InfoResultSerializer, ArchiveRunResultSerializer - -tmpdir_value = os.getenv("TMPDIR", "/tmp") # fallback to /tmp if TMPDIR not set - -class InfoView(APIView): - """ - Return info about the workspace builder service. - This includes version information as well as the lists of available - indexers, enrichers, renderers and settings. - """ - def get(self, request: Request): - settings = get_all_settings() - version_info = get_version_info() - info = InfoResult(version_info['version'], version_info['name'], Stage.INDEXER.components, - Stage.ENRICHER.components, Stage.RENDERER.components, settings) - serializer = InfoResultSerializer(info) - return Response(serializer.data) - - - - -class RunView(APIView): - def post(self, request: Request): - # Import and use health tracker - try: - from .health import get_health_tracker - health_tracker = get_health_tracker() - except Exception as e: - print(f"Warning: Could not initialize health_tracker: {e}") - health_tracker = None - - # Extract the lists of components to run. - components_data = request.data.get("components", "") - if isinstance(components_data, list): - # Components provided as a list - input_component_names = components_data - elif isinstance(components_data, str): - # Components provided as a comma-separated string - input_component_names = components_data.split(",") if components_data else [] - else: - input_component_names = [] - - input_components = [get_component(name) for name in input_component_names if name.strip()] - - components: list[Component] = apply_component_dependencies(input_components) - - # Track the run start (non-blocking, won't interfere with main workflow) - if health_tracker: - try: - health_tracker.start_run(components) - except Exception as health_error: - print(f"Warning: Health tracker start_run failed: {health_error}") - - setting_temp_files: list[tempfile.TemporaryFile] = [] - setting_temp_dirs: list[tempfile.TemporaryDirectory] = [] - try: - active_settings = get_active_settings(components) - setting_values: dict[str, Any] = {} - for setting_dependency in active_settings.values(): - setting = setting_dependency.setting - value_string: str = request.data.get(setting.json_name) - using_default_value = False - if value_string is not None: - value = setting.convert_value(value_string) - elif setting.default_value: - value = setting.default_value - using_default_value = True - elif setting_dependency.required: - raise WorkspaceBuilderUserException(f"Required setting {setting.json_name} must be specified.") - else: - value = None - # Special handling for file-based settings - # Write the data to a temporary file and then specify the path to the temp file - # as the value for the setting. - # If we're using the default value for the file-based setting, though, then that - # means we're referencing a local file for the service, so we can just use the - # path directly without going through the temp file mechanism. - if value is not None: - if setting.type == Setting.Type.FILE and not using_default_value: - try: - # FIXME: Fix the problem with the type inferencing for "value" to address the type warning - tar_stream = io.BytesIO(value) - archive = tarfile.open(fileobj=tar_stream, mode="r") - setting_temp_directory = tempfile.TemporaryDirectory(dir=tmpdir_value) - setting_temp_dirs.append(setting_temp_directory) - archive.extractall(setting_temp_directory.name) - value = setting_temp_directory.name - except Exception as e: - # Assume that the exception was raised from trying to open the tar file, - # because the file is a regular file and not a tar, so in that case just - # treat it as a single, regular file and write it to a temporary named file. - # FIXME: Should catch a narrow exception. - setting_temp_file = tempfile.NamedTemporaryFile(mode="wb+", delete=True) - setting_temp_files.append(setting_temp_file) - setting_temp_file.write(value) - setting_temp_file.flush() - value = setting_temp_file.name - setting_values[setting.name] = value - - # FIXME: For now just support for the archive outputter - # Ideally should be configurable from user to use archive vs. file hierarchy outputter - # Although, practically speaking, with the current mode of operation where the REST - # service is invoked via the run tool, the outputter type is purely an - # implementation detail, so not an issue for now. - outputter = TarFileOutputter() - context = Context(setting_values, outputter) - context.set_property(REGISTRY_PROPERTY_NAME, Registry()) - - # Add configProvidedOverrides to context if present - overrides = request.data.get("overrides", {}) - if overrides: - context.set_property("overrides", overrides) - - - run_components(context, components) - - outputter.close() - archive_bytes = outputter.get_bytes() - - # Count SLXs from context - slx_count = None - try: - slxs = context.get_property("SLXS") - if slxs: - slx_count = len(slxs) - except Exception: - pass - - # Track successful completion (non-blocking) - if health_tracker: - try: - health_tracker.complete_run(warnings=context.warnings, slx_count=slx_count) - except Exception as health_error: - print(f"Warning: Health tracker complete_run failed: {health_error}") - - run_result = ArchiveRunResult("Workspace builder completed successfully.", - context.warnings, - archive_bytes) - except Exception as e: - # Capture full stacktrace - full_stacktrace = traceback.format_exc() - - # Track failed run (non-blocking) with full stacktrace - if health_tracker: - try: - health_tracker.fail_run(e, full_stacktrace) - except Exception as health_error: - print(f"Warning: Health tracker fail_run failed: {health_error}") - - # FIXME: This exception handling block is just for debugging. Can eventually get rid of it. - print(full_stacktrace) - raise e - finally: - for setting_temp_file in setting_temp_files: - setting_temp_file.close() - for setting_temp_dir in setting_temp_dirs: - setting_temp_dir.cleanup() - - serializer = ArchiveRunResultSerializer(run_result) - return Response(serializer.data) - - -class HealthView(APIView): - """ - Health endpoint for liveness and readiness checks. - Returns detailed health information from the HealthTracker. - """ - def get(self, request: Request): - from datetime import datetime, timezone - - try: - from .health import get_health_tracker - health_tracker = get_health_tracker() - - # Get current health info from the tracker - health_info = health_tracker.get_health_info() - is_healthy = health_tracker.is_healthy() - is_ready = health_tracker.is_ready() - - # Build response - response_data = { - 'status': health_info.service_status, - 'service_start_time': health_info.service_start_time, - 'uptime_seconds': health_info.uptime_seconds, - 'is_healthy': is_healthy, - 'is_ready': is_ready, - } - - # Include last run info if available - if health_info.last_run: - last_run = health_info.last_run - response_data['last_run'] = { - 'start_time': last_run.start_time, - 'end_time': last_run.end_time, - 'status': last_run.status, - 'error_message': last_run.error_message, - 'stacktrace': last_run.stacktrace, - 'warnings_count': last_run.warnings_count, - 'parsing_errors_count': last_run.parsing_errors_count, - 'components_run': last_run.components_run, - 'current_stage': last_run.current_stage, - 'current_component': last_run.current_component, - 'slx_count': last_run.slx_count, - 'duration_seconds': last_run.duration_seconds, - } - - return Response(response_data) - except Exception as e: - # If health tracker fails, return a basic healthy response - print(f"Warning: Health tracker failed: {e}") - import traceback - traceback.print_exc() - return Response({ - 'status': 'healthy', - 'timestamp': datetime.now(timezone.utc).isoformat(), - 'is_healthy': True, - 'is_ready': True, - 'error': str(e) - })