From ba94f31fbcf68f0f7dc67793e780b087aff1cbac Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:55:06 +0200 Subject: [PATCH 1/2] fix(aws): a Lake Formation tag the contract does not define is associated, and the module validates A key in a binding's governance.lakeFormation.tags with no governance.lakeFormation.tagDefinitions entry got a depends_on on an aws_lakeformation_lf_tag the module never declared, so tofu validate, fluid plan and fluid apply failed with "Reference to undeclared resource". Such a key is a tag owned outside the contract (a platform's central LF-tag ontology, the model AWS recommends), and it is now associated as written: no edge, no warning, no refusal. - The association waits only for the tags this module creates: the aws_lakeformation_lf_tag of the matching resource name whose key is the association's key. The list comes from the associations that carry a value, and depends_on is omitted when it is empty. A contract that defines every tag it associates emits the same module, byte for byte. - Refused at emit and by fluid validate, through the same functions (lf_tag_definitions, lf_tag_associations), where each used to fail at apply or be dropped: a value the contract's definition of the tag does not allow, compared case-folded as Lake Formation stores keys and values lower-cased (lakeformation-tag-value); a definition with no values, which was skipped, and two definitions that are one resource name, such as a-b and a_b, where the later replaced the earlier (lakeformation-tag-definition); tags on a binding that names no Glue table, which were dropped (lakeformation-tag-association). - The tags and tagDefinitions descriptions in fluid-schema 0.7.3 to 0.7.6, edited in place: a key the contract does not define must already exist in the account, the applying principal needs ASSOCIATE on it, and a key the contract defines must not also be created elsewhere. A test pins the wording in every schema that carries it. - docs/governance-parity.md gets a Lake Formation tags row, the new refusal kinds, and what a platform-owned tag needs. The association's resource key still ignores the database, so two exposures with the same table name in different databases share one association. Changing it changes resource addresses and needs a migration, so it is left for its own change. Prior art: terraform-provider-aws documents aws_lakeformation_resource_lf_tags as attaching existing LF-tags that must exist beforehand, and has no LF-tag data source to check one at plan time; CloudFormation's AWS::LakeFormation::TagAssociation names tag keys literally; AWS's Lake Formation TBAC guidance and its "decentralize LF-tag management" post describe a central steward that creates the tags and domains that hold ASSOCIATE; OpenTofu's depends_on takes only resources declared in the module. In this repo, _wire_aws_deps and the Snowflake _container_deps add an edge only to a resource in the same module, _glue_db_ref inlines the literal name of a container the module does not own, lf_column_exclusions is the shared emit/validate refusal shape, and test_iac_lakeformation_admins.py pinned the last schema correction. Validated: tests/iac/test_iac_lakeformation_tags.py (28 tests, two of them tofu init -backend=false + tofu validate) goes red with the code half of the fix reverted (13 failed), with only the ownership guard removed (6), with the fluid validate mirror removed (6), without case folding (2) and with the schema text reverted (4), and green restored. tests/iac passes with tofu hidden; the AWS tofu validate and moto plan suites pass with tofu on PATH; fluid validate --strict passes a platform-owned contract and refuses the bad ones; fluid generate iac --validate passes. ruff, black 24.10.0, the product guardrail and detect-secrets pass. The full-suite result is in the PR description. --- CHANGELOG.md | 19 + docs/governance-parity.md | 34 +- fluid_build/iac/governance_validation.py | 32 +- fluid_build/iac/providers/aws.py | 228 +++++++++-- fluid_build/schemas/fluid-schema-0.7.3.json | 4 +- fluid_build/schemas/fluid-schema-0.7.4.json | 4 +- fluid_build/schemas/fluid-schema-0.7.5.json | 4 +- fluid_build/schemas/fluid-schema-0.7.6.json | 4 +- tests/iac/test_iac_lakeformation_tags.py | 395 ++++++++++++++++++++ 9 files changed, 669 insertions(+), 55 deletions(-) create mode 100644 tests/iac/test_iac_lakeformation_tags.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7425c398..f491430d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 now builds the tag's commit, also when a manual run starts from `main`, and the quality gate checks `ci.yml` on that commit. +### Fixed + +- **A Lake Formation tag the contract does not define is associated, and the module + validates.** A key in a binding's `governance.lakeFormation.tags` with no + `governance.lakeFormation.tagDefinitions` entry got a `depends_on` on an + `aws_lakeformation_lf_tag` the module never declared, so `tofu validate`, `fluid plan` + and `fluid apply` failed with "Reference to undeclared resource". Such a key is now a + tag the platform owns: associated as written, with no `depends_on`, no warning and no + refusal. It must already exist in the account, and the identity running `fluid apply` + needs `ASSOCIATE` on it. The association still waits for the tags the contract + defines, and a contract that defines every tag it associates emits the same module. + Refused at `fluid validate` and at emit, where each used to fail at apply or be + dropped: a value the contract's definition of the tag does not allow, compared + case-insensitively (`lakeformation-tag-value`); a definition with no values, or two + that would be one resource name, such as `a-b` and `a_b` + (`lakeformation-tag-definition`); and tags on a binding that names no Glue table + (`lakeformation-tag-association`). The `tags` and `tagDefinitions` descriptions in + fluid-schema 0.7.3 to 0.7.6 say so. + ## [0.19.0] — 2026-10-06 A contract can now say that a principal reads a column masked rather than not at all, diff --git a/docs/governance-parity.md b/docs/governance-parity.md index 375efcd3..482afaa2 100644 --- a/docs/governance-parity.md +++ b/docs/governance-parity.md @@ -20,6 +20,7 @@ contract needs `fluidVersion: "0.7.6"` to validate with them. | **Row filters**: `exposes[].policy.authz.rowFilters[] {principal, where, name?}` | A Lake Formation data cells filter per filter (`aws_lakeformation_data_cells_filter`, `row_filter.filter_expression` the predicate, `column_wildcard.excluded_column_names` the principal's excluded columns), and the principal's `SELECT` granted on the filter instead of the table. | A row access policy per filter (`google_bigquery_row_access_policy`, `filter_predicate` the predicate, `grantees` every identity of the principal, whatever it holds: a writer reads rows too), and one more, `fluid_all_rows` (`TRUE`), for every other principal the expose grants anything to, never for a filtered one, since BigQuery shows a principal no policy names no row at all and the union of those that do. | AWS: the column check reads a grant made on a data cells filter, so a filtered principal's excluded columns are still checked. Not checked yet: the filter expression on AWS, and the row access policies on GCP. | | **Governance labels**: `labels`, `exposes[].labels`, `dataClassification`, `sovereignty {jurisdiction, regulatoryFramework, dataResidency}` | The same keys as AWS tags on the bucket and the key, and as parameters on the Glue table. | GCP labels on the dataset, the table and the key: the contract's and the expose's labels, and `fluid_classification`, `fluid_jurisdiction`, `fluid_regulation` (the frameworks joined), `fluid_residency`. | Not checked yet. | | **Access grants**: `accessPolicy.grants[]` | Not emitted: on AWS, access is the binding's `governance.lakeFormation.grants`. `fluid validate` warns only for an aws binding with no Lake Formation grants, where the contract's access intent is unenforced. | One non-authoritative `google_bigquery_dataset_iam_member` per role and member (and `google_storage_bucket_iam_member` for GCS), with the logical principal mapped to its GCP identity. | Not checked yet on either cloud. | +| **Lake Formation tags** (AWS only): `binding.governance.lakeFormation.tags`, and `governance.lakeFormation.tagDefinitions` for the tags the contract creates | One `aws_lakeformation_lf_tag` per defined key, and one `aws_lakeformation_resource_lf_tags` per table with the binding's tags. The association waits for the tags the contract defines (`depends_on`), and a value must be one of its tag's values. A key the contract does not define is a tag the platform owns: associated as written, with no `depends_on` (see "Lake Formation tags the platform owns" below). | Not applicable: these are AWS binding fields. | Not checked yet. | A policy that a binding cannot apply is refused at `fluid validate`, `fluid plan` and `fluid apply`, never dropped: retention, a key or a column restriction on a GCP @@ -28,8 +29,12 @@ restriction on an AWS binding with no Lake Formation grants or on a non-Glue for an AWS key reference on GCP and a Cloud KMS key name on AWS, a mask on an AWS binding with no `maskedViews` (`column-mask-unenforceable`), a text mask on a column that is not a string (`column-mask-type`), a row filter whose principal holds no Lake -Formation read grant (`row-filter-unenforceable`), and a row filter on a BigQuery view -(`row-filter-view`) or on a GCP binding that is not a BigQuery table. +Formation read grant (`row-filter-unenforceable`), a row filter on a BigQuery view +(`row-filter-view`) or on a GCP binding that is not a BigQuery table, a Lake Formation +tag value that the contract's definition of the tag does not allow +(`lakeformation-tag-value`), a tag definition with no values or two that would be one +resource, such as `a-b` and `a_b` (`lakeformation-tag-definition`), and Lake Formation +tags on a binding that names no Glue table (`lakeformation-tag-association`). ## Logical principals and `binding.principals` @@ -276,6 +281,24 @@ partitioning's shape. Changing `retention` later is an in-place update of also refuses without `--allow-data-loss`; BigQuery cannot un-partition a table, so to keep data longer, set a longer `retention` instead. +## Lake Formation tags the platform owns + +AWS recommends one LF-tag ontology per account: a data steward or the platform +creates the tag keys and their values, and each domain is given `ASSOCIATE` on them +to tag its own tables. A contract follows either model, per key: + +* A key in `governance.lakeFormation.tagDefinitions` is a tag the contract creates. + The apply creates it, the association waits for it, and the association's value + must be one of its values (compared case-insensitively, as Lake Formation stores + keys and values lower-cased). Do not also create that key anywhere else in the + account. +* A key in a binding's `governance.lakeFormation.tags` that the contract does not + define is a tag the platform owns. It is associated as written, with no + `depends_on` and no warning. It must already exist in the account, and the identity + running `fluid apply` needs `ASSOCIATE` on it. terraform-provider-aws has no data + source for an LF-tag, so nothing checks the key or the value before apply: Lake + Formation does, at apply. + ## Prerequisites * The Cloud KMS API (`cloudkms.googleapis.com`) and the Data Catalog API @@ -298,6 +321,8 @@ keep data longer, set a longer `retention` instead. caller can see, so it must run as a Lake Formation administrator. When the contract's own grants are not in the listing, it reports an error rather than a pass. +* On AWS, a Lake Formation tag the platform owns must exist before `fluid apply`, + and the identity running it needs `ASSOCIATE` on the tag. ## What is proven, and what is not @@ -313,6 +338,9 @@ keep data longer, set a longer `retention` instead. * The same stand-in shows a revoked reader plans one member destroy that the gate lets through, and that moving from the authoritative access list of 0.16.6 and earlier to member resources revokes a grant removed in the same change and then plans clean. +* `tofu validate` accepts a Lake Formation tag association on a tag the platform owns, + alone and beside a tag the contract defines (`tests/iac/test_iac_lakeformation_tags.py`). + Before, it failed with "Reference to undeclared resource". * **Measured against real Google Cloud**, 4 October 2026, on 0.18.0. In a demo lab, two lineage chains of eleven products were applied with `fluid apply --env gcp` from generated Jenkins pipelines, as a deploy service account reached by Workload Identity @@ -348,6 +376,8 @@ keep data longer, set a longer `retention` instead. moved from `~> 5.0`. * **Not proven yet**: an apply of the masking, row filter and label resources exactly as forge-cli emits them, on either cloud. +* **Not proven**: associating a tag the platform owns on a real account, and how Lake + Formation treats a value whose case differs from the tag's. * **Not proven**: the Lake Formation half against a real account as 0.17.0 derives it from `columnRestrictions`. The grant shape it emits (excluded columns beside `wildcard`) was applied and enforced on a real account from 0.16.6, written by hand diff --git a/fluid_build/iac/governance_validation.py b/fluid_build/iac/governance_validation.py index 4a68db08..bb7bcf9d 100644 --- a/fluid_build/iac/governance_validation.py +++ b/fluid_build/iac/governance_validation.py @@ -16,11 +16,15 @@ ``fluid generate iac``, ``fluid plan`` and ``fluid apply`` refuse a principal that is unmapped or a placeholder, a column restriction nothing on the binding enforces, -a Cloud KMS key on AWS or an AWS key on GCP, and retention or encryption on a GCP -target that is not a BigQuery table. This runs the SAME derivations +a Cloud KMS key on AWS or an AWS key on GCP, retention or encryption on a GCP +target that is not a BigQuery table, and a Lake Formation tag the module cannot +create or associate as declared (a definition with no values, two definitions that +are one resource, a value the contract's definition of the tag does not allow, tags +on a binding with no Glue table). This runs the SAME derivations (``iac/providers/gcp_governance.py``, ``iac/column_access.py``, -``iac/principals.py``) so ``fluid validate`` reports each refusal at stage 2, with -the same message, instead of at apply. The same shape as ``validate_gcp_binding``. +``iac/principals.py``, the AWS emitter's ``lf_*`` functions) so ``fluid validate`` +reports each refusal at stage 2, with the same message, instead of at apply. The +same shape as ``validate_gcp_binding``. """ from __future__ import annotations @@ -45,6 +49,22 @@ def _lf_grants(binding: Mapping[str, Any]) -> List[Any]: return list(grants) if isinstance(grants, list) else [] +def _lf_tag_definition_errors(contract: Mapping[str, Any]) -> List[str]: + """The refusal of the contract's LF-tag definitions, which the AWS emitter creates + once for the whole contract; none for a contract with no expose bound to AWS.""" + from .providers.aws import lf_tag_definitions + + for exposure in contract.get("exposes") or []: + binding = exposure.get("binding") if isinstance(exposure, Mapping) else None + if isinstance(binding, Mapping) and is_cloud(binding, "aws"): + try: + lf_tag_definitions(contract) + except UnsupportedBindingError as exc: + return [_message(exc)] + break + return [] + + def validate_governance(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: """``(errors, warnings)`` for the contract's cloud bindings. @@ -64,6 +84,7 @@ def validate_governance(contract: Mapping[str, Any]) -> Tuple[List[str], List[st _gov.refuse_mixed_dataset_encryption(contract) except UnsupportedBindingError as exc: errors.append(_message(exc)) + errors.extend(_lf_tag_definition_errors(contract)) for index, exposure in enumerate(contract.get("exposes") or []): if not isinstance(exposure, Mapping): continue @@ -73,9 +94,10 @@ def validate_governance(contract: Mapping[str, Any]) -> Tuple[List[str], List[st try: principal_map(binding) if is_cloud(binding, "aws"): - from .providers.aws import lf_column_exclusions + from .providers.aws import lf_column_exclusions, lf_tag_associations lf_column_exclusions(exposure, binding, index) + lf_tag_associations(contract, exposure, binding, index) if not _lf_grants(binding): unenforced.append(str(exposure.get("exposeId") or index)) continue diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index df544c91..3a214c22 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -391,8 +391,8 @@ def emit( # Account-level Lake Formation settings: admins + LF-tag # definitions. Emitted once per contract, before per-exposure - # resources so the LF tag-definitions exist before any - # resource_lf_tags association references them. + # resources, so a resource_lf_tags association can tell the tags + # this module creates (and waits for) from those it does not. _emit_lf_account_settings(resources, contract, cid, base_tags) for index, exposure in enumerate(contract.get("exposes") or []): @@ -411,6 +411,9 @@ def emit( # cannot mask): ``lf_row_filters`` / ``lf_masked_views``. row_filters = lf_row_filters(exposure, binding, index) masked_views = lf_masked_views(exposure, binding, index, exclusions or {}, row_filters) + # The binding's LF-tag associations, each value checked against the + # contract's definition of its tag when it has one: ``lf_tag_associations``. + lf_tags = lf_tag_associations(contract, exposure, binding, index) placement = _placement(packaging, exposure) # The contract's governance labels (classification, jurisdiction, # regulation, its own labels) under fluid's own two, which win. @@ -447,6 +450,7 @@ def emit( exclusions=exclusions or {}, row_filters=row_filters, masked_views=masked_views, + lf_tags=lf_tags, ) # Retention (exposes[].lifecycle) and encryption at rest # (binding.encryption), per bucket this product owns. Nothing is @@ -1525,8 +1529,8 @@ def _wire_aws_deps(resources: Dict[str, Any], cid: str) -> None: # per-exposure emit. Honours top-level ``governance.lakeFormation``: # ``admins`` → ``aws_lakeformation_data_lake_settings``, # ``tagDefinitions`` → one ``aws_lakeformation_lf_tag`` per key. -# Must run before per-resource ``resource_lf_tags`` associations so -# the tag keys exist for the association to reference. +# Must run before per-resource ``resource_lf_tags`` associations, which +# wait for the tags it wrote (see :func:`lf_tag_definitions`). # # * ``_emit_lakeformation`` — fires per AWS exposure. Honours # ``binding.governance.lakeFormation``: @@ -1538,16 +1542,18 @@ def _wire_aws_deps(resources: Dict[str, Any], cid: str) -> None: # :func:`_lf_column_grant_permissions`), # plus, per ``bucketPolicy``, an ``aws_s3_bucket_policy`` for the # grantees in OTHER accounts (see :func:`_lf_bucket_policy`), -# ``tags{}`` → one ``aws_lakeformation_resource_lf_tags`` per table, +# ``tags{}`` → one ``aws_lakeformation_resource_lf_tags`` per table +# (see :func:`lf_tag_associations`), naming a tag the contract defines +# or one owned outside the contract, # ``rowFilter`` → one ``aws_lakeformation_data_cells_filter``. # # Design notes: # - LF resources are emitted alongside the Glue catalog table they # reference; OpenTofu's value-reference edges (``${aws_glue_catalog_table # .{...}.name}``) provide the ordering, no manual ``depends_on`` -# needed. Where a reference would be circular (e.g. tag definitions -# vs tag associations from different exposures), explicit -# ``depends_on`` is set. +# needed. A tag association names its tag keys as literals, so it +# gets an explicit ``depends_on`` on the tags this module defines, and +# none for a tag owned outside the contract (its resource is not here). # - Empty governance blocks emit nothing — every existing contract # stays at zero LF surface area. # - LF is Glue-catalog-backed, so the per-exposure emit only fires for @@ -1598,7 +1604,6 @@ def _emit_lf_account_settings( ) -> None: gov = (contract.get("governance") or {}).get("lakeFormation") or {} admins = gov.get("admins") or [] - tag_defs = gov.get("tagDefinitions") or {} if admins: # ``aws_lakeformation_data_lake_settings`` is a singleton per @@ -1619,15 +1624,133 @@ def _emit_lf_account_settings( "admins": list(admins), } - for tag_key, tag_values in tag_defs.items(): + tag_defs = lf_tag_definitions(contract) + if tag_defs: + resources.setdefault("aws_lakeformation_lf_tag", {}).update(tag_defs) + + +def _lf_tag_name(cid: str, key: Any) -> str: + """The ``aws_lakeformation_lf_tag`` resource name of the contract's tag ``key``.""" + return safe_ident(f"{cid}_lf_tag_{key}") + + +def _lf_tag_definitions(contract: Mapping[str, Any]) -> Mapping[Any, Any]: + """The contract's ``governance.lakeFormation.tagDefinitions`` map, as written.""" + gov = (contract.get("governance") or {}).get("lakeFormation") or {} + tag_defs = gov.get("tagDefinitions") if isinstance(gov, Mapping) else None + return tag_defs if isinstance(tag_defs, Mapping) else {} + + +def lf_tag_definitions(contract: Mapping[str, Any]) -> Dict[str, Dict[str, Any]]: + """``{resource name: aws_lakeformation_lf_tag body}``: the tags this module creates. + + One per key of the contract's ``governance.lakeFormation.tagDefinitions``, named + with the stem :meth:`AwsIacPlugin.emit` uses. Refused where the module could not + create the tag as declared: a key with no values (Lake Formation needs at least + one; the key used to be skipped, and an association to it then named a resource + the module did not declare), and two keys that are one resource name (``a-b`` and + ``a_b``; the later used to replace the earlier). Run here, where ``fluid validate`` + and the emitter both read the definitions, so stage 2 refuses them too. + """ + cid = aws_storage.contract_ident(contract) + out: Dict[str, Dict[str, Any]] = {} + for tag_key, tag_values in _lf_tag_definitions(contract).items(): if not tag_values: + raise UnsupportedBindingError( + "lakeformation-tag-definition", + f"governance.lakeFormation.tagDefinitions.{tag_key} lists no values, and a " + "Lake Formation tag needs at least one, so the module cannot create it.", + ( + "List the values the tag allows.", + "Remove the key if the tag is created outside this contract; " + "associating it then needs no definition here.", + ), + ) + name = _lf_tag_name(cid, tag_key) + if name in out: + raise UnsupportedBindingError( + "lakeformation-tag-definition", + f"governance.lakeFormation.tagDefinitions keys {out[name]['key']!r} and " + f"{str(tag_key)!r} are both the resource aws_lakeformation_lf_tag.{name}, " + "so one definition would replace the other.", + ("Rename one of the keys so that the two differ in a letter or a digit.",), + ) + out[name] = {"key": str(tag_key), "values": list(tag_values)} + return out + + +def lf_tag_associations( + contract: Mapping[str, Any], + exposure: Mapping[str, Any], + binding: Mapping[str, Any], + index: int = 0, +) -> List[Dict[str, str]]: + """The ``lf_tag`` blocks of the binding's ``governance.lakeFormation.tags``. + + One per key that carries a value. A key the contract's ``tagDefinitions`` declare + is a tag this module creates, and the value must be one of its values, compared + case-folded since Lake Formation stores keys and values lower-cased. A key it does + not declare is a tag owned outside the contract (a platform's central LF-tag + ontology): it is associated as written, checked only by Lake Formation at apply, + and :func:`_emit_lf_tag_association` orders nothing on it. Tags on a Glue-catalog + binding that names no Glue table are refused, as they would be dropped. Run here, + where ``fluid validate`` and the emitter both derive the associations, so stage 2 + refuses them too. + """ + gov = (binding.get("governance") or {}).get("lakeFormation") or {} + tags = gov.get("tags") if isinstance(gov, Mapping) else None + if not isinstance(tags, Mapping): + return [] + lf_tags = [{"key": str(k), "value": str(v)} for k, v in tags.items() if v] + fmt = str(binding.get("format") or "parquet") + if not lf_tags or fmt.lower() not in _GLUE_CATALOG_FORMATS: + # Not a Glue-catalog format: _emit_lakeformation ignores the whole + # governance.lakeFormation block of such a binding, by design. + return [] + where = f"exposes[{exposure.get('exposeId') or index}] governance.lakeFormation.tags" + loc = binding.get("location") or {} + if not loc.get("database") or not loc.get("table"): + raise UnsupportedBindingError( + "lakeformation-tag-association", + f"{where} associates {', '.join(t['key'] for t in lf_tags)}, but the binding " + "names no Glue table (location.database and location.table), so the tags " + "would be dropped.", + ( + "Set location.database and location.table to the tags' Glue table.", + "Remove governance.lakeFormation.tags.", + ), + ) + _refuse_a_value_the_tag_does_not_have(contract, lf_tags, where) + return lf_tags + + +def _refuse_a_value_the_tag_does_not_have( + contract: Mapping[str, Any], lf_tags: Sequence[Mapping[str, str]], where: str +) -> None: + """Refuse an association whose value the contract's definition of its tag lacks. + + Only a key the contract defines is checked, case-folded, as Lake Formation stores + keys and values lower-cased; a tag owned outside the contract is Lake Formation's + to check, at apply. + """ + defined = {str(k): v for k, v in _lf_tag_definitions(contract).items() if v} + for tag in lf_tags: + values = defined.get(tag["key"]) + if values is None: continue - resources.setdefault("aws_lakeformation_lf_tag", {})[ - safe_ident(f"{cid}_lf_tag_{tag_key}") - ] = { - "key": str(tag_key), - "values": list(tag_values), - } + if tag["value"].casefold() not in {str(v).casefold() for v in values}: + raise UnsupportedBindingError( + "lakeformation-tag-value", + f"{where}.{tag['key']} is {tag['value']!r}, but " + f"governance.lakeFormation.tagDefinitions.{tag['key']} allows only " + f"{[str(v) for v in values]}, and Lake Formation refuses to associate a " + "value the tag does not have.", + ( + "Associate one of the values the tag allows.", + f"Add {tag['value']!r} to governance.lakeFormation.tagDefinitions." + f"{tag['key']}.", + ), + ) def _lf_location(loc: Mapping[str, Any], placement: _Placement) -> Tuple[Optional[str], str]: @@ -2345,10 +2468,15 @@ def _emit_lakeformation( exclusions: Optional[Mapping[int, Tuple[str, ...]]] = None, row_filters: Optional[Mapping[int, RowFilter]] = None, masked_views: Sequence["_LfMaskedView"] = (), + lf_tags: Sequence[Mapping[str, str]] = (), ) -> None: """Emit per-exposure LF resources. No-op when the binding has no ``governance.lakeFormation`` block. + ``lf_tags`` is :func:`lf_tag_associations`: the table's one + ``aws_lakeformation_resource_lf_tags``, ordered after the tags this module + creates and on nothing for a tag owned outside the contract. + ``row_filters`` is :func:`lf_row_filters`: a grant that carries one reads through a data cells filter (the filter's predicate, and the grant's excluded columns as its column wildcard's exclusions), so its ``SELECT`` is granted on the filter, not on @@ -2494,30 +2622,10 @@ def _emit_lakeformation( # another AWS account) is emitted once per bucket, over every exposure # on it, by :meth:`AwsIacPlugin.emit`. See :func:`_lf_bucket_policies`. - # 3. LF-tag associations on the table (LF-TBAC). - tag_assoc = gov.get("tags") or {} - if tag_assoc and table_key: - lf_tags = [{"key": str(k), "value": str(v)} for k, v in tag_assoc.items() if v] - if lf_tags: - assoc_key = safe_ident(f"{cid}_lf_tags_{table}") - resources.setdefault("aws_lakeformation_resource_lf_tags", {})[assoc_key] = { - "table": [ - { - "database_name": tofu_ref( - f"aws_glue_catalog_table.{table_key}.database_name" - ), - "name": tofu_ref(f"aws_glue_catalog_table.{table_key}.name"), - } - ], - "lf_tag": lf_tags, - # The tag KEYS must exist before this association can be - # applied. The matching ``aws_lakeformation_lf_tag`` - # resources come from the contract-level - # ``governance.lakeFormation.tagDefinitions`` block. - "depends_on": [ - f"aws_lakeformation_lf_tag.{safe_ident(f'{cid}_lf_tag_{k}')}" for k in tag_assoc - ], - } + # 3. LF-tag associations on the table (LF-TBAC): the keys that carry a value, + # from :func:`lf_tag_associations`. + if lf_tags and table_key: + _emit_lf_tag_association(resources, cid, table, table_key, lf_tags) # 4. Row-level (and optional column-level) filter. row_filter = gov.get("rowFilter") @@ -2567,6 +2675,46 @@ def _lf_filter_table_data( } +def _emit_lf_tag_association( + resources: Dict[str, Any], + cid: str, + table: str, + table_key: str, + lf_tags: Sequence[Mapping[str, str]], +) -> None: + """The table's ``aws_lakeformation_resource_lf_tags``: :func:`lf_tag_associations`. + + A tag must exist before it is associated. The association waits only for the + tags this module creates: the ``aws_lakeformation_lf_tag`` of the contract's + ``tagDefinitions``, which :func:`_emit_lf_account_settings` wrote before any + exposure, and only when that resource is this key's (one resource name can + stand for ``a-b`` and ``a_b``). A key the contract does not define is a tag + owned outside the contract, already in the account: an edge to it would name a + resource the module does not declare, and ``tofu validate`` refuses the whole + module. The key stays a plain string, which the renderer escapes. + """ + assoc: Dict[str, Any] = { + "table": [ + { + "database_name": tofu_ref(f"aws_glue_catalog_table.{table_key}.database_name"), + "name": tofu_ref(f"aws_glue_catalog_table.{table_key}.name"), + } + ], + "lf_tag": [dict(t) for t in lf_tags], + } + owned = resources.get("aws_lakeformation_lf_tag") or {} + depends_on: List[str] = [] + for tag in lf_tags: + name = _lf_tag_name(cid, tag["key"]) + if (owned.get(name) or {}).get("key") == tag["key"]: + depends_on.append(f"aws_lakeformation_lf_tag.{name}") + if depends_on: + assoc["depends_on"] = depends_on + resources.setdefault("aws_lakeformation_resource_lf_tags", {})[ + safe_ident(f"{cid}_lf_tags_{table}") + ] = assoc + + def _emit_lf_filtered_grant( resources: Dict[str, Any], cid: str, diff --git a/fluid_build/schemas/fluid-schema-0.7.3.json b/fluid_build/schemas/fluid-schema-0.7.3.json index 7f26de9d..25589ee8 100644 --- a/fluid_build/schemas/fluid-schema-0.7.3.json +++ b/fluid_build/schemas/fluid-schema-0.7.3.json @@ -1386,7 +1386,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": {"type": "string"}, @@ -1470,7 +1470,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": {"type": "string"} }, "rowFilter": { diff --git a/fluid_build/schemas/fluid-schema-0.7.4.json b/fluid_build/schemas/fluid-schema-0.7.4.json index 613ea0ab..8e2a455f 100644 --- a/fluid_build/schemas/fluid-schema-0.7.4.json +++ b/fluid_build/schemas/fluid-schema-0.7.4.json @@ -5289,7 +5289,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5384,7 +5384,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } diff --git a/fluid_build/schemas/fluid-schema-0.7.5.json b/fluid_build/schemas/fluid-schema-0.7.5.json index e21ece28..a0557ccf 100644 --- a/fluid_build/schemas/fluid-schema-0.7.5.json +++ b/fluid_build/schemas/fluid-schema-0.7.5.json @@ -5482,7 +5482,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model \u2014 see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model \u2014 see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5577,7 +5577,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } diff --git a/fluid_build/schemas/fluid-schema-0.7.6.json b/fluid_build/schemas/fluid-schema-0.7.6.json index 1c52b7d9..44c75802 100644 --- a/fluid_build/schemas/fluid-schema-0.7.6.json +++ b/fluid_build/schemas/fluid-schema-0.7.6.json @@ -5894,7 +5894,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5999,7 +5999,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } diff --git a/tests/iac/test_iac_lakeformation_tags.py b/tests/iac/test_iac_lakeformation_tags.py new file mode 100644 index 00000000..23a2c8e1 --- /dev/null +++ b/tests/iac/test_iac_lakeformation_tags.py @@ -0,0 +1,395 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An LF-tag association without a tag definition in the contract emits a valid module. + +``binding.governance.lakeFormation.tags`` associates LF-tags with the table; +``governance.lakeFormation.tagDefinitions`` (top level) is what makes the module +CREATE the tag. A binding that names a tag the contract does not define (the tag +is owned by a platform bootstrap, the account-wide ontology) used to get a +``depends_on`` on an ``aws_lakeformation_lf_tag`` the module never declares, so +``tofu validate`` failed ``Reference to undeclared resource``. Such a key is now a +platform-owned tag: associated as written, with no edge, no warning and no refusal. + +What the module cannot create or associate as declared is refused, at emit and by +``fluid validate`` with the same message: a value the contract's definition of the +tag does not allow, a definition with no values, two definitions that are one +resource name, and tags on a binding that names no Glue table. The schemas say so, +in every version that carries the fields. + +The tests marked ``tofu`` need ``tofu`` on PATH and the provider from the registry +(no AWS credentials), like ``test_iac_aws_validate.py``; they skip without it. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping + +import pytest + +import fluid_build +from fluid_build.iac import build_module, runner +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.governance_validation import validate_governance +from fluid_build.iac.providers.aws import AwsIacPlugin + +pytestmark = [pytest.mark.unit, pytest.mark.provider, pytest.mark.aws] + +CID = "gold_sales_orders" +ASSOC = f"{CID}_lf_tags_orders" +SCHEMAS = Path(fluid_build.__file__).resolve().parent / "schemas" + + +def _contract(tags: Mapping[str, str], definitions: Mapping[str, List[str]] | None = None): + contract: Dict[str, Any] = { + "id": "gold.sales.orders", + "name": "Orders", + "exposes": [ + { + "exposeId": "orders", + "binding": { + "platform": "aws", + "format": "parquet", + "location": { + "database": "sales", + "table": "orders", + "bucket": "acme-sales-lake", + "path": "gold/orders/", + }, + "governance": {"lakeFormation": {"tags": dict(tags)}}, + }, + "contract": {"schema": [{"name": "order_id", "type": "string"}]}, + } + ], + } + if definitions is not None: + contract["governance"] = {"lakeFormation": {"tagDefinitions": copy.deepcopy(definitions)}} + return contract + + +def _declared(document: Mapping[str, Any]) -> set: + """Every address a ``depends_on`` may name: managed resources and data sources.""" + out = set() + for rtype, bodies in (document.get("resource") or {}).items(): + out.update(f"{rtype}.{name}" for name in bodies) + for rtype, bodies in (document.get("data") or {}).items(): + out.update(f"data.{rtype}.{name}" for name in bodies) + return out + + +def _depends_on(document: Mapping[str, Any]) -> Iterator[tuple]: + for rtype, bodies in (document.get("resource") or {}).items(): + for name, body in bodies.items(): + for dep in (body or {}).get("depends_on") or []: + yield f"{rtype}.{name}", dep + + +def _dangling(contract: Mapping[str, Any]) -> List[str]: + document = json.loads(build_module(AwsIacPlugin(), contract)) + declared = _declared(document) + return [ + f"{owner} depends_on {dep}" for owner, dep in _depends_on(document) if dep not in declared + ] + + +def _resources(contract: Mapping[str, Any]) -> Dict[str, Any]: + return json.loads(build_module(AwsIacPlugin(), contract))["resource"] + + +def _association(contract: Mapping[str, Any]) -> Dict[str, Any]: + return _resources(contract)["aws_lakeformation_resource_lf_tags"][ASSOC] + + +def _refusal(contract: Mapping[str, Any]) -> UnsupportedBindingError: + with pytest.raises(UnsupportedBindingError) as raised: + AwsIacPlugin().emit(contract) + return raised.value + + +def _validate_reports(contract: Mapping[str, Any], error: UnsupportedBindingError) -> None: + """``fluid validate`` reports the emit's refusal, with its message and remediation.""" + errors, _ = validate_governance(contract) + assert errors == [f"{error} {' '.join(error.remediation)}"], errors + + +# --------------------------------------------------------------------------- +# A key the contract does not define: a platform-owned tag +# --------------------------------------------------------------------------- + + +def test_a_tag_association_without_a_definition_names_no_missing_resource(): + """The reported defect: tags: {classification: confidential}, no tagDefinitions.""" + assert _dangling(_contract({"classification": "confidential"})) == [] + + +def test_a_platform_owned_tag_is_associated_as_written_with_no_edge(): + resources = _resources(_contract({"classification": "confidential"})) + assert "aws_lakeformation_lf_tag" not in resources + assoc = resources["aws_lakeformation_resource_lf_tags"][ASSOC] + assert assoc["lf_tag"] == [{"key": "classification", "value": "confidential"}] + # Omitted, not an empty list. + assert "depends_on" not in assoc + + +def test_a_platform_owned_tag_validates_with_no_error_and_no_warning(): + """No warning either: the generated pipeline's stage 2 runs ``fluid validate --strict``.""" + assert validate_governance(_contract({"classification": "confidential"})) == ([], []) + + +def test_only_the_keys_this_module_defines_become_dependencies(): + """One key defined by the contract, one owned by the platform.""" + contract = _contract( + {"classification": "confidential", "domain": "sales"}, + {"domain": ["sales", "marketing"]}, + ) + assert _dangling(contract) == [] + assoc = _association(contract) + assert {t["key"] for t in assoc["lf_tag"]} == {"classification", "domain"} + # The defined key is still ordered before the association, and only it. + assert assoc["depends_on"] == [f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain"] + + +def test_a_defined_tag_still_precedes_its_association_control(): + """Control, green on main: with the definition present the module is consistent.""" + contract = _contract({"classification": "confidential"}, {"classification": ["confidential"]}) + assert _dangling(contract) == [] + + +def test_a_contract_that_defines_its_tags_emits_what_it_did(): + """The association body of a contract that defines every key, as main emitted it.""" + contract = _contract( + {"classification": "pii_low", "domain": "sales"}, + {"classification": ["public", "pii_low", "pii_high"], "domain": ["sales", "marketing"]}, + ) + resources = _resources(contract) + table = f"aws_glue_catalog_table.{CID}_sales_orders" + assert resources["aws_lakeformation_lf_tag"] == { + f"{CID}_lf_tag_classification": { + "key": "classification", + "values": ["public", "pii_low", "pii_high"], + }, + f"{CID}_lf_tag_domain": {"key": "domain", "values": ["sales", "marketing"]}, + } + assert resources["aws_lakeformation_resource_lf_tags"] == { + ASSOC: { + "depends_on": [ + f"aws_lakeformation_lf_tag.{CID}_lf_tag_classification", + f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain", + ], + "lf_tag": [ + {"key": "classification", "value": "pii_low"}, + {"key": "domain", "value": "sales"}, + ], + "table": [ + { + "database_name": f"${{{table}.database_name}}", + "name": f"${{{table}.name}}", + } + ], + } + } + + +def test_a_key_with_an_empty_value_adds_no_edge(): + """The edges come from the associations that carry a value, not from every key.""" + contract = _contract( + {"classification": "", "domain": "sales"}, + {"classification": ["public"], "domain": ["sales"]}, + ) + assoc = _association(contract) + assert assoc["lf_tag"] == [{"key": "domain", "value": "sales"}] + assert assoc["depends_on"] == [f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain"] + + +def test_a_key_that_shares_a_defined_tags_resource_name_is_not_ordered_on_it(): + """``a-b`` and ``a_b`` are one resource name but two Lake Formation tags.""" + contract = _contract({"a-b": "x"}, {"a_b": ["y"]}) + resources = _resources(contract) + assert resources["aws_lakeformation_lf_tag"] == { + f"{CID}_lf_tag_a_b": {"key": "a_b", "values": ["y"]} + } + assoc = resources["aws_lakeformation_resource_lf_tags"][ASSOC] + assert assoc["lf_tag"] == [{"key": "a-b", "value": "x"}] + assert "depends_on" not in assoc + assert validate_governance(contract) == ([], []) + + +def test_a_platform_owned_key_is_escaped_as_contract_text(): + """The literal key is a plain string, so the renderer escapes an interpolation in it.""" + text = build_module(AwsIacPlugin(), _contract({"${var.x}": "y"})) + assoc = json.loads(text)["resource"]["aws_lakeformation_resource_lf_tags"][ASSOC] + assert assoc["lf_tag"] == [{"key": "$${var.x}", "value": "y"}] + + +# --------------------------------------------------------------------------- +# Refused, at emit and at ``fluid validate`` +# --------------------------------------------------------------------------- + + +def test_a_value_the_definition_does_not_allow_is_refused(): + contract = _contract({"classification": "confidential"}, {"classification": ["public"]}) + error = _refusal(contract) + assert error.kind == "lakeformation-tag-value" + assert "exposes[orders] governance.lakeFormation.tags.classification" in str(error) + assert "'confidential'" in str(error) and "['public']" in str(error) + _validate_reports(contract, error) + + +@pytest.mark.parametrize("value", ["public", "Public", "PUBLIC"]) +def test_a_value_is_compared_case_folded(value): + """Lake Formation stores keys and values lower-cased.""" + contract = _contract({"classification": value}, {"classification": ["Public", "internal"]}) + assert _association(contract)["lf_tag"] == [{"key": "classification", "value": value}] + assert validate_governance(contract) == ([], []) + + +@pytest.mark.parametrize("values", [[], None]) +def test_a_definition_with_no_values_is_refused(values): + """It used to be skipped, and an association to it named an undeclared resource.""" + for contract in ( + _contract({"classification": "public"}, {"classification": values}), + _contract({}, {"classification": values}), + ): + error = _refusal(contract) + assert error.kind == "lakeformation-tag-definition" + assert "tagDefinitions.classification lists no values" in str(error) + _validate_reports(contract, error) + + +def test_two_definitions_that_are_one_resource_name_are_refused(): + """``a-b`` and ``a_b`` were both ``..._lf_tag_a_b``, and the later replaced the earlier.""" + contract = _contract({}, {"a-b": ["x"], "a_b": ["y"]}) + error = _refusal(contract) + assert error.kind == "lakeformation-tag-definition" + assert "'a-b' and 'a_b'" in str(error) + assert f"aws_lakeformation_lf_tag.{CID}_lf_tag_a_b" in str(error) + _validate_reports(contract, error) + + +@pytest.mark.parametrize("missing", ["table", "database"]) +def test_tags_on_a_binding_with_no_glue_table_are_refused(missing): + """They used to be dropped without a word.""" + contract = _contract({"classification": "confidential"}) + del contract["exposes"][0]["binding"]["location"][missing] + error = _refusal(contract) + assert error.kind == "lakeformation-tag-association" + assert "associates classification" in str(error) + assert "names no Glue table" in str(error) + _validate_reports(contract, error) + + +def test_tags_with_no_value_on_a_binding_with_no_table_are_not_refused(): + contract = _contract({"classification": ""}) + del contract["exposes"][0]["binding"]["location"]["table"] + assert "aws_lakeformation_resource_lf_tags" not in AwsIacPlugin().emit(contract) + assert validate_governance(contract) == ([], []) + + +def test_tags_on_a_format_lake_formation_does_not_govern_are_ignored_as_before(): + """The whole governance.lakeFormation block of a non-Glue binding is ignored, by design.""" + contract = _contract({"classification": "confidential"}) + binding = contract["exposes"][0]["binding"] + binding["format"] = "kinesis_stream" + binding["location"] = {"stream": "orders"} + assert "aws_lakeformation_resource_lf_tags" not in AwsIacPlugin().emit(contract) + assert validate_governance(contract) == ([], []) + + +def test_a_gcp_only_contract_is_not_checked_for_lake_formation_tags(): + """Only the AWS emitter creates the tags; a contract it never runs on is not refused.""" + contract = _contract({}, {"classification": []}) + contract["exposes"][0]["binding"] = { + "platform": "gcp", + "format": "bigquery_table", + "location": {"project": "p", "dataset": "d", "table": "orders"}, + } + errors, _ = validate_governance(contract) + assert not [e for e in errors if "tagDefinitions" in e], errors + + +# --------------------------------------------------------------------------- +# The schemas say so +# --------------------------------------------------------------------------- + + +def _lake_formation_descriptions(): + for path in sorted(SCHEMAS.glob("fluid-schema-*.json")): + defs = json.loads(path.read_text(encoding="utf-8")).get("$defs") or {} + top = ((defs.get("governance") or {}).get("properties") or {}).get("lakeFormation") + per_binding = ((defs.get("bindingGovernance") or {}).get("properties") or {}).get( + "lakeFormation" + ) + if top or per_binding: + yield ( + path.name, + top["properties"]["tagDefinitions"]["description"], + per_binding["properties"]["tags"]["description"], + ) + + +def test_every_schema_that_carries_the_tags_is_covered(): + assert [name for name, _, _ in _lake_formation_descriptions()] == [ + "fluid-schema-0.7.3.json", + "fluid-schema-0.7.4.json", + "fluid-schema-0.7.5.json", + "fluid-schema-0.7.6.json", + ] + + +@pytest.mark.parametrize( + "name,definitions,tags", + list(_lake_formation_descriptions()), + ids=[name for name, _, _ in _lake_formation_descriptions()], +) +def test_every_schema_says_a_key_may_be_owned_outside_the_contract(name, definitions, tags): + # The old claim, which nothing enforced and the emitter assumed. + assert "Keys must reference tag definitions" not in tags + # A key the contract does not define: what the operator must have in place. + assert "must already exist in the account" in tags + assert "needs the Lake Formation ASSOCIATE permission on it" in tags + # A key the contract defines is created by the apply, so in one place only. + assert "must not also be created elsewhere" in tags + assert "must not also be created elsewhere" in definitions + assert "The apply CREATES every tag defined here" in definitions + # The refusals. + assert "must be one of the tag's values" in tags + assert "names no Glue table" in tags + assert "A key with no values is refused" in definitions + assert "one resource name, such as a-b and a_b" in definitions + + +# --------------------------------------------------------------------------- +# ``tofu validate`` (needs tofu on PATH; skips without it) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tags, definitions", + [ + ({"classification": "confidential"}, None), + ({"classification": "confidential", "domain": "sales"}, {"domain": ["sales"]}), + ], + ids=["platform-owned", "platform-owned-beside-a-defined-tag"], +) +def test_the_module_validates_under_tofu(tofu_binary, tofu_env, tmp_path, tags, definitions): + """``tofu init -backend=false`` + ``tofu validate`` on the reported contract.""" + text = build_module(AwsIacPlugin(), _contract(tags, definitions)) + (tmp_path / "main.tf.json").write_text(text, encoding="utf-8") + init = runner.tofu_init(str(tmp_path), backend=False, env=tofu_env) + assert init.ok, f"tofu init failed:\n{init.stderr or init.stdout}" + result = runner.tofu_validate(str(tmp_path), env=tofu_env) + assert result.ok, f"tofu validate failed:\n{result.stderr or result.stdout}" From 889233c39c0495e23506eecd1679986354663cb1 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:18:16 +0200 Subject: [PATCH 2/2] fix(aws): a Lake Formation tag the contract defines is named by reference, not by depends_on The association of a tag the contract defines now names it the way the provider documents, key = aws_lakeformation_lf_tag..key, and carries no depends_on. OpenTofu orders the association after the tag from the reference, and records the tag among its dependencies, so a destroy still removes the association first. A tag the platform owns keeps its literal key and no reference. Every refusal of the previous commit is unchanged. The reference evaluates to the same key, so an association applied from 0.19.0, which emitted literal keys and depends_on, plans no change. Proven with real OpenTofu 1.12.0 and hashicorp/aws 6.67.0 against an in-process moto server, in tests/iac/test_iac_lakeformation_tags_moto.py: - the module 0.19.0 emitted is applied, and the one this commit emits plans "No changes" on the same state; the state then lists both tags as the association's dependencies, and the destroy removes it first; - a tag the platform created beforehand is associated beside one the contract creates, reads back through GetResourceLFTags, plans clean, and is still in the account after the destroy; - a platform-owned key that does not exist passes tofu validate and fails the apply with EntityNotFoundException. moto checks that an associated key exists, not its value, so a wrong value on a platform-owned tag is not proven to fail. iac-tests.yml Stage 1 runs the file and fails if it only skipped; the .secrets.baseline line number of an existing entry in that workflow moves with it. Prior art: the terraform-provider-aws docs for aws_lakeformation_resource_lf_tags reference the tag (key = aws_lakeformation_lf_tag.example.key); OpenTofu's depends_on docs recommend an expression reference over depends_on, which it calls a last resort; cloudposse/terraform-aws-lakeformation wires its associations the same way, with no depends_on on the tags. Validated: the new reference tests go red with the depends_on edges restored (9 failed, the moto no-change test among them), the moto plan gate goes red when the reference names another attribute (the association "must be replaced"), and dropping the ownership guard fails tofu validate and both platform-owned moto tests (12 failed); restored, 35 passed. --- .github/workflows/iac-tests.yml | 11 + .secrets.baseline | 4 +- CHANGELOG.md | 7 +- HONESTLY_TESTED.md | 1 + docs/governance-parity.md | 21 +- fluid_build/iac/providers/aws.py | 47 ++- tests/iac/test_iac_lakeformation_tags.py | 89 ++++- tests/iac/test_iac_lakeformation_tags_moto.py | 378 ++++++++++++++++++ 8 files changed, 510 insertions(+), 48 deletions(-) create mode 100644 tests/iac/test_iac_lakeformation_tags_moto.py diff --git a/.github/workflows/iac-tests.yml b/.github/workflows/iac-tests.yml index 77940923..7ac4bab1 100644 --- a/.github/workflows/iac-tests.yml +++ b/.github/workflows/iac-tests.yml @@ -137,6 +137,17 @@ jobs: -q -p no:randomly --junitxml=state-drift.xml .venv/bin/python scripts/ci/assert_lane_coverage.py state-drift.xml \ --require "state drift=tests/iac/test_iac_state_drift_moto.py" + # Lake Formation tags: a tag the contract defines is named by reference, one + # the platform owns by its literal key. Only a real apply and plan show that + # an association applied with depends_on plans no change under the + # reference, and that a platform-owned tag is associated and survives the + # destroy. Fails if the file only skipped. + - name: Stage 1 — Lake Formation tags apply + plan (tofu vs moto, creds-free) + run: | + .venv/bin/python -m pytest tests/iac/test_iac_lakeformation_tags_moto.py \ + -q -p no:randomly --junitxml=lf-tags-moto.xml + .venv/bin/python scripts/ci/assert_lane_coverage.py lf-tags-moto.xml \ + --require "LF tags moto=tests/iac/test_iac_lakeformation_tags_moto.py" # The default state key names the provider, and the first apply after the # upgrade moves the old key's state with `tofu init -migrate-state`. Only a # real apply, move and plan prove the plan after it changes nothing. diff --git a/.secrets.baseline b/.secrets.baseline index c4cc8d9a..b0b5aff1 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": ".github/workflows/iac-tests.yml", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 247 + "line_number": 258 } ], ".github/workflows/integration.yml": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-10-05T22:13:49Z" + "generated_at": "2026-10-06T14:56:33Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index f491430d..72dcd70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and `fluid apply` failed with "Reference to undeclared resource". Such a key is now a tag the platform owns: associated as written, with no `depends_on`, no warning and no refusal. It must already exist in the account, and the identity running `fluid apply` - needs `ASSOCIATE` on it. The association still waits for the tags the contract - defines, and a contract that defines every tag it associates emits the same module. + needs `ASSOCIATE` on it. A key the contract defines is named by reference, + `key = aws_lakeformation_lf_tag..key`, the provider's documented pattern, + instead of a `depends_on` edge, so the association still waits for the tag. The + reference evaluates to the same key, so an association applied from 0.19.0 plans no + change (applied and planned against moto). Refused at `fluid validate` and at emit, where each used to fail at apply or be dropped: a value the contract's definition of the tag does not allow, compared case-insensitively (`lakeformation-tag-value`); a definition with no values, or two diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 6101fd5d..fa50c4f1 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -45,6 +45,7 @@ Markers declared in `pyproject.toml`. | Stage 3 cross-account proxy | `tests/iac/test_iac_aws_real_cross_account_e2e.py` | ✅ 2/2 before `bucketPolicy` existed — consumer role assumes + LF + S3 bucket policy + Athena SELECT; un-granted principal correctly denied. The consumer is in the deployer's account, so under the new default (`bucketPolicy: cross-account`) it gets **no** bucket-policy statement and the test now asserts that; that version has **not yet been re-run live**. Uses the existing `binding.governance.lakeFormation.grants[]` block. | | Stage 1 LF bucket-policy modes | `tests/iac/test_iac_lakeformation_bucket_policy.py`, `tests/iac/test_iac_lakeformation_bucket_policy_plan.py`, `tests/iac/test_iac_tofu_validate.py::test_lakeformation_bucket_policy_modes_pass_tofu_validate` | ✅ — the three `bucketPolicy` modes rendered (all-grantees byte-identical to the previous emit), `tofu validate` of each, and a real `tofu plan` against moto's STS proving the default keeps a cross-account grantee's statements and plans no bucket policy for a same-account one. | | Stage 1 LF column grants | `tests/iac/test_iac_lakeformation_column_grants.py`, `tests/iac/test_iac_lakeformation_column_permissions.py`, `tests/iac/test_iac_lakeformation_column_grants_plan.py` | ✅ — the three grant shapes rendered (no columns: `table`; `columns`: `table_with_columns.column_names`, no wildcard; `excludedColumns`: `wildcard = true` + `excluded_column_names`; refused at emit: both together, a column name the expose's schema does not declare, an exclusion of every column, and a column limit on a binding with no `location.table`), and a real `tofu plan` against moto of all three. Plan, not validate: the excluded shape without the wildcard passes `tofu validate` and fails `tofu plan` with "Missing required argument", and the file pins that too. A column-limited grant is emitted and planned with `SELECT` only: Lake Formation takes nothing else on `table_with_columns` ("Permissions modification is invalid" on the demo lab's apply, 28 Sept 2026), a `DESCRIBE` beside it is dropped because the column-limited `SELECT` already lets the principal see the table and Lake Formation refuses `DESCRIBE` to a principal holding a partial `SELECT`, and `ALTER`, `DROP`, `DELETE`, `INSERT` or `ALL` beside a column limit is refused at emit, as is a second grant for the same principal on the same table (Lake Formation refuses the table-level permissions to it, a table-level `SELECT` would read the withheld columns, and the provider reads the pair back as one). `DESCRIBE`'s grant option beside a column limit is refused (no resource can carry it). `SELECT`'s grant option is kept beside `columns` and refused beside `excludedColumns`: the guide's pages disagree (the permissions reference: "you can't include the grant option if column filtering is applied"; the console page offers it for simple column-based access), and *Data filtering limitations* is the specific rule followed: "To grant SELECT with the grant option and column filtering, you must use an include list, not an exclude list." **Not proven here**: moto stores any permission on any resource and `tofu plan` checks none, so the permission rules rest on the AWS LF developer guide and that failed apply; the fixed grant has **not yet been applied live**, and `SELECT` with the grant option on an include list has **not been tried live** (a live `GrantPermissions` of it settles the conflict). | +| Stage 1 LF tags | `tests/iac/test_iac_lakeformation_tags.py`, `tests/iac/test_iac_lakeformation_tags_moto.py` | ✅ — a tag the contract defines is named in its association by reference (`key = aws_lakeformation_lf_tag..key`, the provider's documented pattern) with no `depends_on`, and a tag the platform owns by its literal key; the refusals at emit and at `fluid validate`; `tofu validate` of a platform-owned association. A real `tofu apply` / `plan` / `destroy` against moto: an association applied as 0.19.0 emitted it (literal keys and `depends_on`) plans no change under the references, and the state then records the tags as its dependencies; a tag the platform created beforehand is associated beside one the contract creates, reads back through `GetResourceLFTags`, plans clean and survives the destroy; a platform-owned key that does not exist fails the apply with `EntityNotFoundException`. **Not proven here**: moto checks that an associated key exists but not its value, so a wrong value on a platform-owned tag, which Lake Formation refuses, is not proven to fail; moto enforces no `ASSOCIATE` permission; and the platform-owned path has **not been applied live**. | | Stage 1 retention + encryption at rest | `tests/iac/test_iac_aws_retention_encryption.py`, `tests/iac/test_iac_aws_retention_encryption_moto.py`, `tests/iac/test_iac_tofu_validate.py::test_retention_and_kms_pass_tofu_validate`, `tests/cli/test_verify_athena.py` (storage section) | ✅ — `exposes[].lifecycle {retention, expire: true}` and `binding.encryption.kms` (fluid-schema 0.7.6) rendered (prefix-scoped rules, never a whole-bucket rule; key, alias, default SSE-KMS with a bucket key; key policy), `tofu validate` of each shape, and a real `tofu plan` / `apply` / `destroy` against moto: the key policy evaluated against the applying account, an object written with no encryption header landing SSE-KMS under the product key, `fluid verify`'s storage checks passing and then failing on a changed rule and an SSE-S3 object, and dropping the fields deleting the lifecycle configuration and scheduling the key's deletion 7 days out. An existing key is planned against moto too: named by alias, S3 is handed its key ARN; a key pending deletion and an RSA key fail the plan on the SSE configuration's preconditions; the AWS managed key named by its key ARN (which passes the name check) fails it with registerLocation (moto records every key as customer managed, so the test sets its record of alias/aws/s3 to `AWS`). A brownfield bucket carrying an operator's lifecycle rule and SSE-S3 is adopted through `fluid apply`'s own `_adopt_existing`, and the plan shows both configurations as in-place updates. `fluid verify` is covered with botocore stubs, including a key that is not `Enabled` and lifecycle rules filtered by tag, size or a narrower prefix that expire objects sooner. **Not proven here**: moto enforces neither key policies nor Lake Formation, so Athena reading an SSE-KMS location through Lake Formation-vended credentials (the service-linked role's key-policy statement) rests on the AWS documentation and has **not been run live**. | | Stage 1 GCP governance parity (retention, CMEK, column restrictions, mapped principals) | `tests/iac/test_iac_gcp_governance.py`, `tests/iac/test_iac_gcp_governance_plan.py`, `tests/iac/test_iac_aws_column_restrictions.py`, `tests/cli/test_verify_bigquery_governance.py` | ✅ — rendered and `tofu validate`d: partition expiration (never a table TTL), a Cloud KMS key ring, key, service-agent grant and the dataset and table keys, a Data Catalog taxonomy, policy tags and fine-grained readers, and non-authoritative dataset members with the contract's logical principals mapped by `binding.principals` (placeholders and unmapped principals refused). A real `tofu plan` / `apply` with terraform-provider-google against an in-process stand-in for the BigQuery REST API (`tests/iac/_fake_bigquery.py`; the goccy emulator crashes the provider on apply): adding retention or a key to a live table plans its replacement, which the data-loss gate refuses without `--allow-data-loss`; a new retention is in place; the governed table plans clean; the dataset keeps its own access entries. The same stand-in shows a revoked reader is a member destroy the data-loss gate lets through, and that the first apply after the authoritative access list of 0.16.6 and earlier revokes a grant removed in the same change and then plans clean. AWS: the restriction becomes each Lake Formation grant's excluded columns, a real `tofu plan` against moto accepts them, and `fluid verify`'s Lake Formation check runs against moto's stored grants (and against a listed `TableWildcard` grant). `fluid verify` on BigQuery is covered with real `google.cloud.bigquery` Table/Dataset objects and a fake Data Catalog session. **Not proven here**: nothing ran against real BigQuery, Cloud KMS, Data Catalog or Lake Formation; no emulator enforces policy tags, keys or IAM, so a denied principal's refusal, the service agent's use of the key and a load into a tagged column rest on the providers' documentation. | | Stage 2 cross-account (two-account LocalStack) | `tests/integration/test_iac_cross_account_localstack.py` | ✅ 10 tests, gated on `FLUID_IAC_LIVE_XACCT=1` + a second LocalStack Pro carrying `lakeformation`. Producer stack applies in account A (`000000000000`); every grant names a role in account B (`222222222222`) — LocalStack derives the account from the access-key id. Verified live: LF permission + LF prefix-registration + `aws_s3_bucket_policy` all land and read back; the shared-**pool** bucket is referenced via `data.aws_s3_bucket`, its `GetObject` grant is scoped to `location.path` and its `ListBucket` carries the `s3:prefix` condition. Under `ENFORCE_IAM=1` the emitted bucket policy is proven to be the **deciding** access control: with a deliberately broad identity policy on the account-B role, the granted prefix reads and the sibling tenant's prefix is denied — and widening only the bucket policy flips that denial (causal control). **Still NOT proven here**: LF cross-account *authorization* (LocalStack rejects `GrantPermissions` under IAM enforcement even for a registered `DataLakeAdmin` — `docs/upstream-issues/localstack-lakeformation-grant-auth.md`), and cross-account Glue catalog sharing (no AWS RAM in LocalStack; account B gets `EntityNotFoundException`, which is state isolation, not a denial). | diff --git a/docs/governance-parity.md b/docs/governance-parity.md index 482afaa2..23aae546 100644 --- a/docs/governance-parity.md +++ b/docs/governance-parity.md @@ -20,7 +20,7 @@ contract needs `fluidVersion: "0.7.6"` to validate with them. | **Row filters**: `exposes[].policy.authz.rowFilters[] {principal, where, name?}` | A Lake Formation data cells filter per filter (`aws_lakeformation_data_cells_filter`, `row_filter.filter_expression` the predicate, `column_wildcard.excluded_column_names` the principal's excluded columns), and the principal's `SELECT` granted on the filter instead of the table. | A row access policy per filter (`google_bigquery_row_access_policy`, `filter_predicate` the predicate, `grantees` every identity of the principal, whatever it holds: a writer reads rows too), and one more, `fluid_all_rows` (`TRUE`), for every other principal the expose grants anything to, never for a filtered one, since BigQuery shows a principal no policy names no row at all and the union of those that do. | AWS: the column check reads a grant made on a data cells filter, so a filtered principal's excluded columns are still checked. Not checked yet: the filter expression on AWS, and the row access policies on GCP. | | **Governance labels**: `labels`, `exposes[].labels`, `dataClassification`, `sovereignty {jurisdiction, regulatoryFramework, dataResidency}` | The same keys as AWS tags on the bucket and the key, and as parameters on the Glue table. | GCP labels on the dataset, the table and the key: the contract's and the expose's labels, and `fluid_classification`, `fluid_jurisdiction`, `fluid_regulation` (the frameworks joined), `fluid_residency`. | Not checked yet. | | **Access grants**: `accessPolicy.grants[]` | Not emitted: on AWS, access is the binding's `governance.lakeFormation.grants`. `fluid validate` warns only for an aws binding with no Lake Formation grants, where the contract's access intent is unenforced. | One non-authoritative `google_bigquery_dataset_iam_member` per role and member (and `google_storage_bucket_iam_member` for GCS), with the logical principal mapped to its GCP identity. | Not checked yet on either cloud. | -| **Lake Formation tags** (AWS only): `binding.governance.lakeFormation.tags`, and `governance.lakeFormation.tagDefinitions` for the tags the contract creates | One `aws_lakeformation_lf_tag` per defined key, and one `aws_lakeformation_resource_lf_tags` per table with the binding's tags. The association waits for the tags the contract defines (`depends_on`), and a value must be one of its tag's values. A key the contract does not define is a tag the platform owns: associated as written, with no `depends_on` (see "Lake Formation tags the platform owns" below). | Not applicable: these are AWS binding fields. | Not checked yet. | +| **Lake Formation tags** (AWS only): `binding.governance.lakeFormation.tags`, and `governance.lakeFormation.tagDefinitions` for the tags the contract creates | One `aws_lakeformation_lf_tag` per defined key, and one `aws_lakeformation_resource_lf_tags` per table with the binding's tags. The association names each tag the contract defines by reference (`key = aws_lakeformation_lf_tag..key`, the provider's documented pattern), so it waits for the tag with no `depends_on`, and a value must be one of its tag's values. A key the contract does not define is a tag the platform owns: associated by its literal key, with no reference (see "Lake Formation tags the platform owns" below). | Not applicable: these are AWS binding fields. | Not checked yet. | A policy that a binding cannot apply is refused at `fluid validate`, `fluid plan` and `fluid apply`, never dropped: retention, a key or a column restriction on a GCP @@ -288,13 +288,15 @@ creates the tag keys and their values, and each domain is given `ASSOCIATE` on t to tag its own tables. A contract follows either model, per key: * A key in `governance.lakeFormation.tagDefinitions` is a tag the contract creates. - The apply creates it, the association waits for it, and the association's value - must be one of its values (compared case-insensitively, as Lake Formation stores + The apply creates it, and the association names it by reference + (`key = aws_lakeformation_lf_tag..key`), so OpenTofu creates the tag first and + removes the association first on destroy. The association's value must be one of + its values (compared case-insensitively, as Lake Formation stores keys and values lower-cased). Do not also create that key anywhere else in the account. * A key in a binding's `governance.lakeFormation.tags` that the contract does not - define is a tag the platform owns. It is associated as written, with no - `depends_on` and no warning. It must already exist in the account, and the identity + define is a tag the platform owns. It is associated by its literal key, with no + reference and no warning, and a destroy leaves the tag in place. It must already exist in the account, and the identity running `fluid apply` needs `ASSOCIATE` on it. terraform-provider-aws has no data source for an LF-tag, so nothing checks the key or the value before apply: Lake Formation does, at apply. @@ -341,6 +343,15 @@ to tag its own tables. A contract follows either model, per key: * `tofu validate` accepts a Lake Formation tag association on a tag the platform owns, alone and beside a tag the contract defines (`tests/iac/test_iac_lakeformation_tags.py`). Before, it failed with "Reference to undeclared resource". +* A real `tofu apply` and `tofu plan` against moto + (`tests/iac/test_iac_lakeformation_tags_moto.py`): an association applied as 0.19.0 + emitted it, with literal keys and `depends_on`, plans no change once it names its + tags by reference; a tag the platform created beforehand is associated beside one + the contract creates, reads back from Lake Formation and survives the destroy; and a + platform-owned key that does not exist fails the apply with + `EntityNotFoundException`. moto checks that an associated key exists, not that its + value is one of the tag's values, so a wrong value on a platform-owned tag is not + proven to fail. * **Measured against real Google Cloud**, 4 October 2026, on 0.18.0. In a demo lab, two lineage chains of eleven products were applied with `fluid apply --env gcp` from generated Jenkins pipelines, as a deploy service account reached by Workload Identity diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index 3a214c22..b3029112 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -392,7 +392,7 @@ def emit( # Account-level Lake Formation settings: admins + LF-tag # definitions. Emitted once per contract, before per-exposure # resources, so a resource_lf_tags association can tell the tags - # this module creates (and waits for) from those it does not. + # this module creates (and references) from those it does not. _emit_lf_account_settings(resources, contract, cid, base_tags) for index, exposure in enumerate(contract.get("exposes") or []): @@ -1530,7 +1530,7 @@ def _wire_aws_deps(resources: Dict[str, Any], cid: str) -> None: # ``admins`` → ``aws_lakeformation_data_lake_settings``, # ``tagDefinitions`` → one ``aws_lakeformation_lf_tag`` per key. # Must run before per-resource ``resource_lf_tags`` associations, which -# wait for the tags it wrote (see :func:`lf_tag_definitions`). +# reference the tags it wrote (see :func:`lf_tag_definitions`). # # * ``_emit_lakeformation`` — fires per AWS exposure. Honours # ``binding.governance.lakeFormation``: @@ -2474,8 +2474,8 @@ def _emit_lakeformation( ``governance.lakeFormation`` block. ``lf_tags`` is :func:`lf_tag_associations`: the table's one - ``aws_lakeformation_resource_lf_tags``, ordered after the tags this module - creates and on nothing for a tag owned outside the contract. + ``aws_lakeformation_resource_lf_tags``, naming a tag this module creates by + reference to its key and a tag owned outside the contract by its literal key. ``row_filters`` is :func:`lf_row_filters`: a grant that carries one reads through a data cells filter (the filter's predicate, and the grant's excluded columns as its @@ -2684,15 +2684,28 @@ def _emit_lf_tag_association( ) -> None: """The table's ``aws_lakeformation_resource_lf_tags``: :func:`lf_tag_associations`. - A tag must exist before it is associated. The association waits only for the - tags this module creates: the ``aws_lakeformation_lf_tag`` of the contract's - ``tagDefinitions``, which :func:`_emit_lf_account_settings` wrote before any - exposure, and only when that resource is this key's (one resource name can - stand for ``a-b`` and ``a_b``). A key the contract does not define is a tag - owned outside the contract, already in the account: an edge to it would name a - resource the module does not declare, and ``tofu validate`` refuses the whole - module. The key stays a plain string, which the renderer escapes. + A tag must exist before it is associated. A key this module creates names its + tag by reference, ``key = aws_lakeformation_lf_tag..key``, as the provider's + own example does, so OpenTofu orders the association after the tag from the + expression (OpenTofu recommends a reference over ``depends_on``). That is the + ``aws_lakeformation_lf_tag`` of the contract's ``tagDefinitions``, which + :func:`_emit_lf_account_settings` wrote before any exposure, and only when that + resource's key is this key (one resource name can stand for ``a-b`` and + ``a_b``). The reference evaluates to the same string, so a deployed association + plans no change. A key the contract does not define is a tag owned outside the + contract, already in the account: a reference to it would name a resource the + module does not declare, and ``tofu validate`` refuses the whole module. It stays + a plain string, which the renderer escapes. """ + owned = resources.get("aws_lakeformation_lf_tag") or {} + lf_tag: List[Dict[str, Any]] = [] + for tag in lf_tags: + name = _lf_tag_name(cid, tag["key"]) + if (owned.get(name) or {}).get("key") == tag["key"]: + # ``name`` is ``safe_ident`` output, so the expression is emitter text. + lf_tag.append({**tag, "key": tofu_ref(f"aws_lakeformation_lf_tag.{name}.key")}) + else: + lf_tag.append(dict(tag)) assoc: Dict[str, Any] = { "table": [ { @@ -2700,16 +2713,8 @@ def _emit_lf_tag_association( "name": tofu_ref(f"aws_glue_catalog_table.{table_key}.name"), } ], - "lf_tag": [dict(t) for t in lf_tags], + "lf_tag": lf_tag, } - owned = resources.get("aws_lakeformation_lf_tag") or {} - depends_on: List[str] = [] - for tag in lf_tags: - name = _lf_tag_name(cid, tag["key"]) - if (owned.get(name) or {}).get("key") == tag["key"]: - depends_on.append(f"aws_lakeformation_lf_tag.{name}") - if depends_on: - assoc["depends_on"] = depends_on resources.setdefault("aws_lakeformation_resource_lf_tags", {})[ safe_ident(f"{cid}_lf_tags_{table}") ] = assoc diff --git a/tests/iac/test_iac_lakeformation_tags.py b/tests/iac/test_iac_lakeformation_tags.py index 23a2c8e1..a7b2a7e0 100644 --- a/tests/iac/test_iac_lakeformation_tags.py +++ b/tests/iac/test_iac_lakeformation_tags.py @@ -21,6 +21,9 @@ ``depends_on`` on an ``aws_lakeformation_lf_tag`` the module never declares, so ``tofu validate`` failed ``Reference to undeclared resource``. Such a key is now a platform-owned tag: associated as written, with no edge, no warning and no refusal. +A key the contract defines names its tag by reference, +``key = aws_lakeformation_lf_tag..key``, the provider's documented pattern, and +the association carries no ``depends_on``. What the module cannot create or associate as declared is refused, at emit and by ``fluid validate`` with the same message: a value the contract's definition of the @@ -29,13 +32,15 @@ in every version that carries the fields. The tests marked ``tofu`` need ``tofu`` on PATH and the provider from the registry -(no AWS credentials), like ``test_iac_aws_validate.py``; they skip without it. +(no AWS credentials), like ``test_iac_aws_validate.py``; they skip without it. The +apply and plan against moto are in ``test_iac_lakeformation_tags_moto.py``. """ from __future__ import annotations import copy import json +import re from pathlib import Path from typing import Any, Dict, Iterator, List, Mapping @@ -51,6 +56,7 @@ CID = "gold_sales_orders" ASSOC = f"{CID}_lf_tags_orders" +TAG_REF = re.compile(r"^\$\{(aws_lakeformation_lf_tag\.[A-Za-z0-9_]+)\.key\}$") SCHEMAS = Path(fluid_build.__file__).resolve().parent / "schemas" @@ -98,11 +104,26 @@ def _depends_on(document: Mapping[str, Any]) -> Iterator[tuple]: yield f"{rtype}.{name}", dep +def _tag_references(document: Mapping[str, Any]) -> Iterator[tuple]: + """Each association ``lf_tag`` key that is a reference: ``(owner, tag address)``.""" + bodies = (document.get("resource") or {}).get("aws_lakeformation_resource_lf_tags") or {} + for name, body in bodies.items(): + for tag in body.get("lf_tag") or []: + match = TAG_REF.match(tag["key"]) + if match: + yield f"aws_lakeformation_resource_lf_tags.{name}", match.group(1) + + def _dangling(contract: Mapping[str, Any]) -> List[str]: + """Every ``depends_on`` and every tag-key reference that names no declared resource.""" document = json.loads(build_module(AwsIacPlugin(), contract)) declared = _declared(document) return [ f"{owner} depends_on {dep}" for owner, dep in _depends_on(document) if dep not in declared + ] + [ + f"{owner} references {ref}" + for owner, ref in _tag_references(document) + if ref not in declared ] @@ -150,7 +171,7 @@ def test_a_platform_owned_tag_validates_with_no_error_and_no_warning(): assert validate_governance(_contract({"classification": "confidential"})) == ([], []) -def test_only_the_keys_this_module_defines_become_dependencies(): +def test_only_the_keys_this_module_defines_are_references(): """One key defined by the contract, one owned by the platform.""" contract = _contract( {"classification": "confidential", "domain": "sales"}, @@ -158,9 +179,34 @@ def test_only_the_keys_this_module_defines_become_dependencies(): ) assert _dangling(contract) == [] assoc = _association(contract) - assert {t["key"] for t in assoc["lf_tag"]} == {"classification", "domain"} - # The defined key is still ordered before the association, and only it. - assert assoc["depends_on"] == [f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain"] + # The defined key names its tag, so the association waits for it; the + # platform's key is the literal the contract wrote. + assert assoc["lf_tag"] == [ + {"key": "classification", "value": "confidential"}, + {"key": f"${{aws_lakeformation_lf_tag.{CID}_lf_tag_domain.key}}", "value": "sales"}, + ] + assert "depends_on" not in assoc + + +@pytest.mark.parametrize( + "tags, definitions", + [ + ({"classification": "pii_low"}, {"classification": ["pii_low"]}), + ({"classification": "confidential", "domain": "sales"}, {"domain": ["sales"]}), + ({"classification": "confidential"}, None), + ({"a-b": "x"}, {"a_b": ["y"]}), + ], + ids=["defined", "mixed", "platform-owned", "resource-name-collision"], +) +def test_an_association_never_carries_depends_on(tags, definitions): + """OpenTofu prefers a reference to ``depends_on``; the ordering comes from ``key``.""" + document = json.loads(build_module(AwsIacPlugin(), _contract(tags, definitions))) + assert list(_depends_on(document)) == [] + defined = set(definitions or {}) + referenced = {ref for _, ref in _tag_references(document)} + assert referenced == { + f"aws_lakeformation_lf_tag.{CID}_lf_tag_{key}" for key in defined if key in tags + } def test_a_defined_tag_still_precedes_its_association_control(): @@ -169,8 +215,13 @@ def test_a_defined_tag_still_precedes_its_association_control(): assert _dangling(contract) == [] -def test_a_contract_that_defines_its_tags_emits_what_it_did(): - """The association body of a contract that defines every key, as main emitted it.""" +def test_a_contract_that_defines_its_tags_names_each_tag_by_reference(): + """The association body of a contract that defines every key. + + ``main`` (63f3b40) emitted the literal keys and ``depends_on`` on both tags. The + references evaluate to the same keys, so an association deployed from that module + plans no change (``test_iac_lakeformation_tags_moto.py`` applies one and plans this). + """ contract = _contract( {"classification": "pii_low", "domain": "sales"}, {"classification": ["public", "pii_low", "pii_high"], "domain": ["sales", "marketing"]}, @@ -186,13 +237,12 @@ def test_a_contract_that_defines_its_tags_emits_what_it_did(): } assert resources["aws_lakeformation_resource_lf_tags"] == { ASSOC: { - "depends_on": [ - f"aws_lakeformation_lf_tag.{CID}_lf_tag_classification", - f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain", - ], "lf_tag": [ - {"key": "classification", "value": "pii_low"}, - {"key": "domain", "value": "sales"}, + { + "key": f"${{aws_lakeformation_lf_tag.{CID}_lf_tag_classification.key}}", + "value": "pii_low", + }, + {"key": f"${{aws_lakeformation_lf_tag.{CID}_lf_tag_domain.key}}", "value": "sales"}, ], "table": [ { @@ -204,15 +254,17 @@ def test_a_contract_that_defines_its_tags_emits_what_it_did(): } -def test_a_key_with_an_empty_value_adds_no_edge(): - """The edges come from the associations that carry a value, not from every key.""" +def test_a_key_with_an_empty_value_adds_no_reference(): + """The references come from the associations that carry a value, not from every key.""" contract = _contract( {"classification": "", "domain": "sales"}, {"classification": ["public"], "domain": ["sales"]}, ) assoc = _association(contract) - assert assoc["lf_tag"] == [{"key": "domain", "value": "sales"}] - assert assoc["depends_on"] == [f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain"] + assert assoc["lf_tag"] == [ + {"key": f"${{aws_lakeformation_lf_tag.{CID}_lf_tag_domain.key}}", "value": "sales"} + ] + assert "depends_on" not in assoc def test_a_key_that_shares_a_defined_tags_resource_name_is_not_ordered_on_it(): @@ -253,7 +305,8 @@ def test_a_value_the_definition_does_not_allow_is_refused(): def test_a_value_is_compared_case_folded(value): """Lake Formation stores keys and values lower-cased.""" contract = _contract({"classification": value}, {"classification": ["Public", "internal"]}) - assert _association(contract)["lf_tag"] == [{"key": "classification", "value": value}] + key = f"${{aws_lakeformation_lf_tag.{CID}_lf_tag_classification.key}}" + assert _association(contract)["lf_tag"] == [{"key": key, "value": value}] assert validate_governance(contract) == ([], []) diff --git a/tests/iac/test_iac_lakeformation_tags_moto.py b/tests/iac/test_iac_lakeformation_tags_moto.py new file mode 100644 index 00000000..749d51ed --- /dev/null +++ b/tests/iac/test_iac_lakeformation_tags_moto.py @@ -0,0 +1,378 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lake Formation tags, applied and planned with real OpenTofu against moto. + +A tag the contract defines is named by reference in the association, +``key = aws_lakeformation_lf_tag..key``, as the provider's own example does; a +tag the platform owns is named by its literal key. Proven here with a real +``tofu apply`` and ``tofu plan`` against an in-process moto server, which implements +``CreateLFTag``, ``AddLFTagsToResource`` and ``GetResourceLFTags``: + +* an association deployed from the module ``main`` (63f3b40) emitted, literal keys + and ``depends_on`` on the tags, plans no change under the references, and the + state then records the tags as the association's dependencies; +* a tag the platform created beforehand is associated beside one the contract + creates, reads back from Lake Formation, plans clean, and survives a destroy that + removes the contract's own tag; +* a platform-owned key that does not exist fails the apply with Lake Formation's + ``EntityNotFoundException``. + +moto checks that an associated key exists, not that its value is one of the tag's +values, which Lake Formation also refuses ("Tag or tag value does not exist"). The +value of a tag the contract defines is refused at emit +(``test_iac_lakeformation_tags.py``); the value of a platform-owned tag is not proven +here. + +Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is installed; +``iac-tests.yml`` Stage 1 installs both and fails if this file only skipped. +""" + +from __future__ import annotations + +import copy +import json +import os +import re +import subprocess +import time +import urllib.request +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping, Optional +from unittest import mock + +import pytest + +from fluid_build.iac import build_module, get_iac_plugin, runner +from fluid_build.iac.credentials import build_tofu_env + +pytestmark = [pytest.mark.integration, pytest.mark.aws, pytest.mark.provider] + +REGION = "us-east-1" +CID = "gold_sales_orders" +ASSOC = f"aws_lakeformation_resource_lf_tags.{CID}_lf_tags_orders" +TAG_REF = re.compile(r"^\$\{(aws_lakeformation_lf_tag\.[A-Za-z0-9_]+)\.key\}$") + +# The association ``main`` (63f3b40) emitted for the contract that defines both of +# its tags: literal keys, ordered by ``depends_on``. +MAIN_ASSOCIATION = { + "depends_on": [ + f"aws_lakeformation_lf_tag.{CID}_lf_tag_classification", + f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain", + ], + "lf_tag": [ + {"key": "classification", "value": "pii_low"}, + {"key": "domain", "value": "sales"}, + ], + "table": [ + { + "database_name": f"${{aws_glue_catalog_table.{CID}_sales_orders.database_name}}", + "name": f"${{aws_glue_catalog_table.{CID}_sales_orders.name}}", + } + ], +} + + +def _have_moto_server() -> bool: + try: + from moto.server import ThreadedMotoServer # noqa: F401 + + return True + except Exception: # noqa: BLE001 — Flask (the `server` extra) may be absent + return False + + +pytestmark.append( + pytest.mark.skipif( + runner.tofu_path() is None or not _have_moto_server(), + reason="needs `tofu` on PATH + moto server extra (pip install 'moto[glue,server]')", + ) +) + + +@pytest.fixture +def moto_endpoint() -> Iterator[str]: + """A moto server per test, reset first: moto's records are process-wide.""" + from moto.server import ThreadedMotoServer + + server = ThreadedMotoServer(port=0, verbose=False) + server.start() + try: + _, port = server.get_host_and_port() + endpoint = f"http://127.0.0.1:{port}" + reset = urllib.request.Request(f"{endpoint}/moto-api/reset", method="POST") + with urllib.request.urlopen(reset, timeout=30) as answered: # noqa: S310 — local moto + assert answered.status == 200 + yield endpoint + finally: + server.stop() + + +@pytest.fixture(scope="module") +def tofu_env(tmp_path_factory: pytest.TempPathFactory) -> Dict[str, str]: + """No way to reach a real account: every ``AWS_*`` variable is dropped, the + endpoints are moto's and the keys are dummies; a shared plugin cache.""" + env = {k: v for k, v in build_tofu_env().items() if not k.startswith("AWS_")} + env.setdefault("TF_PLUGIN_CACHE_DIR", str(tmp_path_factory.mktemp("tofu-plugin-cache"))) + env["AWS_CONFIG_FILE"] = "/dev/null" + env["AWS_SHARED_CREDENTIALS_FILE"] = "/dev/null" + env["AWS_EC2_METADATA_DISABLED"] = "true" + return env + + +def _provider_override(endpoint: str) -> Dict[str, Any]: + services = ("s3", "sts", "iam", "glue", "lakeformation") + return { + "provider": { + "aws": { + "region": REGION, + "access_key": "testing", + "secret_key": "testing", # pragma: allowlist secret — moto dummy + "skip_credentials_validation": True, + "skip_metadata_api_check": True, + "skip_requesting_account_id": True, + "s3_use_path_style": True, + "endpoints": {svc: endpoint for svc in services}, + } + } + } + + +def _contract( + bucket: str, tags: Mapping[str, str], definitions: Optional[Mapping[str, List[str]]] = None +) -> Dict[str, Any]: + contract: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "id": "gold.sales.orders", + "exposes": [ + { + "exposeId": "orders", + "binding": { + "platform": "aws", + "format": "parquet", + # No region: the sidecar provider block owns it. + "location": { + "database": "sales", + "table": "orders", + "bucket": bucket, + "path": "gold/orders/", + }, + "governance": {"lakeFormation": {"tags": dict(tags)}}, + }, + "contract": {"schema": [{"name": "order_id", "type": "string"}]}, + } + ], + } + if definitions is not None: + contract["governance"] = {"lakeFormation": {"tagDefinitions": copy.deepcopy(definitions)}} + return contract + + +def _emit(contract: Dict[str, Any], endpoint: str) -> Dict[str, Any]: + """The module as forge-cli emits it for an emulator. + + Emitted with ``AWS_ENDPOINT_URL`` set, as an emulator user runs it: the provider + skips requesting the account id, so each Glue resource names its catalog + (``aws._emulator_catalog_id``); provider 6 refuses to read back a Glue id with an + empty one. + """ + with mock.patch.dict(os.environ, {"AWS_ENDPOINT_URL": endpoint}): + return json.loads(build_module(get_iac_plugin("aws"), contract)) + + +def _as_main_emitted(module: Dict[str, Any]) -> Dict[str, Any]: + """``module`` with each association as ``main`` emitted it: every key that names a + tag by reference is that tag's literal key, and the association ``depends_on`` it.""" + out = copy.deepcopy(module) + tags = out["resource"].get("aws_lakeformation_lf_tag") or {} + for body in out["resource"]["aws_lakeformation_resource_lf_tags"].values(): + depends_on = [] + for tag in body["lf_tag"]: + match = TAG_REF.match(tag["key"]) + if match: + depends_on.append(match.group(1)) + tag["key"] = tags[match.group(1).split(".", 1)[1]]["key"] + if depends_on: + body["depends_on"] = depends_on + return out + + +def _write(module: Dict[str, Any], workdir: Path, endpoint: str) -> None: + """``main.tf.json``, and moto's routing as a ``provider_override.tf.json`` sidecar, + which merges into the provider block emitted for an emulator.""" + workdir.mkdir(parents=True, exist_ok=True) + (workdir / "main.tf.json").write_text(json.dumps(module, indent=2)) + (workdir / "provider_override.tf.json").write_text(json.dumps(_provider_override(endpoint))) + + +def _tofu(workdir: Path, env: Dict[str, str], *args: str) -> "subprocess.CompletedProcess[str]": + return subprocess.run( + [str(runner.tofu_path()), *args], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +def _init(workdir: Path, env: Dict[str, str]) -> None: + # A plugin cache shared by pytest-xdist workers is not safe for concurrent + # installs; OpenTofu refuses the lock instead of waiting, so that is retried. + for attempt in range(4): + done = _tofu(workdir, env, "init", "-backend=false", "-input=false", "-no-color") + if done.returncode == 0 or "unable to acquire file lock" not in done.stdout + done.stderr: + break + time.sleep(2 * (attempt + 1)) + assert done.returncode == 0, f"tofu init failed:\n{done.stdout}\n{done.stderr}" + + +def _apply(workdir: Path, env: Dict[str, str]) -> "subprocess.CompletedProcess[str]": + return _tofu(workdir, env, "apply", "-auto-approve", "-input=false", "-no-color") + + +def _applied(workdir: Path, env: Dict[str, str]) -> None: + done = _apply(workdir, env) + assert done.returncode == 0, f"tofu apply failed:\n{done.stdout}\n{done.stderr}" + + +def _plans_no_change(workdir: Path, env: Dict[str, str]) -> None: + """``tofu plan -detailed-exitcode``: 0 is no change, 2 is a change.""" + done = _tofu(workdir, env, "plan", "-detailed-exitcode", "-input=false", "-no-color") + assert done.returncode == 0, f"tofu plan is not clean:\n{done.stdout}\n{done.stderr}" + assert "No changes." in done.stdout + + +def _dependencies(workdir: Path, env: Dict[str, str], address: str) -> List[str]: + """The dependencies the state records for ``address``, which order its destroy.""" + pulled = _tofu(workdir, env, "state", "pull") + assert pulled.returncode == 0, pulled.stderr + for res in json.loads(pulled.stdout)["resources"]: + if f"{res['type']}.{res['name']}" == address: + return list(res["instances"][0].get("dependencies") or []) + raise AssertionError(f"{address} is not in the state") + + +def _lakeformation(endpoint: str) -> Any: + import boto3 + + return boto3.client( + "lakeformation", + endpoint_url=endpoint, + aws_access_key_id="testing", + aws_secret_access_key="testing", # noqa: S106 # pragma: allowlist secret — moto dummy + region_name=REGION, + ) + + +def _table_tags(endpoint: str) -> Dict[str, List[str]]: + """The LF-tags Lake Formation holds on ``sales.orders``.""" + found = _lakeformation(endpoint).get_resource_lf_tags( + Resource={"Table": {"DatabaseName": "sales", "Name": "orders"}} + ) + return {t["TagKey"]: t["TagValues"] for t in found.get("LFTagsOnTable") or []} + + +def _account_tags(endpoint: str) -> Dict[str, List[str]]: + return {t["TagKey"]: t["TagValues"] for t in _lakeformation(endpoint).list_lf_tags()["LFTags"]} + + +def test_a_deployed_contract_that_defines_its_tags_plans_no_change( + tmp_path: Path, moto_endpoint: str, tofu_env: Dict[str, str] +) -> None: + """Applied as ``main`` emitted it, planned as this branch emits it: no change.""" + contract = _contract( + "lf-tags-defined-lake", + {"classification": "pii_low", "domain": "sales"}, + {"classification": ["public", "pii_low", "pii_high"], "domain": ["sales", "marketing"]}, + ) + module = _emit(contract, moto_endpoint) + before = _as_main_emitted(module) + # The module main emitted differs from this one only in the association. + assert before["resource"]["aws_lakeformation_resource_lf_tags"] == { + ASSOC.split(".", 1)[1]: MAIN_ASSOCIATION + } + assert { + k: v for k, v in before["resource"].items() if k != "aws_lakeformation_resource_lf_tags" + } == {k: v for k, v in module["resource"].items() if k != "aws_lakeformation_resource_lf_tags"} + # This one names each tag by reference and has no depends_on. + (assoc,) = module["resource"]["aws_lakeformation_resource_lf_tags"].values() + assert all(TAG_REF.match(t["key"]) for t in assoc["lf_tag"]), assoc + assert "depends_on" not in assoc + + _write(before, tmp_path, moto_endpoint) + _init(tmp_path, tofu_env) + _applied(tmp_path, tofu_env) + assert _table_tags(moto_endpoint) == {"classification": ["pii_low"], "domain": ["sales"]} + + _write(module, tmp_path, moto_endpoint) + _plans_no_change(tmp_path, tofu_env) + _applied(tmp_path, tofu_env) + # The references, not a depends_on, put the tags among the association's + # dependencies, so the destroy removes the association first. + assert { + f"aws_lakeformation_lf_tag.{CID}_lf_tag_classification", + f"aws_lakeformation_lf_tag.{CID}_lf_tag_domain", + } <= set(_dependencies(tmp_path, tofu_env, ASSOC)) + + done = _tofu(tmp_path, tofu_env, "destroy", "-auto-approve", "-input=false", "-no-color") + assert done.returncode == 0, f"tofu destroy failed:\n{done.stdout}\n{done.stderr}" + assert _account_tags(moto_endpoint) == {} + + +def test_a_platform_owned_tag_is_associated_beside_a_defined_one( + tmp_path: Path, moto_endpoint: str, tofu_env: Dict[str, str] +) -> None: + """The platform's tag exists beforehand; the contract creates only its own.""" + _lakeformation(moto_endpoint).create_lf_tag( + TagKey="classification", TagValues=["confidential", "public"] + ) + contract = _contract( + "lf-tags-platform-lake", + {"classification": "confidential", "domain": "sales"}, + {"domain": ["sales", "marketing"]}, + ) + module = _emit(contract, moto_endpoint) + assert list(module["resource"]["aws_lakeformation_lf_tag"]) == [f"{CID}_lf_tag_domain"] + + _write(module, tmp_path, moto_endpoint) + _init(tmp_path, tofu_env) + _applied(tmp_path, tofu_env) + assert _table_tags(moto_endpoint) == {"classification": ["confidential"], "domain": ["sales"]} + _plans_no_change(tmp_path, tofu_env) + + done = _tofu(tmp_path, tofu_env, "destroy", "-auto-approve", "-input=false", "-no-color") + assert done.returncode == 0, f"tofu destroy failed:\n{done.stdout}\n{done.stderr}" + # The contract's tag goes with the state; the platform's tag stays. + assert _account_tags(moto_endpoint) == {"classification": ["confidential", "public"]} + + +def test_a_platform_owned_tag_that_does_not_exist_fails_the_apply( + tmp_path: Path, moto_endpoint: str, tofu_env: Dict[str, str] +) -> None: + """``tofu validate`` cannot know; Lake Formation refuses it at apply.""" + module = _emit( + _contract("lf-tags-missing-lake", {"classification": "confidential"}), moto_endpoint + ) + _write(module, tmp_path, moto_endpoint) + _init(tmp_path, tofu_env) + assert _tofu(tmp_path, tofu_env, "validate", "-no-color").returncode == 0 + + done = _apply(tmp_path, tofu_env) + assert done.returncode != 0, f"tofu apply should have failed:\n{done.stdout}" + said = " ".join((done.stdout + done.stderr).split()) + assert "tag key:classification" in said, said + assert "EntityNotFoundException" in said, said + listed = _tofu(tmp_path, tofu_env, "state", "list") + assert ASSOC not in listed.stdout.split()