diff --git a/README.md b/README.md index b73a746..1ea714e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The site is organised the way a reader uses it (get started, tutorials, how-to g ## Current Versioning -- Current CLI release documented here: `0.18.1` +- Current CLI release documented here: `0.23.0` - Contract schema: `fluidVersion: 0.7.5` is the stable default; `0.7.6` is an opt-in preview Those are different on purpose. `fluid version` reports the installed CLI release, while `fluidVersion` inside a contract selects the contract schema version. diff --git a/docs/.vuepress/cli-version.json b/docs/.vuepress/cli-version.json index 93cc477..6120cc9 100644 --- a/docs/.vuepress/cli-version.json +++ b/docs/.vuepress/cli-version.json @@ -1,6 +1,6 @@ { - "supportedCliVersion": "0.18.1", - "supportedCliInstallSpec": "data-product-forge==0.18.1", + "supportedCliVersion": "0.23.0", + "supportedCliInstallSpec": "data-product-forge==0.23.0", "supportedFluidContractVersion": "0.7.5", "quickstartScaffoldVersion": "0.7.5", "supportedSdkVersion": "0.10.0", diff --git a/docs/advanced/project-dotenv-trust.md b/docs/advanced/project-dotenv-trust.md index 031f0b6..bd70720 100644 --- a/docs/advanced/project-dotenv-trust.md +++ b/docs/advanced/project-dotenv-trust.md @@ -75,13 +75,12 @@ What it may not set is any variable that project file has nothing to supply there. The full list is `fluid_build/credentials/protected_env.py` in forge-cli. -::: tip Two more in the next release after `0.22.0` +::: tip Two more since `0.23.0` `FLUID_ACQUISITION_SECRET_REF_ALLOWLIST`, which allows an acquisition source's `env://` reference against a remote host, and `TESTCONTAINERS_HOST_OVERRIDE`, which rewrites a -contract-declared loopback host to a reachable one, join the list on forge-cli `main` -([#751](https://github.com/Agenticstiger/forge-cli/pull/751)). Both decide where a credential -may be sent, so neither may come from a file beside the contract. The release is a minor bump -from `0.22.0` and its number is not settled yet. +contract-declared loopback host to a reachable one, joined the list in `0.23.0` +([forge-cli #751](https://github.com/Agenticstiger/forge-cli/pull/751)). Both decide where a +credential may be sent, so neither may come from a file beside the contract. ::: ### An unlisted name fails closed diff --git a/docs/cli/README.md b/docs/cli/README.md index 0bd093c..2e23a33 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -1,6 +1,6 @@ # CLI Reference -This section tracks the command surface of `forge-cli` `0.18.1`, the release `pip install data-product-forge` installs. `fluid --help` shows the promoted commands. +This section tracks the command surface of `forge-cli` `0.23.0`, the release `pip install data-product-forge` installs. `fluid --help` shows the promoted commands. ::: tip Looking up *how to do something* (not *what a command does*)? The page you actually want is [**CLI by task →**](./tasks/) — narrative walkthroughs organized around things you might be trying to accomplish, like "deploy to a new cloud", "add quality rules", "debug a failed run", or "add agent governance". Each task page links to the relevant commands. **This** page is the command index, grouped by purpose — better for lookup, worse for onboarding. @@ -8,7 +8,7 @@ The page you actually want is [**CLI by task →**](./tasks/) — narrative walk ## Read this first -- CLI release examples in this section use `0.18.1` +- CLI release examples in this section use `0.18.1` - Contract examples use `fluidVersion: 0.7.5`, the stable schema, which `fluid init --quickstart` emits. `0.7.6` is a preview you opt into per contract - `fluid version` and `fluidVersion` are different things - The pinned CLI version is recorded in [`docs/.vuepress/cli-version.json`](https://github.com/Agenticstiger/forge_docs/blob/main/docs/.vuepress/cli-version.json) and enforced by the [`cli-consistency`](https://github.com/Agenticstiger/forge_docs/actions/workflows/cli-consistency.yml) workflow. diff --git a/docs/cli/contract-tests.md b/docs/cli/contract-tests.md index fc0b862..860bf15 100644 --- a/docs/cli/contract-tests.md +++ b/docs/cli/contract-tests.md @@ -87,7 +87,7 @@ The contract is loaded the way `fluid plan` loads it, with `$ref` fragments and ```yaml - name: Schema gate run: | - pip install data-product-forge==0.18.1 + pip install data-product-forge==0.23.0 fluid contract-tests contract.fluid.yaml --baseline baseline.schema.json ``` diff --git a/docs/cli/mission.md b/docs/cli/mission.md index 5e87caf..054e22b 100644 --- a/docs/cli/mission.md +++ b/docs/cli/mission.md @@ -249,7 +249,7 @@ cosmetics ([`0.13.0` security fixes](../RELEASE_NOTES_0.13.0.md#security)). ```yaml - name: Mission gate run: | - pip install data-product-forge==0.18.1 + pip install data-product-forge==0.23.0 fluid mission check quality-coverage contract.fluid.yaml ``` diff --git a/docs/cli/scaffold-ci.md b/docs/cli/scaffold-ci.md index 1c6b253..907d2e1 100644 --- a/docs/cli/scaffold-ci.md +++ b/docs/cli/scaffold-ci.md @@ -41,7 +41,7 @@ The generated file installs `data-product-forge` from PyPI (`pip install --quiet | publish | On `main` with `DMM_API_URL` set: `fluid publish $CONTRACT --catalog $CATALOG`. | ::: warning Pin the version in a pipeline that holds credentials -An unpinned `pip install data-product-forge` takes whatever release is latest when the job runs, and the same job holds the credentials the apply stage uses. After generating, edit the install line to pin the version you tested, for example `pip install --quiet "data-product-forge==0.18.1"`, and move the pin deliberately when you upgrade. [`fluid generate ci`](./generate.md#fluid-generate-ci) pins the install to the version that generated the file. +An unpinned `pip install data-product-forge` takes whatever release is latest when the job runs, and the same job holds the credentials the apply stage uses. After generating, edit the install line to pin the version you tested, for example `pip install --quiet "data-product-forge==0.23.0"`, and move the pin deliberately when you upgrade. [`fluid generate ci`](./generate.md#fluid-generate-ci) pins the install to the version that generated the file. ::: The settings are `CONTRACT` (the path you passed), `PROVIDER` (default `default`), `BUILD_ID` (default empty; set it to a `builds[].id` to run that build), `CATALOG` (default `datamesh-manager`), `AIRFLOW_DAGS_DEST` and `DMM_API_URL`. GitLab reads them as CI/CD variables, GitHub Actions as `env`, repository `vars` and `secrets`, Jenkins from the `environment` block. diff --git a/docs/cli/version.md b/docs/cli/version.md index c9de18e..718b7ef 100644 --- a/docs/cli/version.md +++ b/docs/cli/version.md @@ -69,7 +69,7 @@ Python: 3.11.15 (darwin) System: Darwin arm64 ``` -`fluid --version` is the top-level flag and prints one line, `FLUID Forge CLI v0.18.1`. +`fluid --version` is the top-level flag and prints one line, `FLUID Forge CLI v0.23.0`. ## Options diff --git a/docs/faq/README.md b/docs/faq/README.md index 16c0ba7..d0aa132 100644 --- a/docs/faq/README.md +++ b/docs/faq/README.md @@ -54,7 +54,7 @@ pip install --upgrade "data-product-forge[local]" fluid version ``` -Then follow the [upgrade guide](../upgrading.md). It has a checklist for each version you cross, because some upgrades need action: `0.17.0`, for example, moves remote state to a per-provider key for a contract that uses a per-contract state key, and needs one `fluid schedule-sync` to retire old Airflow DAGs if you sync Airflow DAGs, and `0.16.3` changed some results on purpose. Pin an exact version in CI (`data-product-forge==0.18.1`), not a range. +Then follow the [upgrade guide](../upgrading.md). It has a checklist for each version you cross, because some upgrades need action: `0.17.0`, for example, moves remote state to a per-provider key for a contract that uses a per-contract state key, and needs one `fluid schedule-sync` to retire old Airflow DAGs if you sync Airflow DAGs, and `0.16.3` changed some results on purpose. Pin an exact version in CI (`data-product-forge==0.23.0`), not a range. Your contracts do not need a new `fluidVersion`: `0.18.1` reads `0.7.1` to `0.7.5`, and `0.7.6` as a preview. diff --git a/docs/providers/README.md b/docs/providers/README.md index 6aaab4d..cb1e186 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -8,7 +8,7 @@ Fluid Forge uses one contract format across local and provider-backed execution ## Docs baseline -- CLI release covered by the primary docs: `0.18.1` +- CLI release covered by the primary docs: `0.23.0` - Contract schema: `0.7.5` is the current stable version; `0.7.6` is a preview you opt into with `fluidVersion: "0.7.6"`. The GCP retention, encryption and `binding.principals` fields need `0.7.6`. - Which `fluidVersion` each scaffolding command writes is listed in [`fluid init`](../cli/init.md#which-fluidversion-each-path-writes). diff --git a/docs/providers/aws.md b/docs/providers/aws.md index 9300c6d..f572b89 100644 --- a/docs/providers/aws.md +++ b/docs/providers/aws.md @@ -2,7 +2,7 @@ Deploy data products to Amazon Web Services (S3, Glue, Athena, Lake Formation) with the same contract and CLI commands you use for the other providers. -**Docs Baseline:** CLI `0.18.1`
+**Docs Baseline:** CLI `0.18.1`
**`fluid apply` creates:** S3 buckets, Glue databases and tables, Lake Formation registrations and grants, KMS keys, S3 lifecycle rules
**`fluid verify` reads:** Glue, Athena, S3, KMS, Lake Formation diff --git a/docs/providers/gcp.md b/docs/providers/gcp.md index 3773fc5..9a17ef4 100644 --- a/docs/providers/gcp.md +++ b/docs/providers/gcp.md @@ -1,7 +1,7 @@ # GCP Provider **Status:** ✅ Production Ready -**Docs Baseline:** CLI `0.18.1`
+**Docs Baseline:** CLI `0.18.1`
**Services:** BigQuery, Cloud Storage, Pub/Sub, IAM, Data Catalog policy tags, Cloud KMS > **Why it matters** diff --git a/docs/providers/local.md b/docs/providers/local.md index 24b09a4..23e08e0 100644 --- a/docs/providers/local.md +++ b/docs/providers/local.md @@ -1,7 +1,7 @@ # Local Provider **Status:** ✅ Production Ready -**Docs Baseline:** CLI `0.18.1`
+**Docs Baseline:** CLI `0.18.1`
**Engine:** DuckDB (1.5.0 or later) > **Why it matters** diff --git a/docs/providers/snowflake.md b/docs/providers/snowflake.md index 9ce55e1..1379ffc 100644 --- a/docs/providers/snowflake.md +++ b/docs/providers/snowflake.md @@ -3,7 +3,7 @@ Deploy data products to Snowflake Data Cloud (databases, schemas and tables) using the same contract format and CLI commands as the other providers. **Status:** ✅ Production -**Docs Baseline:** CLI `0.18.1`
+**Docs Baseline:** CLI `0.18.1`
**Tested Services:** Databases, Schemas, Tables > **Why it matters** diff --git a/docs/recipes/per-environment-overlays.md b/docs/recipes/per-environment-overlays.md index e1be41e..4db67a9 100644 --- a/docs/recipes/per-environment-overlays.md +++ b/docs/recipes/per-environment-overlays.md @@ -250,7 +250,7 @@ jobs: matrix: env: [dev, staging, prod] steps: - - run: pip install "data-product-forge==0.18.1" + - run: pip install "data-product-forge==0.23.0" - run: fluid validate contract.fluid.yaml --env ${{ matrix.env }} - run: fluid plan contract.fluid.yaml --env ${{ matrix.env }} ``` diff --git a/docs/reference/contract-0.7.5.md b/docs/reference/contract-0.7.5.md index eac16ce..18208bb 100644 --- a/docs/reference/contract-0.7.5.md +++ b/docs/reference/contract-0.7.5.md @@ -9,7 +9,7 @@ description: "The fields of the FLUID 0.7.5 contract schema (stable): path, type Status: **stable**. Schema 0.7.5 is the newest stable schema bundled with the CLI. The newer 0.7.6 schema is a preview; see the [0.7.6 preview delta](./contract-0.7.6-preview.md). -Generated from `fluid_build/schemas/fluid-schema-0.7.5.json` in the `data-product-forge` 0.18.1 package (sha256 `763fb786cd73bc4f`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. +Generated from `fluid_build/schemas/fluid-schema-0.7.5.json` in the `data-product-forge` 0.23.0 package (sha256 `a1c33dc003599649`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. Each table row is one field, addressed by its path from the contract root: `.` descends into an object, `[]` marks the items of a list, and `` stands for any key of a map. **Required** means required inside its parent object; the parent can itself be optional. **Allowed values** lists the enum, pattern and range the schema enforces. An object typed plainly `object` rejects keys the schema does not list; `object (open)` accepts extra keys and `object (free-form)` has no listed keys. A row that begins "Only when ..." exists only under that condition. See [how to read this reference](./README.md#how-to-read-a-row). @@ -309,7 +309,7 @@ Each table row is one field, addressed by its path from the contract root: `.` d | `exposes[].binding.governance.lakeFormation.grants[].permissionsWithGrantOption` | array of string | no | unique items | | Subset of permissions the principal can re-grant. Maps to aws_lakeformation_permissions.permissions_with_grant_option. | | `exposes[].binding.governance.lakeFormation.grants[].columns` | array of string | no | unique items | | Optional column-level restriction. When set, the grant is emitted against aws_lakeformation_permissions.table_with_columns instead of .table, allowing the principal to read only the listed columns. | | `exposes[].binding.governance.lakeFormation.grants[].excludedColumns` | array of string | no | unique items | | Opposite of columns: every column EXCEPT these is granted. Maps to table_with_columns.excluded_column_names. Mutually exclusive with columns. | -| `exposes[].binding.governance.lakeFormation.tags` | object (map) | no | | | LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags. | +| `exposes[].binding.governance.lakeFormation.tags` | object (map) | no | | | LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table). | | `exposes[].binding.governance.lakeFormation.tags.` | string | no | | | | | `exposes[].binding.governance.lakeFormation.rowFilter` | object | no | | | Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as aws_lakeformation_data_cells_filter. The filter must then be granted to principals via grants\[\].permissions including SELECT — LF enforces both layers. | | `exposes[].binding.governance.lakeFormation.rowFilter.name` | string | yes | | | Filter name (must be unique per table). | @@ -1139,5 +1139,5 @@ Each table row is one field, addressed by its path from the contract root: `.` d | `governance` | object | no | | | Contract-level governance: account/project-wide settings (e.g. AWS Lake Formation admins and LF-tag definitions). Per-resource grants and tag associations live under binding.governance. | | `governance.lakeFormation` | object | no | | | AWS Lake Formation account-wide settings: data-lake admins and LF-tag (TBAC) definitions. Applies to the AWS account the contract's IaC apply targets. | | `governance.lakeFormation.admins` | array of string | no | unique items
each item: matches `^arn:aws[a-z0-9-]*:iam::` | | IAM principal ARNs (roles or users) granted Lake Formation admin privileges. Emitted as aws_lakeformation_data_lake_settings.admins. AUTHORITATIVE, not additive: Lake Formation's PutDataLakeSettings replaces the account's admin list with the one sent, so an admin NOT listed here is REMOVED on apply. List every principal that must stay a data lake admin, including the role or user that runs the apply: left out, it loses Lake Formation admin on that apply, and the Lake Formation grants and registrations that need an admin can then fail. The terraform-provider-aws resource also clears the data lake settings its configuration omits, and this block sets only the admins, so the account's create-database and create-table default permissions (the IAMAllowedPrincipals default for new databases and tables), trusted resource owners and parameters are cleared too. Destroying the resource empties the admin list and resets CROSS_ACCOUNT_VERSION to 1. Re-applying the same list is a no-op. | -| `governance.lakeFormation.tagDefinitions` | object (map) | no | | | LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec. | +| `governance.lakeFormation.tagDefinitions` | object (map) | no | | | LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec. | | `governance.lakeFormation.tagDefinitions.` | array of string | no | min items 1
unique items | | | diff --git a/docs/reference/contract-0.7.6-preview.md b/docs/reference/contract-0.7.6-preview.md index ed702ec..0b11545 100644 --- a/docs/reference/contract-0.7.6-preview.md +++ b/docs/reference/contract-0.7.6-preview.md @@ -9,27 +9,62 @@ description: "What the FLUID 0.7.6 preview schema adds to, and changes in, the s Status: **preview**. A contract opts in with `fluidVersion: "0.7.6"`; a contract that declares 0.7.5 is unaffected. This page lists the difference between the 0.7.5 schema and the 0.7.6 schema as bundled with the CLI: added fields, fields whose definition changed, and fields removed. A field not listed here has the same type, required flag, allowed values, default and description in both; for those, use the [0.7.5 reference](./contract-0.7.5.md). A definition used at several paths is listed once, at its first path. -Generated from `fluid_build/schemas/fluid-schema-0.7.6.json` in the `data-product-forge` 0.18.1 package (sha256 `772bf80070a9409f`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. +Generated from `fluid_build/schemas/fluid-schema-0.7.6.json` in the `data-product-forge` 0.23.0 package (sha256 `9cef4fbdd5f562e0`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. Each table row is one field, addressed by its path from the contract root: `.` descends into an object, `[]` marks the items of a list, and `` stands for any key of a map. **Required** means required inside its parent object; the parent can itself be optional. **Allowed values** lists the enum, pattern and range the schema enforces. An object typed plainly `object` rejects keys the schema does not list; `object (open)` accepts extra keys and `object (free-form)` has no listed keys. A row that begins "Only when ..." exists only under that condition. See [how to read this reference](./README.md#how-to-read-a-row). ## At a glance -- Added: 48 rows -- Changed: 12 rows +- Added: 93 rows +- Changed: 17 rows - Removed: 0 rows ## Added fields +### metadata + +| Path | Type | Required | Allowed values | Default | Description | +| --- | --- | --- | --- | --- | --- | +| `metadata.team` | object | no | | | NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. `metadata.owner` holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today. | +| `metadata.team.name` | string | no | | | Team name, e.g. 'Sales Analytics'. | +| `metadata.team.members` | array of object | no | | | The people on the team, each with the role they hold on this product. | +| `metadata.team.members[].username` | string | yes | | | The member's identifier in the organisation's directory — the one field Bitol requires. | +| `metadata.team.members[].name` | string | no | | | Display name. | +| `metadata.team.members[].role` | string | no | | | The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it. | +| `metadata.team.members[].email` | string | no | format `email` | | | + ### exposes | Path | Type | Required | Allowed values | Default | Description | | --- | --- | --- | --- | --- | --- | +| `exposes[].contract.id` | string | no | min length 1 | | NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS `id` and as the ODPS output port `contractId`. Without it the id is derived as `.`, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by `fluid validate`. | +| `exposes[].contract.schema[].quality` | array of object | no | each item: min properties 1 | | NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape — the array that is written straight into `schema[].properties[].quality[]` of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's `contract` block into its output-port stub and discards the sub-import's root `extensions`, so rules left in that bucket were dropped outright by a FLUID → ODPS → FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but `$defs.column` is `additionalProperties: false` and did not define it, so a column written the way the exporter supports was REFUSED by `fluid validate`: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive — a column without it exports exactly as it does today, with the rules forge derives from `required`, the primary-key tag and `validationRules`. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 `$defs.DataQuality`, deliberately not re-declared here (a copy would drift from the vendored schema, and `fluid odcs validate` already checks an emitted document against it); the keys below are the common ones. Every rule should carry an `id`: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none. | +| `exposes[].contract.schema[].quality[].id` | string | no | matches `^[A-Za-z0-9_-]+$` | | Stable rule identifier, unique within this column's rules. The pattern is ODCS `$defs.StableId`. Rules forge derives itself are named `-`. | +| `exposes[].contract.schema[].quality[].type` | string | no | one of `text`, `library`, `sql`, `custom` | | ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check. | +| `exposes[].contract.schema[].quality[].metric` | string | no | one of `nullValues`, `missingValues`, `invalidValues`, `duplicateValues`, `rowCount` | | type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule. | +| `exposes[].contract.schema[].quality[].arguments` | object (free-form) | no | | | type: library only. Extra arguments for the metric — `pattern` (a regex) or `validValues` (a list) for invalidValues, as DataHub's ODCS ingestion source reads them. | +| `exposes[].contract.schema[].quality[].query` | string | no | | | type: sql only. The query whose result the comparison operator is applied to. | +| `exposes[].contract.schema[].quality[].engine` | string | no | | | type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt. | +| `exposes[].contract.schema[].quality[].implementation` | any | no | | | type: custom only. The check as that engine expresses it. | +| `exposes[].contract.schema[].quality[].dimension` | string | no | one of `accuracy`, `completeness`, `conformity`, `consistency`, `coverage`, `timeliness`, `uniqueness` | | The data quality dimension the rule measures. | +| `exposes[].contract.schema[].quality[].description` | string | no | | | | +| `exposes[].contract.schema[].quality[].severity` | string | no | | | Severity of a failure. ODCS types this as a free string and documents info / warning / error by example. | +| `exposes[].contract.schema[].quality[].unit` | string | no | | | Unit the comparison is in, usually rows (the default) or percent. | +| `exposes[].policy.authz.columnRestrictions[].mask` | string | no | one of `last_four`, `first_four`, `nullify` | | The masking rule an access: mask restriction applies, required with it and refused without it. last_four: 'XXXXX' and the value's last four characters; first_four: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATA_MASKING_POLICY, LAST_FOUR_CHARACTERS, FIRST_FOUR_CHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6). | +| `exposes[].policy.authz.rowFilters` | array of object | no | | | Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filter_predicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (row_filter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table. | +| `exposes[].policy.authz.rowFilters[].principal` | string | yes | | | Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud. | +| `exposes[].policy.authz.rowFilters[].name` | string | no | matches `^[a-z][a-z0-9_]{0,62}$` | | The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted. | +| `exposes[].policy.authz.rowFilters[].where` | string | yes | min length 1 | | One SQL boolean expression over the expose's columns, e.g. consent_data_analytics = true AND is_suppressed = false. | +| `exposes[].policy.authz.rowFilters[].description` | string | no | | | | | `exposes[].policy.privacy.masking[].params.saltEnv` | string | no | matches `^[A-Z_][A-Z0-9_]{0,127}$` | `"FLUID_PII_HASH_SECRET"` | NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan. | | `exposes[].policy.privacy.masking[].params.keyEnv` | string | no | matches `^[A-Z_][A-Z0-9_]{0,127}$` | | NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUID_PII_TOKENIZATION_KEY and needs at least 32 bytes; encrypt defaults to FLUID_PII_ENCRYPTION_SECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused. | | `exposes[].policy.privacy.masking[].params.keepFirst` | integer | no | min 0
max 64 | `0` | NEW in v0.7.6: mask only. Leading characters left as they are. | | `exposes[].policy.privacy.masking[].params.keepLast` | integer | no | min 0
max 64 | `4` | NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as \*\*\*\*\*\*\*\*4567. A value no longer than keepFirst + keepLast is masked entirely. | +| `exposes[].binding.location.host` | string | no | | | NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads `host` and falls back to `account`, so until now the only valid way to set the bootstrap servers was a field named for something else — and with neither set, the exported ODCS server is the placeholder `${KAFKA_BOOTSTRAP_SERVERS}`. Usually set per environment in an overlay rather than in the base contract. | | `exposes[].binding.governance.lakeFormation.bucketPolicy` | string | no | one of `cross-account`, `none`, `all-grantees` | `"cross-account"` | NEW in v0.7.6: Which grants\[\] principals get a statement in the aws_s3_bucket_policy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted aws_s3_bucket_policy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted. | +| `exposes[].binding.governance.lakeFormation.maskedViews` | object | no | | | Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view. | +| `exposes[].binding.governance.lakeFormation.maskedViews.definer` | string | yes | | | ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole). | +| `exposes[].binding.governance.lakeFormation.maskedViews.validationConnection` | string | yes | | | The Glue connection of type VIEW_VALIDATION_ATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it. | | `exposes[].binding.packaging` | object | no | | | NEW in v0.7.6: Per-exposure packaging override — key-wise precedence over the contract-wide top-level `packaging` block (binding.packaging > packaging > absent-legacy). | | `exposes[].binding.packaging.mode` | string | no | one of `isolated`, `shared` | | NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in `containers`. | | `exposes[].binding.packaging.pool` | string | no | min length 1 | | NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' — a pool must be addressable. Propagated as the `fluid_pool` label/tag for cost attribution. | @@ -45,18 +80,43 @@ Each table row is one field, addressed by its path from the contract root: `.` d | `exposes[].binding.encryption.kms` | string | no | one of `product`, `none`
or matches `^alias/[A-Za-z0-9/_-]{1,250}$`
or matches `^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key\|alias)/[A-Za-z0-9/_-]{1,250}$`
or matches `^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$`
When `platform` is `gcp`: must not match `^(alias/\|arn:)`
When `platform` is `aws`: must not match `^projects/` | `"product"` | NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): `fluid apply` creates a customer managed key for each bucket this product owns (aws_kms_key, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (aws_s3_bucket_server_side_encryption_configuration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. `tofu destroy` deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; `fluid apply` refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data aws_kms_key, so the identity running `fluid apply` needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRIC_DEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. `fluid verify` checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (google_kms_key_ring fluid-<contract id>-<dataset> and google_kms_crypto_key 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's default_encryption_configuration and the table's encryption_configuration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: `tofu destroy` removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so `fluid apply` refuses that plan without --allow-data-loss; the next build lands the data again. `fluid verify` checks kmsKeyName on the dataset and the table. | | `exposes[].binding.principals` | object (map) | no | keys: min length 1
When `platform` is `gcp`: each value is a string matching `^(user\|group\|serviceAccount\|domain):.+$` or an array whose items are each a string matching `^(user\|group\|serviceAccount\|domain):.+$`
When `platform` is `aws`: each value is a string matching `^arn:aws[a-z0-9-]*:iam::` or an array whose items are each a string matching `^arn:aws[a-z0-9-]*:iam::` | | NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants\[\].principal and exposes\[\].policy.authz.columnRestrictions\[\].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; \[\] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles\[\] bound per server (servers\[\].roles) and on dbt grants resolved per target. | | `exposes[].binding.principals.` | string or array of string | no | string: min length 1
array: unique items, each item: min length 1 | | | +| `exposes[].binding.serialization` | object | no | | | NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only `format: kafka_topic`, which tells a consumer nothing about whether the values are Avro or JSON — the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server `format` field; the two `-confluent` values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them. | +| `exposes[].binding.serialization.value` | string | no | one of `json`, `avro`, `protobuf`, `xml`, `avro-confluent`, `json-confluent` | | How message values are serialized. | +| `exposes[].binding.serialization.key` | string | no | one of `json`, `avro`, `protobuf`, `xml`, `avro-confluent`, `json-confluent` | | How message keys are serialized, for a topic whose keys are not opaque bytes. | | `exposes[].lifecycle.expire` | boolean | no | | `false` | NEW in v0.7.6: Delete stored data once it is older than `retention`. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: `fluid apply` writes one rule per expose into the bucket's aws_s3_bucket_lifecycle_configuration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version `retention` after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that `fluid apply` did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and `fluid verify` checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing `expire` later deletes the lifecycle configuration, which `fluid apply` refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); `tofu destroy` deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (time_partitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expiration_ms = `retention` in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than `retention` after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than `retention` lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding `expire` to a live table replaces it: `fluid apply` refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of `retention` is an in-place update. Removing `expire` from a live table removes the partitioning's replacement trigger, which `fluid apply` also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer `retention` instead. `fluid verify` checks the live table's partition expiration. | | `exposes[].semantics.measures[].aggParams` | object | no | | | Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile. | | `exposes[].semantics.measures[].aggParams.percentile` | number | no | min 0
max 1 | | Percentile in \[0, 1\] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted. | | `exposes[].semantics.measures[].aggParams.useDiscretePercentile` | boolean | no | | | Use the discrete percentile (PERCENTILE_DISC / use_discrete_percentile) instead of continuous interpolation. | +### description + +| Path | Type | Required | Allowed values | Default | Description | +| --- | --- | --- | --- | --- | --- | +| `description.purpose` | string | no | | | What this data product is for. | +| `description.usage` | string | no | | | How a consumer is expected to use it. | +| `description.limitations` | string | no | | | What it must not be used for, and what it does not cover. | + ### consumes | Path | Type | Required | Allowed values | Default | Description | | --- | --- | --- | --- | --- | --- | +| `consumes[].contractId` | string | no | | | NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own — a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived `.`; until now `consumes[]` was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it. | | `consumes[].upstreamWorkspace` | string | no | min length 1 | | Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace. | | `consumes[].upstreamDigest` | string | no | matches `^sha256:[0-9a-f]{64}$` | | Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. `fluid apply` re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set. | +### builds + +| Path | Type | Required | Allowed values | Default | Description | +| --- | --- | --- | --- | --- | --- | +| `builds[].properties.sink.catalogAuth` | object | no | When `type` is `oauth2`: requires `credentialEnv`, `oauth2ServerUri`, `scope`
When `type` is `bearer`: requires `tokenEnv` | | Only when `pattern` is `acquisition`. NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:<NAME>, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding client_id:client_secret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is <polaris>/api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization. | +| `builds[].properties.sink.catalogAuth.type` | string | yes | one of `oauth2`, `bearer` | | Only when `pattern` is `acquisition`. oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token. | +| `builds[].properties.sink.catalogAuth.credentialEnv` | string | no | matches `^[A-Za-z_][A-Za-z0-9_]*(?![\s\S])`
must not match `(?:^\|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_\|$)` | | Only when `pattern` is `acquisition`. type oauth2: the NAME of the environment variable holding client_id:client_secret (Iceberg splits it on the first ':'). Emitted as credential=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused. | +| `builds[].properties.sink.catalogAuth.oauth2ServerUri` | string | no | matches `^https?://[^\s/?#@]+(?:[/?#]\S*)?$`
must not match `\$\{\|[\x00-\x1f\x7f]\|^\s*env:` | | Only when `pattern` is `acquisition`. type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's <keycloak>/realms/<realm>/protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens. | +| `builds[].properties.sink.catalogAuth.scope` | string | no | min length 1
must not match `\$\{\|[\x00-\x1f\x7f]\|^\s*env:` | | Only when `pattern` is `acquisition`. type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope. | +| `builds[].properties.sink.catalogAuth.audience` | string | no | min length 1
must not match `\$\{\|[\x00-\x1f\x7f]\|^\s*env:` | | Only when `pattern` is `acquisition`. type oauth2, optional: sent as the audience parameter on the token request. | +| `builds[].properties.sink.catalogAuth.resource` | string | no | min length 1
must not match `\$\{\|[\x00-\x1f\x7f]\|^\s*env:` | | Only when `pattern` is `acquisition`. type oauth2, optional: sent as the resource parameter on the token request. | +| `builds[].properties.sink.catalogAuth.tokenEnv` | string | no | matches `^[A-Za-z_][A-Za-z0-9_]*(?![\s\S])`
must not match `(?:^\|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_\|$)` | | Only when `pattern` is `acquisition`. type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused. | + ### consumers | Path | Type | Required | Allowed values | Default | Description | @@ -97,18 +157,28 @@ Each row names the attribute that differs and shows both values. | --- | --- | --- | --- | | `fluidVersion` | allowed values | one of `0.7.3`, `0.7.4`, `0.7.5`
matches `^\d+\.\d+(\.\d+)?$` | one of `0.7.3`, `0.7.4`, `0.7.5`, `0.7.6`
matches `^\d+\.\d+(\.\d+)?$` | | `fluidVersion` | description | Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 — every 0.7.3 contract validates as 0.7.4 unchanged. | Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 — every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.6' is the current preview. | -| `exposes[].policy.authz.columnRestrictions` | description | Column-level access control. | Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. | +| `description` | type | string | string or object | +| `description` | description | none | NEW in v0.7.6: Either a plain string — unchanged meaning, it becomes `purpose` on export — or the structured object. While this was a string, an export of a valid contract could only ever carry `description.purpose`: `usage` and `limitations` had no home a contract could declare, so every ODCS and ODPS export left them empty. `fluid odps import` restores the object to the root `description` when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in `extensions.odps.metadata.description` so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID → ODPS → FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root `description` is a string still roots `purpose` alone, with the object recoverable from that bucket. string form: Prose description. Exported as `description.purpose`. object form: NEW in v0.7.6: The structured form of the root `description`. The three field names are Bitol's — ODPS v1.1.0 `$defs.Description` and the ODCS v3.1.0 root `description` both use exactly purpose/usage/limitations — so import and export are a pass-through rather than a translation. | +| `exposes[].policy.authz.columnRestrictions` | allowed values | none | Each item: when `access` is `mask`: requires `mask` | +| `exposes[].policy.authz.columnRestrictions` | description | Column-level access control. | Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names. | | `exposes[].policy.authz.columnRestrictions[].principal` | description | none | Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud. | | `exposes[].policy.authz.columnRestrictions[].columns` | description | none | Columns of this expose's schema. | -| `exposes[].policy.authz.columnRestrictions[].access` | description | none | deny: the principal may not read the columns. allow: only the principals an allow names may read them. | +| `exposes[].policy.authz.columnRestrictions[].access` | allowed values | one of `allow`, `deny` | one of `allow`, `deny`, `mask`
When `mask` is set: must be `mask` | +| `exposes[].policy.authz.columnRestrictions[].access` | description | none | deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6). | | `exposes[].policy.privacy.masking` | description | none | Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves. | | `exposes[].policy.privacy.masking[].strategy` | description | none | hash: lowercase hex SHA-256 of salt \|\| value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '\*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenize_pii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce \|\| ciphertext \|\| tag) with the column name as associated data; reversible with the key, not deterministic. k_anonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value. | | `exposes[].policy.privacy.masking[].params` | type | object (free-form) | object (open) | | `exposes[].policy.privacy.masking[].params` | description | none | Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time. | +| `exposes[].binding.location.catalog` | description | NEW in v0.7.5: Iceberg catalog kind/profile selector — maps to iceberg.catalog.type (e.g. glue, rest, nessie, hive). REST-fronted catalogs (polaris / unity / snowflake open catalog) map to 'rest'. Consumed by providers/_iceberg_catalog.py and the streaming-sink validator. | NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'_', so iceberg_rest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/_iceberg_catalog.py. | | `exposes[].binding.location.partitionBy` | description | NEW in v0.7.5: Iceberg partition columns — a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). | NEW in v0.7.5: Iceberg partition columns — a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes\[\].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition `retention` after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written). | | `exposes[].lifecycle` | description | none | NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus `expire`, which is only meaningful where there is storage to expire. | | `exposes[].lifecycle.retention` | description | ISO-8601 duration (e.g., P1D, PT15M, P2Y6M). | How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless `expire` is true. With `expire: true` on an AWS S3 binding, `fluid apply` expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one. | | `consumes` | allowed values | none | each item: if `upstreamWorkspace` is set, `upstreamDigest` is required | +| `builds[].engine` | allowed values | one of `dbt`, `sql`, `python`, `spark`, `glue`, `custom`, `duckdb`, `airbyte`, `meltano`, `dlt`, `kafka-connect`, `debezium`
or matches `^dbt-[a-z0-9]+([_-][a-z0-9]+)*$` | one of `dbt`, `sql`, `python`, `spark`, `glue`, `custom`, `duckdb`, `airbyte`, `meltano`, `dlt`, `kafka-connect`, `debezium`, `flink`
or matches `^dbt-[a-z0-9]+([_-][a-z0-9]+)*$` | +| `builds[].engine` | description | Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). | Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration. | +| `builds[].execution.trigger.type` | allowed values | one of `schedule`, `event`, `manual`, `dependency`, `dataset`, `schedule_and_dataset`, `timetable` | one of `schedule`, `event`, `manual`, `dependency`, `dataset`, `schedule_and_dataset`, `timetable`, `continuous` | +| `builds[].execution.trigger.type` | description | none | NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other. | +| `builds[].properties.sink.catalog` (only when `pattern` is `acquisition`) | allowed values | one of `rest`, `glue`, `nessie`, `unity`, `snowflake-managed`, `hive` | one of `rest`, `glue`, `nessie`, `unity`, `snowflake-managed`, `hive`, `lakekeeper`, `polaris` | ## Removed fields diff --git a/docs/reference/preview-fields.md b/docs/reference/preview-fields.md index d86d3cb..055ac37 100644 --- a/docs/reference/preview-fields.md +++ b/docs/reference/preview-fields.md @@ -80,17 +80,21 @@ unexpected) | `exposes[].binding.encryption.kms` | 0.16.5 | The key that encrypts the data at rest: `product`, `none`, an alias, an AWS key ARN, or a Cloud KMS key name. | | `exposes[].lifecycle.expire` | 0.16.5 | Whether `retention` deletes data. | | `exposes[].binding.principals` | 0.17.0 | A map from the contract's logical principals to the identities they are on this binding's cloud. | - -The [`consumes`](./contract-0.7.6-preview.md#consumes) and [`exposes`](./contract-0.7.6-preview.md#exposes) sections of the preview delta hold the rows for the fields above, in the order the schema declares them. - -*([forge-cli #712](https://github.com/Agenticstiger/forge-cli/pull/712), since 0.20.0)* `builds[].properties.sink.catalogAuth` is a 0.7.6 field too. It has no row above and none in the preview delta: that delta is generated from the 0.7.6 schema bundled in the CLI release this site is pinned to, and the field was added to the schema after it. It names the environment variable that authenticates a streaming sink to an Iceberg REST catalog; see [Authenticate a sink to a REST catalog](../advanced/source-aligned-acquisition.md#authenticate-a-sink-to-a-rest-catalog). - -*(since 0.21.0, [forge-cli #731](https://github.com/Agenticstiger/forge-cli/pull/731))* `builds[].engine: flink` is a 0.7.6 value too, for a job that runs on the Flink Kubernetes Operator, AWS Managed Service for Apache Flink, or another Flink platform. forge-cli validates and plans it but ships no runner; see [Flink jobs](../concepts/builds-exposes-bindings.md#flink-jobs-engine-flink). +| `builds[].properties.sink.catalogAuth` | 0.20.0 | The environment variable that authenticates a streaming sink to an Iceberg REST catalog; see [Authenticate a sink to a REST catalog](../advanced/source-aligned-acquisition.md#authenticate-a-sink-to-a-rest-catalog). | +| `builds[].engine: flink` | 0.21.0 | Apache Flink as a build's engine, for a job that runs on the Flink Kubernetes Operator, AWS Managed Service for Apache Flink, or another Flink platform. forge-cli validates and plans it but ships no runner; see [Flink jobs](../concepts/builds-exposes-bindings.md#flink-jobs-engine-flink). | +| `builds[].execution.trigger.type: continuous` | 0.22.0 | A job its platform keeps running: no scheduler starts it, and it takes no other trigger; see [Continuous builds](../concepts/builds-exposes-bindings.md#continuous-builds-trigger-type-continuous). | +| `metadata.team` | 0.23.0 | The team as a roster — `{name, members[]}`, each member `{username, name, role, email}` — where `metadata.owner` holds one team name, one mailbox and no role. Fills the ODCS and ODPS team block, which came out empty before; `email` has [a caveat](#the-email-caveat). | +| `description` (at the contract root) | 0.23.0 | Either a string, as before, or `{purpose, usage, limitations}` — the shape both Bitol standards define. While it was a string, `usage` and `limitations` had no home a contract could declare, so every ODCS and ODPS export left them empty. | +| `consumes[].contractId` | 0.23.0 | The upstream contract's own id, for an upstream that has one of its own — a product not built with FLUID, registered under an id FLUID did not derive. The canonical-port builder already preferred an explicit id; with `consumes[]` closed there was no way to supply one, so the derived id was exported as if the upstream answered to it. | +| `exposes[].contract.id` | 0.23.0 | An explicit, stable contract id for one output port, used verbatim as the ODCS `id` and the ODPS output port `contractId`. Without it the id is derived from the product and expose names, so renaming either changes it. See [pinning a contract id](#pinning-a-contract-id). | +| `exposes[].binding.location.host` | 0.23.0 | Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already read `host` and fell back to `account`, while the schema accepted only `account`, so the only valid way to set brokers was a field named for something else. Usually set per environment in an overlay, since the brokers differ by environment and the rest of the binding does not. | +| `exposes[].binding.serialization` | 0.23.0 | `{value, key}`, each one of `json`, `avro`, `protobuf`, `xml`, `avro-confluent`, `json-confluent`. The only thing that moves a Kafka server's exported ODCS `format` off `kafka_topic`. See [declaring the wire format](#declaring-the-wire-format). | +| `exposes[].contract.schema[].quality` | 0.23.0 | Column-level data quality rules, in ODCS's own shape, written straight into `schema[].properties[].quality[]` of the exported contract. Setting it replaces the rules forge derives for that column. See [column quality rules](#column-quality-rules). | + +The [0.7.6 preview delta](./contract-0.7.6-preview.md) holds the full row for every field above, grouped by the top-level key it hangs off (`metadata`, `exposes`, `description`, `consumes`, `builds`) and in the order the schema declares them. *(since 0.21.0, [forge-cli #732](https://github.com/Agenticstiger/forge-cli/pull/732))* In a contract that names 0.7.6, an overlay patches `builds`, `exposes` and `consumes` by identity instead of by position; see [How an overlay merges](../concepts/environments-and-overlays.md#how-an-overlay-merges). -*(since 0.22.0, [forge-cli #734](https://github.com/Agenticstiger/forge-cli/pull/734))* `builds[].execution.trigger.type: continuous` is a 0.7.6 value too, for a job its platform keeps running: no scheduler starts it, and it takes no other trigger; see [Continuous builds](../concepts/builds-exposes-bindings.md#continuous-builds-trigger-type-continuous). - ### Federated upstreams `upstreamWorkspace` and `upstreamDigest` travel together. This pair of fields on a `consumes[]` entry validates under 0.7.6: @@ -146,34 +150,16 @@ Replace `` with the domain of your own groups; left as written, val The same refusal applies to a placeholder in a reserved top-level domain (`.example`, `.test`, `.invalid`, `.localhost`), per the `exposes[].binding.principals` row. A value that is not valid for the binding's platform is refused too: an AWS role ARN mapped on a `gcp` binding fails with "is not a GCP IAM member". -## Seven more fields, in the next release after `0.22.0` - -::: tip Not in a released CLI yet -[forge-cli #753](https://github.com/Agenticstiger/forge-cli/pull/753) adds seven optional -fields to the 0.7.6 preview schema, for things a contract could not previously say. All seven -require `fluidVersion: "0.7.6"`, all are additive, and a contract that sets none of them -validates and exports byte for byte as before. The names are taken from Bitol ODCS v3.1.0 and -the released ODPS v1.1.0 rather than invented, so import and export are a pass-through instead -of a translation. +## Four of the 0.23.0 fields need more than a table row -The release is a minor bump from `0.22.0` and its number is not settled yet, so this page names -the release rather than the number. The generated [0.7.6 preview delta](./contract-0.7.6-preview.md) -does not list these rows: it is rendered from the schema bundled in the CLI release this site is -pinned to, and will carry them once that pin moves. -::: +`0.23.0` added the seven rows above +([forge-cli #753](https://github.com/Agenticstiger/forge-cli/pull/753)). All seven are optional +and additive, so a contract that sets none of them validates and exports byte for byte as +before, and their names are taken from Bitol ODCS v3.1.0 and the released ODPS v1.1.0 rather +than invented, so import and export are a pass-through instead of a translation. The generated +[0.7.6 preview delta](./contract-0.7.6-preview.md) carries the full row for each. -| Field | What it declares | What it changes on export | -| --- | --- | --- | -| `metadata.team` | The team as a roster — `{name, members[]}`, each member `{username, name, role, email}` — where `metadata.owner` holds one team name and one mailbox and no role at all | Fills the ODCS and ODPS team block, which came out empty before. See [the `email` caveat](#the-email-caveat) | -| `description` (root) | Either a string, as today, or `{purpose, usage, limitations}` — the shape both Bitol standards define | An export could only ever carry `description.purpose`; `usage` and `limitations` had no home a contract could declare, so every export left them empty | -| `consumes[].contractId` | The upstream contract's own id, for an upstream that has one — a product not built with FLUID, registered in a catalog under an id FLUID did not derive | The canonical-port builder already preferred an explicit id; with `consumes[]` closed there was no way to supply one, so the derived id was exported as if the upstream answered to it | -| `exposes[].contract.id` | An explicit, stable contract id for one output port | Used verbatim as the ODCS `id` and the ODPS output port `contractId`. See [pinning a contract id](#pinning-a-contract-id) | -| `binding.location.host` | Host or endpoint; for a Kafka binding, the bootstrap servers | The ODCS server mapper already read `host` and fell back to `account`, while the schema accepted only `account` — so the only valid way to set bootstrap servers was a field named for something else. With neither set, the exported server is the placeholder `${KAFKA_BOOTSTRAP_SERVERS}` | -| `exposes[].binding.serialization` | `{value, key}`, each one of `json`, `avro`, `protobuf`, `xml`, `avro-confluent`, `json-confluent` | The only thing that moves a Kafka server's ODCS `format` off `kafka_topic`. See [declaring the wire format](#declaring-the-wire-format) | -| `quality` on a column | Column-level data quality rules, in ODCS's own shape | Written straight into `schema[].properties[].quality[]` of the exported ODCS contract. See [column quality rules](#column-quality-rules) | - -`binding.location.host` is usually set per environment in an overlay rather than in the base -contract, since the brokers differ by environment and the rest of the binding does not. +Four of them have something a row cannot hold. ### The `email` caveat diff --git a/docs/sdk-and-plugins/journeys/custom-validator.md b/docs/sdk-and-plugins/journeys/custom-validator.md index bc2111b..2a6921e 100644 --- a/docs/sdk-and-plugins/journeys/custom-validator.md +++ b/docs/sdk-and-plugins/journeys/custom-validator.md @@ -459,7 +459,7 @@ Three places this typically gets installed: ``` 3. **CI** — the bundle from [your-own-ci](./your-own-ci.md) already has a `validate` stage. Add `my-org-validators` to the `pip install` line: ```yaml - - run: pip install --index-url "" "data-product-forge==0.18.1" "my-org-validators==" + - run: pip install --index-url "" "data-product-forge==0.23.0" "my-org-validators==" - run: fluid validate contract.fluid.yaml --strict env: FLUID_PLUGINS_ALLOWLIST: "steward-required,cost-center-required,classification-from-vocab,local" diff --git a/docs/sdk-and-plugins/journeys/your-own-scaffolding.md b/docs/sdk-and-plugins/journeys/your-own-scaffolding.md index 34066c4..014f452 100644 --- a/docs/sdk-and-plugins/journeys/your-own-scaffolding.md +++ b/docs/sdk-and-plugins/journeys/your-own-scaffolding.md @@ -176,7 +176,7 @@ keywords = [ dependencies = [ "pydantic>=2.0", - "data-product-forge==0.18.1", + "data-product-forge==0.23.0", ] [project.optional-dependencies] diff --git a/docs/sdk-and-plugins/reference/companion-packages.md b/docs/sdk-and-plugins/reference/companion-packages.md index 56a7594..13c7bea 100644 --- a/docs/sdk-and-plugins/reference/companion-packages.md +++ b/docs/sdk-and-plugins/reference/companion-packages.md @@ -48,7 +48,7 @@ The PyPI name reflects the product brand (`data-product-forge`). The import path ```toml dependencies = [ - "data-product-forge==0.18.1", # pin exact for reproducibility + "data-product-forge==0.23.0", # pin exact for reproducibility ] ``` diff --git a/docs/sdk-and-plugins/reference/trust-model.md b/docs/sdk-and-plugins/reference/trust-model.md index 8b8b3c6..d2b4759 100644 --- a/docs/sdk-and-plugins/reference/trust-model.md +++ b/docs/sdk-and-plugins/reference/trust-model.md @@ -167,7 +167,7 @@ Plugins are tested, scanned, and approved by your platform team before showing u ```toml # In your deploy environment's pyproject.toml or requirements: -data-product-forge==0.18.1 +data-product-forge==0.23.0 data-product-forge-sdk==0.10.0 data-product-forge-custom-scaffold==0.4.1 my-org-validators==1.2.0 diff --git a/docs/upgrading.md b/docs/upgrading.md index 935ba12..70cbb72 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -97,13 +97,11 @@ prints). Some fields added in these releases exist only in `0.7.6`: ## Read this first if you consume the exported artifacts -::: warning Shipping in the next release after `0.22.0` +::: warning New in `0.23.0` Exported ODCS and ODPS documents change shape. Every change below corrects output that was wrong, but the old values may be baked into whatever reads them, so this is the section to read before you upgrade rather than after. The work is -[forge-cli #753](https://github.com/Agenticstiger/forge-cli/pull/753); the release is a minor -bump from `0.22.0` and its number is not settled yet, so this page names the release rather -than the number. +[forge-cli #753](https://github.com/Agenticstiger/forge-cli/pull/753). Nothing here changes a FLUID contract. If you do not consume `fluid generate standard`, `fluid odcs export`, `fluid odps export` or `fluid generate artifacts` output, skip it. @@ -509,10 +507,9 @@ cover the SQL changes in full. ### The two drift defects, and what fixes them -::: tip Shipping in the next release after `0.22.0` -Both fixes are on forge-cli `main`. The release is a minor bump from `0.22.0` and its number -is not settled yet, so this page names the release rather than the number. Their pull request -is [forge-cli #749](https://github.com/Agenticstiger/forge-cli/pull/749). +::: tip Fixed in `0.23.0` +Both fixes shipped in `0.23.0`, in +[forge-cli #749](https://github.com/Agenticstiger/forge-cli/pull/749). ::: **An incremental cursor written on 1.5 and read on 2.0 silently dropped or duplicated rows.** diff --git a/docs/vision.md b/docs/vision.md index 108677d..1249524 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -147,7 +147,7 @@ fluid market --search "customer analytics" ## Versioning in the docs -- Current CLI release baseline: `0.18.1` +- Current CLI release baseline: `0.23.0` - Current scaffolded contract examples: `fluidVersion: 0.7.5` That split is intentional. The CLI release and the contract schema version move on related but different timelines. diff --git a/docs/walkthrough/export-orchestration.md b/docs/walkthrough/export-orchestration.md index 19b79ba..b55b852 100644 --- a/docs/walkthrough/export-orchestration.md +++ b/docs/walkthrough/export-orchestration.md @@ -1,6 +1,6 @@ # Generating Orchestration Code from Contracts -**Docs baseline:** CLI `0.18.1` +**Docs baseline:** CLI `0.18.1` `fluid export` reads the tasks you declare under `orchestration` in a contract and writes orchestration code for Airflow, Dagster or Prefect. This page exports the same kind of contract for AWS, GCP and Snowflake, shows what comes out, and lists what the generated code does not do. Everything here was run on 0.18.1; paths are shortened to `...`. diff --git a/docs/walkthrough/universal-pipeline.md b/docs/walkthrough/universal-pipeline.md index 0f75740..156759f 100644 --- a/docs/walkthrough/universal-pipeline.md +++ b/docs/walkthrough/universal-pipeline.md @@ -2,7 +2,7 @@ One hand-written Jenkinsfile for every provider, with no provider branches in it: the contract's `binding.platform` selects the provider. -**Docs baseline:** CLI `0.18.1` +**Docs baseline:** CLI `0.18.1` ::: tip Starting a new pipeline? `fluid generate ci --system jenkins` writes a pipeline for you that runs the bundle chain, checks the digests between stages and gives every parameter a default, so the first build runs with them. See [Jenkins CI/CD](./jenkins-cicd.md) and [The 11-stage pipeline](./11-stage-pipeline.md). This page is for teams that keep a Jenkinsfile of their own and want one that does not change per cloud. diff --git a/scripts/forge-cli-releases.json b/scripts/forge-cli-releases.json index 4f32104..4fd6435 100644 --- a/scripts/forge-cli-releases.json +++ b/scripts/forge-cli-releases.json @@ -21,7 +21,7 @@ ], "repo": "Agenticstiger/forge-cli", "changelogUrl": "https://raw.githubusercontent.com/Agenticstiger/forge-cli/main/CHANGELOG.md", - "reviewedThroughRelease": "0.22.0", + "reviewedThroughRelease": "0.23.0", "releaseOfPr": { "707": "0.20.0", "709": "0.20.0", diff --git a/scripts/gen_contract_reference.py b/scripts/gen_contract_reference.py index bcdf721..9bc77b3 100644 --- a/scripts/gen_contract_reference.py +++ b/scripts/gen_contract_reference.py @@ -62,8 +62,8 @@ "$comment", "$defs", "$id", "$ref", "$schema", "additionalProperties", "allOf", "anyOf", "const", "default", "dependentRequired", "description", "enum", "errorMessage", "examples", "format", "if", "items", "maximum", - "minItems", "minLength", "minimum", "not", "oneOf", "pattern", "properties", - "propertyNames", "required", "then", "title", "type", "uniqueItems", + "minItems", "minLength", "minProperties", "minimum", "not", "oneOf", "pattern", + "properties", "propertyNames", "required", "then", "title", "type", "uniqueItems", } # Keywords whose value is data or a name->schema map, not a schema to audit. DATA_KEYWORDS = {"examples", "enum", "const", "default", "dependentRequired", "required"} @@ -359,6 +359,11 @@ def constraint_lines(self, eff: dict[str, Any], *, with_alts: bool = True) -> li lines.append(f"min items {eff['minItems']}") if eff.get("uniqueItems"): lines.append("unique items") + if "minProperties" in eff: + # The object counterpart of minItems. 0.7.6 uses it on a column's + # quality rule, where every key is optional and an empty `{}` would + # otherwise validate as a rule that says nothing. + lines.append(f"min properties {eff['minProperties']}") if "not" in eff: neg = self.resolve(eff["not"]) if set(neg) == {"pattern"}: