fix(duckdb): run on DuckDB 2.x as well as 1.5.x

.github/workflows/ci.yml

@@ -623,6 +623,79 @@ jobs:
         if: always()
         run: python scripts/cleanup_local_test_artifacts.py
 
+  # ──────────────────────────────────────────────────────────────
+  # DuckDB floor. pyproject.toml takes duckdb>=1.5.0 with no ceiling, and
+  # DuckDB 2.0 needs Python 3.11, so the only other job on 1.5.x is the
+  # Python 3.10 test leg, which gets the newest 1.5 patch, never the floor.
+  # This installs exactly the floor the sandbox enforces (MIN_DUCKDB_VERSION,
+  # read from the code, so the two cannot drift) on a supported Python and
+  # runs what differs between DuckDB versions: the compat seam
+  # (providers/_duckdb_compat.py), the sandbox, the MCP DuckDB driver, the
+  # DuckDB acquisition runner (its database-source attaches) and every suite
+  # the DuckDB end-to-end job runs. tests/test_ci_duckdb_floor.py fails when a
+  # file the end-to-end job runs, or a test that branches on the DuckDB
+  # version, is missing here.
+  # ──────────────────────────────────────────────────────────────
+  duckdb-floor:
+    name: DuckDB floor (Python 3.12)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install with local extras, then the DuckDB floor
+        env:
+          # Same pip timeout and retried install as the test matrix (see the
+          # comments there).
+          PIP_TIMEOUT: "30"
+        run: |
+          python -m pip install --upgrade pip
+          for attempt in 1 2 3; do
+            python -m pip install -e ".[dev,local]" && break
+            if [ "$attempt" -eq 3 ]; then
+              echo "::error::pip install -e .[dev,local] failed 3 times; this is an install failure, not a test failure."
+              exit 1
+            fi
+            echo "::warning::pip install -e .[dev,local] failed (attempt ${attempt}/3); retrying in 30s."
+            sleep 30
+          done
+          floor=$(python -c "from fluid_build.providers._duckdb_sandbox import MIN_DUCKDB_VERSION as v; print('.'.join(map(str, v)))")
+          for attempt in 1 2 3; do
+            python -m pip install "duckdb==${floor}" && break
+            if [ "$attempt" -eq 3 ]; then
+              echo "::error::pip install duckdb==${floor} failed 3 times; this is an install failure, not a test failure."
+              exit 1
+            fi
+            echo "::warning::pip install duckdb==${floor} failed (attempt ${attempt}/3); retrying in 30s."
+            sleep 30
+          done
+          python -c "import duckdb, sys; print('duckdb', duckdb.__version__); sys.exit(duckdb.__version__ != '${floor}')"
+      - name: Run the DuckDB-version-sensitive tests on the floor
+        env:
+          # The quickstart and example suites fail, rather than skip, without
+          # duckdb (as in the DuckDB end-to-end job).
+          FLUID_REQUIRE_DUCKDB: "1"
+        run: |
+          python -m pytest -q \
+            tests/providers/test_duckdb_compat.py \
+            tests/providers/test_duckdb_sandbox.py \
+            tests/providers/test_duckdb_sandbox_workspace_root.py \
+            tests/output_ports/test_drivers_duckdb.py \
+            tests/build_runners/test_duckdb_runner.py \
+            tests/build_runners/test_duckdb_runner_mysql_attach.py \
+            tests/test_e2e_local.py \
+            tests/test_e2e_local_apply.py \
+            tests/test_e2e_local_negative.py \
+            tests/providers/test_local_live_happy_path.py \
+            tests/test_quickstart_templates_execute.py \
+            tests/test_examples_execute.py
+      - name: Cleanup local artifacts (defensive)
+        if: always()
+        run: python scripts/cleanup_local_test_artifacts.py
+
   # ──────────────────────────────────────────────────────────────
   # dbt v2 canary. The `test` and `coverage-and-audit` jobs install
   # `dbt-core<2` + dbt-duckdb, so they exercise the 1.x line only. Every

CHANGELOG.md

@@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Upgrade notes
 
+- forge-cli runs on DuckDB 2.x as well as 1.5.x (`duckdb>=1.5.0`, no ceiling; DuckDB 2.0
+  needs Python 3.11, so a Python 3.10 install stays on 1.5.x). On 2.x, a DuckDB database
+  file forge-cli creates (`fluid demo`'s `.fluid/db.duckdb`, the local provider's session
+  and `persist` databases, a Meltano acquisition's DuckDB destination) is written in the
+  format DuckDB 1.5 writes, which DuckDB 1.0 and later open, so a type that needs a newer
+  format, such as a `VARIANT` column, is refused in it on 2.x as it is on 1.5; a file
+  that already exists keeps its format.
+  DuckDB 2.x's own default format cannot be read by 1.5: a file another 2.x tool created
+  (dbt-duckdb and dlt included), or forge-cli 0.21.0 or older on DuckDB 2.0, opens only
+  on 2.x. To convert one, run on 2.x: `ATTACH 'old.duckdb' AS src (READ_ONLY); ATTACH
+  'new.duckdb' AS dst (STORAGE_VERSION 'v1.0.0'); COPY FROM DATABASE src TO dst;`.
+- Contract SQL runs on the DuckDB that is installed, and DuckDB 2.0 changed some SQL: the
+  `x -> x + 1` lambda is an error (write `lambda x: x + 1`), and integer `//` and `%` by
+  zero raise an error instead of returning NULL (wrap them in `TRY(...)`). See
+  https://duckdb.org/docs/2.0/operations_manual/migration.html.
+- A DuckDB acquisition build now resolves a Postgres source's `connection.secretRef`. It
+  ignored it, and libpq's `PGPASSWORD` or `~/.pgpass` supplied the password, so a
+  `secretRef` that does not resolve where `fluid apply` runs now fails the build. To keep
+  using `PGPASSWORD`, remove the `secretRef`; an inline `password` still wins over it.
 - `fluid validate --env <env> --strict` (on one contract or a whole workspace), `fluid
   contract-validation --env <env> --strict` and `fluid test --env <env> --strict` (the
   native engine; `--engine soda` does not read `--strict`) now fail when an overlay patch
@@ -52,6 +71,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   here`, or the rule's own `description`, instead of quoting the whole object and the
   subschema it must not match.
 
+### Fixed
+
+- **DuckDB 2.0.** `fluid verify` reported every local file as `error` ("Relations no
+  longer have fetchone()"); `fluid apply`'s local SQL and materialize steps, and the
+  actions of the legacy `fluid_build.contract_tests` module, lost their row counts (a
+  materialize step logged `local_materialize_fallback`, a `contract_tests` action
+  reported `-1` rows). DuckDB 2.0 relations have no `fetchone()`, and a query result
+  keeps its connection busy until it is read to the end, which fails the next step on
+  that connection. Every DuckDB result is now read to the end, the same way on 1.5.x and
+  2.x.
+- **DuckDB 2.0: Postgres and MySQL sources.** A DuckDB acquisition build reading
+  Postgres failed on DuckDB 2.0 (`Table Function with name postgres_scan does not
+  exist!`): the postgres extension dropped `postgres_scan`. A MySQL or MariaDB source
+  failed on every DuckDB version, before connecting (`Unknown secret storage found:
+  'local_file'`): its `ATTACH` looked for a default secret in the persistent storage the
+  sandbox turns off. Both are now attached read-only before the sandbox locks, each with
+  an empty temporary secret named in the `ATTACH` (the DSN carries the credentials), and
+  read as `<alias>.<schema>.<table>` on 1.5.x and 2.x; a Postgres source's
+  `connection.secretRef` is resolved into the password, as a MySQL source's already was.
+  A Postgres enum column still reads, and lands, as text. A stream name is looked up as
+  DuckDB looks up names, ignoring case: where two tables' names differ only in case, the
+  build can read the other one.
+  A source the runner cannot attach (unreachable, refused, a `secretRef` that does not
+  resolve) fails the build with the reason, its password masked, and writes a run
+  record; a MySQL or MariaDB source used to end `fluid apply` with `Unexpected error
+  during execution` instead.
+- A local SQL action, or a legacy `fluid_build.contract_tests` action, whose SQL ends in
+  a statement that returns no rows (`CREATE TABLE ... AS`, an `INSERT` without
+  `RETURNING`) now fails with a message saying to write just the `SELECT`. DuckDB 1.5
+  failed there on `'NoneType' object has no attribute 'write_csv'`, and DuckDB 2.0
+  reported success with a one-row `Count` file, an empty `Success` file (a `CREATE TABLE
+  ... AS PIVOT`) or the rows of an earlier `SELECT` in the same SQL as the output.
+- **DuckDB 2.0: contract SQL.** A local SQL action whose SQL ends in a comment after its
+  last `;` failed to write its output (`syntax error at or near ";"`); SQL with a
+  statement that does not parse ran the statements ahead of it before failing, where
+  DuckDB 1.5 runs none; and the result of a `PIVOT` without an `IN` list, or of a query
+  calling `nextval()`, could not be written to an `s3://` output or, in
+  `fluid_build.contract_tests`, to Parquet (`Cannot create the view ...: it reads rows
+  held in this process`). Each now behaves as on 1.5.
+
 ## [0.21.0] — 2026-10-19
 
 ### Upgrade notes

examples/source-aligned-postgres-duckdb/README.md

@@ -1,8 +1,8 @@
 # Source-aligned Postgres → DuckDB → Parquet
 
 A minimal source-aligned data product (`Bronze` / `SDP`) that acquires
-`public.orders` from a Postgres source via DuckDB's `postgres_scan` and
-lands it as Parquet.
+`public.orders` from a Postgres source through DuckDB's postgres extension
+(the source is attached read-only) and lands it as Parquet.
 
 **Vocabulary note.** v0.7.3 carries two interchangeable classifications:
 the medallion `metadata.layer` (`Bronze`/`Silver`/`Gold`) and the Data
@@ -34,7 +34,7 @@ The pipeline:
 
 ```
 fluid validate contract.fluid.yaml
-fluid apply --build ingest_orders contract.fluid.yaml
+fluid apply contract.fluid.yaml --mode amend-and-build --build-id ingest_orders
 ```
 
 `apply` detects `pattern: acquisition` + `engine: duckdb`, dispatches to
@@ -45,7 +45,10 @@ table to `out/orders.parquet`, and persists a run record under
 ## What this exercises
 
 - v0.7.3 `acquisition` build pattern + `duckdb` engine
-- Postgres source via `postgres_scan` (no Airbyte server needed)
+- Postgres source attached by DuckDB's postgres extension (no Airbyte server needed).
+  The attach is `READ_ONLY`, which refuses writes through DuckDB's catalog but
+  is no write barrier on the server; outside this demo, connect as a role that
+  can only read (for example one granted `pg_read_all_data`)
 - Parquet sink via `COPY (...) TO ... (FORMAT 'parquet')`
 - Run-record persistence + lock acquisition
 - Pre-land hook chain (`dlp_scan`, `quality_gate`) — non-blocking on this

examples/source-aligned-postgres-duckdb/contract.fluid.yaml

@@ -4,7 +4,7 @@ id: bronze.crm_orders
 name: CRM Orders Bronze
 description: |
   Source-aligned Bronze data product. Acquires raw orders from a Postgres
-  source via DuckDB's postgres_scan and lands them as Parquet for downstream
+  source through DuckDB's postgres extension and lands them as Parquet for downstream
   Silver/Gold consumption.
 domain: sales
 

fluid_build/build_runners/_embedded_sql_io.py

@@ -471,12 +471,13 @@ def relations_read(sql: str) -> Optional[FrozenSet[str]]:
     if not str(sql or "").strip():
         return None
     try:
+        from fluid_build.providers._duckdb_compat import first_row
         from fluid_build.providers._duckdb_sandbox import DuckDBAllowlist, secure_duckdb_connect
 
         # Parsed only, never bound or run: no file, URL or extension at all.
         con = secure_duckdb_connect(":memory:", allow=DuckDBAllowlist.none())
         try:
-            row = con.execute("SELECT json_serialize_sql(?)", [str(sql)]).fetchone()
+            row = first_row(con.execute("SELECT json_serialize_sql(?)", [str(sql)]))
         finally:
             con.close()
         doc = json.loads(row[0]) if row and row[0] else {}

fluid_build/build_runners/_late_arrival.py

@@ -370,6 +370,7 @@ def split_late_events_in_duckdb(
     can call this too.
     """
     # ── input validation ────────────────────────────────────────────
+    from fluid_build.providers._duckdb_compat import first_row, one_row
     from fluid_build.providers._sql_safety import validate_ident
 
     validate_ident(source_relation)
@@ -380,9 +381,9 @@ def split_late_events_in_duckdb(
         return {"on_time": 0, "late": 0}
 
     # ── compute watermark ────────────────────────────────────────────
-    watermark_row = con.execute(
-        f"SELECT max({event_time_column}) FROM {source_relation}"
-    ).fetchone()
+    watermark_row = first_row(
+        con.execute(f"SELECT max({event_time_column}) FROM {source_relation}")
+    )
     watermark = watermark_row[0] if watermark_row else None
     if watermark is None:
         # Empty source — no enforcement possible.
@@ -401,17 +402,17 @@ def split_late_events_in_duckdb(
     )
 
     # ── append late rows to side-output ──────────────────────────────
-    late_count = con.execute(
-        f"SELECT count(*) FROM {source_relation} WHERE {threshold_expr}"
-    ).fetchone()[0]
+    late_count = one_row(
+        con.execute(f"SELECT count(*) FROM {source_relation} WHERE {threshold_expr}")
+    )[0]
     if late_count > 0:
         con.execute(
             f"INSERT INTO {side_output_relation} "
             f"SELECT * FROM {source_relation} WHERE {threshold_expr}"
         )
         con.execute(f"DELETE FROM {source_relation} WHERE {threshold_expr}")
 
-    on_time_count = con.execute(f"SELECT count(*) FROM {source_relation}").fetchone()[0]
+    on_time_count = one_row(con.execute(f"SELECT count(*) FROM {source_relation}"))[0]
 
     return {"on_time": int(on_time_count), "late": int(late_count)}