Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Security

- **An acquisition source no longer receives this host's ambient database credentials.** A contract
names the host its postgres or mysql source lives on, and libpq and mysql_scanner read
credentials from the process environment for whatever host they are handed — so a contract naming
a host of its own choosing could be authenticated with this machine's `PGPASSWORD`, `PGPASSFILE`,
`PGSERVICE` or `MYSQL_PWD`: a credential the contract was never given, belonging to a different
server. `fluid init --discover` has suppressed these since its JDBC introspection landed; the
acquisition runners had not, so the same contract fields reached the same client libraries with
the environment intact.
The duckdb runner's `ATTACH` now runs inside the same suppression window, which is shared from
`credentials/host_db_env.py` rather than copied (`build_runners` may not import `cli`). 14
variables are covered, and `cli/discover/_jdbc_introspect` keeps its original private names as
aliases, so nothing that referenced them changes.
**This route is independent of a contract's own `connection.secretRef`** and no allowlist over
`secretRef` would close it, because the credential never passes through `secretRef` at all. The
declared DSN still carries its own credentials, so a legitimate source is unaffected and no
contract change is needed.


### Fixed

- **An incremental cursor written on DuckDB 1.5 and read on 2.0 silently dropped or
Expand Down
8 changes: 7 additions & 1 deletion fluid_build/build_runners/duckdb/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
)
from fluid_build.api.schema import SchemaFingerprint
from fluid_build.api.source import AcquisitionMode
from fluid_build.credentials.host_db_env import host_credential_env_suppressed
from fluid_build.providers._duckdb_compat import first_row
from fluid_build.providers._duckdb_sandbox import (
DuckDBAllowlist,
Expand Down Expand Up @@ -883,7 +884,12 @@ def _attach_external_databases(self, con: Any, ctx: RunContext) -> None:
raise _SourceSetupError(str(exc)) from exc
except duckdb.Error as exc: # the extension did not load
raise _SourceAttachError(str(exc)) from exc
_attach_with_password_masked(con, attach_sql, conn.get("password"))
# The contract chose this host, and libpq / mysql_scanner would
# read this machine's credentials for it (see host_db_env). The
# declared DSN still carries its own, so a real source is
# unaffected.
with host_credential_env_suppressed():
_attach_with_password_masked(con, attach_sql, conn.get("password"))
elif kind == "sqlite":
conn = dict(ctx.source.connection.raw)
# SQLite uses a path, not a libpq-style DSN. The path comes
Expand Down
78 changes: 16 additions & 62 deletions fluid_build/cli/discover/_jdbc_introspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@
from typing import Any, Dict, Iterator, List, Optional, Set, Tuple
from urllib.parse import urlparse

from fluid_build.credentials.host_db_env import (
HOST_CREDENTIAL_ENV,
NO_CLIENT_CERT,
host_credential_env_suppressed,
)
from fluid_build.providers._sql_safety import (
build_libpq_dsn,
build_mysql_dsn,
Expand Down Expand Up @@ -115,39 +120,16 @@
# MySQL client's connection defaults (MySQL 8.4 Reference Manual, "Environment
# Variables"), cleared so none of them stands in for a part the DSN leaves out.
# MYSQL_GROUP_SUFFIX only names extra option-file groups. DuckDB 1.5.6's
# mysql_scanner was seen to send no password from an option file (~/.my.cnf,
# ~/.mylogin.cnf, ~/my.cnf, /etc/my.cnf, /etc/mysql/my.cnf, $MYSQL_HOME/my.cnf),
# with or without that suffix, so clearing it is a precaution: there is no
# option-file fallback for it to close.
_HOST_CREDENTIAL_ENV = (
"PGPASSWORD",
"PGPASSFILE",
"PGSERVICE",
"PGSERVICEFILE",
"PGSYSCONFDIR",
"PGSSLCERT",
"PGSSLKEY",
"PGOAUTHTOKEN",
"PGGSSDELEGATION",
"MYSQL_PWD",
"MYSQL_HOST",
"MYSQL_TCP_PORT",
"MYSQL_UNIX_PORT",
"MYSQL_GROUP_SUFFIX",
)

# libpq skips a client certificate whose file does not exist (ENOENT or
# ENOTDIR). Pointing a pinned connection's certificate and key below os.devnull
# means it never presents this host's ~/.postgresql/postgresql.crt to a server
# that asks for one.
_NO_CLIENT_CERT = os.path.join(os.devnull, "fluid-no-client-certificate")

# The suppression below mutates the process environment. forge_run's in-process
# worker thread can run alongside forge_from_source, so windows are serialised
# (and re-entrant within one thread) rather than interleaved: two overlapping
# windows could otherwise restore the host credentials while the second attach
# is still inside its own.
_HOST_CREDENTIAL_ENV_LOCK = threading.RLock()
# The one implementation now lives in ``credentials/host_db_env`` so the
# acquisition runners can use it too -- ``build_runners`` may not import
# ``cli``. These aliases keep this module's original private names, which its
# own tests and the patch points around them refer to.
# Only the names something actually refers to: _HOST_CREDENTIAL_ENV from this
# module's own tests, _NO_CLIENT_CERT from build_attach_dsn below. The lock
# needs no alias -- nothing outside host_db_env touches it, and it is reachable
# there as HOST_CREDENTIAL_ENV_LOCK if that ever changes.
_HOST_CREDENTIAL_ENV = HOST_CREDENTIAL_ENV
Comment thread
github-code-quality[bot] marked this conversation as resolved.
Fixed
Comment thread
fas89 marked this conversation as resolved.
Resolved
_NO_CLIENT_CERT = NO_CLIENT_CERT


# The mysql_scanner quoting lives beside the libpq one in _sql_safety, which
Expand Down Expand Up @@ -341,35 +323,7 @@ def _uri_scheme(uri: str) -> str:
)


@contextlib.contextmanager
def _host_credential_env_suppressed() -> Iterator[None]:
"""Hide this host's ambient DB credential env vars for a pinned ATTACH.

The saved values are put back when the window closes, also when the attach
raises. Windows opened through this function in different threads are
serialised by :data:`_HOST_CREDENTIAL_ENV_LOCK`, so they never interleave
their save and restore; within one thread a nested window is safe when it
closes before the window around it.

That is all the lock covers. Code that does not go through this function
is not serialised with it, so there is no guarantee against:

* another thread reading these variables, which sees them unset while a
window is open;
* ``os.environ.copy()`` in another thread, which can race the pop and the
restore (and raise ``KeyError``);
* an ``engine._run_sync_stage(env_snapshot=True)`` whose snapshot and
restore overlap a window, which can delete the variables for good or
bring them back inside the window;
* same-thread windows that do not close in LIFO order, where the first to
close restores the variables inside the other.
"""
with _HOST_CREDENTIAL_ENV_LOCK:
saved = {k: os.environ.pop(k) for k in _HOST_CREDENTIAL_ENV if k in os.environ}
try:
yield
finally:
os.environ.update(saved)
_host_credential_env_suppressed = host_credential_env_suppressed


@dataclass
Expand Down
107 changes: 107 additions & 0 deletions fluid_build/credentials/host_db_env.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# Copyright 2024-2026 Agentics Transformation Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This host's ambient database credentials, hidden for the length of one attach.

A contract names the host its source lives on. libpq and mysql_scanner then read
credentials from the PROCESS environment for whatever host they are handed, so a
contract naming a host of its own choosing could be authenticated with this
machine's ``PGPASSWORD`` -- a credential the contract was never given, belonging
to a different server. No allowlist over the contract's own ``secretRef`` closes
that, because the credential never passes through ``secretRef`` at all.

``fluid init --discover`` has suppressed these since its JDBC introspection
landed. The acquisition runners had not, so the same contract fields reached the
same client libraries with the environment intact.

This is where the one implementation lives so both use it. It sits under
``credentials`` rather than beside the discover code because ``build_runners``
may not import ``cli`` (the import-linter contract "build_runners must not
depend on cli"), and because hiding a credential is a credentials concern rather
than a discover detail. ``cli/discover/_jdbc_introspect`` re-exports these under
their original private spellings, so the tests and patch points that grew up
there keep working unchanged.
"""

from __future__ import annotations

import contextlib
import os
import threading
from typing import Iterator

# mysql_scanner was seen to send no password from an option file (~/.my.cnf,
# ~/.mylogin.cnf, ~/my.cnf, /etc/my.cnf, /etc/mysql/my.cnf, $MYSQL_HOME/my.cnf),
# with or without that suffix, so clearing it is a precaution: there is no
# option-file fallback for it to close.
HOST_CREDENTIAL_ENV = (
"PGPASSWORD",
"PGPASSFILE",
"PGSERVICE",
"PGSERVICEFILE",
"PGSYSCONFDIR",
"PGSSLCERT",
"PGSSLKEY",
"PGOAUTHTOKEN",
"PGGSSDELEGATION",
"MYSQL_PWD",
"MYSQL_HOST",
"MYSQL_TCP_PORT",
"MYSQL_UNIX_PORT",
"MYSQL_GROUP_SUFFIX",
)

# libpq skips a client certificate whose file does not exist (ENOENT or
# ENOTDIR). Pointing a pinned connection's certificate and key below os.devnull
# means it never presents this host's ~/.postgresql/postgresql.crt to a server
# that asks for one.
NO_CLIENT_CERT = os.path.join(os.devnull, "fluid-no-client-certificate")

# The suppression below mutates the process environment. forge_run's in-process
# worker thread can run alongside forge_from_source, so windows are serialised
# (and re-entrant within one thread) rather than interleaved: two overlapping
# windows could otherwise restore the host credentials while the second attach
# is still inside its own.
HOST_CREDENTIAL_ENV_LOCK = threading.RLock()


@contextlib.contextmanager
def host_credential_env_suppressed() -> Iterator[None]:
"""Hide this host's ambient DB credential env vars for a pinned ATTACH.

The saved values are put back when the window closes, also when the attach
raises. Windows opened through this function in different threads are
serialised by :data:`HOST_CREDENTIAL_ENV_LOCK`, so they never interleave
their save and restore; within one thread a nested window is safe when it
closes before the window around it.

That is all the lock covers. Code that does not go through this function
is not serialised with it, so there is no guarantee against:

* another thread reading these variables, which sees them unset while a
window is open;
* ``os.environ.copy()`` in another thread, which can race the pop and the
restore (and raise ``KeyError``);
* an ``engine._run_sync_stage(env_snapshot=True)`` whose snapshot and
restore overlap a window, which can delete the variables for good or
bring them back inside the window;
* same-thread windows that do not close in LIFO order, where the first to
close restores the variables inside the other.
"""
with HOST_CREDENTIAL_ENV_LOCK:
saved = {k: os.environ.pop(k) for k in HOST_CREDENTIAL_ENV if k in os.environ}
try:
yield
finally:
os.environ.update(saved)
Loading
Loading