diff --git a/CHANGELOG.d/keyverse-dotenv-dependency-rules.md b/CHANGELOG.d/keyverse-dotenv-dependency-rules.md new file mode 100644 index 00000000..77206b6d --- /dev/null +++ b/CHANGELOG.d/keyverse-dotenv-dependency-rules.md @@ -0,0 +1,3 @@ +### Added + +- Six packaged regex rules report direct dotenv-loading and container-file dependencies for the CWL Keyverse migration, with 43 syntax/negative-control cases and six full-scanner integration cases. Findings remain advisory syntax evidence; central migration enforcement, full scanner acceptance and organization-wide cutover are separate gates. See `docs/doctoring/dotenv_dependency_detection.md` and ContextualWisdomLab/.github#2063. diff --git a/docs/doctoring/dotenv_dependency_detection.md b/docs/doctoring/dotenv_dependency_detection.md new file mode 100644 index 00000000..40495afe --- /dev/null +++ b/docs/doctoring/dotenv_dependency_detection.md @@ -0,0 +1,86 @@ +# Dotenv dependency detection for the Keyverse migration + +Date: 2026-09-09. Status: proposed declarative detector; not organization-wide +enforcement or proof of credential migration. + +## Responsibility and implementation decision + +Central migration: ContextualWisdomLab/.github#2063. +Keyverse owner repair: ContextualWisdomLab/keyverse#151, stacked on #129. + +Keyverse owns credential lifecycle and workload resolution; AppGuardrail owns +source-pattern detection. This change adds YAML regex rules to the existing +packaged engine rather than copying scanners into consumers or `.github`. +It adds no new Python runtime or alternative vault. New security-runtime +capabilities remain subject to the organization Rust policy. + +The exact engine read was `e71d37e7c58118e6764c96ab7c4492fe33eed6f8`: +`scanner/cli/appguardrail.py` parses supported `pattern-regex` entries, +compiles them and loads `scanner/rules/*.yml`. A Semgrep structural `pattern:` +file alone would not execute in this built-in path, so it was not used. + +## Detected syntax + +Six IDs cover direct Python load_dotenv/dotenv_values calls, simple Pydantic +`env_file` assignments, direct Rust dotenv/dotenvy calls, direct Node dotenv +loading, shell source/dot of `.env`, and literal Compose/container dotenv-file +transport. Single and double quotes, home/relative paths, common one-line +settings and simple multiline Compose lists are included in the test corpus. + +The rules do not read a referenced dotenv file. They detect code syntax, not +secret values. Ordinary imports, comments in the tested forms, disabling dotenv, +`env_file=None`, non-dotenv configuration and `/dev/null` are negative controls. +Transitive lockfile dependencies and the mere existence of an example are not +claims that a runtime loader executed. + +## Interpretation and limits + +WARNING is intentional: use of dotenv by arbitrary AppGuardrail users is not +proof of an exploitable vulnerability. CWL's central migration gate must adopt +these IDs explicitly for new production regressions while tracking existing +legacy findings to their owner. This PR does not turn all warnings into errors, +change deployment protections or break unrelated standalone consumers. + +The matcher is lexical and bounded, not an AST/alias/taint engine. Dynamic file +variables, wrapper functions, unusual inline control flow, long/multiline calls, +Docker implicit `.env` interpolation, Java/Go/C# sources, framework-native +implicit loading, complex YAML anchors/lists and strings containing code may +need separate evidence or structural detectors. Absence of a finding is not +proof that a repository does not depend on dotenv or that it uses Keyverse. + +Organization inventory must project only rule ID, repository, exact revision, +path, line and reviewed disposition. Do not export generic scanner source +snippets or raw settings as migration evidence; the surrounding code could +contain confidential data. Preserve all unreadable/unscanned repositories in +the denominator. Do not run credential-bearing application source just to scan +it, and do not follow a matched dotenv locator. + +## Verification and remaining gates + +An initial local run failed all 43 syntax/contract tests with the rules absent. +After the YAML was added, 43 passed using the retrieved production parser and +compiler logic in a local excerpt harness. The harness is not committed: the +repository test imports the real owner engine. This is not a complete scanner +checkout and does not establish end-to-end packaging or full coverage. + +Six additional integration cases require actual SCAN_RULES discovery and +`_scan_file` execution in the complete repository. They have been added but +have not been executed in the local excerpt environment. Full repository CI, +coverage/docstrings, packaging, security checks and independent review must +pass before merge/release. This PR adds no workflow copies or model credentials. + +Reproduction in the complete repository: + +```sh +python -m pytest -q tests/test_dotenv_dependency_rules.py tests/test_dotenv_scanner_integration.py +``` + +## Primary references — APA 7th + +OWASP Foundation. (n.d.). *Secrets management cheat sheet*. https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html + +GitHub. (n.d.). *OpenID Connect reference*. https://docs.github.com/en/actions/reference/security/oidc + +The former motivates explicit custody/lifecycle rather than simply renaming +configuration transport. The latter informs the owner workload-authentication +contract, not this source-pattern detector. Neither is a claim of certification. diff --git a/scanner/rules/dotenv_dependencies.yml b/scanner/rules/dotenv_dependencies.yml new file mode 100644 index 00000000..9b33892b --- /dev/null +++ b/scanner/rules/dotenv_dependencies.yml @@ -0,0 +1,55 @@ +rules: + - id: python-dotenv-runtime-load + patterns: + - pattern-regex: '^[ \t]*(?:[A-Za-z_]\w*(?:[ \t]*:[ \t]*[^=\r\n]{1,128})?[ \t]*=[ \t]*)?(?:dotenv\.)?(?:load_dotenv|dotenv_values)[ \t]*\(' + message: | + Runtime dotenv loading is present. For the CWL migration, resolve credentials + through a released Keyverse contract and keep non-secret settings separate. + This is bounded syntax evidence, not proof that secret values were exposed. + severity: WARNING + languages: [python] + - id: python-dotenv-settings-source + patterns: + - pattern-regex: '^[ \t]*(?:[A-Za-z_]\w*[ \t]*=[ \t]*SettingsConfigDict[ \t]*\([ \t]*)?env_file[ \t]*=[ \t]*["\x27][^"\x27\r\n]{0,256}\.env(?:\.[A-Za-z0-9_-]+)?["\x27]' + message: | + A settings source explicitly loads a dotenv file. Replace this credential + authority with Keyverse and retain non-secret settings in typed configuration. + Validate startup and rollback before removing an existing deployment source. + severity: WARNING + languages: [python] + - id: rust-dotenv-runtime-load + patterns: + - pattern-regex: '^[ \t]*(?:let[ \t]+(?:mut[ \t]+)?[A-Za-z_]\w*[ \t]*=[ \t]*)?(?:dotenvy|dotenv)::(?:dotenv|from_filename|from_path)[ \t]*\(' + message: | + A Rust runtime loads dotenv content. Use the released Keyverse credential + port with workload identity; do not substitute environment fallback on failure. + Imported libraries alone are not an executed loader finding. + severity: WARNING + languages: [rust] + - id: node-dotenv-runtime-load + patterns: + - pattern-regex: '^[ \t]*(?:dotenv[ \t]*\.[ \t]*config[ \t]*\(|require[ \t]*\([ \t]*["\x27]dotenv["\x27][ \t]*\)[ \t]*\.[ \t]*config[ \t]*\(|require[ \t]*\([ \t]*["\x27]dotenv/config["\x27][ \t]*\)|import[ \t]*["\x27]dotenv/config["\x27])' + message: | + A Node runtime invokes dotenv or imports its automatic loader. Move server + credentials to Keyverse; never place resolved values in browser bundles. + This rule covers the declared direct syntax, not arbitrary import aliases. + severity: WARNING + languages: [javascript, typescript] + - id: shell-dotenv-source + patterns: + - pattern-regex: '^[ \t]*(?:source|\.)[ \t]+["\x27]?(?:[^"\x27\r\n#;]{0,256}/)?\.env(?:\.[A-Za-z0-9_-]+)?(?=["\x27\s;]|$)' + message: | + A shell sources dotenv content as executable configuration. Use approved + Keyverse bootstrap and credential resolution rather than sourcing secret files. + Confirm the deployment owner and recovery path before retiring the loader. + severity: WARNING + languages: [generic] + - id: compose-dotenv-transport + patterns: + - pattern-regex: '^[ \t]*(?:env_file[ \t]*:[ \t]*(?:\[[ \t]*|\r?\n[ \t]*-[ \t]*)?["\x27]?[^"\x27\r\n#]{0,256}\.env(?:\.[A-Za-z0-9_-]+)?(?=["\x27\s\],]|$)|(?:[A-Za-z_]\w*[ \t]*:?=[ \t]*)?(?:docker|podman)[ \t]+(?:compose|run)[^\r\n#]{0,256}--env-file(?:[ \t]+|=)["\x27]?[^"\x27\r\n#]{0,256}\.env(?:\.[A-Za-z0-9_-]+)?(?=["\x27\s;]|$))' + message: | + A container invocation explicitly depends on a dotenv file. Adopt the released + Keyverse credential contract and a non-secret deployment descriptor instead. + This is migration inventory; it does not establish complete secret coverage. + severity: WARNING + languages: [generic] diff --git a/tests/test_dotenv_dependency_rules.py b/tests/test_dotenv_dependency_rules.py new file mode 100644 index 00000000..49e46eab --- /dev/null +++ b/tests/test_dotenv_dependency_rules.py @@ -0,0 +1,97 @@ +"""Bounded syntax evidence for the CWL Keyverse credential migration.""" +from pathlib import Path + +import pytest + +from scanner.cli.appguardrail import _compile_yaml_regex_rule, _parse_yaml_regex_rules + +RULE_PATH = Path(__file__).resolve().parents[1] / "scanner/rules/dotenv_dependencies.yml" +RULE_IDS = { + "python-dotenv-runtime-load", "python-dotenv-settings-source", + "rust-dotenv-runtime-load", "node-dotenv-runtime-load", + "shell-dotenv-source", "compose-dotenv-transport", +} + + +def loaded_rules(): + """Compile through the owner's supported YAML engine, not Semgrep fixtures.""" + assert RULE_PATH.is_file(), "the executable dotenv migration rules are missing" + parsed_rules = _parse_yaml_regex_rules(RULE_PATH.read_text(encoding="utf-8")) + compiled_rules = [compiled for parsed in parsed_rules + for compiled in _compile_yaml_regex_rule(parsed)] + assert {rule["id"] for rule in compiled_rules} == RULE_IDS + assert len(compiled_rules) == len(RULE_IDS), "one executable pattern per ID" + return {rule["id"]: rule for rule in compiled_rules} + + +@pytest.mark.parametrize("rule_id,source", [ + ("python-dotenv-runtime-load", "load_dotenv()"), + ("python-dotenv-runtime-load", " dotenv.load_dotenv(override=True)"), + ("python-dotenv-runtime-load", 'settings = dotenv_values(".env")'), + ("python-dotenv-runtime-load", 'values: dict[str, str] = dotenv_values(".env")'), + ("python-dotenv-runtime-load", "values = dotenv.dotenv_values()"), + ("python-dotenv-settings-source", ' env_file=".env",'), + ("python-dotenv-settings-source", 'model_config = SettingsConfigDict(env_file="../.env")'), + ("python-dotenv-settings-source", 'model_config = SettingsConfigDict(\n env_file=".env",\n)'), + ("python-dotenv-settings-source", ' env_file = "~/.env.production"'), + ("rust-dotenv-runtime-load", " dotenvy::dotenv().ok();"), + ("rust-dotenv-runtime-load", 'let _ = dotenv::from_filename(".env");'), + ("rust-dotenv-runtime-load", 'let mut loaded = dotenvy::dotenv();'), + ("rust-dotenv-runtime-load", 'dotenvy::from_path("private.env")?;'), + ("node-dotenv-runtime-load", "dotenv.config();"), + ("node-dotenv-runtime-load", "require('dotenv').config();"), + ("node-dotenv-runtime-load", 'require("dotenv/config");'), + ("node-dotenv-runtime-load", 'import "dotenv/config";'), + ("shell-dotenv-source", 'source "$HOME/.env"'), + ("shell-dotenv-source", ". .env"), + ("shell-dotenv-source", ". '/run/app/.env.production'"), + ("compose-dotenv-transport", ' env_file: .env'), + ("compose-dotenv-transport", ' env_file:\n - .env.production'), + ("compose-dotenv-transport", ' env_file: ["../.env"]'), + ("compose-dotenv-transport", 'docker compose --env-file "$HOME/.env" up'), + ("compose-dotenv-transport", 'COMPOSE := docker compose --env-file "$$HOME/.env"'), + ("compose-dotenv-transport", 'podman run --env-file .env application'), +]) +def test_declared_runtime_dependency_is_detected(rule_id, source): + """The tested literal loader/transport syntax produces evidence.""" + assert loaded_rules()[rule_id]["pattern"].search(source) + + +@pytest.mark.parametrize("rule_id,source", [ + ("python-dotenv-runtime-load", "# load_dotenv()"), + ("python-dotenv-runtime-load", "from dotenv import load_dotenv"), + ("python-dotenv-runtime-load", 'description = "load_dotenv()"'), + ("python-dotenv-runtime-load", "def load_dotenv():"), + ("python-dotenv-runtime-load", 'os.environ["PYTHON_DOTENV_DISABLED"] = "1"'), + ("python-dotenv-settings-source", "env_file = None"), + ("python-dotenv-settings-source", 'env_file = "settings.toml"'), + ("python-dotenv-settings-source", '# env_file = ".env"'), + ("rust-dotenv-runtime-load", "// dotenvy::dotenv().ok();"), + ("rust-dotenv-runtime-load", "use dotenvy::dotenv;"), + ("node-dotenv-runtime-load", "// dotenv.config();"), + ("node-dotenv-runtime-load", 'import dotenv from "dotenv";'), + ("node-dotenv-runtime-load", 'const note = "dotenv.config()";'), + ("shell-dotenv-source", "# source ~/.env"), + ("shell-dotenv-source", "source /run/configuration/settings.sh"), + ("compose-dotenv-transport", "# env_file: .env"), + ("compose-dotenv-transport", 'env_file: settings.config'), + ("compose-dotenv-transport", "docker compose --env-file /dev/null up"), +]) +def test_negative_control_is_not_a_migration_finding(rule_id, source): + """Imports, comments, disabled dotenv and non-dotenv config are distinct.""" + assert not loaded_rules()[rule_id]["pattern"].search(source) + + +def test_migration_rules_do_not_claim_a_proven_secret_leak(): + """Syntax evidence is advisory until an explicit organization gate adopts it.""" + for rule in loaded_rules().values(): + assert rule["severity"] == "WARNING" + assert "Keyverse" in rule["message"] + assert "secret leak" not in rule["message"].lower() + + +def test_patterns_stay_bounded_on_long_nonmatching_lines(): + """Do not introduce unbounded wildcard alternatives into migration scanning.""" + for rule in loaded_rules().values(): + assert not rule["pattern"].search("x" * 100_000) + assert ".*" not in rule["pattern"].pattern diff --git a/tests/test_dotenv_scanner_integration.py b/tests/test_dotenv_scanner_integration.py new file mode 100644 index 00000000..6ce31042 --- /dev/null +++ b/tests/test_dotenv_scanner_integration.py @@ -0,0 +1,30 @@ +"""The complete installed scanner must load and execute migration rules.""" +import pytest + +from scanner.cli.appguardrail import SCAN_RULES, _scan_file + + +@pytest.mark.parametrize("rule_id,file_name,source", [ + ("python-dotenv-runtime-load", "runtime_config.py", "load_dotenv()\n"), + ( + "python-dotenv-runtime-load", + "typed_runtime_config.py", + 'values: dict[str, str] = dotenv_values(".env")\n', + ), + ("python-dotenv-settings-source", "runtime_settings.py", 'env_file=".env"\n'), + ("rust-dotenv-runtime-load", "runtime_config.rs", "dotenvy::dotenv().ok();\n"), + ("node-dotenv-runtime-load", "runtime_config.js", "require('dotenv').config();\n"), + ("shell-dotenv-source", "runtime_start.sh", 'source "$HOME/.env"\n'), + ("compose-dotenv-transport", "compose_runtime.yml", "env_file: .env\n"), +]) +def test_packaged_scanner_executes_dotenv_rule(tmp_path, rule_id, file_name, source): + """A rule file alone is insufficient: production discovery must execute it.""" + registered = [rule for rule in SCAN_RULES if rule["id"] == rule_id] + assert len(registered) == 1 + source_file = tmp_path / file_name + source_file.write_text(source, encoding="utf-8") + findings = [finding for finding in _scan_file(source_file, tmp_path) + if finding["rule_id"] == rule_id] + assert len(findings) == 1 + assert findings[0]["line"] == 1 + assert findings[0]["severity"] == "WARNING"