From a27686da6cf7ebb307ef379e262d4783eda0d49d Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 15:59:46 +0200 Subject: [PATCH 01/31] feat(api,sdk): write-only vault secrets (values never readable back by users) --- api/oss/src/apis/fastapi/access/router.py | 12 +- api/oss/src/apis/fastapi/vault/router.py | 55 +- api/oss/src/core/secrets/dtos.py | 80 ++- api/oss/src/core/secrets/redaction.py | 80 +++ api/oss/src/core/secrets/services.py | 100 +++- api/oss/src/core/workflows/service.py | 7 +- api/oss/src/dbs/postgres/secrets/mappings.py | 42 +- api/oss/src/middlewares/auth.py | 26 +- .../unit/middlewares/test_auth_grants.py | 141 +++++ .../pytest/unit/secrets/test_write_only.py | 516 ++++++++++++++++++ .../unit/vault/test_write_only_routes.py | 257 +++++++++ docs/design/write-only-secrets/README.md | 99 ++++ .../agenta/sdk/agents/connections/__init__.py | 2 + .../agenta/sdk/agents/connections/errors.py | 26 + .../agenta/sdk/agents/platform/connections.py | 15 + .../agenta/sdk/agents/platform/secrets.py | 20 +- .../agenta/sdk/middlewares/running/vault.py | 51 ++ .../platform/test_write_only_secrets.py | 169 ++++++ 18 files changed, 1663 insertions(+), 35 deletions(-) create mode 100644 api/oss/src/core/secrets/redaction.py create mode 100644 api/oss/tests/pytest/unit/middlewares/test_auth_grants.py create mode 100644 api/oss/tests/pytest/unit/secrets/test_write_only.py create mode 100644 api/oss/tests/pytest/unit/vault/test_write_only_routes.py create mode 100644 docs/design/write-only-secrets/README.md create mode 100644 sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py diff --git a/api/oss/src/apis/fastapi/access/router.py b/api/oss/src/apis/fastapi/access/router.py index 521fe2fdac..505262db56 100644 --- a/api/oss/src/apis/fastapi/access/router.py +++ b/api/oss/src/apis/fastapi/access/router.py @@ -4,7 +4,10 @@ from fastapi import APIRouter, Query, HTTPException, Request from fastapi.responses import JSONResponse -from oss.src.middlewares.auth import sign_secret_token +from oss.src.middlewares.auth import ( + SECRET_RESOLVE_GRANT, + sign_secret_token, +) from oss.src.utils.logging import get_module_logger from oss.src.utils.caching import get_cache, set_cache from oss.src.utils.context import get_auth_context, get_auth_scope @@ -135,6 +138,12 @@ async def check_permissions( # Always re-mint a fresh ephemeral Secret token (same scope, new expiry) so the # returned credential is uniformly short-lived and renewable — never echoing an # ApiKey/Bearer. Callers (services, the runner) re-check periodically to refresh. + # + # A run_service exchange is "I am about to execute a workload that needs the stored + # keys", so that credential — and only that one — carries the secret-resolve grant + # letting the vault return write-only secret values in plaintext. This is the + # GitHub-secrets line: a member who can run workloads can reach the values through + # a run either way; direct reads with a session/ApiKey stay redacted. secret_token = await sign_secret_token( user_id=user_id, user_email=getattr(request.state, "user_email", None), @@ -142,6 +151,7 @@ async def check_permissions( workspace_id=str(ctx.scope.workspace_id), organization_id=str(ctx.scope.organization_id), organization_name=getattr(request.state, "organization_name", None), + grants=[SECRET_RESOLVE_GRANT] if action == "run_service" else None, ) credentials_header = f"Secret {secret_token}" diff --git a/api/oss/src/apis/fastapi/vault/router.py b/api/oss/src/apis/fastapi/vault/router.py index df12b051b0..b559deadcb 100644 --- a/api/oss/src/apis/fastapi/vault/router.py +++ b/api/oss/src/apis/fastapi/vault/router.py @@ -15,11 +15,15 @@ CreateSecretDTO, UpdateSecretDTO, SecretResponseDTO, + WriteOnlyCannotBeDisabledError, ) +from oss.src.core.secrets.redaction import redact_secret_response from oss.src.core.access.permissions.types import Permission from oss.src.core.access.permissions.service import check_action_access +from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT, request_has_grant + log = get_module_logger(__name__) @@ -104,6 +108,21 @@ def __init__( operation_id="delete_secret", ) + @staticmethod + def _for_caller( + request: Request, secret_dto: SecretResponseDTO + ) -> SecretResponseDTO: + """The response shape ``request``'s principal may see. + + Only the platform runtime (a Secret token carrying the ``secret-resolve`` grant) + receives write-only values in plaintext; every user principal — session, ApiKey, + unscoped Secret token — gets the redacted shape. + """ + if request_has_grant(request, SECRET_RESOLVE_GRANT): + return secret_dto + + return redact_secret_response(secret_dto) + @intercept_exceptions() async def create_secret(self, request: Request, body: CreateSecretDTO): has_permission = await check_action_access( @@ -126,7 +145,7 @@ async def create_secret(self, request: Request, body: CreateSecretDTO): await invalidate_cache( project_id=request.state.project_id, ) - return vault_secret + return self._for_caller(request, vault_secret) @intercept_exceptions() async def list_secrets(self, request: Request): @@ -143,6 +162,13 @@ async def list_secrets(self, request: Request): status_code=403, ) + # The runtime (secret-resolve grant) needs plaintext, and the shared cache only + # ever stores the redacted shape, so grant-bearing reads go straight to the DB. + if request_has_grant(request, SECRET_RESOLVE_GRANT): + return await self.service.list_secrets( + project_id=UUID(request.state.project_id), + ) + cache_key = {} secrets_dtos = await get_cache( @@ -154,12 +180,16 @@ async def list_secrets(self, request: Request): ) if secrets_dtos is not None: - return secrets_dtos + # Entries are stored redacted; re-redacting is idempotent and shields against + # entries written before write-only secrets existed. + return [redact_secret_response(dto) for dto in secrets_dtos] secrets_dtos = await self.service.list_secrets( project_id=UUID(request.state.project_id), ) + secrets_dtos = [redact_secret_response(dto) for dto in secrets_dtos] + await set_cache( project_id=request.state.project_id, namespace="list_secrets", @@ -207,7 +237,7 @@ async def read_secret(self, request: Request, secret_id_or_slug: str): raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Secret not found" ) - return secrets_dto + return self._for_caller(request, secrets_dto) @intercept_exceptions() async def update_secret( @@ -226,12 +256,17 @@ async def update_secret( status_code=403, ) - secrets_dto = await self.service.update_secret( - project_id=UUID(request.state.project_id), - secret_id=UUID(secret_id), - update_secret_dto=body, - user_id=UUID(request.state.user_id), - ) + try: + secrets_dto = await self.service.update_secret( + project_id=UUID(request.state.project_id), + secret_id=UUID(secret_id), + update_secret_dto=body, + user_id=UUID(request.state.user_id), + ) + except WriteOnlyCannotBeDisabledError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=e.message + ) from e if secrets_dto is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Secret not found" @@ -239,7 +274,7 @@ async def update_secret( await invalidate_cache( project_id=request.state.project_id, ) - return secrets_dto + return self._for_caller(request, secrets_dto) @intercept_exceptions() async def delete_secret(self, request: Request, secret_id: str): diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index d7de1b31df..a677387f2b 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -1,4 +1,4 @@ -from typing import Optional, Union, List, Dict, Any +from typing import ClassVar, Optional, Union, List, Dict, Any from pydantic import BaseModel, Field, model_validator @@ -17,8 +17,29 @@ from oss.src.core.webhooks.utils import validate_url_format_and_literal_ip +class WriteOnlyCannotBeDisabledError(Exception): + """Raised when an update tries to turn `write_only` off. + + Turning it off would make the stored value readable again, defeating the flag's whole + guarantee. The transition is one-way: off -> on only. + """ + + def __init__( + self, + message: str = "A write-only secret cannot be made readable again. " + "Delete it and create a new secret instead.", + ): + self.message = message + super().__init__(message) + + +# The value-bearing fields below are Optional so that read models can carry a redacted +# (value-less) shape and updates can omit a value to mean "keep the stored one". Presence +# at CREATE time is still enforced, by `SecretDTO.validate_secret_data_based_on_kind`. + + class StandardProviderSettingsDTO(BaseModel): - key: str + key: Optional[str] = None class CustomModelSettingsDTO(BaseModel): @@ -55,7 +76,7 @@ class CustomProviderDTO(BaseModel): class SSOProviderSettingsDTO(BaseModel): client_id: str - client_secret: str + client_secret: Optional[str] = None issuer_url: str scopes: List[str] extra: Dict[str, Any] = Field(default_factory=dict) @@ -66,7 +87,7 @@ class SSOProviderDTO(BaseModel): class WebhookProviderSettingsDTO(BaseModel): - key: str + key: Optional[str] = None class WebhookProviderDTO(BaseModel): @@ -75,7 +96,7 @@ class WebhookProviderDTO(BaseModel): class CustomSecretSettingsDTO(BaseModel): format: CustomSecretFormat - content: Union[str, Dict[str, Union[str, int, float, bool, None]]] + content: Optional[Union[str, Dict[str, Union[str, int, float, bool, None]]]] = None # text -> content is a str (stored verbatim); json -> a flat {str: primitive} map. @@ -93,6 +114,11 @@ class SecretDTO(BaseModel): CustomSecretDTO, ] + # Whether the kind's value field (provider key, custom-secret content, ...) must be + # present. True on the create path; the update payload and the response model turn it + # off so a value-less shape (keep-stored-on-omit, write-only redaction) validates. + VALUE_REQUIRED: ClassVar[bool] = True + @model_validator(mode="before") def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): kind = values.get("kind") @@ -112,7 +138,9 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): "The provided request secret dto is not a valid type for StandardProviderDTO" ) provider = data.get("provider") - if not isinstance(provider, dict) or "key" not in provider: + if not isinstance(provider, dict) or ( + cls.VALUE_REQUIRED and provider.get("key") is None + ): raise ValueError( "The provided request secret dto is missing required fields for StandardProviderSettingsDTO" ) @@ -160,7 +188,9 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): raise ValueError( "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" ) - required_fields = {"client_id", "client_secret", "issuer_url", "scopes"} + required_fields = {"client_id", "issuer_url", "scopes"} + if cls.VALUE_REQUIRED: + required_fields.add("client_secret") if not required_fields.issubset(provider.keys()): raise ValueError( "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" @@ -171,7 +201,9 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): "The provided request secret dto is not a valid type for WebhookProviderDTO" ) provider = data.get("provider") - if not isinstance(provider, dict) or "key" not in provider: + if not isinstance(provider, dict) or ( + cls.VALUE_REQUIRED and provider.get("key") is None + ): raise ValueError( "The provided request secret dto is missing required fields for WebhookProviderSettingsDTO" ) @@ -184,13 +216,15 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): if ( not isinstance(secret, dict) or "format" not in secret - or "content" not in secret + or (cls.VALUE_REQUIRED and secret.get("content") is None) ): raise ValueError( "The provided request secret dto requires data.secret.{format, content} for CustomSecretDTO" ) - fmt, content = secret["format"], secret["content"] - if fmt == CustomSecretFormat.TEXT.value: + fmt, content = secret["format"], secret.get("content") + if content is None: + pass # Value-less shape allowed when VALUE_REQUIRED is off; nothing to type-check. + elif fmt == CustomSecretFormat.TEXT.value: if not isinstance(content, str): raise ValueError("A text custom_secret requires a string content") # Stored verbatim; do NOT re-serialize a JSON-looking string. @@ -213,6 +247,9 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): class CreateSecretDTO(Slug, BaseModel): header: Header secret: SecretDTO + # None means "platform default", which is write-only. An explicit False is the + # compatibility escape hatch for callers that still need to read the value back. + write_only: Optional[bool] = None @model_validator(mode="before") def ensure_header_exists(cls, values): @@ -258,9 +295,19 @@ def update_provider_slug_with_header_name(cls, values): return values +class UpdateSecretPayloadDTO(SecretDTO): + """The update-path secret payload: same shape as `SecretDTO`, but a value field may be + omitted to mean "keep the stored value" (see `VaultService.update_secret`).""" + + VALUE_REQUIRED: ClassVar[bool] = False + + class UpdateSecretDTO(BaseModel): header: Optional[Header] = None - secret: Optional[SecretDTO] = None + secret: Optional[UpdateSecretPayloadDTO] = None + # None keeps the stored flag. True tightens a readable secret to write-only. + # False on a write-only secret is rejected (`WriteOnlyCannotBeDisabledError`). + write_only: Optional[bool] = None @model_validator(mode="before") def update_provider_slug_with_header_name(cls, values): @@ -279,6 +326,15 @@ class SecretResponseDTO(Identifier, Slug, SecretDTO): header: Header lifecycle: Optional[LegacyLifecycleDTO] = None + write_only: bool = False + # Server-computed, set only on redacted (write-only) user-facing responses so a client + # can show that a value exists, and which one, without ever receiving it. + has_key: Optional[bool] = None + key_preview: Optional[str] = None + + # A read model may carry a redacted, value-less payload. + VALUE_REQUIRED: ClassVar[bool] = False + @model_validator(mode="before") def build_up_model_keys(cls, values: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/api/oss/src/core/secrets/redaction.py b/api/oss/src/core/secrets/redaction.py new file mode 100644 index 0000000000..cd8707f871 --- /dev/null +++ b/api/oss/src/core/secrets/redaction.py @@ -0,0 +1,80 @@ +"""Redaction of write-only vault secrets for user-facing responses. + +A secret with ``write_only=True`` can be created, replaced, and deleted, but its value is +never returned to a user: every vault route strips the value and attaches ``has_key`` and a +``key_preview`` instead. Only the platform runtime — a caller whose verified Secret token +carries the ``secret-resolve`` grant — receives the plaintext, because the workload it runs +needs the real key. In-process readers (`VaultService` and below) are untouched: redaction +happens strictly at the API response boundary. +""" + +from typing import Any, Optional + +from oss.src.core.secrets.enums import SecretKind +from oss.src.core.secrets.dtos import SecretResponseDTO + + +# Keys inside a custom provider's free-form `extras` that hold credential material (the +# SDK's connection resolver consumes `extras.api_key` as the key, and the AWS trio is +# injected as credentials). Redaction strips these; plain config (region, api_version) +# stays readable. `VaultService`'s update carry-over fills the same set back in when a +# replace-only form omits them. +CREDENTIAL_EXTRAS_KEYS = ( + "api_key", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", +) + + +def mask_secret_value(value: str) -> str: + """A short, non-reversible display preview like ``sk-****9Qa``. + + Values shorter than 12 characters mask entirely: revealing 6 of their characters + would give away most of the secret. + """ + if len(value) >= 12: + return f"{value[:3]}****{value[-3:]}" + + return "****" + + +def redact_secret_response(secret: SecretResponseDTO) -> SecretResponseDTO: + """The user-facing shape of ``secret``: value stripped when it is write-only. + + Returns the input unchanged for readable (``write_only=False``) secrets, so legacy + records keep their exact response. Never mutates the input. + """ + if not secret.write_only: + return secret + + redacted = secret.model_copy(deep=True) + + value: Optional[Any] = None + data = redacted.data + + if redacted.kind in ( + SecretKind.PROVIDER_KEY, + SecretKind.CUSTOM_PROVIDER, + SecretKind.WEBHOOK_PROVIDER, + ): + value = data.provider.key + data.provider.key = None + extras = getattr(data.provider, "extras", None) + if extras: + value = value or extras.get("api_key") + for extras_key in CREDENTIAL_EXTRAS_KEYS: + extras.pop(extras_key, None) + elif redacted.kind == SecretKind.SSO_PROVIDER: + value = data.provider.client_secret + data.provider.client_secret = None + elif redacted.kind == SecretKind.CUSTOM_SECRET: + value = data.secret.content + data.secret.content = None + + redacted.has_key = bool(value) + redacted.key_preview = ( + mask_secret_value(value) if isinstance(value, str) and value else None + ) + + return redacted diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 1c2087c758..d6dbbbeca7 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -10,7 +10,12 @@ ) from oss.src.core.secrets.interfaces import SecretsDAOInterface from oss.src.core.secrets.context import set_data_encryption_key -from oss.src.core.secrets.dtos import CreateSecretDTO, UpdateSecretDTO +from oss.src.core.secrets.redaction import CREDENTIAL_EXTRAS_KEYS +from oss.src.core.secrets.dtos import ( + CreateSecretDTO, + UpdateSecretDTO, + WriteOnlyCannotBeDisabledError, +) def next_provider_key_name( @@ -35,6 +40,70 @@ def next_provider_key_name( return f"{title} {index}" +# The value-bearing (credential) field of each payload shape, as (container, field) pairs. +# `provider.key` covers standard providers, custom providers, and webhooks; the other two +# cover SSO and custom secrets. `_carry_over_saved_value` probes with hasattr, so pairs a +# given kind does not have are simply skipped. +_VALUE_FIELDS = ( + ("provider", "key"), + ("provider", "client_secret"), + ("secret", "content"), +) + + +def _carry_over_saved_value(*, stored_data: Any, update_data: Any) -> None: + """Fill an update payload's omitted value field from the stored record. + + An update that omits the value means "keep the stored one" — the contract replace-only + forms rely on for write-only secrets, applied uniformly so update semantics do not fork + on the flag. An empty string counts as omitted: an empty credential is never a + meaningful value, and replace-only forms submit empty for "unchanged". + """ + for container_name, field in _VALUE_FIELDS: + update_container = getattr(update_data, container_name, None) + stored_container = getattr(stored_data, container_name, None) + + if update_container is None or stored_container is None: + continue + if not hasattr(update_container, field): + continue + + current_value = getattr(update_container, field) + if current_value is not None and current_value != "": + continue + + stored_value = getattr(stored_container, field, None) + if stored_value is not None: + setattr(update_container, field, stored_value) + + _carry_over_saved_extras(stored_data=stored_data, update_data=update_data) + + +def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: + """Same keep-on-omit contract for the credential keys of a custom provider's extras.""" + update_container = getattr(update_data, "provider", None) + stored_container = getattr(stored_data, "provider", None) + + if update_container is None or stored_container is None: + return + if not hasattr(update_container, "extras"): + return + + stored_extras = getattr(stored_container, "extras", None) or {} + if not stored_extras: + return + + update_extras = update_container.extras + if update_extras is None: + update_container.extras = dict(stored_extras) + return + + for extras_key in CREDENTIAL_EXTRAS_KEYS: + stored_value = stored_extras.get(extras_key) + if stored_value is not None and not update_extras.get(extras_key): + update_extras[extras_key] = stored_value + + def _carry_over_saved_policy(*, stored_data: Any, update_data: Any) -> None: """Fill an update payload's omitted ``models``/``harnesses`` from the stored record. @@ -78,6 +147,11 @@ async def create_secret( uuid4(), ) + # Write-only is the platform default for NEW secrets; an explicit False is the + # compatibility escape hatch. Existing rows are untouched (they carry no flag). + if create_secret_dto.write_only is None: + create_secret_dto.write_only = True + if create_secret_dto.secret.kind == SecretKind.PROVIDER_KEY: await self._name_and_slug_provider_key( project_id=project_id, @@ -192,17 +266,31 @@ async def update_secret( with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): - if update_secret_dto.secret is not None: + if ( + update_secret_dto.secret is not None + or update_secret_dto.write_only is not None + ): stored_secret_dto = await self.secrets_dao.get_by_id( secret_id=secret_id, project_id=project_id, organization_id=organization_id, ) if stored_secret_dto is not None: - _carry_over_saved_policy( - stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, - ) + if ( + stored_secret_dto.write_only + and update_secret_dto.write_only is False + ): + raise WriteOnlyCannotBeDisabledError() + + if update_secret_dto.secret is not None: + _carry_over_saved_policy( + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) + _carry_over_saved_value( + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) secret_dto = await self.secrets_dao.update( secret_id=secret_id, diff --git a/api/oss/src/core/workflows/service.py b/api/oss/src/core/workflows/service.py index 2c4c5cead3..8e83495fe9 100644 --- a/api/oss/src/core/workflows/service.py +++ b/api/oss/src/core/workflows/service.py @@ -141,7 +141,7 @@ find_string_embeds, ) -from oss.src.middlewares.auth import sign_secret_token +from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT, sign_secret_token from oss.src.services.db_manager import get_project_by_id from agenta.sdk.decorators.running import ( @@ -2804,11 +2804,15 @@ async def _prepare_invoke( project_id=str(project_id), ) + # The grant lets the run's vault reads receive write-only secret values in + # plaintext. It normally rides the credential `/access/permissions/check` re-mints, + # but a service running with auth middleware disabled uses this token directly. secret_token = await sign_secret_token( user_id=str(user_id), project_id=str(project_id), workspace_id=str(project.workspace_id), organization_id=str(project.organization_id), + grants=[SECRET_RESOLVE_GRANT], ) credentials = f"Secret {secret_token}" @@ -2929,6 +2933,7 @@ async def inspect_workflow( project_id=str(project_id), workspace_id=str(project.workspace_id), organization_id=str(project.organization_id), + grants=[SECRET_RESOLVE_GRANT], ) credentials = f"Secret {secret_token}" diff --git a/api/oss/src/dbs/postgres/secrets/mappings.py b/api/oss/src/dbs/postgres/secrets/mappings.py index 6b59732563..2a266080d4 100644 --- a/api/oss/src/dbs/postgres/secrets/mappings.py +++ b/api/oss/src/dbs/postgres/secrets/mappings.py @@ -13,6 +13,21 @@ ) +# The flag rides inside the (encrypted) `data` JSON, as a sibling of the payload fields, +# so no schema migration is needed. It is popped back out in `map_secrets_dbe_to_dto`, so +# payload DTOs never see it; rows without the key read as write_only=False (legacy). +_WRITE_ONLY_KEY = "write_only" + + +def _data_payload(data_json: dict, *, write_only: bool) -> str: + if write_only: + data_json[_WRITE_ONLY_KEY] = True + else: + data_json.pop(_WRITE_ONLY_KEY, None) + + return json.dumps(data_json) + + def map_secrets_dto_to_dbe( *, project_id: uuid.UUID | None, @@ -26,7 +41,10 @@ def map_secrets_dto_to_dbe( project_id=project_id, organization_id=organization_id, kind=secret_dto.secret.kind.value, - data=json.dumps(secret_dto.secret.data.model_dump(exclude_none=True)), + data=_data_payload( + secret_dto.secret.data.model_dump(exclude_none=True), + write_only=bool(secret_dto.write_only), + ), ) return vault_secret_dbe @@ -46,27 +64,45 @@ def map_secrets_dto_to_dbe_update( if hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) + # Resolve the effective flag BEFORE overwriting data: a None on the update DTO means + # "keep the stored flag" (the service only sets it for explicit transitions). + write_only = update_secret_dto.write_only + if write_only is None: + write_only = bool(json.loads(secrets_dbe.data).get(_WRITE_ONLY_KEY)) + if update_secret_dto.secret: for key, value in update_secret_dto.secret.model_dump( exclude_none=True ).items(): if key == "data" and hasattr(secrets_dbe, key): - secrets_dbe.data = update_secret_dto.secret.data.model_dump_json() + secrets_dbe.data = _data_payload( + update_secret_dto.secret.data.model_dump(), + write_only=write_only, + ) elif hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) + elif update_secret_dto.write_only is not None: + secrets_dbe.data = _data_payload( + json.loads(secrets_dbe.data), + write_only=write_only, + ) def map_secrets_dbe_to_dto(*, secrets_dbe: SecretsDBE) -> SecretResponseDTO: + data = json.loads(secrets_dbe.data) # type: ignore + write_only = bool(data.pop(_WRITE_ONLY_KEY, False)) + vault_secret_dto = SecretResponseDTO( id=secrets_dbe.id, # type: ignore slug=secrets_dbe.slug, kind=SecretKind(secrets_dbe.kind).value, - data=json.loads(secrets_dbe.data), # type: ignore + data=data, header=Header(name=secrets_dbe.name, description=secrets_dbe.description), lifecycle=LegacyLifecycleDTO( created_at=str(secrets_dbe.created_at), updated_at=str(secrets_dbe.updated_at), ), + write_only=write_only, ) return vault_secret_dto diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index 3a277ea919..f51100bd32 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from uuid import UUID from datetime import datetime, timezone import asyncio @@ -92,6 +92,22 @@ # reject a token minted a moment "in the future". Tolerate that much drift on `iat` and `exp`. _SECRET_LEEWAY = 30 # seconds +# A grant ADDS one capability to an otherwise general-purpose token, instead of +# confining that token to a narrower use. The runtime's credential must stay +# general-purpose (it authenticates workflows, tools, and vault reads alike), so the +# vault's plaintext-read capability rides a grant. +SECRET_RESOLVE_GRANT = "secret-resolve" + + +def request_has_grant(request: Request, grant: str) -> bool: + """Whether the request's verified credential carries ``grant``. + + Only `verify_secret_token` populates grants; session and ApiKey principals never + carry any, so this is False for them by construction. + """ + return grant in getattr(request.state, "token_grants", ()) + + _ZERO_UUID = "00000000-0000-0000-0000-000000000000" _NULL_UUID = "null" @@ -916,6 +932,8 @@ async def verify_secret_token( leeway=_SECRET_LEEWAY, ) + request.state.token_grants = tuple(auth_context.get("grants") or ()) + request.state.user_id = auth_context.get("user_id") request.state.user_email = auth_context.get("user_email") request.state.project_id = auth_context.get("project_id") @@ -1000,6 +1018,7 @@ async def sign_secret_token( workspace_id: Optional[str] = None, organization_id: Optional[str] = None, organization_name: Optional[str] = None, + grants: Optional[List[str]] = None, ): try: if not _SECRET_KEY: @@ -1019,6 +1038,11 @@ async def sign_secret_token( "exp": _exp, } + # A token with no grants carries no `grants` key at all, rather than a null or + # empty one, so its payload stays the shape every existing holder was issued. + if grants: + auth_context["grants"] = list(grants) + secret_token = encode( payload=auth_context, key=_SECRET_KEY, diff --git a/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py new file mode 100644 index 0000000000..4043790109 --- /dev/null +++ b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py @@ -0,0 +1,141 @@ +"""A grant ADDS one capability to a general-purpose Secret token; a scope CONFINES one. + +The secret-resolve grant is what lets the platform runtime read write-only vault secrets in +plaintext. These pin the axis separation: a granted token stays valid everywhere (unlike a +scoped one), the claim round-trips into ``request.state.token_grants``, and principals +without it — including plain unscoped tokens — read as grant-less. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from jwt import decode, encode +from starlette.requests import Request + +from oss.src.middlewares import auth + + +SECRET_KEY = "unit-test-secret-key-with-32-bytes" + + +class _RecordingLog: + def __init__(self): + self.calls = [] + + def __getattr__(self, level): + def log(event, *args, **fields): + self.calls.append((level, event, fields)) + + return log + + +@pytest.fixture(name="log") +def _log(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(auth, "log", recorder) + monkeypatch.setattr(auth, "_SECRET_KEY", SECRET_KEY) + return recorder + + +def _request(path: str = "/vault/v1/secrets/") -> Request: + return Request( + { + "type": "http", + "method": "GET", + "path": path, + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "root_path": "", + } + ) + + +def _claims(token: str) -> dict: + return decode( + jwt=token, + key=SECRET_KEY, + algorithms=["HS256"], + options={"verify_exp": False}, + ) + + +@pytest.mark.asyncio +async def test_granted_token_carries_the_claim_and_stays_valid_on_any_path(log): + token = await auth.sign_secret_token( + user_id="u", + project_id="p", + grants=[auth.SECRET_RESOLVE_GRANT], + ) + + assert _claims(token)["grants"] == [auth.SECRET_RESOLVE_GRANT] + # A grant must never confine the token the way a scope does: the runtime uses this same + # credential for workflows, tools, and session coordination. + for path in ( + "/vault/v1/secrets/", + "/workflows/123/revisions/commit", + "/access/permissions/check", + ): + request = _request(path=path) + await auth.verify_secret_token(request=request, secret_token=token) + assert auth.request_has_grant(request, auth.SECRET_RESOLVE_GRANT) + + assert log.calls == [] + + +@pytest.mark.asyncio +async def test_plain_token_carries_no_grants_claim_and_reads_grant_less(log): + token = await auth.sign_secret_token(user_id="u", project_id="p") + + assert "grants" not in _claims(token) + + request = _request() + await auth.verify_secret_token(request=request, secret_token=token) + + assert request.state.token_grants == () + assert not auth.request_has_grant(request, auth.SECRET_RESOLVE_GRANT) + + +def test_request_without_verified_token_has_no_grants(): + # Session and ApiKey principals never populate token_grants at all. + assert not auth.request_has_grant(_request(), auth.SECRET_RESOLVE_GRANT) + + +@pytest.mark.asyncio +async def test_foreign_grant_names_do_not_confer_secret_resolve(log): + token = await auth.sign_secret_token(user_id="u", grants=["something-else"]) + + request = _request() + await auth.verify_secret_token(request=request, secret_token=token) + + assert request.state.token_grants == ("something-else",) + assert not auth.request_has_grant(request, auth.SECRET_RESOLVE_GRANT) + + +@pytest.mark.asyncio +async def test_grants_ride_expiry_unchanged(log): + # The grant changes what the token may READ, never how long it lives. + now = datetime.now(timezone.utc).timestamp() + claims = _claims( + await auth.sign_secret_token(user_id="u", grants=[auth.SECRET_RESOLVE_GRANT]) + ) + + assert now + 15 * 60 - 5 < claims["exp"] < now + 15 * 60 + 5 + + +@pytest.mark.asyncio +async def test_forged_grants_on_an_unsigned_token_are_rejected(log): + expiry = datetime.now(timezone.utc) + timedelta(seconds=600) + forged = encode( + payload={ + "user_id": "u", + "grants": [auth.SECRET_RESOLVE_GRANT], + "exp": int(expiry.timestamp()), + }, + key="some-other-key-entirely-not-ours!", + algorithm="HS256", + ) + + with pytest.raises(Exception): + await auth.verify_secret_token(request=_request(), secret_token=forged) diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py new file mode 100644 index 0000000000..1b056bc25c --- /dev/null +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -0,0 +1,516 @@ +"""Write-only secrets: the value can be set and replaced, never read back by a user. + +Covers the three layers below the router: the service (default-on at create, value +carry-over on update, the one-way flag), the redaction helper (per-kind value stripping, +has_key/key_preview), and the postgres mappings (the flag rides inside the encrypted data +JSON and never leaks into the payload DTOs). +""" + +from uuid import uuid4 + +import pytest + +from oss.src.core.secrets.dtos import ( + CreateSecretDTO, + SecretResponseDTO, + UpdateSecretDTO, + WriteOnlyCannotBeDisabledError, +) +from oss.src.core.secrets.redaction import ( + mask_secret_value, + redact_secret_response, +) +from oss.src.core.secrets.services import VaultService +from oss.src.dbs.postgres.secrets.mappings import ( + map_secrets_dbe_to_dto, + map_secrets_dto_to_dbe, + map_secrets_dto_to_dbe_update, +) + + +PROJECT_ID = uuid4() + + +class _FakeSecretsDAO: + """In-memory DAO: stores what the service hands it, like the real mapping would.""" + + def __init__(self): + self.records: dict = {} + + async def create(self, project_id, organization_id, create_secret_dto): + record = SecretResponseDTO( + id=uuid4(), + slug=create_secret_dto.slug, + kind=create_secret_dto.secret.kind, + data=create_secret_dto.secret.data.model_dump(exclude_none=True), + header=create_secret_dto.header, + write_only=bool(create_secret_dto.write_only), + ) + self.records[record.id] = record + return record + + async def list(self, project_id, organization_id): + return list(self.records.values()) + + async def get_by_id(self, secret_id, project_id, organization_id): + return self.records.get(secret_id) + + async def update( + self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + ): + stored = self.records.get(secret_id) + if stored is None: + return None + + write_only = update_secret_dto.write_only + if write_only is None: + write_only = stored.write_only + + updated = stored.model_copy( + update={ + "header": update_secret_dto.header or stored.header, + "write_only": write_only, + } + ) + if update_secret_dto.secret is not None: + updated.kind = update_secret_dto.secret.kind + updated.data = update_secret_dto.secret.data + + self.records[secret_id] = updated + return updated + + +@pytest.fixture(name="service") +def _service(): + return VaultService(_FakeSecretsDAO()) + + +def _provider_key_create(key="sk-test-openai-key-bc", write_only=None): + return CreateSecretDTO( + header={"name": "OpenAI"}, + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": key}}, + }, + write_only=write_only, + ) + + +# --- service: create ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_create_defaults_to_write_only(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + assert created.write_only is True + + +@pytest.mark.asyncio +async def test_create_accepts_explicit_false_as_escape_hatch(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=_provider_key_create(write_only=False), + ) + + assert created.write_only is False + + +# --- service: keep-stored-on-omit ------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("omitted_key", [None, ""]) +async def test_update_without_provider_key_keeps_the_stored_one(service, omitted_key): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + update = UpdateSecretDTO( + header={"name": "OpenAI (renamed)"}, + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": omitted_key}}, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.provider.key == "sk-test-openai-key-bc" + assert updated.header.name == "OpenAI (renamed)" + + +@pytest.mark.asyncio +async def test_update_with_a_new_provider_key_replaces_the_stored_one(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + update = UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": "sk-test-rotated"}}, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.provider.key == "sk-test-rotated" + + +@pytest.mark.asyncio +async def test_update_without_custom_provider_key_and_extras_keeps_stored_values( + service, +): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "my-gateway"}, + secret={ + "kind": "custom_provider", + "data": { + "kind": "openai", + "provider": { + "url": "https://gateway.example.com/v1", + "key": "gw-test-key", + "extras": { + "api_key": "extra-key-123456", + "region": "eu-west-1", + }, + }, + "models": [{"slug": "gpt-5"}], + }, + }, + ), + ) + + update = UpdateSecretDTO( + secret={ + "kind": "custom_provider", + "data": { + "kind": "openai", + "provider": {"url": "https://gateway.example.com/v2"}, + "models": [{"slug": "gpt-5"}], + }, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.provider.url == "https://gateway.example.com/v2" + assert updated.data.provider.key == "gw-test-key" + # Omitted extras carry over whole: replace-only forms must not wipe them. + assert updated.data.provider.extras["api_key"] == "extra-key-123456" + assert updated.data.provider.extras["region"] == "eu-west-1" + + +@pytest.mark.asyncio +async def test_update_with_partial_extras_refills_credential_keys_only(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "my-gateway"}, + secret={ + "kind": "custom_provider", + "data": { + "kind": "openai", + "provider": { + "url": "https://gateway.example.com/v1", + "key": "gw-test-key", + "extras": { + "api_key": "extra-key-123456", + "region": "eu-west-1", + }, + }, + "models": [{"slug": "gpt-5"}], + }, + }, + ), + ) + + update = UpdateSecretDTO( + secret={ + "kind": "custom_provider", + "data": { + "kind": "openai", + "provider": { + "url": "https://gateway.example.com/v1", + "extras": {"region": "us-east-1"}, + }, + "models": [{"slug": "gpt-5"}], + }, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + # The submitted config wins; only the credential keys refill from storage. + assert updated.data.provider.extras["region"] == "us-east-1" + assert updated.data.provider.extras["api_key"] == "extra-key-123456" + + +@pytest.mark.asyncio +async def test_update_without_custom_secret_content_keeps_the_stored_one(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "gh-token"}, + secret={ + "kind": "custom_secret", + "data": { + "secret": {"format": "text", "content": "ghp_example_token_xyz"} + }, + }, + ), + ) + + update = UpdateSecretDTO( + header={"name": "gh-token (renamed)"}, + secret={ + "kind": "custom_secret", + "data": {"secret": {"format": "text"}}, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.secret.content == "ghp_example_token_xyz" + + +# --- service: the flag is one-way ------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_write_only_cannot_be_turned_off(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + with pytest.raises(WriteOnlyCannotBeDisabledError): + await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(write_only=False), + ) + + +@pytest.mark.asyncio +async def test_readable_secret_can_be_tightened_to_write_only(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=_provider_key_create(write_only=False), + ) + + updated = await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(write_only=True), + ) + + assert updated.write_only is True + + +# --- redaction ------------------------------------------------------------------------- + + +def _response(kind, data, write_only=True): + return SecretResponseDTO( + id=uuid4(), + slug="s", + kind=kind, + data=data, + header={"name": "n"}, + write_only=write_only, + ) + + +def test_mask_hides_short_values_entirely_and_previews_long_ones(): + assert mask_secret_value("short") == "****" + assert mask_secret_value("elevenchars") == "****" + assert mask_secret_value("sk-live-1234567890abc9Qa") == "sk-****9Qa" + + +def test_redacts_provider_key_and_reports_presence(): + secret = _response( + "provider_key", + {"kind": "openai", "provider": {"key": "sk-live-1234567890abc"}}, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.provider.key is None + assert redacted.has_key is True + assert redacted.key_preview == "sk-****abc" + # The input is never mutated: internal readers keep their plaintext DTO. + assert secret.data.provider.key == "sk-test-openai-key-bc" + + +def test_redacts_custom_provider_key_and_credential_extras(): + secret = _response( + "custom_provider", + { + "kind": "openai", + "provider": { + "url": "https://gateway.example.com/v1", + "key": None, + "extras": {"api_key": "extra-key-123456", "region": "eu-west-1"}, + }, + "models": [{"slug": "gpt-5"}], + }, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.provider.key is None + assert "api_key" not in redacted.data.provider.extras + assert redacted.data.provider.extras["region"] == "eu-west-1" + assert redacted.data.provider.url == "https://gateway.example.com/v1" + assert redacted.has_key is True + assert redacted.key_preview == "ext****456" + + +def test_redacts_text_custom_secret_content(): + secret = _response( + "custom_secret", + {"secret": {"format": "text", "content": "ghp_abcdef1234567890"}}, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.secret.content is None + assert redacted.has_key is True + assert redacted.key_preview == "ghp****890" + + +def test_redacts_json_custom_secret_without_a_preview(): + secret = _response( + "custom_secret", + {"secret": {"format": "json", "content": {"token": "abc", "user": "x"}}}, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.secret.content is None + assert redacted.has_key is True + # A structured value has no single previewable string. + assert redacted.key_preview is None + + +def test_readable_secret_passes_through_unchanged(): + secret = _response( + "provider_key", + {"kind": "openai", "provider": {"key": "sk-test-openai-key-bc"}}, + write_only=False, + ) + + redacted = redact_secret_response(secret) + + assert redacted is secret + assert redacted.data.provider.key == "sk-test-openai-key-bc" + assert redacted.has_key is None + assert redacted.key_preview is None + + +def test_write_only_without_a_value_reports_has_key_false(): + secret = _response( + "custom_provider", + { + "kind": "openai", + "provider": {"url": "https://gateway.example.com/v1"}, + "models": [], + }, + ) + + redacted = redact_secret_response(secret) + + assert redacted.has_key is False + assert redacted.key_preview is None + + +# --- postgres mappings ----------------------------------------------------------------- + + +def test_mapping_round_trips_the_flag_through_the_data_json(): + import json + + dbe = map_secrets_dto_to_dbe( + project_id=PROJECT_ID, + organization_id=None, + secret_dto=_provider_key_create(write_only=True), + ) + + stored = json.loads(dbe.data) + assert stored["write_only"] is True + assert stored["provider"]["key"] == "sk-test-openai-key-bc" + + dbe.id = uuid4() + dto = map_secrets_dbe_to_dto(secrets_dbe=dbe) + + assert dto.write_only is True + # The flag never leaks into the payload shape. + assert not hasattr(dto.data, "write_only") + assert dto.data.provider.key == "sk-test-openai-key-bc" + + +def test_mapping_reads_legacy_rows_as_readable(): + dbe = map_secrets_dto_to_dbe( + project_id=PROJECT_ID, + organization_id=None, + secret_dto=_provider_key_create(write_only=False), + ) + dbe.id = uuid4() + + assert map_secrets_dbe_to_dto(secrets_dbe=dbe).write_only is False + + +def test_update_mapping_preserves_the_stored_flag_when_unspecified(): + import json + + dbe = map_secrets_dto_to_dbe( + project_id=PROJECT_ID, + organization_id=None, + secret_dto=_provider_key_create(write_only=True), + ) + + map_secrets_dto_to_dbe_update( + secrets_dbe=dbe, + update_secret_dto=UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": { + "kind": "openai", + "provider": {"key": "sk-test-rotated"}, + }, + }, + ), + ) + + stored = json.loads(dbe.data) + assert stored["write_only"] is True + assert stored["provider"]["key"] == "sk-test-rotated" + + +def test_update_mapping_applies_a_tightening_flag(monkeypatch): + import json + + dbe = map_secrets_dto_to_dbe( + project_id=PROJECT_ID, + organization_id=None, + secret_dto=_provider_key_create(write_only=False), + ) + + map_secrets_dto_to_dbe_update( + secrets_dbe=dbe, + update_secret_dto=UpdateSecretDTO(write_only=True), + ) + + stored = json.loads(dbe.data) + assert stored["write_only"] is True + assert stored["provider"]["key"] == "sk-live-1234567890abc" diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py new file mode 100644 index 0000000000..e99232d152 --- /dev/null +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -0,0 +1,257 @@ +"""Every vault route redacts write-only values for users; only the runtime grant reads them. + +Drives the real `VaultRouter` + `VaultService` over an in-memory DAO, with the permission +check and the Redis cache monkeypatched. The caller's principal is simulated by a test +middleware: requests with the `x-test-grant` header carry the secret-resolve grant (the +platform runtime); requests without it are ordinary user principals (session/ApiKey). +""" + +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from oss.src.apis.fastapi.vault import router as vault_router_module +from oss.src.apis.fastapi.vault.router import VaultRouter +from oss.src.core.secrets.dtos import SecretResponseDTO +from oss.src.core.secrets.services import VaultService +from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT + + +PROJECT_ID = str(uuid4()) +USER_ID = str(uuid4()) + +KEY = "sk-test-openai-secret" + + +class _FakeSecretsDAO: + def __init__(self): + self.records: dict = {} + + async def create(self, project_id, organization_id, create_secret_dto): + record = SecretResponseDTO( + id=uuid4(), + slug=create_secret_dto.slug, + kind=create_secret_dto.secret.kind, + data=create_secret_dto.secret.data.model_dump(exclude_none=True), + header=create_secret_dto.header, + write_only=bool(create_secret_dto.write_only), + ) + self.records[str(record.id)] = record + return record + + async def list(self, project_id, organization_id): + return list(self.records.values()) + + async def get_by_id(self, secret_id, project_id, organization_id): + return self.records.get(str(secret_id)) + + async def get_by_slug(self, secret_slug, project_id, organization_id): + return next( + (r for r in self.records.values() if r.slug == secret_slug), + None, + ) + + async def update( + self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + ): + stored = self.records.get(str(secret_id)) + if stored is None: + return None + + write_only = update_secret_dto.write_only + if write_only is None: + write_only = stored.write_only + + updated = stored.model_copy( + update={ + "header": update_secret_dto.header or stored.header, + "write_only": write_only, + } + ) + if update_secret_dto.secret is not None: + updated.kind = update_secret_dto.secret.kind + updated.data = update_secret_dto.secret.data + + self.records[str(secret_id)] = updated + return updated + + async def delete(self, secret_id, project_id, organization_id): + self.records.pop(str(secret_id), None) + + +class _FakeCache: + def __init__(self): + self.store = {} + + async def get_cache(self, *, project_id, namespace, key, model=None, is_list=False): + return self.store.get(namespace) + + async def set_cache(self, *, project_id, namespace, key, value): + self.store[namespace] = value + + async def invalidate_cache(self, *, project_id): + self.store.clear() + + +@pytest.fixture(name="harness") +def _harness(monkeypatch): + cache = _FakeCache() + dao = _FakeSecretsDAO() + + async def _allow(**kwargs): + return True + + monkeypatch.setattr(vault_router_module, "check_action_access", _allow) + monkeypatch.setattr(vault_router_module, "get_cache", cache.get_cache) + monkeypatch.setattr(vault_router_module, "set_cache", cache.set_cache) + monkeypatch.setattr(vault_router_module, "invalidate_cache", cache.invalidate_cache) + + app = FastAPI() + + @app.middleware("http") + async def _principal(request, call_next): + request.state.user_id = USER_ID + request.state.project_id = PROJECT_ID + if request.headers.get("x-test-grant"): + request.state.token_grants = (SECRET_RESOLVE_GRANT,) + return await call_next(request) + + app.include_router(VaultRouter(vault_service=VaultService(dao)).router) + + return TestClient(app), cache + + +GRANT = {"x-test-grant": "1"} + + +def _create(client, write_only=None, key=KEY): + body = { + "header": {"name": "OpenAI"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": key}}, + }, + } + if write_only is not None: + body["write_only"] = write_only + response = client.post("/secrets/", json=body) + assert response.status_code == 200, response.text + return response.json() + + +def test_create_echo_is_redacted_and_write_only_by_default(harness): + client, _ = harness + + created = _create(client) + + assert created["write_only"] is True + assert "key" not in created["data"]["provider"] + assert created["has_key"] is True + assert created["key_preview"] == "sk-****abc" + assert KEY not in str(created) + + +def test_create_with_explicit_false_keeps_todays_response(harness): + client, _ = harness + + created = _create(client, write_only=False) + + assert created["write_only"] is False + assert created["data"]["provider"]["key"] == KEY + assert "has_key" not in created + assert "key_preview" not in created + + +def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): + client, _ = harness + created = _create(client) + + user_read = client.get(f"/secrets/{created['id']}") + assert user_read.status_code == 200 + assert "key" not in user_read.json()["data"]["provider"] + + runtime_read = client.get(f"/secrets/{created['id']}", headers=GRANT) + assert runtime_read.status_code == 200 + assert runtime_read.json()["data"]["provider"]["key"] == KEY + + +def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): + client, cache = harness + _create(client) + + listed = client.get("/secrets/") + assert listed.status_code == 200 + (secret,) = listed.json() + assert "key" not in secret["data"]["provider"] + assert secret["has_key"] is True + + # What went into Redis is the redacted DTO: no plaintext at rest in the cache. + (cached,) = cache.store["list_secrets"] + assert cached.data.provider.key is None + assert cached.has_key is True + + +def test_grant_list_bypasses_the_redacted_cache_and_gets_plaintext(harness): + client, _ = harness + _create(client) + + # A user listing first populates the cache with the redacted shape. + client.get("/secrets/") + + runtime_list = client.get("/secrets/", headers=GRANT) + assert runtime_list.status_code == 200 + (secret,) = runtime_list.json() + assert secret["data"]["provider"]["key"] == KEY + + +def test_update_echo_is_redacted_and_omitted_key_keeps_the_stored_value(harness): + client, _ = harness + created = _create(client) + + updated = client.put( + f"/secrets/{created['id']}", + json={ + "header": {"name": "OpenAI (renamed)"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {}}, + }, + }, + ) + assert updated.status_code == 200, updated.text + assert "key" not in updated.json()["data"]["provider"] + assert updated.json()["header"]["name"] == "OpenAI (renamed)" + + # The grant read proves the stored value survived the value-less update. + runtime_read = client.get(f"/secrets/{created['id']}", headers=GRANT) + assert runtime_read.json()["data"]["provider"]["key"] == KEY + + +def test_write_only_cannot_be_disabled_over_the_api(harness): + client, _ = harness + created = _create(client) + + response = client.put(f"/secrets/{created['id']}", json={"write_only": False}) + + assert response.status_code == 400 + assert "write-only" in response.json()["detail"] + + +def test_readable_secret_lists_with_its_value_as_today(harness): + client, _ = harness + _create(client, write_only=False) + + (secret,) = client.get("/secrets/").json() + + assert secret["data"]["provider"]["key"] == KEY + assert secret["write_only"] is False + + +def test_delete_still_works(harness): + client, _ = harness + created = _create(client) + + assert client.delete(f"/secrets/{created['id']}").status_code == 204 + assert client.get(f"/secrets/{created['id']}").status_code == 404 diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md new file mode 100644 index 0000000000..7bb2fa33aa --- /dev/null +++ b/docs/design/write-only-secrets/README.md @@ -0,0 +1,99 @@ +# Write-only vault secrets + +A vault secret's value can be created, replaced, and deleted — but never read back by a +user. The platform runtime keeps reading it through a granted internal path so runs still +work. This is the GitHub-secrets model. + +Status: backend landed (API + Python SDK). Frontend and Fern client regeneration follow in +a second PR. + +## The contract + +### The flag + +- `write_only: bool` on every secret. **New secrets default to `true`.** An explicit + `write_only: false` at creation is the compatibility escape hatch. +- Existing rows carry no flag and read as `write_only: false`; their behavior is unchanged. +- The flag is **one-way**: an update may tighten `false → true`, but `true → false` is + rejected with HTTP 400 (`WriteOnlyCannotBeDisabledError`). Making a value readable again + would defeat the guarantee; delete and recreate instead. +- Storage: the flag rides inside the existing encrypted `data` JSON as a sibling key + (`"write_only": true`), popped out at the mapping layer. **No schema migration.** + +### Redaction (user-facing responses) + +For `write_only: true`, every user-facing vault response (create echo, list, get, update +echo) strips the value and adds: + +- `has_key: bool` — whether a value is stored. +- `key_preview: str | null` — masked preview like `sk-****9Qa` (first 3 + last 3 characters, + only for string values of 12+ characters; shorter values and JSON content show no + preview). One helper: `oss/src/core/secrets/redaction.py`. + +Stripped fields per kind: `provider.key` (provider_key, custom_provider, webhook_provider), +`provider.client_secret` (sso_provider), `secret.content` (custom_secret), plus the +credential keys of a custom provider's `extras` (`api_key`, `aws_access_key_id`, +`aws_secret_access_key`, `aws_session_token`). Non-credential config (URL, region, +api_version, models, harnesses) stays readable. + +Redaction happens once, at the API response boundary (`VaultRouter`). In-process readers +(`VaultService` and below: webhooks, SSO overrides, EE organizations) are untouched and +keep plaintext. + +The Redis list cache stores the **redacted** shape — which also removes the previous +plaintext-at-rest in Redis for write-only secrets. + +### Updates: keep-stored-on-omit + +On update, an omitted value field means "keep the stored value" (extends the existing +`_carry_over_saved_policy` pattern for `models`/`harnesses`). This covers the standard +provider key, the custom provider key and its credential `extras`, and custom secret +content. **An empty string counts as omitted**: an empty credential is never a meaningful +value, and replace-only forms submit empty for "unchanged". Values are therefore +replace-only — they cannot be cleared in place. + +This applies to all secrets, not only write-only ones, so update semantics do not fork on +the flag. + +### The runtime plaintext path: the `secret-resolve` grant + +- Constant: `SECRET_RESOLVE_GRANT = "secret-resolve"` (`oss/src/middlewares/auth.py`). +- It is a **grant**: an additive claim, not a restriction. The runtime's credential is + general-purpose — it authenticates workflows, tools, session coordination, and vault + reads alike — so the plaintext capability rides a `grants` claim that adds one ability + and never narrows what the token can otherwise do. +- Minted in two places: + - `GET /access/permissions/check` attaches the grant to the re-minted `credentials` for + `action=run_service` exchanges — the credential every workflow service and sandbox run + actually uses for its vault reads. + - The workflow invoke/inspect prelude (`sign_secret_token` in + `core/workflows/service.py`) — covers services running with auth middleware disabled, + which use that token directly. +- A verified Secret token carrying the grant receives plaintext from all vault read routes + (write_only is ignored for it). Everyone else — session, ApiKey, unscoped Secret token — + gets the redacted shape. **Strict stance: no transition period for ApiKey callers.** + +Trust line (same as GitHub's): anyone who can run a workload can reach the values through a +run, so the `run_service` exchange hands out the grant. What the flag removes is the casual +read: no session, ApiKey, or list/get call ever returns the value. + +## Consumer impact + +| Consumer | Path | Impact | +| --- | --- | --- | +| Frontend forms | vault routes, session auth | Redacted for write-only secrets; needs replace-only forms (follow-up) | +| Direct API users (ApiKey) | vault routes | Redacted for write-only secrets; no escape hatch besides `write_only: false` at creation | +| Platform runs (playground, deployments, agents) | granted credential via `permissions/check` | Unchanged — plaintext | +| Standalone SDK runs (ApiKey) | `VaultConnectionResolver` | Fail loud: `WriteOnlySecretError` with instructions to use env vars | +| Standalone SDK legacy services | `VaultMiddleware.get_secrets` | Redacted entries dropped with a clear `log.error`; env-var keys are not shadowed by them | +| Named tool secrets | `resolve_named_secrets` | Redacted entries skipped with a clear `log.error` (best-effort contract kept) | +| In-process readers (webhooks, SSO, EE orgs) | `VaultService` direct | Unchanged — plaintext | + +## Frontend follow-up (second PR) + +- Replace-only secret forms: no value prefill; show `key_preview`/`has_key`; a "Replace + key" action instead of an editable field. +- Surface `write_only` in the connections/secrets lists. +- Optional "readable" toggle at creation only (maps to `write_only: false`), if product + wants the escape hatch exposed. +- Regenerate the Fern client for the new `write_only`, `has_key`, `key_preview` fields. diff --git a/sdks/python/agenta/sdk/agents/connections/__init__.py b/sdks/python/agenta/sdk/agents/connections/__init__.py index fe89d186aa..7249b6814c 100644 --- a/sdks/python/agenta/sdk/agents/connections/__init__.py +++ b/sdks/python/agenta/sdk/agents/connections/__init__.py @@ -19,6 +19,7 @@ UnsupportedConnectionModeError, UnsupportedDeploymentError, UnsupportedProviderError, + WriteOnlySecretError, ) from .interfaces import ConnectionResolver from .models import ( @@ -64,4 +65,5 @@ "UnsupportedProviderError", "UnsupportedConnectionModeError", "UnsupportedDeploymentError", + "WriteOnlySecretError", ] diff --git a/sdks/python/agenta/sdk/agents/connections/errors.py b/sdks/python/agenta/sdk/agents/connections/errors.py index 1acb184f37..ab46b361e3 100644 --- a/sdks/python/agenta/sdk/agents/connections/errors.py +++ b/sdks/python/agenta/sdk/agents/connections/errors.py @@ -55,6 +55,32 @@ def __init__(self, *, provider: str, slug: Optional[str] = None) -> None: self.slug = slug +class WriteOnlySecretError(ConnectionResolutionError): + """Raised when the chosen connection's key exists but came back redacted. + + The vault holds a write-only secret for this connection: the platform runtime reads it + through a granted credential, but this caller's credential (typically an ApiKey in a + standalone run) only receives the redacted shape. Passing the redacted (empty) key to a + provider would fail with a misleading auth error, so this fails loud with instructions. + """ + + # A standalone run against a write-only secret is a config situation, not a server fault. + status_code = 422 + + def __init__(self, *, slug: Optional[str] = None, provider: str = "") -> None: + subject = ( + f"connection '{slug}'" if slug else f"provider '{provider}' connection" + ) + # Engineering copy; adjust freely. + super().__init__( + f"{subject} uses a write-only secret: its value cannot be read back " + "outside the platform runtime. For standalone runs, provide the provider " + "key via its environment variable (for example OPENAI_API_KEY) instead." + ) + self.slug = slug + self.provider = provider + + class InvalidConnectionConfigurationError(AgentConnectionError): """Raised when resolved routing and credentials form an unsafe combination.""" diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index c3f01a45c2..6475d103c3 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -38,6 +38,7 @@ ResolvedConnection, RuntimeAuthContext, UnsupportedConnectionModeError, + WriteOnlySecretError, ) from ..model_catalog import model_input_modalities from .connection import PlatformConnection @@ -260,6 +261,9 @@ class _ConnectionCandidate: # harness intersection). Neither field filters resolution here yet. models: Optional[List[str]] = None harnesses: Optional[List[str]] = None + # True when the vault says a key exists (write_only + has_key) but this caller's + # credential received the redacted, value-less shape. + write_only_redacted: bool = False def matches_provider(self, provider: Optional[str]) -> bool: return bool( @@ -363,6 +367,11 @@ def _model_lookup_values(model: ModelRef, deployment: str) -> Set[str]: return {value for value in values if value} +def _write_only_redacted(secret: Dict[str, Any], value: Optional[str]) -> bool: + """Whether the vault redacted this record's value for the current caller.""" + return bool(secret.get("write_only")) and bool(secret.get("has_key")) and not value + + def _provider_key_candidate(secret: Dict[str, Any]) -> Optional[_ConnectionCandidate]: data = _data(secret) provider = _stripped(data.get("kind")) @@ -380,6 +389,7 @@ def _provider_key_candidate(secret: Dict[str, Any]) -> Optional[_ConnectionCandi api_key=key, models=_saved_models(data), harnesses=_saved_harnesses(data), + write_only_redacted=_write_only_redacted(secret, key), ) @@ -445,6 +455,7 @@ def _custom_provider_candidate( ), models=_saved_models(data), harnesses=_saved_harnesses(data), + write_only_redacted=_write_only_redacted(secret, api_key), ) @@ -598,6 +609,10 @@ def _resolve_from_secrets( env = chosen.resolved_env(provider) resolved_model = chosen.selected_model_id(model) if not env: + # A key that EXISTS but was redacted must not surface as "add your key" — the key + # is already in the vault; this caller's credential just may not read it. + if chosen.write_only_redacted: + raise WriteOnlySecretError(slug=chosen.slug, provider=provider) raise MissingCredentialError(provider=provider, slug=chosen.slug) return build_resolved_connection( provider=provider, diff --git a/sdks/python/agenta/sdk/agents/platform/secrets.py b/sdks/python/agenta/sdk/agents/platform/secrets.py index 2ade52aca4..1198f1d8f3 100644 --- a/sdks/python/agenta/sdk/agents/platform/secrets.py +++ b/sdks/python/agenta/sdk/agents/platform/secrets.py @@ -60,9 +60,18 @@ async def resolve_named_secrets( "agent: named-secret read HTTP %s", response.status_code ) continue - value = _text_custom_secret_value(response.json()) + payload = response.json() + value = _text_custom_secret_value(payload) if value is not None: resolved[name] = value + elif _is_write_only_redacted(payload): + # Engineering copy; adjust freely. + log.error( + "agent: secret %r is write-only: its value cannot be read back " + "outside the platform runtime. For standalone runs, provide it " + "via the tool's environment instead.", + name, + ) except Exception: # pylint: disable=broad-except log.warning("agent: named-secret read failed", exc_info=True) @@ -72,6 +81,15 @@ async def resolve_named_secrets( return resolved +def _is_write_only_redacted(payload: Any) -> bool: + """Whether the vault says a value exists but redacted it for this caller.""" + return ( + isinstance(payload, dict) + and bool(payload.get("write_only")) + and bool(payload.get("has_key")) + ) + + def _text_custom_secret_value(payload: Any) -> Optional[str]: """Extract only the vault shape MCP headers can safely consume.""" if not isinstance(payload, dict) or payload.get("kind") != "custom_secret": diff --git a/sdks/python/agenta/sdk/middlewares/running/vault.py b/sdks/python/agenta/sdk/middlewares/running/vault.py index 565de15cc9..70921cceac 100644 --- a/sdks/python/agenta/sdk/middlewares/running/vault.py +++ b/sdks/python/agenta/sdk/middlewares/running/vault.py @@ -362,6 +362,18 @@ async def get_secrets( except Exception: # pylint: disable=bare-except display_exception("Vault: Vault Secrets Exception") + vault_secrets, redacted_names = _split_write_only_redacted(vault_secrets) + if redacted_names: + # Engineering copy; adjust freely. + log.error( + "Vault: %d secret(s) are write-only and were returned without their values " + "(%s). Their values cannot be read back outside the platform runtime; for " + "standalone runs, provide the provider key via its environment variable " + "(for example OPENAI_API_KEY) instead.", + len(redacted_names), + ", ".join(redacted_names), + ) + local_standard = {} # A project may hold several connections per provider family, so vault provider_key records # are kept as a list. Keying them by family (as the locals still are) would drop every @@ -408,6 +420,45 @@ async def get_secrets( return secrets, combined_vault, local_secrets +def _split_write_only_redacted( + vault_secrets: List[Dict[str, Any]], +) -> tuple[List[Dict[str, Any]], List[str]]: + """Drop entries whose value the vault redacted (write-only secret, non-granted caller). + + A redacted entry has no usable credential, so keeping it would either pass an empty key + to a provider or shadow a working env-var key of the same provider family. Returns the + usable entries and the display names of the dropped ones. + """ + usable: List[Dict[str, Any]] = [] + redacted_names: List[str] = [] + + for secret in vault_secrets or []: + if ( + isinstance(secret, dict) + and secret.get("write_only") + and secret.get("has_key") + ): + data = secret.get("data") or {} + kind = secret.get("kind") + value = None + if kind in ("provider_key", "custom_provider"): + provider = data.get("provider") or {} + value = provider.get("key") or (provider.get("extras") or {}).get( + "api_key" + ) + elif kind == "custom_secret": + value = (data.get("secret") or {}).get("content") + + if not value: + header = secret.get("header") or {} + redacted_names.append(header.get("name") or secret.get("slug") or kind) + continue + + usable.append(secret) + + return usable, redacted_names + + def _has_invalid_secrets_error(response: Any) -> bool: status = getattr(response, "status", None) status_type = getattr(status, "type", None) diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py new file mode 100644 index 0000000000..04e193d75b --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -0,0 +1,169 @@ +"""SDK behavior when the vault redacts a write-only secret for this caller. + +The platform runtime reads write-only secrets in plaintext through its granted credential, +so in-platform runs never see the redacted shape. A standalone run (ApiKey credential) +does — and must fail loud with instructions, never pass an empty key to a provider. +""" + +from __future__ import annotations + +import pytest + +from agenta.sdk.agents.connections import ( + MissingCredentialError, + ModelRef, + WriteOnlySecretError, +) +from agenta.sdk.agents.platform import connections +from agenta.sdk.agents.platform.secrets import _is_write_only_redacted +from agenta.sdk.middlewares.running.vault import _split_write_only_redacted + + +def _model(model: str = "gpt-5.5", provider: str = "openai") -> ModelRef: + return ModelRef(provider=provider, model=model, connection={"mode": "agenta"}) + + +def _redacted_provider_key(name: str = "OpenAI", provider: str = "openai") -> dict: + """The list-response shape a non-granted caller receives for a write-only secret.""" + return { + "kind": "provider_key", + "slug": f"{provider}-abc123", + "header": {"name": name}, + "data": {"kind": provider, "provider": {}}, + "write_only": True, + "has_key": True, + "key_preview": "sk-****abc", + } + + +def _plaintext_provider_key(provider: str = "openai", key: str = "sk-live-123") -> dict: + """The same secret as the granted runtime sees it: write-only, value present.""" + return { + "kind": "provider_key", + "slug": f"{provider}-abc123", + "header": {"name": "OpenAI"}, + "data": {"kind": provider, "provider": {"key": key}}, + "write_only": True, + } + + +def test_redacted_write_only_key_fails_loud_with_instructions(): + with pytest.raises(WriteOnlySecretError) as raised: + connections._resolve_from_secrets( + secrets=[_redacted_provider_key()], model=_model(), harness="pi_core" + ) + + message = str(raised.value) + assert "write-only" in message + assert "environment variable" in message + # Never the misleading "add your key" error: the key exists, it is just unreadable here. + assert not isinstance(raised.value, MissingCredentialError) + + +def test_granted_plaintext_write_only_key_resolves_normally(): + resolved = connections._resolve_from_secrets( + secrets=[_plaintext_provider_key()], model=_model(), harness="pi_core" + ) + + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["OPENAI_API_KEY"] == "sk-live-123" + + +def test_ordinary_keyless_secret_still_reports_missing_credential(): + keyless = { + "kind": "provider_key", + "slug": "openai-abc123", + "header": {"name": "OpenAI"}, + "data": {"kind": "openai", "provider": {}}, + } + + with pytest.raises(MissingCredentialError): + connections._resolve_from_secrets( + secrets=[keyless], model=_model(), harness="pi_core" + ) + + +def test_redacted_custom_provider_fails_loud_too(): + redacted = { + "kind": "custom_provider", + "slug": "my-gateway", + "header": {"name": "my-gateway"}, + "data": { + "kind": "openai", + "provider": {"url": "https://gateway.example.com/v1"}, + "models": [{"slug": "gpt-5.5"}], + "provider_slug": "my-gateway", + }, + "write_only": True, + "has_key": True, + } + + with pytest.raises(WriteOnlySecretError): + connections._resolve_from_secrets( + secrets=[redacted], + model=ModelRef( + provider="openai", + model="gpt-5.5", + connection={"mode": "agenta", "slug": "my-gateway"}, + ), + harness="pi_core", + ) + + +# --- the vault middleware's list partition --------------------------------------------- + + +def test_partition_drops_redacted_entries_and_names_them(): + usable, redacted_names = _split_write_only_redacted( + [ + _redacted_provider_key(name="Prod OpenAI"), + _plaintext_provider_key(provider="anthropic"), + { + "kind": "provider_key", + "header": {"name": "Legacy"}, + "data": {"kind": "mistral", "provider": {"key": "m-key"}}, + }, + ] + ) + + assert redacted_names == ["Prod OpenAI"] + assert [s["data"]["kind"] for s in usable] == ["anthropic", "mistral"] + + +def test_partition_keeps_write_only_entries_whose_value_came_through(): + usable, redacted_names = _split_write_only_redacted([_plaintext_provider_key()]) + + assert redacted_names == [] + assert len(usable) == 1 + + +def test_partition_drops_redacted_custom_secret_content(): + usable, redacted_names = _split_write_only_redacted( + [ + { + "kind": "custom_secret", + "slug": "gh-token", + "header": {"name": "gh-token"}, + "data": {"secret": {"format": "text"}}, + "write_only": True, + "has_key": True, + } + ] + ) + + assert usable == [] + assert redacted_names == ["gh-token"] + + +# --- named-secret redaction detection -------------------------------------------------- + + +def test_named_secret_redaction_is_detected(): + assert _is_write_only_redacted( + {"kind": "custom_secret", "write_only": True, "has_key": True} + ) + assert not _is_write_only_redacted( + {"kind": "custom_secret", "write_only": True, "has_key": False} + ) + assert not _is_write_only_redacted({"kind": "custom_secret"}) + assert not _is_write_only_redacted(None) From 26b8a15ab5fe27badb970547b573d67388d9df6b Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 16:05:19 +0200 Subject: [PATCH 02/31] feat(api): gate the write-only default behind AGENTA_VAULT_WRITE_ONLY_DEFAULT --- api/oss/src/core/secrets/services.py | 7 +- api/oss/src/utils/env.py | 17 +++++ .../pytest/unit/secrets/test_write_only.py | 41 ++++++++++-- .../unit/vault/test_write_only_routes.py | 65 ++++++++++++++++--- docs/design/write-only-secrets/README.md | 23 +++++-- 5 files changed, 131 insertions(+), 22 deletions(-) diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index d6dbbbeca7..321e5b73a1 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -147,10 +147,11 @@ async def create_secret( uuid4(), ) - # Write-only is the platform default for NEW secrets; an explicit False is the - # compatibility escape hatch. Existing rows are untouched (they carry no flag). + # The write-only default for NEW secrets is env-gated (off until the web UI ships + # replace-only secret forms); an explicit request value always wins. Existing rows + # are untouched (they carry no flag). if create_secret_dto.write_only is None: - create_secret_dto.write_only = True + create_secret_dto.write_only = env.agenta.vault.write_only_default if create_secret_dto.secret.kind == SecretKind.PROVIDER_KEY: await self._name_and_slug_provider_key( diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index c338f41150..c64b68e3be 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -570,6 +570,22 @@ class SessionsConfig(BaseModel): model_config = ConfigDict(extra="ignore") +# --------------------------------------------------------------------------- +# agenta.vault — vault (secrets) behavior. +# --------------------------------------------------------------------------- + + +class VaultConfig(BaseModel): + """Vault (secrets) behavior.""" + + # Whether NEW secrets default to write-only (value never readable back by users). + # Off until the web UI ships replace-only secret forms; an explicit `write_only` + # on the create request always wins over this default. + write_only_default: bool = _parse_bool_env("AGENTA_VAULT_WRITE_ONLY_DEFAULT", False) + + model_config = ConfigDict(extra="ignore") + + # --------------------------------------------------------------------------- # agenta — top-level Agenta core config. # --------------------------------------------------------------------------- @@ -598,6 +614,7 @@ class AgentaConfig(BaseModel): redaction: RedactionConfig = RedactionConfig() services: ServicesConfig = ServicesConfig() sessions: SessionsConfig = SessionsConfig() + vault: VaultConfig = VaultConfig() webhooks: WebhooksConfig = WebhooksConfig() workers: WorkersConfig = WorkersConfig() diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index 1b056bc25c..c5d949b796 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -21,6 +21,7 @@ redact_secret_response, ) from oss.src.core.secrets.services import VaultService +from oss.src.utils.env import env from oss.src.dbs.postgres.secrets.mappings import ( map_secrets_dbe_to_dto, map_secrets_dto_to_dbe, @@ -99,8 +100,32 @@ def _provider_key_create(key="sk-test-openai-key-bc", write_only=None): # --- service: create ------------------------------------------------------------------ +@pytest.fixture(name="write_only_gate") +def _write_only_gate(monkeypatch): + def set_gate(value: bool): + monkeypatch.setattr(env.agenta.vault, "write_only_default", value) + + return set_gate + + @pytest.mark.asyncio -async def test_create_defaults_to_write_only(service): +async def test_create_defaults_off_while_the_gate_is_off(service, write_only_gate): + # Today's behavior until the web UI ships replace-only forms. + write_only_gate(False) + + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + assert created.write_only is False + + +@pytest.mark.asyncio +async def test_create_defaults_to_write_only_when_the_gate_is_on( + service, write_only_gate +): + write_only_gate(True) + created = await service.create_secret( project_id=PROJECT_ID, create_secret_dto=_provider_key_create() ) @@ -109,13 +134,19 @@ async def test_create_defaults_to_write_only(service): @pytest.mark.asyncio -async def test_create_accepts_explicit_false_as_escape_hatch(service): +@pytest.mark.parametrize("gate", [False, True]) +@pytest.mark.parametrize("explicit", [False, True]) +async def test_an_explicit_request_value_always_wins_over_the_gate( + service, write_only_gate, gate, explicit +): + write_only_gate(gate) + created = await service.create_secret( project_id=PROJECT_ID, - create_secret_dto=_provider_key_create(write_only=False), + create_secret_dto=_provider_key_create(write_only=explicit), ) - assert created.write_only is False + assert created.write_only is explicit # --- service: keep-stored-on-omit ------------------------------------------------------ @@ -290,7 +321,7 @@ async def test_update_without_custom_secret_content_keeps_the_stored_one(service @pytest.mark.asyncio async def test_write_only_cannot_be_turned_off(service): created = await service.create_secret( - project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + project_id=PROJECT_ID, create_secret_dto=_provider_key_create(write_only=True) ) with pytest.raises(WriteOnlyCannotBeDisabledError): diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index e99232d152..300d7636df 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -17,6 +17,7 @@ from oss.src.core.secrets.dtos import SecretResponseDTO from oss.src.core.secrets.services import VaultService from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT +from oss.src.utils.env import env PROJECT_ID = str(uuid4()) @@ -141,10 +142,10 @@ def _create(client, write_only=None, key=KEY): return response.json() -def test_create_echo_is_redacted_and_write_only_by_default(harness): +def test_create_echo_is_redacted_for_a_write_only_secret(harness): client, _ = harness - created = _create(client) + created = _create(client, write_only=True) assert created["write_only"] is True assert "key" not in created["data"]["provider"] @@ -153,6 +154,31 @@ def test_create_echo_is_redacted_and_write_only_by_default(harness): assert KEY not in str(created) +def test_create_without_the_flag_keeps_todays_response_while_the_gate_is_off(harness): + # The current frontend sends no flag; until AGENTA_VAULT_WRITE_ONLY_DEFAULT flips on, + # its creates must behave exactly as today. + client, _ = harness + + created = _create(client) + + assert created["write_only"] is False + assert created["data"]["provider"]["key"] == KEY + assert "has_key" not in created + assert "key_preview" not in created + + +def test_create_without_the_flag_is_write_only_once_the_gate_is_on( + harness, monkeypatch +): + monkeypatch.setattr(env.agenta.vault, "write_only_default", True) + client, _ = harness + + created = _create(client) + + assert created["write_only"] is True + assert "key" not in created["data"]["provider"] + + def test_create_with_explicit_false_keeps_todays_response(harness): client, _ = harness @@ -166,7 +192,7 @@ def test_create_with_explicit_false_keeps_todays_response(harness): def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): client, _ = harness - created = _create(client) + created = _create(client, write_only=True) user_read = client.get(f"/secrets/{created['id']}") assert user_read.status_code == 200 @@ -179,7 +205,7 @@ def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): client, cache = harness - _create(client) + _create(client, write_only=True) listed = client.get("/secrets/") assert listed.status_code == 200 @@ -195,7 +221,7 @@ def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): def test_grant_list_bypasses_the_redacted_cache_and_gets_plaintext(harness): client, _ = harness - _create(client) + _create(client, write_only=True) # A user listing first populates the cache with the redacted shape. client.get("/secrets/") @@ -208,7 +234,7 @@ def test_grant_list_bypasses_the_redacted_cache_and_gets_plaintext(harness): def test_update_echo_is_redacted_and_omitted_key_keeps_the_stored_value(harness): client, _ = harness - created = _create(client) + created = _create(client, write_only=True) updated = client.put( f"/secrets/{created['id']}", @@ -229,9 +255,32 @@ def test_update_echo_is_redacted_and_omitted_key_keeps_the_stored_value(harness) assert runtime_read.json()["data"]["provider"]["key"] == KEY +def test_todays_edit_form_shape_empty_string_key_keeps_the_stored_value(harness): + # The CURRENT frontend cannot prefill a redacted value, so its edit form re-sends + # `key: ""`. If "" cleared the credential, every edit through today's UI would wipe a + # write-only secret — so empty string must mean "keep the stored value". + client, _ = harness + created = _create(client, write_only=True) + + updated = client.put( + f"/secrets/{created['id']}", + json={ + "header": {"name": "OpenAI (edited in today's UI)"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": ""}}, + }, + }, + ) + assert updated.status_code == 200, updated.text + + runtime_read = client.get(f"/secrets/{created['id']}", headers=GRANT) + assert runtime_read.json()["data"]["provider"]["key"] == KEY + + def test_write_only_cannot_be_disabled_over_the_api(harness): client, _ = harness - created = _create(client) + created = _create(client, write_only=True) response = client.put(f"/secrets/{created['id']}", json={"write_only": False}) @@ -251,7 +300,7 @@ def test_readable_secret_lists_with_its_value_as_today(harness): def test_delete_still_works(harness): client, _ = harness - created = _create(client) + created = _create(client, write_only=True) assert client.delete(f"/secrets/{created['id']}").status_code == 204 assert client.get(f"/secrets/{created['id']}").status_code == 404 diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 7bb2fa33aa..d4c9f63ecf 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -4,15 +4,24 @@ A vault secret's value can be created, replaced, and deleted — but never read user. The platform runtime keeps reading it through a granted internal path so runs still work. This is the GitHub-secrets model. -Status: backend landed (API + Python SDK). Frontend and Fern client regeneration follow in -a second PR. +Status: backend landed (API + Python SDK), inert by default behind +`AGENTA_VAULT_WRITE_ONLY_DEFAULT=false`. The web half is deferred until the frontend +refactor (PR #6065) lands; frontend and Fern client regeneration follow in a second PR, +after which the gate flips on. Until then, an explicitly created `write_only: true` secret +shows cosmetically as "not configured" in today's Settings (the UI does not read `has_key` +yet) — accepted; the run path is unaffected either way. ## The contract ### The flag -- `write_only: bool` on every secret. **New secrets default to `true`.** An explicit - `write_only: false` at creation is the compatibility escape hatch. +- `write_only: bool` on every secret. The default for NEW secrets is env-gated: + **`AGENTA_VAULT_WRITE_ONLY_DEFAULT` (bool, default `false`)**. While off, flag-less + creates behave exactly as today (`write_only: false`); once the web UI ships + replace-only forms, the gate flips to `true` and new secrets default to write-only. + An explicit `write_only` on the create request always wins over the gate, in both + directions — so a caller can opt in to write-only immediately regardless of the + default, and `write_only: false` remains the escape hatch after the flip. - Existing rows carry no flag and read as `write_only: false`; their behavior is unchanged. - The flag is **one-way**: an update may tighten `false → true`, but `true → false` is rejected with HTTP 400 (`WriteOnlyCannotBeDisabledError`). Making a value readable again @@ -48,8 +57,10 @@ plaintext-at-rest in Redis for write-only secrets. On update, an omitted value field means "keep the stored value" (extends the existing `_carry_over_saved_policy` pattern for `models`/`harnesses`). This covers the standard provider key, the custom provider key and its credential `extras`, and custom secret -content. **An empty string counts as omitted**: an empty credential is never a meaningful -value, and replace-only forms submit empty for "unchanged". Values are therefore +content. **An empty string counts as omitted — this is mandatory, not a convenience**: the +CURRENT frontend's edit form re-sends `key: ""` when it cannot prefill a value, so if `""` +cleared the credential, every edit of a write-only secret through today's UI would wipe +it. An empty credential is never a meaningful value anyway. Values are therefore replace-only — they cannot be cleared in place. This applies to all secrets, not only write-only ones, so update semantics do not fork on From 22fc179e98e3222dcef8aa1ccdfbe3ab11033a51 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 16:51:32 +0200 Subject: [PATCH 03/31] fix(api,sdk): close write-only leak paths from review (shared classifier, outward surfaces, cache generation, atomic one-way, identity-local carry-over) --- api/ee/src/core/organizations/service.py | 12 + .../unit/test_write_only_provider_settings.py | 78 ++++++ api/oss/src/apis/fastapi/vault/router.py | 38 +-- api/oss/src/core/secrets/dtos.py | 20 +- api/oss/src/core/secrets/redaction.py | 97 +++---- api/oss/src/core/secrets/services.py | 103 +++++--- api/oss/src/core/webhooks/service.py | 53 +++- api/oss/src/dbs/postgres/secrets/dao.py | 20 +- api/oss/src/dbs/postgres/secrets/mappings.py | 12 +- api/oss/src/utils/caching.py | 19 +- .../pytest/unit/access/test_grant_exchange.py | 159 +++++++++++ .../pytest/unit/secrets/test_write_only.py | 239 ++++++++++++++++- .../unit/utils/test_cache_key_tenancy.py | 129 +++++++++ .../unit/vault/test_write_only_routes.py | 248 +++++++++++++++++- .../unit/webhooks/test_write_only_outward.py | 193 ++++++++++++++ docs/design/write-only-secrets/README.md | 75 ++++-- .../sdk/agents/connections/credentials.py | 87 ++++++ .../agenta/sdk/agents/connections/errors.py | 9 +- .../agenta/sdk/agents/platform/connections.py | 31 ++- .../agenta/sdk/agents/platform/secrets.py | 11 +- .../agenta/sdk/middlewares/running/vault.py | 5 +- .../connections/test_credentials_parity.py | 53 ++++ .../platform/test_write_only_secrets.py | 54 ++++ 23 files changed, 1550 insertions(+), 195 deletions(-) create mode 100644 api/ee/tests/pytest/unit/test_write_only_provider_settings.py create mode 100644 api/oss/tests/pytest/unit/access/test_grant_exchange.py create mode 100644 api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py create mode 100644 api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py create mode 100644 sdks/python/agenta/sdk/agents/connections/credentials.py create mode 100644 sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py diff --git a/api/ee/src/core/organizations/service.py b/api/ee/src/core/organizations/service.py index d338660cb1..38658bd5a7 100644 --- a/api/ee/src/core/organizations/service.py +++ b/api/ee/src/core/organizations/service.py @@ -28,6 +28,7 @@ SSOProviderDTO, SSOProviderSettingsDTO, ) +from oss.src.core.secrets.redaction import redact_secret_response from oss.src.core.secrets.services import VaultService from oss.src.dbs.postgres.secrets.dao import SecretsDAO from oss.src.core.shared.dtos import Header @@ -879,12 +880,23 @@ async def _get_provider_settings( if not secret: raise HTTPException(status_code=404, detail="Provider secret not found") + # This feeds USER-facing provider responses, so a write-only secret loses its + # client_secret here. The login-time reader (the SuperTokens overrides) resolves + # the secret through VaultService directly and keeps plaintext. + secret = redact_secret_response(secret) + data = secret.data if hasattr(data, "provider"): return data.provider.model_dump() if isinstance(data, dict): provider = data.get("provider") or {} if isinstance(provider, dict): + if getattr(secret, "write_only", False): + provider = { + key: value + for key, value in provider.items() + if key != "client_secret" + } return provider raise HTTPException(status_code=500, detail="Invalid provider secret format") diff --git a/api/ee/tests/pytest/unit/test_write_only_provider_settings.py b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py new file mode 100644 index 0000000000..76d59ec925 --- /dev/null +++ b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py @@ -0,0 +1,78 @@ +"""EE organization-provider responses respect write-only SSO secrets. + +`_get_provider_settings` feeds the user-facing provider serialization; once the vault +record is write-only it must drop `client_secret` while keeping the non-secret settings. +The login-time reader (SuperTokens overrides) resolves through `VaultService` directly and +is unaffected. +""" + +from uuid import uuid4 + +import pytest + +from ee.src.core.organizations.service import OrganizationProvidersService +from oss.src.core.secrets.dtos import SecretResponseDTO + + +ORGANIZATION_ID = uuid4() +SECRET_ID = uuid4() + + +class _StubVaultService: + def __init__(self, secret): + self._secret = secret + + async def get_secret_by_id( + self, *, secret_id, organization_id=None, project_id=None + ): + return self._secret + + +def _sso_secret(write_only: bool) -> SecretResponseDTO: + return SecretResponseDTO( + id=SECRET_ID, + slug="sso", + kind="sso_provider", + data={ + "provider": { + "client_id": "client-1", + "client_secret": "super-secret-value-123", + "issuer_url": "https://issuer.example.com", + "scopes": ["openid"], + } + }, + header={"name": "okta"}, + write_only=write_only, + ) + + +@pytest.mark.asyncio +async def test_write_only_sso_secret_drops_client_secret_from_settings(monkeypatch): + monkeypatch.setattr( + OrganizationProvidersService, + "_vault_service", + staticmethod(lambda: _StubVaultService(_sso_secret(write_only=True))), + ) + + settings = await OrganizationProvidersService()._get_provider_settings( + str(ORGANIZATION_ID), str(SECRET_ID) + ) + + assert settings.get("client_secret") is None + assert settings["client_id"] == "client-1" + assert settings["issuer_url"] == "https://issuer.example.com" + + +@pytest.mark.asyncio +async def test_readable_sso_secret_keeps_todays_settings(monkeypatch): + monkeypatch.setattr( + OrganizationProvidersService, + "_vault_service", + staticmethod(lambda: _StubVaultService(_sso_secret(write_only=False))), + ) + + settings = await OrganizationProvidersService()._get_provider_settings( + str(ORGANIZATION_ID), str(SECRET_ID) + ) + + assert settings["client_secret"] == "super-secret-value-123" diff --git a/api/oss/src/apis/fastapi/vault/router.py b/api/oss/src/apis/fastapi/vault/router.py index b559deadcb..a42c36348b 100644 --- a/api/oss/src/apis/fastapi/vault/router.py +++ b/api/oss/src/apis/fastapi/vault/router.py @@ -8,11 +8,12 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.exceptions import intercept_exceptions -from oss.src.utils.caching import get_cache, set_cache, invalidate_cache +from oss.src.utils.caching import invalidate_cache from oss.src.core.secrets.services import VaultService from oss.src.core.secrets.dtos import ( CreateSecretDTO, + SecretValueRequiredError, UpdateSecretDTO, SecretResponseDTO, WriteOnlyCannotBeDisabledError, @@ -162,42 +163,11 @@ async def list_secrets(self, request: Request): status_code=403, ) - # The runtime (secret-resolve grant) needs plaintext, and the shared cache only - # ever stores the redacted shape, so grant-bearing reads go straight to the DB. - if request_has_grant(request, SECRET_RESOLVE_GRANT): - return await self.service.list_secrets( - project_id=UUID(request.state.project_id), - ) - - cache_key = {} - - secrets_dtos = await get_cache( - project_id=request.state.project_id, - namespace="list_secrets", - key=cache_key, - model=SecretResponseDTO, - is_list=True, - ) - - if secrets_dtos is not None: - # Entries are stored redacted; re-redacting is idempotent and shields against - # entries written before write-only secrets existed. - return [redact_secret_response(dto) for dto in secrets_dtos] - secrets_dtos = await self.service.list_secrets( project_id=UUID(request.state.project_id), ) - secrets_dtos = [redact_secret_response(dto) for dto in secrets_dtos] - - await set_cache( - project_id=request.state.project_id, - namespace="list_secrets", - key=cache_key, - value=secrets_dtos, - ) - - return secrets_dtos + return [self._for_caller(request, secret_dto) for secret_dto in secrets_dtos] @intercept_exceptions() async def read_secret(self, request: Request, secret_id_or_slug: str): @@ -263,7 +233,7 @@ async def update_secret( update_secret_dto=body, user_id=UUID(request.state.user_id), ) - except WriteOnlyCannotBeDisabledError as e: + except (SecretValueRequiredError, WriteOnlyCannotBeDisabledError) as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=e.message ) from e diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index a677387f2b..2b1084fb25 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -17,6 +17,22 @@ from oss.src.core.webhooks.utils import validate_url_format_and_literal_ip +class SecretValueRequiredError(Exception): + """Raised when an update changes a secret's kind or provider family without a new value. + + Keep-on-omit is identity-local: carrying a stored credential across a kind or provider + change would silently hand one provider's key to another. + """ + + def __init__( + self, + message: str = "Changing a secret's kind or provider requires a new " + "credential value; the stored value is never carried across identities.", + ): + self.message = message + super().__init__(message) + + class WriteOnlyCannotBeDisabledError(Exception): """Raised when an update tries to turn `write_only` off. @@ -247,8 +263,8 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): class CreateSecretDTO(Slug, BaseModel): header: Header secret: SecretDTO - # None means "platform default", which is write-only. An explicit False is the - # compatibility escape hatch for callers that still need to read the value back. + # None means "platform default": env-gated via AGENTA_VAULT_WRITE_ONLY_DEFAULT + # (currently False). An explicit value always wins over the gate, in both directions. write_only: Optional[bool] = None @model_validator(mode="before") diff --git a/api/oss/src/core/secrets/redaction.py b/api/oss/src/core/secrets/redaction.py index cd8707f871..25e8fc6dcb 100644 --- a/api/oss/src/core/secrets/redaction.py +++ b/api/oss/src/core/secrets/redaction.py @@ -1,46 +1,57 @@ """Redaction of write-only vault secrets for user-facing responses. A secret with ``write_only=True`` can be created, replaced, and deleted, but its value is -never returned to a user: every vault route strips the value and attaches ``has_key`` and a -``key_preview`` instead. Only the platform runtime — a caller whose verified Secret token -carries the ``secret-resolve`` grant — receives the plaintext, because the workload it runs -needs the real key. In-process readers (`VaultService` and below) are untouched: redaction -happens strictly at the API response boundary. +never returned to a user: every outward route strips the credential material and attaches +``has_key`` and a ``key_preview`` instead. Only the platform runtime — a caller whose +verified Secret token carries the ``secret-resolve`` grant — receives the plaintext, +because the workload it runs needs the real key. In-process readers (`VaultService` and +below) are untouched: redaction happens strictly at the response boundary. + +WHAT counts as credential material is not decided here: the canonical classifier lives in +the SDK (``agenta.sdk.agents.connections.credentials``) and is imported, so the fields the +SDK resolver consumes as credentials and the fields this module strips can never drift. """ from typing import Any, Optional -from oss.src.core.secrets.enums import SecretKind -from oss.src.core.secrets.dtos import SecretResponseDTO - - -# Keys inside a custom provider's free-form `extras` that hold credential material (the -# SDK's connection resolver consumes `extras.api_key` as the key, and the AWS trio is -# injected as credentials). Redaction strips these; plain config (region, api_version) -# stays readable. `VaultService`'s update carry-over fills the same set back in when a -# replace-only form omits them. -CREDENTIAL_EXTRAS_KEYS = ( - "api_key", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", +from agenta.sdk.agents.connections.credentials import ( + CREDENTIAL_EXTRAS_KEYS, + PRIMARY_CREDENTIAL_FIELDS, ) +from oss.src.core.secrets.dtos import SecretResponseDTO + def mask_secret_value(value: str) -> str: """A short, non-reversible display preview like ``sk-****9Qa``. - Values shorter than 12 characters mask entirely: revealing 6 of their characters - would give away most of the secret. + Policy: values under 20 characters mask entirely; longer ones disclose at most 3+3 + characters and never more than 25% of the value (so a 20-character value shows 5). """ - if len(value) >= 12: - return f"{value[:3]}****{value[-3:]}" + if len(value) < 20: + return "****" + + disclosed = min(6, len(value) // 4) + prefix = disclosed - disclosed // 2 + suffix = disclosed // 2 + + return f"{value[:prefix]}****{value[-suffix:]}" - return "****" + +def primary_credential_value(secret: SecretResponseDTO) -> Optional[Any]: + """The kind's primary value field (key, client_secret, content), or None.""" + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get( + str(secret.kind.value), (None, None) + ) + if container_name is None: + return None + + container = getattr(secret.data, container_name, None) + return getattr(container, field, None) if container is not None else None def redact_secret_response(secret: SecretResponseDTO) -> SecretResponseDTO: - """The user-facing shape of ``secret``: value stripped when it is write-only. + """The user-facing shape of ``secret``: credential material stripped when write-only. Returns the input unchanged for readable (``write_only=False``) secrets, so legacy records keep their exact response. Never mutates the input. @@ -50,29 +61,25 @@ def redact_secret_response(secret: SecretResponseDTO) -> SecretResponseDTO: redacted = secret.model_copy(deep=True) + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get( + str(redacted.kind.value), (None, None) + ) value: Optional[Any] = None - data = redacted.data - - if redacted.kind in ( - SecretKind.PROVIDER_KEY, - SecretKind.CUSTOM_PROVIDER, - SecretKind.WEBHOOK_PROVIDER, - ): - value = data.provider.key - data.provider.key = None - extras = getattr(data.provider, "extras", None) + has_credential_extras = False + + if container_name is not None: + container = getattr(redacted.data, container_name, None) + if container is not None and hasattr(container, field): + value = getattr(container, field) + setattr(container, field, None) + + extras = getattr(container, "extras", None) if container is not None else None if extras: - value = value or extras.get("api_key") for extras_key in CREDENTIAL_EXTRAS_KEYS: - extras.pop(extras_key, None) - elif redacted.kind == SecretKind.SSO_PROVIDER: - value = data.provider.client_secret - data.provider.client_secret = None - elif redacted.kind == SecretKind.CUSTOM_SECRET: - value = data.secret.content - data.secret.content = None - - redacted.has_key = bool(value) + if extras.pop(extras_key, None) not in (None, ""): + has_credential_extras = True + + redacted.has_key = bool(value) or has_credential_extras redacted.key_preview = ( mask_secret_value(value) if isinstance(value, str) and value else None ) diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 321e5b73a1..69861da13b 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Optional from uuid import UUID, uuid4 from oss.src.utils.env import env @@ -10,9 +10,13 @@ ) from oss.src.core.secrets.interfaces import SecretsDAOInterface from oss.src.core.secrets.context import set_data_encryption_key -from oss.src.core.secrets.redaction import CREDENTIAL_EXTRAS_KEYS +from oss.src.core.secrets.redaction import ( + CREDENTIAL_EXTRAS_KEYS, + PRIMARY_CREDENTIAL_FIELDS, +) from oss.src.core.secrets.dtos import ( CreateSecretDTO, + SecretValueRequiredError, UpdateSecretDTO, WriteOnlyCannotBeDisabledError, ) @@ -40,43 +44,65 @@ def next_provider_key_name( return f"{title} {index}" -# The value-bearing (credential) field of each payload shape, as (container, field) pairs. -# `provider.key` covers standard providers, custom providers, and webhooks; the other two -# cover SSO and custom secrets. `_carry_over_saved_value` probes with hasattr, so pairs a -# given kind does not have are simply skipped. -_VALUE_FIELDS = ( - ("provider", "key"), - ("provider", "client_secret"), - ("secret", "content"), -) +def _provider_family(data: Any) -> Optional[str]: + """The provider family (`data.kind`) as a canonical string; None for family-less kinds.""" + kind = getattr(data, "kind", None) + if kind is None: + return None + return str(getattr(kind, "value", kind)) -def _carry_over_saved_value(*, stored_data: Any, update_data: Any) -> None: +def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> None: """Fill an update payload's omitted value field from the stored record. An update that omits the value means "keep the stored one" — the contract replace-only forms rely on for write-only secrets, applied uniformly so update semantics do not fork on the flag. An empty string counts as omitted: an empty credential is never a meaningful value, and replace-only forms submit empty for "unchanged". + + Only called when the update keeps the stored kind AND provider family — a credential + must never silently cross identities (see `update_secret`). """ - for container_name, field in _VALUE_FIELDS: + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get(kind, (None, None)) + + if container_name is not None: update_container = getattr(update_data, container_name, None) stored_container = getattr(stored_data, container_name, None) - if update_container is None or stored_container is None: - continue - if not hasattr(update_container, field): - continue + if ( + update_container is not None + and stored_container is not None + and hasattr(update_container, field) + ): + current_value = getattr(update_container, field) + if current_value is None or current_value == "": + stored_value = getattr(stored_container, field, None) + if stored_value is not None: + setattr(update_container, field, stored_value) - current_value = getattr(update_container, field) - if current_value is not None and current_value != "": - continue + _carry_over_saved_extras(stored_data=stored_data, update_data=update_data) - stored_value = getattr(stored_container, field, None) - if stored_value is not None: - setattr(update_container, field, stored_value) - _carry_over_saved_extras(stored_data=stored_data, update_data=update_data) +def _require_explicit_value(*, secret: Any) -> None: + """Reject a kind/family-changing update that carries no new credential value.""" + kind = str(secret.kind.value) + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get(kind, (None, None)) + if container_name is None: + return + + container = getattr(secret.data, container_name, None) + value = getattr(container, field, None) if container is not None else None + has_value = value is not None and value != "" + + if not has_value and container is not None: + extras = getattr(container, "extras", None) or {} + has_value = any( + extras.get(extras_key) not in (None, "") + for extras_key in CREDENTIAL_EXTRAS_KEYS + ) + + if not has_value: + raise SecretValueRequiredError() def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: @@ -284,14 +310,27 @@ async def update_secret( raise WriteOnlyCannotBeDisabledError() if update_secret_dto.secret is not None: - _carry_over_saved_policy( - stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, - ) - _carry_over_saved_value( - stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, - ) + # Keep-on-omit is an identity-local contract: a stored credential + # never silently becomes another kind's or another provider's + # credential. Changing either requires an explicit new value. + same_identity = stored_secret_dto.kind == ( + update_secret_dto.secret.kind + ) and _provider_family( + stored_secret_dto.data + ) == _provider_family(update_secret_dto.secret.data) + + if same_identity: + _carry_over_saved_policy( + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) + _carry_over_saved_value( + kind=str(stored_secret_dto.kind.value), + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) + else: + _require_explicit_value(secret=update_secret_dto.secret) secret_dto = await self.secrets_dao.update( secret_id=secret_id, diff --git a/api/oss/src/core/webhooks/service.py b/api/oss/src/core/webhooks/service.py index 2a7d7d313e..11830db600 100644 --- a/api/oss/src/core/webhooks/service.py +++ b/api/oss/src/core/webhooks/service.py @@ -15,6 +15,7 @@ WebhookProviderSettingsDTO, ) from oss.src.core.secrets.enums import SecretKind +from oss.src.core.secrets.redaction import redact_secret_response from oss.src.core.secrets.services import VaultService from oss.src.core.shared.dtos import Status, Windowing from oss.src.core.webhooks.delivery import ( @@ -59,6 +60,34 @@ def _generate_secret(self) -> str: return "".join(secrets.choice(alphabet) for _ in range(32)) + async def _resolve_outward_secret( + self, + *, + project_id: UUID, + # + secret_id: UUID, + ) -> Optional[str]: + """The signing secret as a USER response may carry it: None once write-only. + + Only for response shaping. Internal signing paths (`_resolve_secret` here, the + dispatcher's own resolver) stay plaintext regardless of the flag. + """ + try: + secret_dto = await self.vault_service.get_secret_by_id( + secret_id=secret_id, + project_id=project_id, + ) + + if secret_dto is None: + return None + + return redact_secret_response(secret_dto).data.provider.key + + except Exception as e: # pylint: disable=broad-exception-caught + log.warning(f"Failed to resolve webhook secret {secret_id}: {e}") + + return None + async def _resolve_secret( self, *, @@ -147,9 +176,11 @@ async def create_subscription( secret_id=secret_dto.id, ) + # The create echo respects write-only: once the stored secret is write-only, no + # response carries the value again — not even the creating one. return self._with_secret( subscription=result, - secret=secret_value, + secret=redact_secret_response(secret_dto).data.provider.key, ) async def test_subscription( @@ -325,7 +356,7 @@ async def fetch_subscription( return None if result.secret_id: - secret_value = await self._resolve_secret( + secret_value = await self._resolve_outward_secret( project_id=project_id, secret_id=result.secret_id, ) @@ -418,18 +449,12 @@ async def edit_subscription( if result is None: return None - if subscription.secret is not None: - result = self._with_secret( - subscription=result, - secret=subscription.secret, - ) - - return result - - if result.secret_id: - secret_value = await self._resolve_secret( + # Even a just-provided secret echoes through the outward resolver, so a + # write-only record never comes back — the caller already knows what it sent. + if result.secret_id or secret_id: + secret_value = await self._resolve_outward_secret( project_id=project_id, - secret_id=result.secret_id, + secret_id=result.secret_id or secret_id, ) result = self._with_secret( subscription=result, @@ -489,7 +514,7 @@ async def set_subscription_active( return None if result.secret_id: - secret_value = await self._resolve_secret( + secret_value = await self._resolve_outward_secret( project_id=project_id, secret_id=result.secret_id, ) diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index 78fc46259b..099bee7603 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -1,6 +1,8 @@ +import json from uuid import UUID from oss.src.dbs.postgres.secrets.dbes import SecretsDBE +from oss.src.core.secrets.dtos import WriteOnlyCannotBeDisabledError from oss.src.core.secrets.interfaces import SecretsDAOInterface from oss.src.dbs.postgres.shared.engine import ( @@ -123,9 +125,16 @@ async def update( ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) - stmt = select(SecretsDBE).filter_by( - id=secret_id, - **scope_filter, + # FOR UPDATE serializes concurrent updates so the one-way write_only check + # below always sees the latest committed flag — two racing updates cannot + # both observe False and let a stale explicit False win. + stmt = ( + select(SecretsDBE) + .filter_by( + id=secret_id, + **scope_filter, + ) + .with_for_update() ) result = await session.execute(stmt) secrets_dbe = result.scalar() @@ -133,6 +142,11 @@ async def update( if secrets_dbe is None: return None + if update_secret_dto.write_only is False and bool( + json.loads(secrets_dbe.data).get("write_only") + ): + raise WriteOnlyCannotBeDisabledError() + map_secrets_dto_to_dbe_update( secrets_dbe=secrets_dbe, update_secret_dto=update_secret_dto, diff --git a/api/oss/src/dbs/postgres/secrets/mappings.py b/api/oss/src/dbs/postgres/secrets/mappings.py index 2a266080d4..b3cba52215 100644 --- a/api/oss/src/dbs/postgres/secrets/mappings.py +++ b/api/oss/src/dbs/postgres/secrets/mappings.py @@ -64,11 +64,13 @@ def map_secrets_dto_to_dbe_update( if hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) - # Resolve the effective flag BEFORE overwriting data: a None on the update DTO means - # "keep the stored flag" (the service only sets it for explicit transitions). - write_only = update_secret_dto.write_only - if write_only is None: - write_only = bool(json.loads(secrets_dbe.data).get(_WRITE_ONLY_KEY)) + # Resolve the effective flag BEFORE overwriting data. The transition is one-way, so + # the mapper NEVER clears a stored flag: an explicit False can only reach it stale + # (the DAO rejects true->false under the row lock), and trusting it would resurrect + # readability. + write_only = bool(update_secret_dto.write_only) or bool( + json.loads(secrets_dbe.data).get(_WRITE_ONLY_KEY) + ) if update_secret_dto.secret: for key, value in update_secret_dto.secret.model_dump( diff --git a/api/oss/src/utils/caching.py b/api/oss/src/utils/caching.py index b2c6f97b62..9c63bc8107 100644 --- a/api/oss/src/utils/caching.py +++ b/api/oss/src/utils/caching.py @@ -46,12 +46,17 @@ def _pack( project_id: Optional[str] = None, user_id: Optional[str] = None, pattern: Optional[bool] = False, + full_project_id: bool = False, ) -> str: + # Security-sensitive namespaces pass full_project_id=True: the 12-character suffix + # is a display-length compromise, and two projects sharing a suffix must never share + # a cache entry that guards tenant data. if project_id: - project_id = project_id[-12:] if len(project_id) > 12 else project_id + if not full_project_id: + project_id = project_id[-12:] if len(project_id) > 12 else project_id else: project_id = "" - project_id = project_id + "-" * (12 - len(project_id)) + project_id = project_id + "-" * max(0, 12 - len(project_id)) if user_id: user_id = user_id[-12:] if len(user_id) > 12 else user_id @@ -81,6 +86,7 @@ def pack( project_id: Optional[str] = None, user_id: Optional[str] = None, pattern: Optional[bool] = False, + full_project_id: bool = False, ) -> str: return _pack( namespace=namespace, @@ -88,6 +94,7 @@ def pack( project_id=project_id, user_id=user_id, pattern=pattern, + full_project_id=full_project_id, ) @@ -227,6 +234,7 @@ async def _maybe_retry_get( model: Optional[Type[BaseModel]] = None, is_list: Optional[bool] = False, retry: Optional[bool] = True, + full_project_id: bool = False, *, ttl: Optional[int] = None, lock_ttl: int, @@ -241,6 +249,7 @@ async def _maybe_retry_get( key=key, project_id=project_id, user_id=user_id, + full_project_id=full_project_id, ) if CACHE_DEBUG: @@ -307,6 +316,7 @@ async def _maybe_retry_get( model=model, is_list=is_list, retry=retry, + full_project_id=full_project_id, # ttl=ttl, lock=lock_ttl, @@ -328,6 +338,7 @@ async def set_cache( key: Optional[Union[str, dict]] = None, value: Optional[Any] = None, ttl: Optional[int] = AGENTA_CACHE_TTL, + full_project_id: bool = False, ) -> Optional[bool]: # Noop if caching is disabled if not env.agenta.api.caching.enabled: @@ -339,6 +350,7 @@ async def set_cache( key=key, project_id=project_id, user_id=user_id, + full_project_id=full_project_id, ) cache_value: bytes = _serialize(value) cache_px = int(ttl * 1000) @@ -394,6 +406,7 @@ async def get_cache( model: Optional[Type[BaseModel]] = None, is_list: Optional[bool] = False, retry: Optional[bool] = True, + full_project_id: bool = False, *, ttl: Optional[int] = None, lock: Optional[int] = AGENTA_CACHE_LOCK_TTL, @@ -413,6 +426,7 @@ async def get_cache( key=key, project_id=project_id, user_id=user_id, + full_project_id=full_project_id, ) data = await _try_get_and_maybe_renew(cache_name, model, is_list, ttl) @@ -429,6 +443,7 @@ async def get_cache( model=model, is_list=is_list, retry=retry, + full_project_id=full_project_id, # ttl=ttl, lock_ttl=lock, diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py new file mode 100644 index 0000000000..9948774399 --- /dev/null +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -0,0 +1,159 @@ +"""The `/access/permissions/check` exchange is the only place users' credentials become +grant-bearing runtime credentials — and only for an ALLOWED `run_service` exchange. + +Drives the real `AccessRouter.check_permissions` handler with real token minting (the +returned credentials decode with the real key); only the permission verdict and the Redis +cache are faked. +""" + +from uuid import uuid4 + +import pytest +from jwt import decode +from starlette.requests import Request + +import oss.src.middlewares.auth as auth_module +from oss.src.apis.fastapi.access import router as access_router_module +from oss.src.apis.fastapi.access.router import AccessRouter +from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT +from oss.src.utils.context import ( + AuthContext, + AuthScope, + SecretCredentials, + reset_auth_context, + set_auth_context, +) + + +SECRET_KEY = "unit-test-secret-key-with-32-bytes" + +ORGANIZATION_ID = uuid4() +WORKSPACE_ID = uuid4() +PROJECT_ID = uuid4() +USER_ID = uuid4() + + +@pytest.fixture(name="exchange") +def _exchange(monkeypatch): + monkeypatch.setattr(auth_module, "_SECRET_KEY", SECRET_KEY) + + verdict = {"allow": True} + + async def _check_action_access(**kwargs): + return verdict["allow"] + + async def _get_cache(**kwargs): + return None + + async def _set_cache(**kwargs): + return True + + monkeypatch.setattr( + access_router_module, "check_action_access", _check_action_access + ) + monkeypatch.setattr(access_router_module, "get_cache", _get_cache) + monkeypatch.setattr(access_router_module, "set_cache", _set_cache) + + router = AccessRouter() + + async def run(action, resource_type="service"): + request = Request( + { + "type": "http", + "method": "GET", + "path": "/access/permissions/check", + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "root_path": "", + } + ) + + token = set_auth_context( + AuthContext( + credentials=SecretCredentials(value="caller-token"), + scope=AuthScope( + organization_id=ORGANIZATION_ID, + workspace_id=WORKSPACE_ID, + project_id=PROJECT_ID, + user_id=USER_ID, + ), + ) + ) + try: + return await router.check_permissions( + request, + action=action, + scope_type=None, + scope_id=None, + resource_type=resource_type, + resource_id=None, + ) + finally: + reset_auth_context(token) + + return run, verdict + + +def _claims(header_value: str) -> dict: + assert header_value.startswith("Secret ") + return decode( + jwt=header_value[len("Secret ") :], + key=SECRET_KEY, + algorithms=["HS256"], + options={"verify_exp": False}, + ) + + +def _body(response) -> dict: + import json + + return json.loads(response.body) + + +@pytest.mark.asyncio +async def test_allowed_run_service_exchange_returns_a_granted_credential(exchange): + run, _ = exchange + + body = _body(await run("run_service")) + + assert body["effect"] == "allow" + claims = _claims(body["credentials"]) + assert claims["grants"] == [SECRET_RESOLVE_GRANT] + assert claims["project_id"] == str(PROJECT_ID) + + +@pytest.mark.asyncio +async def test_non_run_actions_never_receive_the_grant(exchange): + run, _ = exchange + + body = _body(await run("view_secret", resource_type="local_secrets")) + + assert body["effect"] == "allow" + assert "grants" not in _claims(body["credentials"]) + + +@pytest.mark.asyncio +async def test_denied_exchange_returns_no_credential_at_all(exchange): + run, verdict = exchange + verdict["allow"] = False + + from fastapi import HTTPException + + with pytest.raises(HTTPException) as raised: + await run("run_service") + + assert raised.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_missing_action_is_denied(exchange): + run, _ = exchange + + from fastapi import HTTPException + + with pytest.raises(HTTPException) as raised: + await run(None) + + assert raised.value.status_code == 403 diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index c5d949b796..98ee2d26c8 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -13,6 +13,7 @@ from oss.src.core.secrets.dtos import ( CreateSecretDTO, SecretResponseDTO, + SecretValueRequiredError, UpdateSecretDTO, WriteOnlyCannotBeDisabledError, ) @@ -348,6 +349,112 @@ async def test_readable_secret_can_be_tightened_to_write_only(service): assert updated.write_only is True +# --- service: keep-on-omit is identity-local ------------------------------------------- + + +@pytest.mark.asyncio +async def test_provider_family_change_with_omitted_key_is_rejected(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + # OpenAI -> Anthropic without a new key must never reuse the OpenAI credential. + update = UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": {"kind": "anthropic", "provider": {"key": ""}}, + }, + ) + + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + +@pytest.mark.asyncio +async def test_provider_family_change_with_a_new_key_is_allowed(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + update = UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": {"kind": "anthropic", "provider": {"key": "sk-ant-new-key-123"}}, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.provider.key == "sk-ant-new-key-123" + + +@pytest.mark.asyncio +async def test_kind_change_with_omitted_content_is_rejected(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + # provider_key -> custom_secret with no content would irreversibly replace the + # stored credential with nothing. + update = UpdateSecretDTO( + secret={ + "kind": "custom_secret", + "data": {"secret": {"format": "text"}}, + }, + ) + + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + +@pytest.mark.asyncio +async def test_family_change_does_not_carry_credential_extras(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "my-gateway"}, + secret={ + "kind": "custom_provider", + "data": { + "kind": "openai", + "provider": { + "url": "https://gateway.example.com/v1", + "extras": {"api_key": "extra-key-123456"}, + }, + "models": [{"slug": "gpt-5"}], + }, + }, + ), + ) + + # Family change WITH an explicit new key: allowed, but the old family's extras + # credentials must not ride along. + update = UpdateSecretDTO( + secret={ + "kind": "custom_provider", + "data": { + "kind": "anthropic", + "provider": { + "url": "https://gateway.example.com/v1", + "key": "sk-ant-new-key-123", + }, + "models": [{"slug": "claude"}], + }, + }, + ) + updated = await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + assert updated.data.provider.key == "sk-ant-new-key-123" + assert not (updated.data.provider.extras or {}).get("api_key") + + # --- redaction ------------------------------------------------------------------------- @@ -362,23 +469,27 @@ def _response(kind, data, write_only=True): ) -def test_mask_hides_short_values_entirely_and_previews_long_ones(): - assert mask_secret_value("short") == "****" - assert mask_secret_value("elevenchars") == "****" - assert mask_secret_value("sk-live-1234567890abc9Qa") == "sk-****9Qa" +def test_mask_boundaries_pin_the_preview_policy(): + # Under 20 characters: fully masked. From 20: at most 3+3, never more than 25%. + assert mask_secret_value("x" * 11) == "****" + assert mask_secret_value("x" * 12) == "****" + assert mask_secret_value("x" * 19) == "****" + assert mask_secret_value("abcdefghijklmnopqrst") == "abc****st" # 20 chars -> 5 + assert mask_secret_value("sk-example-credential9Qa") == "sk-****9Qa" # 24 -> 3+3 + assert mask_secret_value("x" * 400) == "xxx****xxx" # cap stays 3+3 def test_redacts_provider_key_and_reports_presence(): secret = _response( "provider_key", - {"kind": "openai", "provider": {"key": "sk-live-1234567890abc"}}, + {"kind": "openai", "provider": {"key": "sk-test-openai-key-bc"}}, ) redacted = redact_secret_response(secret) assert redacted.data.provider.key is None assert redacted.has_key is True - assert redacted.key_preview == "sk-****abc" + assert redacted.key_preview == "sk-****bc" # The input is never mutated: internal readers keep their plaintext DTO. assert secret.data.provider.key == "sk-test-openai-key-bc" @@ -404,20 +515,98 @@ def test_redacts_custom_provider_key_and_credential_extras(): assert redacted.data.provider.extras["region"] == "eu-west-1" assert redacted.data.provider.url == "https://gateway.example.com/v1" assert redacted.has_key is True - assert redacted.key_preview == "ext****456" + # Only the primary value field gets a preview; extras credentials never do. + assert redacted.key_preview is None + + +def test_redacts_every_sdk_credential_extras_key(): + # The classifier is shared with the SDK resolver, so everything the resolver would + # inject as a credential must come back stripped — including uppercase env-style + # keys and the bedrock/azure/anthropic tokens the first pass missed. + extras = { + "ANTHROPIC_AUTH_TOKEN": "tok-a", + "AWS_BEARER_TOKEN_BEDROCK": "tok-b", + "AWS_SECRET_ACCESS_KEY": "tok-c", + "AZURE_OPENAI_API_KEY": "tok-d", + "aws_bearer_token_bedrock": "tok-e", + "vertex_ai_credentials": '{"type": "service_account"}', + # Config survives. + "AWS_REGION": "eu-west-1", + "vertex_ai_project": "my-project", + } + secret = _response( + "custom_provider", + { + "kind": "bedrock", + "provider": {"url": None, "extras": dict(extras)}, + "models": [{"slug": "claude"}], + }, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.provider.extras == { + "AWS_REGION": "eu-west-1", + "vertex_ai_project": "my-project", + } + assert redacted.has_key is True + assert redacted.key_preview is None + + +def test_aws_only_secret_reports_has_key_true(): + secret = _response( + "custom_provider", + { + "kind": "bedrock", + "provider": { + "extras": { + "aws_access_key_id": "AKIA123", + "aws_secret_access_key": "shhh", + "aws_region_name": "eu-west-1", + } + }, + "models": [], + }, + ) + + redacted = redact_secret_response(secret) + + assert redacted.has_key is True + assert "aws_secret_access_key" not in redacted.data.provider.extras + assert redacted.data.provider.extras["aws_region_name"] == "eu-west-1" + + +def test_redacts_sso_client_secret(): + secret = _response( + "sso_provider", + { + "provider": { + "client_id": "client-1", + "client_secret": "super-secret-value-123", + "issuer_url": "https://issuer.example.com", + "scopes": ["openid"], + } + }, + ) + + redacted = redact_secret_response(secret) + + assert redacted.data.provider.client_secret is None + assert redacted.data.provider.client_id == "client-1" + assert redacted.has_key is True def test_redacts_text_custom_secret_content(): secret = _response( "custom_secret", - {"secret": {"format": "text", "content": "ghp_abcdef1234567890"}}, + {"secret": {"format": "text", "content": "ghp_example_token_xyz"}}, ) redacted = redact_secret_response(secret) assert redacted.data.secret.content is None assert redacted.has_key is True - assert redacted.key_preview == "ghp****890" + assert redacted.key_preview == "ghp****yz" def test_redacts_json_custom_secret_without_a_preview(): @@ -528,7 +717,7 @@ def test_update_mapping_preserves_the_stored_flag_when_unspecified(): assert stored["provider"]["key"] == "sk-test-rotated" -def test_update_mapping_applies_a_tightening_flag(monkeypatch): +def test_update_mapping_applies_a_tightening_flag(): import json dbe = map_secrets_dto_to_dbe( @@ -545,3 +734,33 @@ def test_update_mapping_applies_a_tightening_flag(monkeypatch): stored = json.loads(dbe.data) assert stored["write_only"] is True assert stored["provider"]["key"] == "sk-live-1234567890abc" + + +def test_update_mapping_never_clears_the_flag_on_a_stale_explicit_false(): + # Concurrency guard: a racing update that read write_only=False before another + # request tightened the secret must not resurrect readability at the mapper. + import json + + dbe = map_secrets_dto_to_dbe( + project_id=PROJECT_ID, + organization_id=None, + secret_dto=_provider_key_create(write_only=True), + ) + + map_secrets_dto_to_dbe_update( + secrets_dbe=dbe, + update_secret_dto=UpdateSecretDTO( + write_only=False, + secret={ + "kind": "provider_key", + "data": { + "kind": "openai", + "provider": {"key": "sk-test-rotated"}, + }, + }, + ), + ) + + stored = json.loads(dbe.data) + assert stored["write_only"] is True + assert stored["provider"]["key"] == "sk-rotated-9876543210xyz" diff --git a/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py b/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py new file mode 100644 index 0000000000..6d56de80bb --- /dev/null +++ b/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py @@ -0,0 +1,129 @@ +"""Security-sensitive cache keys must carry the FULL tenant id. + +The default key packing truncates project ids to their last 12 characters — fine for +display-length economy, but two projects sharing a UUID suffix would share a cache entry. +`full_project_id=True` is the opt-out the vault list uses; these pin both the key shape +and the end-to-end isolation through the real (de)serialization path. +""" + +import pytest +from pydantic import BaseModel + +from oss.src.utils import caching +from oss.src.utils.caching import get_cache, pack, set_cache + + +# Same 12-character suffix, different projects. +PROJECT_A = "aaaaaaaa-aaaa-4aaa-8aaa-123456789012" +PROJECT_B = "bbbbbbbb-bbbb-4bbb-8bbb-123456789012" + + +def test_truncated_packing_collides_on_a_shared_suffix_and_full_packing_does_not(): + truncated_a = pack(namespace="list_secrets", key={}, project_id=PROJECT_A) + truncated_b = pack(namespace="list_secrets", key={}, project_id=PROJECT_B) + assert truncated_a == truncated_b # the hazard full_project_id exists to remove + + full_a = pack( + namespace="list_secrets", key={}, project_id=PROJECT_A, full_project_id=True + ) + full_b = pack( + namespace="list_secrets", key={}, project_id=PROJECT_B, full_project_id=True + ) + assert full_a != full_b + assert PROJECT_A in full_a + assert PROJECT_B in full_b + + +class _FakeRedis: + """Just enough of the engine surface for set/get, storing by exact key.""" + + def __init__(self): + self.store = {} + + async def set(self, name, value, px=None, nx=False, ex=None): + if nx and name in self.store: + return False + self.store[name] = value + return True + + async def get(self, name): + return self.store.get(name) + + async def expire(self, name, ttl): + return True + + async def delete(self, name): + return self.store.pop(name, None) is not None + + +class _Entry(BaseModel): + owner: str + + +@pytest.mark.asyncio +async def test_full_id_keys_isolate_same_suffix_projects_through_real_serialization( + monkeypatch, +): + # The unit conftest flips caching off (no live Redis); the fake engine stands in. + monkeypatch.setattr(caching.env.agenta.api.caching, "enabled", True) + monkeypatch.setattr(caching, "_cache_engine", _FakeRedis()) + + await set_cache( + namespace="list_secrets", + project_id=PROJECT_A, + key={}, + value=_Entry(owner="a"), + full_project_id=True, + ) + await set_cache( + namespace="list_secrets", + project_id=PROJECT_B, + key={}, + value=_Entry(owner="b"), + full_project_id=True, + ) + + read_a = await get_cache( + namespace="list_secrets", + project_id=PROJECT_A, + key={}, + model=_Entry, + retry=False, + full_project_id=True, + ) + read_b = await get_cache( + namespace="list_secrets", + project_id=PROJECT_B, + key={}, + model=_Entry, + retry=False, + full_project_id=True, + ) + + assert read_a.owner == "a" + assert read_b.owner == "b" + + +@pytest.mark.asyncio +async def test_truncated_keys_do_cross_projects_which_is_why_the_vault_opts_out( + monkeypatch, +): + monkeypatch.setattr(caching.env.agenta.api.caching, "enabled", True) + monkeypatch.setattr(caching, "_cache_engine", _FakeRedis()) + + await set_cache( + namespace="list_secrets", + project_id=PROJECT_A, + key={}, + value=_Entry(owner="a"), + ) + + crossed = await get_cache( + namespace="list_secrets", + project_id=PROJECT_B, + key={}, + model=_Entry, + retry=False, + ) + + assert crossed is not None and crossed.owner == "a" diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index 300d7636df..99c8e563a5 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -6,12 +6,16 @@ platform runtime); requests without it are ordinary user principals (session/ApiKey). """ +from datetime import datetime, timedelta, timezone from uuid import uuid4 import pytest -from fastapi import FastAPI +from jwt import encode +from fastapi import FastAPI, HTTPException +from fastapi.responses import JSONResponse from fastapi.testclient import TestClient +import oss.src.middlewares.auth as auth_module from oss.src.apis.fastapi.vault import router as vault_router_module from oss.src.apis.fastapi.vault.router import VaultRouter from oss.src.core.secrets.dtos import SecretResponseDTO @@ -83,17 +87,52 @@ async def delete(self, secret_id, project_id, organization_id): class _FakeCache: + """Keyed like the real cache: (namespace, project, logical key) -> value.""" + def __init__(self): self.store = {} - async def get_cache(self, *, project_id, namespace, key, model=None, is_list=False): - return self.store.get(namespace) + @staticmethod + def _name(project_id, namespace, key): + return (namespace, str(project_id), str(sorted((key or {}).items()))) + + async def get_cache( + self, + *, + project_id, + namespace, + key, + model=None, + is_list=False, + retry=True, + full_project_id=False, + ): + return self.store.get(self._name(project_id, namespace, key)) - async def set_cache(self, *, project_id, namespace, key, value): - self.store[namespace] = value + async def set_cache( + self, *, project_id, namespace, key, value, full_project_id=False + ): + self.store[self._name(project_id, namespace, key)] = value async def invalidate_cache(self, *, project_id): - self.store.clear() + # The real pattern invalidation does not reach the full-id generation keys. + self.store = { + name: value + for name, value in self.store.items() + if name[0] == "list_secrets_generation" + } + + def entries(self, namespace): + return [value for name, value in self.store.items() if name[0] == namespace] + + def generation(self, project_id): + return self.store.get(self._name(project_id, "list_secrets_generation", {})) + + def poison(self, project_id, generation, value): + """Simulate a stale reader writing its snapshot under an old generation.""" + self.store[ + self._name(project_id, "list_secrets", {"generation": generation}) + ] = value @pytest.fixture(name="harness") @@ -150,7 +189,7 @@ def test_create_echo_is_redacted_for_a_write_only_secret(harness): assert created["write_only"] is True assert "key" not in created["data"]["provider"] assert created["has_key"] is True - assert created["key_preview"] == "sk-****abc" + assert created["key_preview"] == "sk-****et" assert KEY not in str(created) @@ -214,7 +253,7 @@ def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): assert secret["has_key"] is True # What went into Redis is the redacted DTO: no plaintext at rest in the cache. - (cached,) = cache.store["list_secrets"] + ((cached,),) = cache.entries("list_secrets") assert cached.data.provider.key is None assert cached.has_key is True @@ -304,3 +343,196 @@ def test_delete_still_works(harness): assert client.delete(f"/secrets/{created['id']}").status_code == 204 assert client.get(f"/secrets/{created['id']}").status_code == 404 + + +def test_kind_or_family_change_without_a_new_value_is_400(harness): + client, _ = harness + created = _create(client, write_only=True) + + response = client.put( + f"/secrets/{created['id']}", + json={ + "secret": { + "kind": "provider_key", + "data": {"kind": "anthropic", "provider": {}}, + } + }, + ) + + assert response.status_code == 400 + assert "credential value" in response.json()["detail"] + + +# --- the cache generation prevents stale plaintext repopulation ------------------------ + + +def test_stale_snapshot_cannot_repopulate_the_cache_after_tightening(harness): + client, cache = harness + created = _create(client, write_only=False) + + # A first list materializes the generation and a readable cache entry. + client.get("/secrets/") + stale_generation = cache.generation(PROJECT_ID) + assert stale_generation + + # Tightening the secret bumps the generation. + tightened = client.put(f"/secrets/{created['id']}", json={"write_only": True}) + assert tightened.status_code == 200 + assert cache.generation(PROJECT_ID) != stale_generation + + # A stale reader (whose DB snapshot predates the tightening) now writes its + # plaintext list — it can only land under the DEAD generation. + plaintext_snapshot = [ + SecretResponseDTO( + id=created["id"], + slug=created["slug"], + kind="provider_key", + data={"kind": "openai", "provider": {"key": KEY}}, + header={"name": "OpenAI"}, + write_only=False, + ) + ] + cache.poison(PROJECT_ID, stale_generation, plaintext_snapshot) + + # No later reader consults the dead generation: the list stays redacted. + (secret,) = client.get("/secrets/").json() + assert "key" not in secret["data"]["provider"] + + +# --- real signed tokens through the real verifier -------------------------------------- + +SECRET_KEY = "unit-test-secret-key-with-32-bytes" + + +@pytest.fixture(name="token_client") +def _token_client(monkeypatch): + """Same router harness, but the principal comes from a REAL `Secret` token run + through the real `verify_secret_token` — nothing injects `token_grants` directly.""" + monkeypatch.setattr(auth_module, "_SECRET_KEY", SECRET_KEY) + + cache = _FakeCache() + dao = _FakeSecretsDAO() + + async def _allow(**kwargs): + return True + + monkeypatch.setattr(vault_router_module, "check_action_access", _allow) + monkeypatch.setattr(vault_router_module, "get_cache", cache.get_cache) + monkeypatch.setattr(vault_router_module, "set_cache", cache.set_cache) + monkeypatch.setattr(vault_router_module, "invalidate_cache", cache.invalidate_cache) + + app = FastAPI() + + @app.middleware("http") + async def _authenticate(request, call_next): + header = request.headers.get("authorization", "") + if not header.startswith("Secret "): + return JSONResponse({"detail": "Unauthorized"}, status_code=401) + try: + await auth_module.verify_secret_token(request, header[len("Secret ") :]) + except HTTPException as exc: + return JSONResponse({"detail": "Unauthorized"}, status_code=exc.status_code) + return await call_next(request) + + app.include_router(VaultRouter(vault_service=VaultService(dao)).router) + + return TestClient(app) + + +def _auth(token): + return {"Authorization": f"Secret {token}"} + + +@pytest.mark.asyncio +async def test_real_tokens_grant_and_deny_plaintext(token_client): + plain = await auth_module.sign_secret_token(user_id=USER_ID, project_id=PROJECT_ID) + granted = await auth_module.sign_secret_token( + user_id=USER_ID, + project_id=PROJECT_ID, + grants=[SECRET_RESOLVE_GRANT], + ) + + created = token_client.post( + "/secrets/", + json={ + "header": {"name": "OpenAI"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": KEY}}, + }, + "write_only": True, + }, + headers=_auth(plain), + ) + assert created.status_code == 200, created.text + assert "key" not in created.json()["data"]["provider"] + secret_id = created.json()["id"] + + ungranted_read = token_client.get(f"/secrets/{secret_id}", headers=_auth(plain)) + assert ungranted_read.status_code == 200 + assert "key" not in ungranted_read.json()["data"]["provider"] + + granted_read = token_client.get(f"/secrets/{secret_id}", headers=_auth(granted)) + assert granted_read.status_code == 200 + assert granted_read.json()["data"]["provider"]["key"] == KEY + + granted_list = token_client.get("/secrets/", headers=_auth(granted)) + (listed,) = granted_list.json() + assert listed["data"]["provider"]["key"] == KEY + + +@pytest.mark.asyncio +async def test_expired_granted_token_is_rejected(token_client): + # Encoded here rather than through `sign_secret_token`, which only ever issues a live + # token; what is under test is the middleware refusing an expired one, grant or not. + expired = encode( + payload={ + "user_id": USER_ID, + "project_id": PROJECT_ID, + "grants": [SECRET_RESOLVE_GRANT], + "exp": int( + (datetime.now(timezone.utc) - timedelta(seconds=60)).timestamp() + ), + }, + key=auth_module._SECRET_KEY, + algorithm="HS256", + ) + + response = token_client.get("/secrets/", headers=_auth(expired)) + + assert response.status_code == 401 + + +# --- schema + never-echo --------------------------------------------------------------- + + +def test_openapi_documents_the_write_only_contract(harness): + client, _ = harness + + schemas = client.app.openapi()["components"]["schemas"] + + for field in ("write_only", "has_key", "key_preview"): + assert field in schemas["SecretResponseDTO"]["properties"] + assert "write_only" in schemas["CreateSecretDTO"]["properties"] + assert "write_only" in schemas["UpdateSecretDTO"]["properties"] + + +CANARY = "sk-CANARY-DO-NOT-ECHO-abc123" + + +def test_malformed_create_never_echoes_the_submitted_key(harness): + client, _ = harness + + response = client.post( + "/secrets/", + json={ + "header": {"name": "x"}, + "secret": { + "kind": "invalid_kind", + "data": {"kind": "openai", "provider": {"key": CANARY}}, + }, + }, + ) + + assert response.status_code == 422 + assert CANARY not in response.text diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py new file mode 100644 index 0000000000..027507f483 --- /dev/null +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -0,0 +1,193 @@ +"""Webhook responses are write-only-aware; internal signing keeps plaintext. + +The signing secret lives in the vault. Once that record is write-only (via the env gate or +a manual tighten), no USER-facing webhook response — create echo, fetch, edit echo — may +carry the value again, while the internal resolver the signer uses stays plaintext. +""" + +from uuid import UUID, uuid4 + +import pytest + +from oss.src.core.secrets.dtos import SecretResponseDTO, UpdateSecretDTO +from oss.src.core.secrets.services import VaultService +from oss.src.core.webhooks.service import WebhooksService +from oss.src.core.webhooks.types import ( + WebhookSubscription, + WebhookSubscriptionCreate, + WebhookSubscriptionData, +) +from oss.src.utils.env import env + + +PROJECT_ID = uuid4() +USER_ID = uuid4() + + +class _FakeSecretsDAO: + def __init__(self): + self.records: dict = {} + + async def create(self, project_id, organization_id, create_secret_dto): + record = SecretResponseDTO( + id=uuid4(), + slug=create_secret_dto.slug, + kind=create_secret_dto.secret.kind, + data=create_secret_dto.secret.data.model_dump(exclude_none=True), + header=create_secret_dto.header, + write_only=bool(create_secret_dto.write_only), + ) + self.records[record.id] = record + return record + + async def get_by_id(self, secret_id, project_id, organization_id): + return self.records.get(secret_id) + + async def list(self, project_id, organization_id): + return list(self.records.values()) + + async def update( + self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + ): + stored = self.records.get(secret_id) + if stored is None: + return None + write_only = update_secret_dto.write_only + if write_only is None: + write_only = stored.write_only + updated = stored.model_copy(update={"write_only": write_only}) + if update_secret_dto.secret is not None: + updated.data = update_secret_dto.secret.data + self.records[secret_id] = updated + return updated + + +class _FakeWebhooksDAO: + def __init__(self): + self.subscriptions: dict = {} + + async def create_subscription( + self, *, project_id, user_id, subscription, secret_id + ): + record = WebhookSubscription( + id=uuid4(), + name=subscription.name, + data=subscription.data, + secret_id=secret_id, + ) + self.subscriptions[record.id] = record + return record + + async def fetch_subscription(self, *, project_id, subscription_id): + return self.subscriptions.get(subscription_id) + + +@pytest.fixture(name="services") +def _services(): + secrets_dao = _FakeSecretsDAO() + vault_service = VaultService(secrets_dao) + webhooks_service = WebhooksService( + webhooks_dao=_FakeWebhooksDAO(), + vault_service=vault_service, + ) + return webhooks_service, vault_service + + +def _subscription_create(): + return WebhookSubscriptionCreate( + name="notify", + data=WebhookSubscriptionData(url="https://example.com/hook"), + secret="whsec_provided_by_user_12345", + ) + + +@pytest.mark.asyncio +async def test_gate_on_create_and_fetch_never_return_the_signing_secret( + services, monkeypatch +): + monkeypatch.setattr(env.agenta.vault, "write_only_default", True) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + assert created.secret is None + + fetched = await webhooks_service.fetch_subscription( + project_id=PROJECT_ID, + subscription_id=created.id, + ) + assert fetched.secret is None + + +@pytest.mark.asyncio +async def test_gate_off_keeps_todays_responses(services, monkeypatch): + monkeypatch.setattr(env.agenta.vault, "write_only_default", False) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + assert created.secret == "whsec_provided_by_user_12345" + + fetched = await webhooks_service.fetch_subscription( + project_id=PROJECT_ID, + subscription_id=created.id, + ) + assert fetched.secret == "whsec_provided_by_user_12345" + + +@pytest.mark.asyncio +async def test_manually_tightened_secret_stops_appearing_in_fetches( + services, monkeypatch +): + monkeypatch.setattr(env.agenta.vault, "write_only_default", False) + webhooks_service, vault_service = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + assert created.secret is not None + + stored = await webhooks_service.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + await vault_service.update_secret( + secret_id=UUID(str(stored.secret_id)), + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(write_only=True), + ) + + fetched = await webhooks_service.fetch_subscription( + project_id=PROJECT_ID, + subscription_id=created.id, + ) + assert fetched.secret is None + + +@pytest.mark.asyncio +async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypatch): + monkeypatch.setattr(env.agenta.vault, "write_only_default", True) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + signing_value = await webhooks_service._resolve_secret( + project_id=PROJECT_ID, + secret_id=stored.secret_id, + ) + + assert signing_value == "whsec_provided_by_user_12345" diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index d4c9f63ecf..50fa76dcad 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -25,7 +25,10 @@ yet) — accepted; the run path is unaffected either way. - Existing rows carry no flag and read as `write_only: false`; their behavior is unchanged. - The flag is **one-way**: an update may tighten `false → true`, but `true → false` is rejected with HTTP 400 (`WriteOnlyCannotBeDisabledError`). Making a value readable again - would defeat the guarantee; delete and recreate instead. + would defeat the guarantee; delete and recreate instead. The transition is enforced + atomically: the DAO checks under a `SELECT ... FOR UPDATE` row lock, and the mapper + never clears a stored flag even when handed a stale explicit `false` — concurrent + updates cannot resurrect readability. - Storage: the flag rides inside the existing encrypted `data` JSON as a sibling key (`"write_only": true`), popped out at the mapping layer. **No schema migration.** @@ -34,23 +37,43 @@ yet) — accepted; the run path is unaffected either way. For `write_only: true`, every user-facing vault response (create echo, list, get, update echo) strips the value and adds: -- `has_key: bool` — whether a value is stored. -- `key_preview: str | null` — masked preview like `sk-****9Qa` (first 3 + last 3 characters, - only for string values of 12+ characters; shorter values and JSON content show no - preview). One helper: `oss/src/core/secrets/redaction.py`. - -Stripped fields per kind: `provider.key` (provider_key, custom_provider, webhook_provider), -`provider.client_secret` (sso_provider), `secret.content` (custom_secret), plus the -credential keys of a custom provider's `extras` (`api_key`, `aws_access_key_id`, -`aws_secret_access_key`, `aws_session_token`). Non-credential config (URL, region, -api_version, models, harnesses) stays readable. - -Redaction happens once, at the API response boundary (`VaultRouter`). In-process readers -(`VaultService` and below: webhooks, SSO overrides, EE organizations) are untouched and -keep plaintext. - -The Redis list cache stores the **redacted** shape — which also removes the previous -plaintext-at-rest in Redis for write-only secrets. +- `has_key: bool` — whether any credential material is stored (the primary value OR a + credential extra: an AWS-only secret reports `true`). +- `key_preview: str | null` — masked preview of the PRIMARY value only. Policy: values + under 20 characters mask entirely (`****`); from 20 on, at most first 3 + last 3 + characters and never more than 25% of the value (a 20-character value shows 5). + Extras credentials and JSON content never get a preview. One helper: + `oss/src/core/secrets/redaction.py`. + +**One credential classifier.** What counts as credential material is defined once, in the +SDK (`agenta.sdk.agents.connections.credentials`): the primary value field per kind +(`provider.key`; `provider.client_secret` for sso_provider; `secret.content` for +custom_secret) plus the full credential-extras set the SDK resolver consumes (`api_key`, +the `aws_*`/`AWS_*` credential trio and bearer tokens, `ANTHROPIC_AUTH_TOKEN` and the +other provider tokens, `AZURE_OPENAI_API_KEY`, `GOOGLE_APPLICATION_CREDENTIALS`, ...). +The API imports that module for redaction, `has_key`, and update carry-over; a parity +test fails if a resolver-accepted extras key is ever left unclassified. Non-credential +config (URL, region, api_version, project, models, harnesses) stays readable. + +Redaction happens at the response boundary, in every outward surface: + +- the vault routes (`VaultRouter`), for all five endpoints; +- webhook subscription responses (create echo, fetch, edit echo) — the signing value + disappears from responses once its vault record is write-only, while the delivery + signers (the service-internal resolver and the dispatcher's own) keep plaintext; +- the EE organization-provider serialization drops `client_secret`; the SuperTokens + login-time reader keeps plaintext. + +In-process runtime readers (`VaultService` and below) are untouched. + +**Cache.** The Redis list cache stores the **redacted** shape (which also removes the +previous plaintext-at-rest in Redis). The list cache key carries a per-project +**generation** that every secret write bumps, so a reader holding a pre-write DB snapshot +can only write its entry under a dead generation no later reader consults — a stale +reader can never repopulate the cache with plaintext after a tighten. Both the list and +generation keys use the **full project id** (`full_project_id=True` in the cache helper); +the default 12-character-suffix key would let two projects with the same suffix share a +security-sensitive entry. ### Updates: keep-stored-on-omit @@ -66,6 +89,14 @@ replace-only — they cannot be cleared in place. This applies to all secrets, not only write-only ones, so update semantics do not fork on the flag. +**Keep-on-omit is identity-local.** An update that changes the secret's kind or its +provider family (`data.kind`) must carry an explicit new credential value; omitted or +empty values are rejected with HTTP 400 (`SecretValueRequiredError`), and the old +identity's credential extras never carry over. A stored OpenAI key can never silently +become an Anthropic key, and a kind change can never silently erase the stored value. +(Consequence: a credential-less record — for example an endpoint-only custom provider — +cannot be the target of a kind/family change; delete and recreate it.) + ### The runtime plaintext path: the `secret-resolve` grant - Constant: `SECRET_RESOLVE_GRANT = "secret-resolve"` (`oss/src/middlewares/auth.py`). @@ -95,10 +126,12 @@ read: no session, ApiKey, or list/get call ever returns the value. | Frontend forms | vault routes, session auth | Redacted for write-only secrets; needs replace-only forms (follow-up) | | Direct API users (ApiKey) | vault routes | Redacted for write-only secrets; no escape hatch besides `write_only: false` at creation | | Platform runs (playground, deployments, agents) | granted credential via `permissions/check` | Unchanged — plaintext | -| Standalone SDK runs (ApiKey) | `VaultConnectionResolver` | Fail loud: `WriteOnlySecretError` with instructions to use env vars | +| Standalone SDK runs (ApiKey) | `VaultConnectionResolver` | Fail loud: `WriteOnlySecretError`, raised even when config extras survive; remediation = switch the connection to `self_managed` AND set the env variable | | Standalone SDK legacy services | `VaultMiddleware.get_secrets` | Redacted entries dropped with a clear `log.error`; env-var keys are not shadowed by them | -| Named tool secrets | `resolve_named_secrets` | Redacted entries skipped with a clear `log.error` (best-effort contract kept) | -| In-process readers (webhooks, SSO, EE orgs) | `VaultService` direct | Unchanged — plaintext | +| Named tool secrets | `resolve_named_secrets` | Redacted entries skipped with a clear `log.error` (no secret names in logs) | +| Webhook subscribers (UI/API responses) | webhook routes | Signing secret disappears from create/fetch/edit responses once its record is write-only; deliveries keep signing | +| EE SSO provider settings | organization-provider routes | `client_secret` dropped once write-only; login flow unaffected | +| In-process runtime readers (SuperTokens login, delivery signing, EE orgs internals) | `VaultService` direct | Unchanged — plaintext | ## Frontend follow-up (second PR) diff --git a/sdks/python/agenta/sdk/agents/connections/credentials.py b/sdks/python/agenta/sdk/agents/connections/credentials.py new file mode 100644 index 0000000000..76094e8b50 --- /dev/null +++ b/sdks/python/agenta/sdk/agents/connections/credentials.py @@ -0,0 +1,87 @@ +"""The canonical classification of credential material inside vault secrets. + +One list, consumed by BOTH sides of the write-only contract so they can never drift: + +- the SDK's connection resolver (``platform/connections.py``), which decides which extras + are credentials to inject and whether a redacted record still holds usable material; +- the API's redaction and update carry-over (``oss/src/core/secrets/redaction.py``), which + must strip, refill, and report presence for exactly the same fields. + +Every key the resolver accepts in a custom provider's ``extras`` must appear in exactly one +of the two sets below; a parity test enforces that, so adding an extras key to the resolver +without classifying it here fails the build. +""" + +from __future__ import annotations + +from typing import Dict, FrozenSet, Tuple + +# The primary value field per secret kind, as (container attribute, field name). +PRIMARY_CREDENTIAL_FIELDS: Dict[str, Tuple[str, str]] = { + "provider_key": ("provider", "key"), + "custom_provider": ("provider", "key"), + "webhook_provider": ("provider", "key"), + "sso_provider": ("provider", "client_secret"), + "custom_secret": ("secret", "content"), +} + +# Extras keys (as stored: the UI's snake_case aliases plus raw env-style names) that hold +# credential material. `vertex_ai_credentials`/`GOOGLE_APPLICATION_CREDENTIALS` are +# included because the stored value may be pasted service-account material, not a path. +CREDENTIAL_EXTRAS_KEYS: FrozenSet[str] = frozenset( + { + # UI snake_case aliases. + "api_key", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_bearer_token_bedrock", + "vertex_ai_credentials", + # Raw env-style keys: API keys / auth tokens. + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_OAUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "MISTRAL_API_KEY", + "MINIMAX_API_KEY", + "GROQ_API_KEY", + "TOGETHERAI_API_KEY", + "TOGETHER_API_KEY", + "OPENROUTER_API_KEY", + # Raw env-style keys: AWS. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_BEARER_TOKEN_BEDROCK", + # Raw env-style keys: GCP / Azure. + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_API_KEY", + "AZURE_OPENAI_API_KEY", + } +) + +# Extras keys that are plain configuration: safe to keep readable, never carried as +# credential material. The parity test requires resolver-accepted keys to be here or above. +CONFIG_EXTRAS_KEYS: FrozenSet[str] = frozenset( + { + "aws_region_name", + "vertex_ai_project", + "vertex_ai_location", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + } +) + + +def credential_extras(extras: Dict[str, object]) -> Dict[str, object]: + """The subset of ``extras`` holding non-empty credential material.""" + return { + key: value + for key, value in extras.items() + if key in CREDENTIAL_EXTRAS_KEYS and value not in (None, "") + } diff --git a/sdks/python/agenta/sdk/agents/connections/errors.py b/sdks/python/agenta/sdk/agents/connections/errors.py index ab46b361e3..31f5a4a6b9 100644 --- a/sdks/python/agenta/sdk/agents/connections/errors.py +++ b/sdks/python/agenta/sdk/agents/connections/errors.py @@ -71,11 +71,14 @@ def __init__(self, *, slug: Optional[str] = None, provider: str = "") -> None: subject = ( f"connection '{slug}'" if slug else f"provider '{provider}' connection" ) - # Engineering copy; adjust freely. + # Engineering copy; adjust freely. The remediation must change the connection + # MODE, not only the environment: an `agenta`-mode connection never reads env + # keys, so "set the env var" alone loops the user straight back to this error. super().__init__( f"{subject} uses a write-only secret: its value cannot be read back " - "outside the platform runtime. For standalone runs, provide the provider " - "key via its environment variable (for example OPENAI_API_KEY) instead." + "outside the platform runtime. For standalone runs, switch this " + "connection's authentication to self_managed and provide the provider " + "key via its environment variable (for example OPENAI_API_KEY)." ) self.slug = slug self.provider = provider diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index 6475d103c3..cc45887073 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -24,6 +24,7 @@ HARNESS_CONNECTION_CAPABILITIES, PROVIDER_ENV_VARS, ) +from ..connections.credentials import credential_extras from ..connections.endpoints import build_resolved_connection from ..connections import ( AmbiguousConnectionError, @@ -367,9 +368,18 @@ def _model_lookup_values(model: ModelRef, deployment: str) -> Set[str]: return {value for value in values if value} -def _write_only_redacted(secret: Dict[str, Any], value: Optional[str]) -> bool: - """Whether the vault redacted this record's value for the current caller.""" - return bool(secret.get("write_only")) and bool(secret.get("has_key")) and not value +def _write_only_redacted(secret: Dict[str, Any], has_credential: bool) -> bool: + """Whether the vault redacted this record's value for the current caller. + + ``has_credential`` must consider EVERY credential channel the candidate could use (the + primary key and the credential extras): a surviving config extra like ``AWS_REGION`` + must not read as "credentialed". + """ + return ( + bool(secret.get("write_only")) + and bool(secret.get("has_key")) + and not has_credential + ) def _provider_key_candidate(secret: Dict[str, Any]) -> Optional[_ConnectionCandidate]: @@ -389,7 +399,7 @@ def _provider_key_candidate(secret: Dict[str, Any]) -> Optional[_ConnectionCandi api_key=key, models=_saved_models(data), harnesses=_saved_harnesses(data), - write_only_redacted=_write_only_redacted(secret, key), + write_only_redacted=_write_only_redacted(secret, bool(key)), ) @@ -455,7 +465,9 @@ def _custom_provider_candidate( ), models=_saved_models(data), harnesses=_saved_harnesses(data), - write_only_redacted=_write_only_redacted(secret, api_key), + write_only_redacted=_write_only_redacted( + secret, bool(api_key) or bool(credential_extras(extras)) + ), ) @@ -598,6 +610,11 @@ def _resolve_from_secrets( else _choose_default(candidates, model, harness) ) provider = chosen.resolved_provider(model) + # Checked BEFORE the endpoint and env checks: a redacted write-only key is the deeper + # cause, and surviving config extras (AWS_REGION) can make `env` non-empty, which would + # otherwise let the run proceed mis-credentialed. + if chosen.write_only_redacted: + raise WriteOnlySecretError(slug=chosen.slug, provider=provider) # A chosen custom connection must carry a usable base URL. Failing here (rather than # returning endpoint=None) keeps the harness from falling back to a provider default and # silently ignoring the user's routing choice (design Decision 4). The error names the slug @@ -609,10 +626,6 @@ def _resolve_from_secrets( env = chosen.resolved_env(provider) resolved_model = chosen.selected_model_id(model) if not env: - # A key that EXISTS but was redacted must not surface as "add your key" — the key - # is already in the vault; this caller's credential just may not read it. - if chosen.write_only_redacted: - raise WriteOnlySecretError(slug=chosen.slug, provider=provider) raise MissingCredentialError(provider=provider, slug=chosen.slug) return build_resolved_connection( provider=provider, diff --git a/sdks/python/agenta/sdk/agents/platform/secrets.py b/sdks/python/agenta/sdk/agents/platform/secrets.py index 1198f1d8f3..c8a5001151 100644 --- a/sdks/python/agenta/sdk/agents/platform/secrets.py +++ b/sdks/python/agenta/sdk/agents/platform/secrets.py @@ -65,12 +65,13 @@ async def resolve_named_secrets( if value is not None: resolved[name] = value elif _is_write_only_redacted(payload): - # Engineering copy; adjust freely. + # Engineering copy; adjust freely. No secret name in the log (module + # policy above); the caller knows which names it requested. log.error( - "agent: secret %r is write-only: its value cannot be read back " - "outside the platform runtime. For standalone runs, provide it " - "via the tool's environment instead.", - name, + "agent: a requested named secret is write-only: its value " + "cannot be read back outside the platform runtime. For " + "standalone runs, set the value directly in the tool's " + "environment configuration instead." ) except Exception: # pylint: disable=broad-except log.warning("agent: named-secret read failed", exc_info=True) diff --git a/sdks/python/agenta/sdk/middlewares/running/vault.py b/sdks/python/agenta/sdk/middlewares/running/vault.py index 70921cceac..3a7f8469c5 100644 --- a/sdks/python/agenta/sdk/middlewares/running/vault.py +++ b/sdks/python/agenta/sdk/middlewares/running/vault.py @@ -10,6 +10,7 @@ from agenta.sdk.utils.exceptions import suppress, display_exception from agenta.sdk.utils.providers import normalize_provider_kind +from agenta.sdk.agents.connections.credentials import credential_extras from agenta.sdk.models.workflows import WorkflowServiceRequest from agenta.sdk.contexts.running import RunningContext @@ -443,8 +444,8 @@ def _split_write_only_redacted( value = None if kind in ("provider_key", "custom_provider"): provider = data.get("provider") or {} - value = provider.get("key") or (provider.get("extras") or {}).get( - "api_key" + value = provider.get("key") or credential_extras( + provider.get("extras") or {} ) elif kind == "custom_secret": value = (data.get("secret") or {}).get("content") diff --git a/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py b/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py new file mode 100644 index 0000000000..1365664abe --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py @@ -0,0 +1,53 @@ +"""The credential classifier and the resolver's extras vocabulary cannot drift. + +`credentials.py` is the single classification both the SDK resolver and the API's +write-only redaction consume. Every extras key the resolver accepts must be classified as +credential or config — adding a key to the resolver without classifying it fails here. +""" + +from agenta.sdk.agents.connections.credentials import ( + CONFIG_EXTRAS_KEYS, + CREDENTIAL_EXTRAS_KEYS, + PRIMARY_CREDENTIAL_FIELDS, + credential_extras, +) +from agenta.sdk.agents.platform.connections import ( + _ALLOWED_EXTRA_ENV_KEYS, + _SNAKE_EXTRA_ENV_ALIASES, +) + + +def test_every_resolver_extras_key_is_classified(): + resolver_keys = set(_SNAKE_EXTRA_ENV_ALIASES) | set(_ALLOWED_EXTRA_ENV_KEYS) + unclassified = resolver_keys - (CREDENTIAL_EXTRAS_KEYS | CONFIG_EXTRAS_KEYS) + + assert not unclassified, ( + f"extras keys accepted by the resolver but unclassified in credentials.py: " + f"{sorted(unclassified)} — classify each as credential or config" + ) + + +def test_credential_and_config_classifications_are_disjoint(): + assert not (CREDENTIAL_EXTRAS_KEYS & CONFIG_EXTRAS_KEYS) + + +def test_primary_fields_cover_every_secret_kind(): + assert set(PRIMARY_CREDENTIAL_FIELDS) == { + "provider_key", + "custom_provider", + "webhook_provider", + "sso_provider", + "custom_secret", + } + + +def test_credential_extras_keeps_only_non_empty_credential_material(): + extras = { + "api_key": "k", + "AWS_SECRET_ACCESS_KEY": "s", + "ANTHROPIC_AUTH_TOKEN": "", + "aws_region_name": "eu-west-1", + "unknown_key": "x", + } + + assert credential_extras(extras) == {"api_key": "k", "AWS_SECRET_ACCESS_KEY": "s"} diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py index 04e193d75b..0c223720e8 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -83,6 +83,60 @@ def test_ordinary_keyless_secret_still_reports_missing_credential(): ) +def test_redacted_aws_only_secret_fails_loud_despite_surviving_config_extras(): + # After redaction an AWS-credentialed secret keeps only config extras (region). The + # resulting env is NON-empty, so an env-emptiness check alone would let the run + # proceed mis-credentialed; the write-only check must fire first. + redacted = { + "kind": "custom_provider", + "slug": "bedrock-conn", + "header": {"name": "bedrock-conn"}, + "data": { + "kind": "bedrock", + "provider": {"extras": {"aws_region_name": "eu-west-1"}}, + "models": [{"slug": "claude-opus-5"}], + "provider_slug": "bedrock-conn", + }, + "write_only": True, + "has_key": True, + } + + with pytest.raises(WriteOnlySecretError): + connections._resolve_from_secrets( + secrets=[redacted], + model=ModelRef( + provider="anthropic", + model="claude-opus-5", + connection={"mode": "agenta", "slug": "bedrock-conn"}, + ), + harness="claude_code", + ) + + +def test_plaintext_aws_only_secret_is_not_treated_as_redacted(): + plaintext = { + "kind": "custom_provider", + "slug": "bedrock-conn", + "header": {"name": "bedrock-conn"}, + "data": { + "kind": "bedrock", + "provider": { + "extras": { + "aws_access_key_id": "AKIA123", + "aws_secret_access_key": "shhh", + "aws_region_name": "eu-west-1", + } + }, + "models": [{"slug": "claude-opus-5"}], + "provider_slug": "bedrock-conn", + }, + "write_only": True, + } + + candidates = connections._catalog([plaintext]) + assert candidates[0].write_only_redacted is False + + def test_redacted_custom_provider_fails_loud_too(): redacted = { "kind": "custom_provider", From f470762ae422618e8ac77f979b57394cd3225826 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 18:01:56 +0200 Subject: [PATCH 04/31] fix(api): build the update-path secret payload at every vault update call site CodeRabbit review round on #6164: - Webhook secret rotation and EE SSO provider updates built the parent SecretDTO for UpdateSecretDTO.secret, which pydantic rejects; both now build UpdateSecretPayloadDTO. A source walk pins every call site, and the webhook rotation path gets real coverage. - The forged-token test asserts UnauthorizedException/401/invalid_token instead of any exception, and is renamed for what it forges (a foreign-signed token, not an unsigned one). - The cache-key tenancy test no longer reads as if the truncated-id collision were an invariant: it records the tracked exception (issue #6166) that flipping the default needs invalidate_cache to carry the flag and a deploy plan for the evaluation lock keys. - The vault route fake now models production invalidation faithfully: it reaches neither full-id namespace, so the stale-snapshot test passes on the generation, not on the fake. - The design note names #6065 (frontend refactor, merged) and #6135 (stacking base) separately, and records the cache-tenancy gap. --- api/ee/src/core/organizations/service.py | 3 +- api/oss/src/core/webhooks/service.py | 3 +- api/oss/src/utils/caching.py | 5 +- .../unit/middlewares/test_auth_grants.py | 10 ++- .../pytest/unit/secrets/test_write_only.py | 48 +++++++++++ .../unit/utils/test_cache_key_tenancy.py | 17 +++- .../unit/vault/test_write_only_routes.py | 10 ++- .../unit/webhooks/test_write_only_outward.py | 81 +++++++++++++++++++ docs/design/write-only-secrets/README.md | 27 +++++-- 9 files changed, 188 insertions(+), 16 deletions(-) diff --git a/api/ee/src/core/organizations/service.py b/api/ee/src/core/organizations/service.py index 38658bd5a7..fcabd4c6b4 100644 --- a/api/ee/src/core/organizations/service.py +++ b/api/ee/src/core/organizations/service.py @@ -23,6 +23,7 @@ from oss.src.core.secrets.dtos import ( CreateSecretDTO, UpdateSecretDTO, + UpdateSecretPayloadDTO, SecretDTO, SecretKind, SSOProviderDTO, @@ -723,7 +724,7 @@ async def update_provider( if settings_changed: updated_secret = UpdateSecretDTO( header=Header(name=provider.slug, description=provider.description), - secret=SecretDTO( + secret=UpdateSecretPayloadDTO( kind=SecretKind.SSO_PROVIDER, data=SSOProviderDTO( provider=SSOProviderSettingsDTO( diff --git a/api/oss/src/core/webhooks/service.py b/api/oss/src/core/webhooks/service.py index 11830db600..c1ab3b87e1 100644 --- a/api/oss/src/core/webhooks/service.py +++ b/api/oss/src/core/webhooks/service.py @@ -11,6 +11,7 @@ CreateSecretDTO, SecretDTO, UpdateSecretDTO, + UpdateSecretPayloadDTO, WebhookProviderDTO, WebhookProviderSettingsDTO, ) @@ -409,7 +410,7 @@ async def edit_subscription( secret_id=existing.secret_id, project_id=project_id, update_secret_dto=UpdateSecretDTO( - secret=SecretDTO( + secret=UpdateSecretPayloadDTO( kind=SecretKind.WEBHOOK_PROVIDER, data=WebhookProviderDTO( provider=WebhookProviderSettingsDTO( diff --git a/api/oss/src/utils/caching.py b/api/oss/src/utils/caching.py index 9c63bc8107..f11b2ac2d1 100644 --- a/api/oss/src/utils/caching.py +++ b/api/oss/src/utils/caching.py @@ -50,7 +50,10 @@ def _pack( ) -> str: # Security-sensitive namespaces pass full_project_id=True: the 12-character suffix # is a display-length compromise, and two projects sharing a suffix must never share - # a cache entry that guards tenant data. + # a cache entry that guards tenant data. The truncated default is a tracked exception + # (issue #6166): flipping it needs `invalidate_cache` to carry the flag as well, and a + # deploy plan for the evaluation lock keys, whose shape must not change under a + # rolling deploy. if project_id: if not full_project_id: project_id = project_id[-12:] if len(project_id) > 12 else project_id diff --git a/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py index 4043790109..05aaf236f1 100644 --- a/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py +++ b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py @@ -13,6 +13,7 @@ from starlette.requests import Request from oss.src.middlewares import auth +from oss.src.utils.exceptions import UnauthorizedException SECRET_KEY = "unit-test-secret-key-with-32-bytes" @@ -125,7 +126,7 @@ async def test_grants_ride_expiry_unchanged(log): @pytest.mark.asyncio -async def test_forged_grants_on_an_unsigned_token_are_rejected(log): +async def test_forged_grants_on_a_foreign_signed_token_are_rejected(log): expiry = datetime.now(timezone.utc) + timedelta(seconds=600) forged = encode( payload={ @@ -137,5 +138,10 @@ async def test_forged_grants_on_an_unsigned_token_are_rejected(log): algorithm="HS256", ) - with pytest.raises(Exception): + # The concrete rejection, not any failure: a signature check that broke into an + # AttributeError would otherwise still read as "rejected". + with pytest.raises(UnauthorizedException) as rejected: await auth.verify_secret_token(request=_request(), secret_token=forged) + + assert rejected.value.status_code == 401 + assert rejected.value.detail["reason"] == "invalid_token" diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index 98ee2d26c8..75ba347fd0 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -764,3 +764,51 @@ def test_update_mapping_never_clears_the_flag_on_a_stale_explicit_false(): stored = json.loads(dbe.data) assert stored["write_only"] is True assert stored["provider"]["key"] == "sk-rotated-9876543210xyz" + + +# --- the update-path payload type, at every call site ---------------------------------- + + +def test_update_call_sites_build_the_update_path_payload(): + """`UpdateSecretDTO.secret` is `UpdateSecretPayloadDTO`, and pydantic rejects the + parent `SecretDTO` there — a caller that builds the parent breaks that write path at + runtime, not at import time. This walks the source so a new call site cannot + reintroduce the mismatch (it caught webhook secret rotation and SSO provider updates). + """ + import ast + from pathlib import Path + + source_roots = [ + Path(__file__).resolve().parents[5] / "oss" / "src", + Path(__file__).resolve().parents[5] / "ee" / "src", + ] + + offenders = [] + + for root in source_roots: + if not root.exists(): # EE is absent in an OSS-only checkout. + continue + + for path in root.rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if getattr(node.func, "id", None) != "UpdateSecretDTO": + continue + + for keyword in node.keywords: + if keyword.arg != "secret": + continue + if not isinstance(keyword.value, ast.Call): + continue # a dict or a variable: validated by pydantic as data. + + built = getattr(keyword.value.func, "id", None) + if built != "UpdateSecretPayloadDTO": + offenders.append(f"{path}:{node.lineno} builds {built}") + + assert not offenders, ( + "UpdateSecretDTO(secret=...) must be built with UpdateSecretPayloadDTO: " + + "; ".join(offenders) + ) diff --git a/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py b/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py index 6d56de80bb..b77a327199 100644 --- a/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py +++ b/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py @@ -2,8 +2,15 @@ The default key packing truncates project ids to their last 12 characters — fine for display-length economy, but two projects sharing a UUID suffix would share a cache entry. -`full_project_id=True` is the opt-out the vault list uses; these pin both the key shape -and the end-to-end isolation through the real (de)serialization path. +`full_project_id=True` is the opt-in the vault namespaces use; these pin both the key +shape and the end-to-end isolation through the real (de)serialization path. + +The truncated default is a TRACKED EXCEPTION, not the intended end state: every other +namespace — `check_permissions` and `check_action_access` included — still keys on the +short id. Flipping the default needs `invalidate_cache` to carry the flag too, and a +deploy plan for the evaluation lock keys (a key-shape change mid rolling deploy loses +mutual exclusion). Tracked in issue #6166; the test below pins the hazard so the +exception stays visible instead of reading as an invariant. """ import pytest @@ -18,10 +25,12 @@ PROJECT_B = "bbbbbbbb-bbbb-4bbb-8bbb-123456789012" -def test_truncated_packing_collides_on_a_shared_suffix_and_full_packing_does_not(): +def test_the_truncated_default_is_a_known_collision_hazard_full_packing_removes(): + # Not a property worth keeping: a live record of what issue #6166 has to fix. When the + # default flips, this assertion inverts and the namespaces below stop needing the flag. truncated_a = pack(namespace="list_secrets", key={}, project_id=PROJECT_A) truncated_b = pack(namespace="list_secrets", key={}, project_id=PROJECT_B) - assert truncated_a == truncated_b # the hazard full_project_id exists to remove + assert truncated_a == truncated_b full_a = pack( namespace="list_secrets", key={}, project_id=PROJECT_A, full_project_id=True diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index 99c8e563a5..90cc8b4707 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -114,12 +114,18 @@ async def set_cache( ): self.store[self._name(project_id, namespace, key)] = value + # Both vault namespaces are packed with the FULL project id, while the router's + # blanket `invalidate_cache(project_id=...)` scans the TRUNCATED-id pattern — so + # production invalidation reaches neither of them. Modelling that here is what keeps + # the stale-snapshot test honest: it must pass because the generation is dead, not + # because a fake wiped the entry. Anything else the project caches is still swept. + _FULL_ID_NAMESPACES = ("list_secrets", "list_secrets_generation") + async def invalidate_cache(self, *, project_id): - # The real pattern invalidation does not reach the full-id generation keys. self.store = { name: value for name, value in self.store.items() - if name[0] == "list_secrets_generation" + if name[0] in self._FULL_ID_NAMESPACES } def entries(self, namespace): diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py index 027507f483..59d940a468 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -16,6 +16,7 @@ WebhookSubscription, WebhookSubscriptionCreate, WebhookSubscriptionData, + WebhookSubscriptionEdit, ) from oss.src.utils.env import env @@ -81,6 +82,20 @@ async def create_subscription( async def fetch_subscription(self, *, project_id, subscription_id): return self.subscriptions.get(subscription_id) + async def edit_subscription(self, *, project_id, user_id, subscription, secret_id): + record = self.subscriptions.get(subscription.id) + if record is None: + return None + record = record.model_copy( + update={ + "name": subscription.name or record.name, + "data": subscription.data, + "secret_id": secret_id or record.secret_id, + } + ) + self.subscriptions[record.id] = record + return record + @pytest.fixture(name="services") def _services(): @@ -191,3 +206,69 @@ async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypat ) assert signing_value == "whsec_provided_by_user_12345" + + +@pytest.mark.asyncio +async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_value( + services, monkeypatch +): + # The rotation path builds an update-path payload DTO; the parent `SecretDTO` does not + # validate there, so this pins that editing a subscription's secret still works. + monkeypatch.setattr(env.agenta.vault, "write_only_default", False) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + + edited = await webhooks_service.edit_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=WebhookSubscriptionEdit( + id=created.id, + name="notify", + data=WebhookSubscriptionData(url="https://example.com/hook"), + secret="whsec_test_rotated", + ), + ) + + assert edited is not None + + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + signing_value = await webhooks_service._resolve_secret( + project_id=PROJECT_ID, + secret_id=stored.secret_id, + ) + + assert signing_value == "whsec_test_rotated" + + +@pytest.mark.asyncio +async def test_rotation_echo_stays_redacted_for_a_write_only_secret( + services, monkeypatch +): + monkeypatch.setattr(env.agenta.vault, "write_only_default", True) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=_subscription_create(), + ) + + edited = await webhooks_service.edit_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=WebhookSubscriptionEdit( + id=created.id, + name="notify", + data=WebhookSubscriptionData(url="https://example.com/hook"), + secret="whsec_test_rotated", + ), + ) + + assert edited.secret is None diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 50fa76dcad..5e345fffd6 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -5,11 +5,15 @@ user. The platform runtime keeps reading it through a granted internal path so r work. This is the GitHub-secrets model. Status: backend landed (API + Python SDK), inert by default behind -`AGENTA_VAULT_WRITE_ONLY_DEFAULT=false`. The web half is deferred until the frontend -refactor (PR #6065) lands; frontend and Fern client regeneration follow in a second PR, -after which the gate flips on. Until then, an explicitly created `write_only: true` secret -shows cosmetically as "not configured" in today's Settings (the UI does not read `has_key` -yet) — accepted; the run path is unaffected either way. +`AGENTA_VAULT_WRITE_ONLY_DEFAULT=false`. Two PR numbers appear around this work and mean +different things: **#6065** is the frontend package-extraction refactor (merged) that the +web half of this feature builds on, and **#6135** is the branch this backend PR is +stacked on (the per-turn trace-export credential fix), which is a stacking base only and +has nothing to do with secrets. The web half (replace-only forms plus Fern client +regeneration) follows in a second PR, after which the gate flips on. Until then, an +explicitly created `write_only: true` secret shows cosmetically as "not configured" in +today's Settings (the UI does not read `has_key` yet) — accepted; the run path is +unaffected either way. ## The contract @@ -141,3 +145,16 @@ read: no session, ApiKey, or list/get call ever returns the value. - Optional "readable" toggle at creation only (maps to `write_only: false`), if product wants the escape hatch exposed. - Regenerate the Fern client for the new `write_only`, `has_key`, `key_preview` fields. + +## Known gap: cache-key tenancy + +Both vault namespaces (`list_secrets`, `list_secrets_generation`) pack their Redis keys +with the FULL project id, because they cache secret payloads. The platform default still +truncates a project id to its last 12 characters, so every other namespace, including +`check_permissions` and `check_action_access`, would share an entry between two projects +whose UUIDs end the same way. Server-generated UUID4s make that remote, and unreachable by +a caller who cannot pick their own project id, but it is a default worth removing. + +Not fixed here: flipping it needs `invalidate_cache` to carry the same flag (its scan +pattern is packed short today) and a deploy plan for the evaluation lock keys, whose shape +must not change under a rolling deploy. Tracked in issue #6166. From 9fb88ef1f20fd523814c69cf04889c503b7c6d96 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 18:30:11 +0200 Subject: [PATCH 05/31] fix(api): keep webhook signing secrets readable to the subscriber A signing secret is a shared secret: the subscriber verifies our signature with the same value. When Agenta generates it, the create response is the only place they can ever read it, so the vault-wide write-only default would ship a subscription nobody can verify. Both webhook create paths now opt out of the default explicitly. --- api/oss/src/core/webhooks/service.py | 14 +++- .../unit/webhooks/test_write_only_outward.py | 76 ++++++++++++++++--- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/api/oss/src/core/webhooks/service.py b/api/oss/src/core/webhooks/service.py index c1ab3b87e1..6e1dbb32fd 100644 --- a/api/oss/src/core/webhooks/service.py +++ b/api/oss/src/core/webhooks/service.py @@ -165,6 +165,12 @@ async def create_subscription( ), ), ), + # Opted out of the write-only default on purpose. A signing secret is a + # SHARED secret: the subscriber verifies our signature with the same + # value, and when we generated it here that response is the only place + # they can ever read it. Redacting it would ship a subscription nobody + # can verify. + write_only=False, ), ) @@ -177,8 +183,9 @@ async def create_subscription( secret_id=secret_dto.id, ) - # The create echo respects write-only: once the stored secret is write-only, no - # response carries the value again — not even the creating one. + # The create echo goes through redaction anyway: the row is created readable, so + # the value comes back here, and if a user later tightens it by hand every later + # response — this one included, on a re-create — redacts. return self._with_secret( subscription=result, secret=redact_secret_response(secret_dto).data.provider.key, @@ -436,6 +443,9 @@ async def edit_subscription( ), ), ), + # Same shared-secret reasoning as the create path: a subscription + # that first gets a secret on edit must stay verifiable. + write_only=False, ), ) secret_id = secret_dto.id diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py index 59d940a468..811e897769 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -1,8 +1,10 @@ """Webhook responses are write-only-aware; internal signing keeps plaintext. -The signing secret lives in the vault. Once that record is write-only (via the env gate or -a manual tighten), no USER-facing webhook response — create echo, fetch, edit echo — may -carry the value again, while the internal resolver the signer uses stays plaintext. +The signing secret lives in the vault, but it is a SHARED secret: the subscriber verifies +our signature with the same value, so webhook records are created readable regardless of +the env gate. Once a record IS write-only (only a manual tighten gets it there), no +USER-facing webhook response — create echo, fetch, edit echo — may carry the value again, +while the internal resolver the signer uses stays plaintext. """ from uuid import UUID, uuid4 @@ -117,9 +119,9 @@ def _subscription_create(): @pytest.mark.asyncio -async def test_gate_on_create_and_fetch_never_return_the_signing_secret( - services, monkeypatch -): +async def test_gate_on_still_leaves_the_signing_secret_readable(services, monkeypatch): + # The vault-wide write-only default must not reach webhook signing secrets: the + # subscriber needs the value to verify signatures. monkeypatch.setattr(env.agenta.vault, "write_only_default", True) webhooks_service, _ = services @@ -128,13 +130,45 @@ async def test_gate_on_create_and_fetch_never_return_the_signing_secret( user_id=USER_ID, subscription=_subscription_create(), ) - assert created.secret is None + assert created.secret == "whsec_provided_by_user_12345" fetched = await webhooks_service.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id, ) - assert fetched.secret is None + assert fetched.secret == "whsec_provided_by_user_12345" + + +@pytest.mark.asyncio +async def test_gate_on_returns_a_generated_secret_on_the_create_echo( + services, monkeypatch +): + # The create echo is the ONLY place a caller can read a secret Agenta generated for + # them. Redacting it would ship a subscription that can never be verified. + monkeypatch.setattr(env.agenta.vault, "write_only_default", True) + webhooks_service, _ = services + + created = await webhooks_service.create_subscription( + project_id=PROJECT_ID, + user_id=USER_ID, + subscription=WebhookSubscriptionCreate( + name="notify", + data=WebhookSubscriptionData(url="https://example.com/hook"), + ), + ) + + assert created.secret + + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + signing_value = await webhooks_service._resolve_secret( + project_id=PROJECT_ID, + secret_id=stored.secret_id, + ) + + # What the subscriber was handed is what we sign with. + assert created.secret == signing_value @pytest.mark.asyncio @@ -188,8 +222,8 @@ async def test_manually_tightened_secret_stops_appearing_in_fetches( @pytest.mark.asyncio async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypatch): - monkeypatch.setattr(env.agenta.vault, "write_only_default", True) - webhooks_service, _ = services + monkeypatch.setattr(env.agenta.vault, "write_only_default", False) + webhooks_service, vault_service = services created = await webhooks_service.create_subscription( project_id=PROJECT_ID, @@ -197,6 +231,15 @@ async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypat subscription=_subscription_create(), ) + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + await vault_service.update_secret( + secret_id=UUID(str(stored.secret_id)), + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(write_only=True), + ) + stored = await webhooks_service.dao.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id ) @@ -251,8 +294,8 @@ async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_valu async def test_rotation_echo_stays_redacted_for_a_write_only_secret( services, monkeypatch ): - monkeypatch.setattr(env.agenta.vault, "write_only_default", True) - webhooks_service, _ = services + monkeypatch.setattr(env.agenta.vault, "write_only_default", False) + webhooks_service, vault_service = services created = await webhooks_service.create_subscription( project_id=PROJECT_ID, @@ -260,6 +303,15 @@ async def test_rotation_echo_stays_redacted_for_a_write_only_secret( subscription=_subscription_create(), ) + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, subscription_id=created.id + ) + await vault_service.update_secret( + secret_id=UUID(str(stored.secret_id)), + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(write_only=True), + ) + edited = await webhooks_service.edit_subscription( project_id=PROJECT_ID, user_id=USER_ID, From 58b74c97135944f53522ac7d99fa4dbc02b7966c Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 19:01:38 +0200 Subject: [PATCH 06/31] refactor(api): stop caching the vault secrets list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list is small and only the settings page reads it, and the runtime path already went straight to the database. Caching it bought little while costing a whole class of question — what a shared Redis entry holds, and whether a stale reader can repopulate it with plaintext after a tighten — which the generation counter and the full-project-id cache keys existed only to answer. The route now reads the database on every request and redacts at the response boundary, so what a caller sees is what the row says. The per-project sweep each secret write already fired is untouched: it predates write-only secrets and serves the other namespaces. --- api/oss/src/utils/caching.py | 22 +-- .../unit/utils/test_cache_key_tenancy.py | 138 ----------------- .../unit/vault/test_write_only_routes.py | 146 +++--------------- docs/design/write-only-secrets/README.md | 32 ++-- 4 files changed, 35 insertions(+), 303 deletions(-) delete mode 100644 api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py diff --git a/api/oss/src/utils/caching.py b/api/oss/src/utils/caching.py index f11b2ac2d1..b2c6f97b62 100644 --- a/api/oss/src/utils/caching.py +++ b/api/oss/src/utils/caching.py @@ -46,20 +46,12 @@ def _pack( project_id: Optional[str] = None, user_id: Optional[str] = None, pattern: Optional[bool] = False, - full_project_id: bool = False, ) -> str: - # Security-sensitive namespaces pass full_project_id=True: the 12-character suffix - # is a display-length compromise, and two projects sharing a suffix must never share - # a cache entry that guards tenant data. The truncated default is a tracked exception - # (issue #6166): flipping it needs `invalidate_cache` to carry the flag as well, and a - # deploy plan for the evaluation lock keys, whose shape must not change under a - # rolling deploy. if project_id: - if not full_project_id: - project_id = project_id[-12:] if len(project_id) > 12 else project_id + project_id = project_id[-12:] if len(project_id) > 12 else project_id else: project_id = "" - project_id = project_id + "-" * max(0, 12 - len(project_id)) + project_id = project_id + "-" * (12 - len(project_id)) if user_id: user_id = user_id[-12:] if len(user_id) > 12 else user_id @@ -89,7 +81,6 @@ def pack( project_id: Optional[str] = None, user_id: Optional[str] = None, pattern: Optional[bool] = False, - full_project_id: bool = False, ) -> str: return _pack( namespace=namespace, @@ -97,7 +88,6 @@ def pack( project_id=project_id, user_id=user_id, pattern=pattern, - full_project_id=full_project_id, ) @@ -237,7 +227,6 @@ async def _maybe_retry_get( model: Optional[Type[BaseModel]] = None, is_list: Optional[bool] = False, retry: Optional[bool] = True, - full_project_id: bool = False, *, ttl: Optional[int] = None, lock_ttl: int, @@ -252,7 +241,6 @@ async def _maybe_retry_get( key=key, project_id=project_id, user_id=user_id, - full_project_id=full_project_id, ) if CACHE_DEBUG: @@ -319,7 +307,6 @@ async def _maybe_retry_get( model=model, is_list=is_list, retry=retry, - full_project_id=full_project_id, # ttl=ttl, lock=lock_ttl, @@ -341,7 +328,6 @@ async def set_cache( key: Optional[Union[str, dict]] = None, value: Optional[Any] = None, ttl: Optional[int] = AGENTA_CACHE_TTL, - full_project_id: bool = False, ) -> Optional[bool]: # Noop if caching is disabled if not env.agenta.api.caching.enabled: @@ -353,7 +339,6 @@ async def set_cache( key=key, project_id=project_id, user_id=user_id, - full_project_id=full_project_id, ) cache_value: bytes = _serialize(value) cache_px = int(ttl * 1000) @@ -409,7 +394,6 @@ async def get_cache( model: Optional[Type[BaseModel]] = None, is_list: Optional[bool] = False, retry: Optional[bool] = True, - full_project_id: bool = False, *, ttl: Optional[int] = None, lock: Optional[int] = AGENTA_CACHE_LOCK_TTL, @@ -429,7 +413,6 @@ async def get_cache( key=key, project_id=project_id, user_id=user_id, - full_project_id=full_project_id, ) data = await _try_get_and_maybe_renew(cache_name, model, is_list, ttl) @@ -446,7 +429,6 @@ async def get_cache( model=model, is_list=is_list, retry=retry, - full_project_id=full_project_id, # ttl=ttl, lock_ttl=lock, diff --git a/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py b/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py deleted file mode 100644 index b77a327199..0000000000 --- a/api/oss/tests/pytest/unit/utils/test_cache_key_tenancy.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Security-sensitive cache keys must carry the FULL tenant id. - -The default key packing truncates project ids to their last 12 characters — fine for -display-length economy, but two projects sharing a UUID suffix would share a cache entry. -`full_project_id=True` is the opt-in the vault namespaces use; these pin both the key -shape and the end-to-end isolation through the real (de)serialization path. - -The truncated default is a TRACKED EXCEPTION, not the intended end state: every other -namespace — `check_permissions` and `check_action_access` included — still keys on the -short id. Flipping the default needs `invalidate_cache` to carry the flag too, and a -deploy plan for the evaluation lock keys (a key-shape change mid rolling deploy loses -mutual exclusion). Tracked in issue #6166; the test below pins the hazard so the -exception stays visible instead of reading as an invariant. -""" - -import pytest -from pydantic import BaseModel - -from oss.src.utils import caching -from oss.src.utils.caching import get_cache, pack, set_cache - - -# Same 12-character suffix, different projects. -PROJECT_A = "aaaaaaaa-aaaa-4aaa-8aaa-123456789012" -PROJECT_B = "bbbbbbbb-bbbb-4bbb-8bbb-123456789012" - - -def test_the_truncated_default_is_a_known_collision_hazard_full_packing_removes(): - # Not a property worth keeping: a live record of what issue #6166 has to fix. When the - # default flips, this assertion inverts and the namespaces below stop needing the flag. - truncated_a = pack(namespace="list_secrets", key={}, project_id=PROJECT_A) - truncated_b = pack(namespace="list_secrets", key={}, project_id=PROJECT_B) - assert truncated_a == truncated_b - - full_a = pack( - namespace="list_secrets", key={}, project_id=PROJECT_A, full_project_id=True - ) - full_b = pack( - namespace="list_secrets", key={}, project_id=PROJECT_B, full_project_id=True - ) - assert full_a != full_b - assert PROJECT_A in full_a - assert PROJECT_B in full_b - - -class _FakeRedis: - """Just enough of the engine surface for set/get, storing by exact key.""" - - def __init__(self): - self.store = {} - - async def set(self, name, value, px=None, nx=False, ex=None): - if nx and name in self.store: - return False - self.store[name] = value - return True - - async def get(self, name): - return self.store.get(name) - - async def expire(self, name, ttl): - return True - - async def delete(self, name): - return self.store.pop(name, None) is not None - - -class _Entry(BaseModel): - owner: str - - -@pytest.mark.asyncio -async def test_full_id_keys_isolate_same_suffix_projects_through_real_serialization( - monkeypatch, -): - # The unit conftest flips caching off (no live Redis); the fake engine stands in. - monkeypatch.setattr(caching.env.agenta.api.caching, "enabled", True) - monkeypatch.setattr(caching, "_cache_engine", _FakeRedis()) - - await set_cache( - namespace="list_secrets", - project_id=PROJECT_A, - key={}, - value=_Entry(owner="a"), - full_project_id=True, - ) - await set_cache( - namespace="list_secrets", - project_id=PROJECT_B, - key={}, - value=_Entry(owner="b"), - full_project_id=True, - ) - - read_a = await get_cache( - namespace="list_secrets", - project_id=PROJECT_A, - key={}, - model=_Entry, - retry=False, - full_project_id=True, - ) - read_b = await get_cache( - namespace="list_secrets", - project_id=PROJECT_B, - key={}, - model=_Entry, - retry=False, - full_project_id=True, - ) - - assert read_a.owner == "a" - assert read_b.owner == "b" - - -@pytest.mark.asyncio -async def test_truncated_keys_do_cross_projects_which_is_why_the_vault_opts_out( - monkeypatch, -): - monkeypatch.setattr(caching.env.agenta.api.caching, "enabled", True) - monkeypatch.setattr(caching, "_cache_engine", _FakeRedis()) - - await set_cache( - namespace="list_secrets", - project_id=PROJECT_A, - key={}, - value=_Entry(owner="a"), - ) - - crossed = await get_cache( - namespace="list_secrets", - project_id=PROJECT_B, - key={}, - model=_Entry, - retry=False, - ) - - assert crossed is not None and crossed.owner == "a" diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index 90cc8b4707..a611e3f082 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -1,7 +1,7 @@ """Every vault route redacts write-only values for users; only the runtime grant reads them. Drives the real `VaultRouter` + `VaultService` over an in-memory DAO, with the permission -check and the Redis cache monkeypatched. The caller's principal is simulated by a test +check monkeypatched. The caller's principal is simulated by a test middleware: requests with the `x-test-grant` header carry the secret-resolve grant (the platform runtime); requests without it are ordinary user principals (session/ApiKey). """ @@ -86,73 +86,14 @@ async def delete(self, secret_id, project_id, organization_id): self.records.pop(str(secret_id), None) -class _FakeCache: - """Keyed like the real cache: (namespace, project, logical key) -> value.""" - - def __init__(self): - self.store = {} - - @staticmethod - def _name(project_id, namespace, key): - return (namespace, str(project_id), str(sorted((key or {}).items()))) - - async def get_cache( - self, - *, - project_id, - namespace, - key, - model=None, - is_list=False, - retry=True, - full_project_id=False, - ): - return self.store.get(self._name(project_id, namespace, key)) - - async def set_cache( - self, *, project_id, namespace, key, value, full_project_id=False - ): - self.store[self._name(project_id, namespace, key)] = value - - # Both vault namespaces are packed with the FULL project id, while the router's - # blanket `invalidate_cache(project_id=...)` scans the TRUNCATED-id pattern — so - # production invalidation reaches neither of them. Modelling that here is what keeps - # the stale-snapshot test honest: it must pass because the generation is dead, not - # because a fake wiped the entry. Anything else the project caches is still swept. - _FULL_ID_NAMESPACES = ("list_secrets", "list_secrets_generation") - - async def invalidate_cache(self, *, project_id): - self.store = { - name: value - for name, value in self.store.items() - if name[0] in self._FULL_ID_NAMESPACES - } - - def entries(self, namespace): - return [value for name, value in self.store.items() if name[0] == namespace] - - def generation(self, project_id): - return self.store.get(self._name(project_id, "list_secrets_generation", {})) - - def poison(self, project_id, generation, value): - """Simulate a stale reader writing its snapshot under an old generation.""" - self.store[ - self._name(project_id, "list_secrets", {"generation": generation}) - ] = value - - @pytest.fixture(name="harness") def _harness(monkeypatch): - cache = _FakeCache() dao = _FakeSecretsDAO() async def _allow(**kwargs): return True monkeypatch.setattr(vault_router_module, "check_action_access", _allow) - monkeypatch.setattr(vault_router_module, "get_cache", cache.get_cache) - monkeypatch.setattr(vault_router_module, "set_cache", cache.set_cache) - monkeypatch.setattr(vault_router_module, "invalidate_cache", cache.invalidate_cache) app = FastAPI() @@ -166,7 +107,7 @@ async def _principal(request, call_next): app.include_router(VaultRouter(vault_service=VaultService(dao)).router) - return TestClient(app), cache + return TestClient(app) GRANT = {"x-test-grant": "1"} @@ -188,7 +129,7 @@ def _create(client, write_only=None, key=KEY): def test_create_echo_is_redacted_for_a_write_only_secret(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) @@ -202,7 +143,7 @@ def test_create_echo_is_redacted_for_a_write_only_secret(harness): def test_create_without_the_flag_keeps_todays_response_while_the_gate_is_off(harness): # The current frontend sends no flag; until AGENTA_VAULT_WRITE_ONLY_DEFAULT flips on, # its creates must behave exactly as today. - client, _ = harness + client = harness created = _create(client) @@ -216,7 +157,7 @@ def test_create_without_the_flag_is_write_only_once_the_gate_is_on( harness, monkeypatch ): monkeypatch.setattr(env.agenta.vault, "write_only_default", True) - client, _ = harness + client = harness created = _create(client) @@ -225,7 +166,7 @@ def test_create_without_the_flag_is_write_only_once_the_gate_is_on( def test_create_with_explicit_false_keeps_todays_response(harness): - client, _ = harness + client = harness created = _create(client, write_only=False) @@ -236,7 +177,7 @@ def test_create_with_explicit_false_keeps_todays_response(harness): def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) user_read = client.get(f"/secrets/{created['id']}") @@ -248,8 +189,8 @@ def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): assert runtime_read.json()["data"]["provider"]["key"] == KEY -def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): - client, cache = harness +def test_list_is_redacted_for_users(harness): + client = harness _create(client, write_only=True) listed = client.get("/secrets/") @@ -257,20 +198,13 @@ def test_list_is_redacted_and_the_cache_stores_the_redacted_shape(harness): (secret,) = listed.json() assert "key" not in secret["data"]["provider"] assert secret["has_key"] is True + assert KEY not in listed.text - # What went into Redis is the redacted DTO: no plaintext at rest in the cache. - ((cached,),) = cache.entries("list_secrets") - assert cached.data.provider.key is None - assert cached.has_key is True - -def test_grant_list_bypasses_the_redacted_cache_and_gets_plaintext(harness): - client, _ = harness +def test_grant_list_gets_plaintext(harness): + client = harness _create(client, write_only=True) - # A user listing first populates the cache with the redacted shape. - client.get("/secrets/") - runtime_list = client.get("/secrets/", headers=GRANT) assert runtime_list.status_code == 200 (secret,) = runtime_list.json() @@ -278,7 +212,7 @@ def test_grant_list_bypasses_the_redacted_cache_and_gets_plaintext(harness): def test_update_echo_is_redacted_and_omitted_key_keeps_the_stored_value(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) updated = client.put( @@ -304,7 +238,7 @@ def test_todays_edit_form_shape_empty_string_key_keeps_the_stored_value(harness) # The CURRENT frontend cannot prefill a redacted value, so its edit form re-sends # `key: ""`. If "" cleared the credential, every edit through today's UI would wipe a # write-only secret — so empty string must mean "keep the stored value". - client, _ = harness + client = harness created = _create(client, write_only=True) updated = client.put( @@ -324,7 +258,7 @@ def test_todays_edit_form_shape_empty_string_key_keeps_the_stored_value(harness) def test_write_only_cannot_be_disabled_over_the_api(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) response = client.put(f"/secrets/{created['id']}", json={"write_only": False}) @@ -334,7 +268,7 @@ def test_write_only_cannot_be_disabled_over_the_api(harness): def test_readable_secret_lists_with_its_value_as_today(harness): - client, _ = harness + client = harness _create(client, write_only=False) (secret,) = client.get("/secrets/").json() @@ -344,7 +278,7 @@ def test_readable_secret_lists_with_its_value_as_today(harness): def test_delete_still_works(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) assert client.delete(f"/secrets/{created['id']}").status_code == 204 @@ -352,7 +286,7 @@ def test_delete_still_works(harness): def test_kind_or_family_change_without_a_new_value_is_400(harness): - client, _ = harness + client = harness created = _create(client, write_only=True) response = client.put( @@ -369,42 +303,6 @@ def test_kind_or_family_change_without_a_new_value_is_400(harness): assert "credential value" in response.json()["detail"] -# --- the cache generation prevents stale plaintext repopulation ------------------------ - - -def test_stale_snapshot_cannot_repopulate_the_cache_after_tightening(harness): - client, cache = harness - created = _create(client, write_only=False) - - # A first list materializes the generation and a readable cache entry. - client.get("/secrets/") - stale_generation = cache.generation(PROJECT_ID) - assert stale_generation - - # Tightening the secret bumps the generation. - tightened = client.put(f"/secrets/{created['id']}", json={"write_only": True}) - assert tightened.status_code == 200 - assert cache.generation(PROJECT_ID) != stale_generation - - # A stale reader (whose DB snapshot predates the tightening) now writes its - # plaintext list — it can only land under the DEAD generation. - plaintext_snapshot = [ - SecretResponseDTO( - id=created["id"], - slug=created["slug"], - kind="provider_key", - data={"kind": "openai", "provider": {"key": KEY}}, - header={"name": "OpenAI"}, - write_only=False, - ) - ] - cache.poison(PROJECT_ID, stale_generation, plaintext_snapshot) - - # No later reader consults the dead generation: the list stays redacted. - (secret,) = client.get("/secrets/").json() - assert "key" not in secret["data"]["provider"] - - # --- real signed tokens through the real verifier -------------------------------------- SECRET_KEY = "unit-test-secret-key-with-32-bytes" @@ -416,16 +314,12 @@ def _token_client(monkeypatch): through the real `verify_secret_token` — nothing injects `token_grants` directly.""" monkeypatch.setattr(auth_module, "_SECRET_KEY", SECRET_KEY) - cache = _FakeCache() dao = _FakeSecretsDAO() async def _allow(**kwargs): return True monkeypatch.setattr(vault_router_module, "check_action_access", _allow) - monkeypatch.setattr(vault_router_module, "get_cache", cache.get_cache) - monkeypatch.setattr(vault_router_module, "set_cache", cache.set_cache) - monkeypatch.setattr(vault_router_module, "invalidate_cache", cache.invalidate_cache) app = FastAPI() @@ -513,7 +407,7 @@ async def test_expired_granted_token_is_rejected(token_client): def test_openapi_documents_the_write_only_contract(harness): - client, _ = harness + client = harness schemas = client.app.openapi()["components"]["schemas"] @@ -527,7 +421,7 @@ def test_openapi_documents_the_write_only_contract(harness): def test_malformed_create_never_echoes_the_submitted_key(harness): - client, _ = harness + client = harness response = client.post( "/secrets/", diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 5e345fffd6..da3058cfec 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -70,14 +70,12 @@ Redaction happens at the response boundary, in every outward surface: In-process runtime readers (`VaultService` and below) are untouched. -**Cache.** The Redis list cache stores the **redacted** shape (which also removes the -previous plaintext-at-rest in Redis). The list cache key carries a per-project -**generation** that every secret write bumps, so a reader holding a pre-write DB snapshot -can only write its entry under a dead generation no later reader consults — a stale -reader can never repopulate the cache with plaintext after a tighten. Both the list and -generation keys use the **full project id** (`full_project_id=True` in the cache helper); -the default 12-character-suffix key would let two projects with the same suffix share a -security-sensitive entry. +**No cache.** The list route reads the database on every request. The list is small and +only the settings page reads it, and the runtime path already bypassed the cache, so +caching bought little while making the redaction guarantee depend on what a shared Redis +entry holds and on how a stale reader is kept from repopulating it. Removing the cache +removes that whole class of question: what a caller sees is what the row says, redacted +at the response boundary for every principal without the grant. ### Updates: keep-stored-on-omit @@ -146,15 +144,11 @@ read: no session, ApiKey, or list/get call ever returns the value. wants the escape hatch exposed. - Regenerate the Fern client for the new `write_only`, `has_key`, `key_preview` fields. -## Known gap: cache-key tenancy +## Known gap: cache-key tenancy (elsewhere) -Both vault namespaces (`list_secrets`, `list_secrets_generation`) pack their Redis keys -with the FULL project id, because they cache secret payloads. The platform default still -truncates a project id to its last 12 characters, so every other namespace, including -`check_permissions` and `check_action_access`, would share an entry between two projects -whose UUIDs end the same way. Server-generated UUID4s make that remote, and unreachable by -a caller who cannot pick their own project id, but it is a default worth removing. - -Not fixed here: flipping it needs `invalidate_cache` to carry the same flag (its scan -pattern is packed short today) and a deploy plan for the evaluation lock keys, whose shape -must not change under a rolling deploy. Tracked in issue #6166. +The platform cache truncates a project id to its last 12 characters, so two projects whose +UUIDs end the same way share an entry in every namespace that caches per project, +including `check_permissions` and `check_action_access`. Server-generated UUID4s make that +remote, and unreachable by a caller who cannot pick their own project id, but it is a +default worth removing. Nothing here depends on it — the vault caches nothing — and the +platform-wide fix is tracked in issue #6166. From 876233d1fef995acc427a3bfb843631bd7c2bc4f Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 19:01:46 +0200 Subject: [PATCH 07/31] fix(sdk): use this run's own provider key when the vault redacts a write-only secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error told the user to provide the provider key in the environment, but nothing read it: an agenta-mode connection resolved from the vault alone, so a standalone run against a write-only secret failed even with OPENAI_API_KEY exported. Resolution now reads the variable the harness itself would use for that connection — the provider family's key, or Bedrock's and Azure's own channels, so one service's credential is never sent to another — and raises only when that variable is empty too. The error text says what the resolver already tried. --- .../agenta/sdk/agents/connections/errors.py | 20 +-- .../agenta/sdk/agents/platform/connections.py | 46 ++++++- .../platform/test_write_only_secrets.py | 122 ++++++++++++++++-- 3 files changed, 168 insertions(+), 20 deletions(-) diff --git a/sdks/python/agenta/sdk/agents/connections/errors.py b/sdks/python/agenta/sdk/agents/connections/errors.py index 31f5a4a6b9..5f615f0ff5 100644 --- a/sdks/python/agenta/sdk/agents/connections/errors.py +++ b/sdks/python/agenta/sdk/agents/connections/errors.py @@ -60,8 +60,10 @@ class WriteOnlySecretError(ConnectionResolutionError): The vault holds a write-only secret for this connection: the platform runtime reads it through a granted credential, but this caller's credential (typically an ApiKey in a - standalone run) only receives the redacted shape. Passing the redacted (empty) key to a - provider would fail with a misleading auth error, so this fails loud with instructions. + standalone run) only receives the redacted shape. The resolver falls back to the + provider's standard environment variable first; this is raised only when that key is + absent too, because passing the redacted (empty) key to a provider would fail with a + misleading auth error. """ # A standalone run against a write-only secret is a config situation, not a server fault. @@ -71,14 +73,14 @@ def __init__(self, *, slug: Optional[str] = None, provider: str = "") -> None: subject = ( f"connection '{slug}'" if slug else f"provider '{provider}' connection" ) - # Engineering copy; adjust freely. The remediation must change the connection - # MODE, not only the environment: an `agenta`-mode connection never reads env - # keys, so "set the env var" alone loops the user straight back to this error. + # The remediation the resolver itself already tried: it reads the provider's + # standard environment variable before raising, so this error means that key is + # missing too. Naming it keeps the instruction actionable and true. super().__init__( - f"{subject} uses a write-only secret: its value cannot be read back " - "outside the platform runtime. For standalone runs, switch this " - "connection's authentication to self_managed and provide the provider " - "key via its environment variable (for example OPENAI_API_KEY)." + f"{subject} uses a write-only secret: Agenta stores the value but never " + "returns it, so only runs on the Agenta platform can use it. To run " + "outside the platform, provide the provider key in this run's environment " + "(for example OPENAI_API_KEY)." ) self.slug = slug self.provider = provider diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index cc45887073..fa5ee59653 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -11,7 +11,8 @@ from __future__ import annotations -from dataclasses import dataclass, field +import os +from dataclasses import dataclass, field, replace from typing import Any, Dict, Iterable, List, Optional, Sequence, Set import httpx @@ -178,6 +179,34 @@ def _provider_env_var(provider: Optional[str]) -> Optional[str]: return _PROVIDER_ENV_VARS.get(provider.lower()) if provider else None +def _credential_env_var( + provider: str, candidate: "_ConnectionCandidate" +) -> Optional[str]: + """The environment variable this candidate's credential would ride. + + Deliberately the variable the harness itself would read, not merely the provider + family's: a Bedrock or Azure candidate authenticates through its own channel, and + reading a family key (say ``OPENAI_API_KEY``) for it would send one service's + credential to another. + """ + if candidate.deployment == "bedrock": + return "AWS_BEARER_TOKEN_BEDROCK" + if candidate.deployment == "azure": + return "AZURE_OPENAI_API_KEY" + return _provider_env_var(provider) or _provider_env_var(candidate.provider) + + +def _environment_credential( + provider: str, candidate: "_ConnectionCandidate" +) -> Optional[Dict[str, str]]: + """This run's own credential for the candidate, or ``None`` when it has none.""" + env_var = _credential_env_var(provider, candidate) + if not env_var: + return None + value = (os.environ.get(env_var) or "").strip() + return {env_var: value} if value else None + + def _header_name(secret: Dict[str, Any]) -> Optional[str]: return _stripped(_as_dict(secret.get("header")).get("name")) @@ -614,7 +643,20 @@ def _resolve_from_secrets( # cause, and surviving config extras (AWS_REGION) can make `env` non-empty, which would # otherwise let the run proceed mis-credentialed. if chosen.write_only_redacted: - raise WriteOnlySecretError(slug=chosen.slug, provider=provider) + # The vault will never hand this caller the value, so the connection cannot supply + # the credential here. A provider key in this run's own environment is the + # documented way to run outside the platform, and it is what the error tells the + # user to do — so use it when it is there, and fail loud only when it is not. The + # key rides the variable it was read from, never a different channel. + fallback = _environment_credential(provider, chosen) + if fallback is None: + raise WriteOnlySecretError(slug=chosen.slug, provider=provider) + chosen = replace( + chosen, + api_key=None, + write_only_redacted=False, + env={**chosen.env, **fallback}, + ) # A chosen custom connection must carry a usable base URL. Failing here (rather than # returning endpoint=None) keeps the harness from falling back to a provider default and # silently ignoring the user's routing choice (design Decision 4). The error names the slug diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py index 0c223720e8..9ffa9c25a2 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -2,7 +2,8 @@ The platform runtime reads write-only secrets in plaintext through its granted credential, so in-platform runs never see the redacted shape. A standalone run (ApiKey credential) -does — and must fail loud with instructions, never pass an empty key to a provider. +does — and then falls back to this run's own provider key from the environment, failing +loud with instructions when there is none. It never passes an empty key to a provider. """ from __future__ import annotations @@ -14,11 +15,25 @@ ModelRef, WriteOnlySecretError, ) +from agenta.sdk.agents.capabilities import PROVIDER_ENV_VARS from agenta.sdk.agents.platform import connections from agenta.sdk.agents.platform.secrets import _is_write_only_redacted from agenta.sdk.middlewares.running.vault import _split_write_only_redacted +@pytest.fixture(autouse=True) +def _no_ambient_provider_keys(monkeypatch): + """A developer machine exports provider keys; the fallback must not read them here. + + Every case below states its own environment, so the ambient one is cleared first. + """ + for name in set(PROVIDER_ENV_VARS.values()) | { + "AWS_BEARER_TOKEN_BEDROCK", + "AZURE_OPENAI_API_KEY", + }: + monkeypatch.delenv(name, raising=False) + + def _model(model: str = "gpt-5.5", provider: str = "openai") -> ModelRef: return ModelRef(provider=provider, model=model, connection={"mode": "agenta"}) @@ -55,11 +70,33 @@ def test_redacted_write_only_key_fails_loud_with_instructions(): message = str(raised.value) assert "write-only" in message - assert "environment variable" in message + assert "OPENAI_API_KEY" in message + assert "this run's environment" in message # Never the misleading "add your key" error: the key exists, it is just unreadable here. assert not isinstance(raised.value, MissingCredentialError) +def test_redacted_write_only_key_uses_this_runs_own_provider_key(monkeypatch): + # What the error text instructs, made true: the run's own key resolves the connection. + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + + resolved = connections._resolve_from_secrets( + secrets=[_redacted_provider_key()], model=_model(), harness="pi_core" + ) + + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["OPENAI_API_KEY"] == "sk-from-env" + + +def test_a_key_for_another_provider_family_is_not_a_fallback(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-env") + + with pytest.raises(WriteOnlySecretError): + connections._resolve_from_secrets( + secrets=[_redacted_provider_key()], model=_model(), harness="pi_core" + ) + + def test_granted_plaintext_write_only_key_resolves_normally(): resolved = connections._resolve_from_secrets( secrets=[_plaintext_provider_key()], model=_model(), harness="pi_core" @@ -113,6 +150,43 @@ def test_redacted_aws_only_secret_fails_loud_despite_surviving_config_extras(): ) +def test_a_bedrock_connection_never_falls_back_to_the_family_api_key(monkeypatch): + # Bedrock authenticates with a bearer token of its own; an Anthropic API key in the + # environment is a credential for a different service and must not be sent instead. + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-env") + redacted = { + "kind": "custom_provider", + "slug": "bedrock-conn", + "header": {"name": "bedrock-conn"}, + "data": { + "kind": "bedrock", + "provider": {"extras": {"aws_region_name": "eu-west-1"}}, + "models": [{"slug": "claude-opus-5"}], + "provider_slug": "bedrock-conn", + }, + "write_only": True, + "has_key": True, + } + model = ModelRef( + provider="anthropic", + model="claude-opus-5", + connection={"mode": "agenta", "slug": "bedrock-conn"}, + ) + + with pytest.raises(WriteOnlySecretError): + connections._resolve_from_secrets( + secrets=[redacted], model=model, harness="claude_code" + ) + + # Its own channel does resolve it. + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "aws-bearer-env") + resolved = connections._resolve_from_secrets( + secrets=[redacted], model=model, harness="claude_code" + ) + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["AWS_BEARER_TOKEN_BEDROCK"] == "aws-bearer-env" + + def test_plaintext_aws_only_secret_is_not_treated_as_redacted(): plaintext = { "kind": "custom_provider", @@ -152,18 +226,48 @@ def test_redacted_custom_provider_fails_loud_too(): "has_key": True, } + model = ModelRef( + provider="openai", + model="gpt-5.5", + connection={"mode": "agenta", "slug": "my-gateway"}, + ) + with pytest.raises(WriteOnlySecretError): connections._resolve_from_secrets( - secrets=[redacted], - model=ModelRef( - provider="openai", - model="gpt-5.5", - connection={"mode": "agenta", "slug": "my-gateway"}, - ), - harness="pi_core", + secrets=[redacted], model=model, harness="pi_core" ) +def test_a_redacted_gateway_resolves_with_this_runs_key(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-gateway-env") + redacted = { + "kind": "custom_provider", + "slug": "my-gateway", + "header": {"name": "my-gateway"}, + "data": { + "kind": "openai", + "provider": {"url": "https://gateway.example.com/v1"}, + "models": [{"slug": "gpt-5.5"}], + "provider_slug": "my-gateway", + }, + "write_only": True, + "has_key": True, + } + + resolved = connections._resolve_from_secrets( + secrets=[redacted], + model=ModelRef( + provider="openai", + model="gpt-5.5", + connection={"mode": "agenta", "slug": "my-gateway"}, + ), + harness="pi_core", + ) + + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["OPENAI_API_KEY"] == "sk-gateway-env" + + # --- the vault middleware's list partition --------------------------------------------- From c7ed707c00e43e860a37476f98f68dbdbd022ed8 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 22:46:16 +0200 Subject: [PATCH 08/31] fix(api): stop the permissions exchange from minting the secret-resolve grant The exchange attached the grant whenever a caller asked with action=run_service, and any member who may run a service can ask: they could call it with their own session or ApiKey, take the returned credential to the vault routes, and read every write-only value in plaintext. That is the whole guarantee, self-serve. The exchange now only carries forward a grant the caller already holds on a verified Secret token, and never creates one. Creation stays where a run actually starts, in-process at the invoke and inspect hops, so the credential still travels with the run: the workflow service and the runner both re-exchange the granted token they were handed, and refresh keeps working with no new credential to distribute and no deployment change. --- api/oss/src/apis/fastapi/access/router.py | 21 +++++++--- .../pytest/unit/access/test_grant_exchange.py | 39 +++++++++++++++++-- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/api/oss/src/apis/fastapi/access/router.py b/api/oss/src/apis/fastapi/access/router.py index 505262db56..d775d7bff2 100644 --- a/api/oss/src/apis/fastapi/access/router.py +++ b/api/oss/src/apis/fastapi/access/router.py @@ -139,11 +139,20 @@ async def check_permissions( # returned credential is uniformly short-lived and renewable — never echoing an # ApiKey/Bearer. Callers (services, the runner) re-check periodically to refresh. # - # A run_service exchange is "I am about to execute a workload that needs the stored - # keys", so that credential — and only that one — carries the secret-resolve grant - # letting the vault return write-only secret values in plaintext. This is the - # GitHub-secrets line: a member who can run workloads can reach the values through - # a run either way; direct reads with a session/ApiKey stay redacted. + # This exchange NEVER creates the secret-resolve grant; it only carries forward one + # the caller already holds on a verified Secret token. Minting on `action` alone + # made the grant self-serve: any member who may run a service could ask for it with + # their own session or ApiKey and then spend it on the vault routes, which is the + # write-only guarantee gone. The grant is created in-process where a run actually + # starts (`WorkflowsService._prepare_invoke` / `inspect_workflow`) and then travels + # with the run — the workflow service and the runner both re-exchange the granted + # credential they were handed, so refresh keeps working and nothing else can bootstrap + # a grant it was not given. + carried_grants = [ + grant + for grant in getattr(request.state, "token_grants", ()) or () + if grant == SECRET_RESOLVE_GRANT + ] secret_token = await sign_secret_token( user_id=user_id, user_email=getattr(request.state, "user_email", None), @@ -151,7 +160,7 @@ async def check_permissions( workspace_id=str(ctx.scope.workspace_id), organization_id=str(ctx.scope.organization_id), organization_name=getattr(request.state, "organization_name", None), - grants=[SECRET_RESOLVE_GRANT] if action == "run_service" else None, + grants=carried_grants or None, ) credentials_header = f"Secret {secret_token}" diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index 9948774399..8d88de0c03 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -56,7 +56,12 @@ async def _set_cache(**kwargs): router = AccessRouter() - async def run(action, resource_type="service"): + async def run(action, resource_type="service", carried_grants=()): + """Run the exchange as a principal whose credential carries ``carried_grants``. + + A session or ApiKey principal never has any: `verify_secret_token` is the only + path that populates `token_grants`, and only from a verified Secret token. + """ request = Request( { "type": "http", @@ -69,6 +74,7 @@ async def run(action, resource_type="service"): "root_path": "", } ) + request.state.token_grants = tuple(carried_grants) token = set_auth_context( AuthContext( @@ -113,10 +119,13 @@ def _body(response) -> dict: @pytest.mark.asyncio -async def test_allowed_run_service_exchange_returns_a_granted_credential(exchange): +async def test_a_granted_caller_keeps_the_grant_through_the_exchange(exchange): + # The refresh path: the workflow service and the runner re-exchange the granted + # credential a run was started with, and must get one back or the run loses its + # ability to read the secrets it was authorized to use. run, _ = exchange - body = _body(await run("run_service")) + body = _body(await run("run_service", carried_grants=(SECRET_RESOLVE_GRANT,))) assert body["effect"] == "allow" claims = _claims(body["credentials"]) @@ -124,6 +133,30 @@ async def test_allowed_run_service_exchange_returns_a_granted_credential(exchang assert claims["project_id"] == str(PROJECT_ID) +@pytest.mark.asyncio +async def test_a_plain_caller_cannot_mint_the_grant_by_asking_for_it(exchange): + # The escalation this closes: a member who may run a service could call the exchange + # with their own session or ApiKey — neither of which carries a grant — and spend the + # returned credential on the vault routes to read every write-only value in plaintext. + run, _ = exchange + + body = _body(await run("run_service")) + + assert body["effect"] == "allow" + claims = _claims(body["credentials"]) + assert "grants" not in claims + + +@pytest.mark.asyncio +async def test_an_unrelated_carried_grant_is_not_forwarded(exchange): + run, _ = exchange + + body = _body(await run("run_service", carried_grants=("some-other-grant",))) + + claims = _claims(body["credentials"]) + assert "grants" not in claims + + @pytest.mark.asyncio async def test_non_run_actions_never_receive_the_grant(exchange): run, _ = exchange From 8c103052ad72f3fd61b26913d0e3cc3d5eed96ab Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 22:48:42 +0200 Subject: [PATCH 09/31] fix(api): carry a secret's kept credential over from the locked row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep-on-omit filled an omitted credential from a snapshot the service read before the DAO took its row lock, so a rotation that committed in between was silently undone: the update wrote the older value back over the newer one, and nothing reported it. The DAO now resolves the carry-over inside the locked transaction, against the row as it actually stands — the shape GitDAO.commit_revision already uses for the same reason. The identity check travels with it, because deciding same-identity against a stale row is how a credential crosses identities. The fake DAOs call the resolver at the same point, so keep-on-omit stays exercised rather than skipped, and a new test rotates the stored row inside that window and pins that the value carried over is the rotated one. --- api/oss/src/core/secrets/interfaces.py | 8 +- api/oss/src/core/secrets/services.py | 74 +++++++++++------ api/oss/src/dbs/postgres/secrets/dao.py | 15 +++- .../pytest/unit/secrets/test_services.py | 6 ++ .../pytest/unit/secrets/test_write_only.py | 81 ++++++++++++++++++- .../unit/vault/test_write_only_routes.py | 13 ++- .../unit/webhooks/test_write_only_outward.py | 13 ++- 7 files changed, 180 insertions(+), 30 deletions(-) diff --git a/api/oss/src/core/secrets/interfaces.py b/api/oss/src/core/secrets/interfaces.py index 38275a6e6d..07f894ae44 100644 --- a/api/oss/src/core/secrets/interfaces.py +++ b/api/oss/src/core/secrets/interfaces.py @@ -1,5 +1,5 @@ from uuid import UUID -from typing import Optional, List +from typing import Callable, List, Optional from oss.src.core.secrets.dtos import ( CreateSecretDTO, @@ -48,6 +48,12 @@ async def update( update_secret_dto: UpdateSecretDTO, project_id: Optional[UUID] = None, organization_id: Optional[UUID] = None, + user_id: Optional[UUID] = None, + # Called with the row as it stands under the write lock, before the update is + # applied. Every decision that reads stored state belongs here: a check made + # against a snapshot read earlier is a check against a row another writer can + # still have replaced. It may raise to refuse the update. + resolve_update: Optional[Callable[[SecretResponseDTO], None]] = None, ) -> Optional[SecretResponseDTO]: raise NotImplementedError diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 69861da13b..5130ef9cbe 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -1,4 +1,5 @@ from typing import Any, Optional +from functools import partial from uuid import UUID, uuid4 from oss.src.utils.env import env @@ -16,6 +17,7 @@ ) from oss.src.core.secrets.dtos import ( CreateSecretDTO, + SecretResponseDTO, SecretValueRequiredError, UpdateSecretDTO, WriteOnlyCannotBeDisabledError, @@ -130,6 +132,45 @@ def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: update_extras[extras_key] = stored_value +def _resolve_credential_carry_over( + stored_secret_dto: SecretResponseDTO, + *, + update_secret_dto: UpdateSecretDTO, +) -> None: + """Fill this update's omitted credential from the row UNDER THE WRITE LOCK. + + Called by the DAO inside the locked transaction rather than by the service before it, + because a value carried over from a snapshot read earlier is a value another writer + may already have replaced: a rotation that commits in between would be silently + undone, the update writing the older credential back over the newer one. + + Keep-on-omit is also identity-local — a stored credential never silently becomes + another kind's or another provider's credential — and that decision reads the same + stored row, so it belongs under the same lock. + """ + if update_secret_dto.secret is None: + return + + same_identity = stored_secret_dto.kind == ( + update_secret_dto.secret.kind + ) and _provider_family(stored_secret_dto.data) == _provider_family( + update_secret_dto.secret.data + ) + + if same_identity: + _carry_over_saved_policy( + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) + _carry_over_saved_value( + kind=str(stored_secret_dto.kind.value), + stored_data=stored_secret_dto.data, + update_data=update_secret_dto.secret.data, + ) + else: + _require_explicit_value(secret=update_secret_dto.secret) + + def _carry_over_saved_policy(*, stored_data: Any, update_data: Any) -> None: """Fill an update payload's omitted ``models``/``harnesses`` from the stored record. @@ -309,35 +350,22 @@ async def update_secret( ): raise WriteOnlyCannotBeDisabledError() - if update_secret_dto.secret is not None: - # Keep-on-omit is an identity-local contract: a stored credential - # never silently becomes another kind's or another provider's - # credential. Changing either requires an explicit new value. - same_identity = stored_secret_dto.kind == ( - update_secret_dto.secret.kind - ) and _provider_family( - stored_secret_dto.data - ) == _provider_family(update_secret_dto.secret.data) - - if same_identity: - _carry_over_saved_policy( - stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, - ) - _carry_over_saved_value( - kind=str(stored_secret_dto.kind.value), - stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, - ) - else: - _require_explicit_value(secret=update_secret_dto.secret) - secret_dto = await self.secrets_dao.update( secret_id=secret_id, update_secret_dto=update_secret_dto, project_id=project_id, organization_id=organization_id, user_id=user_id, + # Resolved against the LOCKED row, not the snapshot above: see + # `_resolve_credential_carry_over`. + resolve_update=( + partial( + _resolve_credential_carry_over, + update_secret_dto=update_secret_dto, + ) + if update_secret_dto.secret is not None + else None + ), ) return secret_dto diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index 099bee7603..5003bfba5e 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -1,4 +1,5 @@ import json +from typing import Callable, Optional from uuid import UUID from oss.src.dbs.postgres.secrets.dbes import SecretsDBE @@ -10,7 +11,11 @@ get_transactions_engine, ) -from oss.src.core.secrets.dtos import CreateSecretDTO, UpdateSecretDTO +from oss.src.core.secrets.dtos import ( + CreateSecretDTO, + SecretResponseDTO, + UpdateSecretDTO, +) from oss.src.dbs.postgres.secrets.mappings import ( map_secrets_dto_to_dbe, map_secrets_dbe_to_dto, @@ -122,6 +127,7 @@ async def update( project_id: UUID | None, organization_id: UUID | None, user_id: UUID | None = None, + resolve_update: Optional[Callable[[SecretResponseDTO], None]] = None, ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) @@ -142,6 +148,13 @@ async def update( if secrets_dbe is None: return None + # Every decision that reads stored state runs HERE, against the locked row. + # A caller that read the row before this transaction may be holding a + # snapshot another writer has already replaced; the keep-on-omit carry-over + # in particular would then write a rotated credential back to its old value. + if resolve_update is not None: + resolve_update(map_secrets_dbe_to_dto(secrets_dbe=secrets_dbe)) + if update_secret_dto.write_only is False and bool( json.loads(secrets_dbe.data).get("write_only") ): diff --git a/api/oss/tests/pytest/unit/secrets/test_services.py b/api/oss/tests/pytest/unit/secrets/test_services.py index f7031b9f61..b730b66b71 100644 --- a/api/oss/tests/pytest/unit/secrets/test_services.py +++ b/api/oss/tests/pytest/unit/secrets/test_services.py @@ -63,6 +63,7 @@ async def update( project_id, organization_id, user_id=None, + resolve_update=None, ): del user_id scope = (project_id, organization_id) @@ -72,6 +73,11 @@ async def update( ) if stored is None: return None + + # Production resolves the update against the row under the write lock; the fake + # does the same at the same point, so keep-on-omit is exercised, not skipped. + if resolve_update is not None: + resolve_update(stored) # Like the postgres mapping, the whole data blob is replaced: whatever the payload # omits is gone unless the service carried it over first. record = SecretResponseDTO( diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index 75ba347fd0..de8cd4aa10 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -58,12 +58,23 @@ async def get_by_id(self, secret_id, project_id, organization_id): return self.records.get(secret_id) async def update( - self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + self, + secret_id, + update_secret_dto, + project_id, + organization_id, + user_id=None, + resolve_update=None, ): stored = self.records.get(secret_id) if stored is None: return None + # Production resolves the update against the row under the write lock; the fake + # does the same at the same point, so keep-on-omit is exercised, not skipped. + if resolve_update is not None: + resolve_update(stored) + write_only = update_secret_dto.write_only if write_only is None: write_only = stored.write_only @@ -153,6 +164,70 @@ async def test_an_explicit_request_value_always_wins_over_the_gate( # --- service: keep-stored-on-omit ------------------------------------------------------ +class _RotatingDAO(_FakeSecretsDAO): + """A DAO where another writer commits a rotation while this update waits for the lock. + + The real DAO takes ``SELECT ... FOR UPDATE`` and only then resolves the update, so a + writer that committed while we waited is visible. This fake reproduces that window by + rotating the stored row immediately before it resolves. + """ + + def __init__(self, rotated_key: str): + super().__init__() + self.rotated_key = rotated_key + + async def update( + self, + secret_id, + update_secret_dto, + project_id, + organization_id, + user_id=None, + resolve_update=None, + ): + stored = self.records.get(secret_id) + if stored is not None: + rotated = stored.model_dump() + rotated["data"]["provider"]["key"] = self.rotated_key + self.records[secret_id] = SecretResponseDTO(**rotated) + + return await super().update( + secret_id=secret_id, + update_secret_dto=update_secret_dto, + project_id=project_id, + organization_id=organization_id, + user_id=user_id, + resolve_update=resolve_update, + ) + + +@pytest.mark.asyncio +async def test_an_omitted_key_keeps_the_value_a_racing_rotation_just_stored(): + # A rotation that lands between "read the row" and "write the row" must not be undone. + # Resolving the omitted credential against a snapshot taken before the lock wrote the + # OLD key back over the new one, silently reverting the rotation. + dao = _RotatingDAO(rotated_key="sk-test-rotated") + service = VaultService(dao) + + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=_provider_key_create(write_only=True), + ) + + updated = await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": ""}}, + } + ), + ) + + assert updated.data.provider.key == "sk-test-rotated" + + @pytest.mark.asyncio @pytest.mark.parametrize("omitted_key", [None, ""]) async def test_update_without_provider_key_keeps_the_stored_one(service, omitted_key): @@ -733,7 +808,7 @@ def test_update_mapping_applies_a_tightening_flag(): stored = json.loads(dbe.data) assert stored["write_only"] is True - assert stored["provider"]["key"] == "sk-live-1234567890abc" + assert stored["provider"]["key"] == "sk-test-openai-key-bc" def test_update_mapping_never_clears_the_flag_on_a_stale_explicit_false(): @@ -763,7 +838,7 @@ def test_update_mapping_never_clears_the_flag_on_a_stale_explicit_false(): stored = json.loads(dbe.data) assert stored["write_only"] is True - assert stored["provider"]["key"] == "sk-rotated-9876543210xyz" + assert stored["provider"]["key"] == "sk-test-rotated" # --- the update-path payload type, at every call site ---------------------------------- diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index a611e3f082..7b0f709a75 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -59,12 +59,23 @@ async def get_by_slug(self, secret_slug, project_id, organization_id): ) async def update( - self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + self, + secret_id, + update_secret_dto, + project_id, + organization_id, + user_id=None, + resolve_update=None, ): stored = self.records.get(str(secret_id)) if stored is None: return None + # Production resolves the update against the row under the write lock; the fake + # does the same at the same point, so keep-on-omit is exercised, not skipped. + if resolve_update is not None: + resolve_update(stored) + write_only = update_secret_dto.write_only if write_only is None: write_only = stored.write_only diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py index 811e897769..0930353368 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -50,11 +50,22 @@ async def list(self, project_id, organization_id): return list(self.records.values()) async def update( - self, secret_id, update_secret_dto, project_id, organization_id, user_id=None + self, + secret_id, + update_secret_dto, + project_id, + organization_id, + user_id=None, + resolve_update=None, ): stored = self.records.get(secret_id) if stored is None: return None + + # Production resolves the update against the row under the write lock; the fake + # does the same at the same point, so keep-on-omit is exercised, not skipped. + if resolve_update is not None: + resolve_update(stored) write_only = update_secret_dto.write_only if write_only is None: write_only = stored.write_only From 0604378baf78357516a9328a0fc9a20a247cc792 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 22:48:53 +0200 Subject: [PATCH 10/31] fix(sdk): accept every credential channel a redacted connection could use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The environment fallback for a write-only connection read one variable per candidate, so a Bedrock connection could only be credentialed by AWS_BEARER_TOKEN_BEDROCK and a Vertex one not at all — even though the vault path accepts an AWS key pair or service-account material for exactly those connections. The fallback now offers the same channels the stored credential could have used, and a channel counts only when every variable in it is set: half an AWS key pair authenticates nothing, and passing it on would fail at the provider with a misleading error instead of here with an actionable one. --- .../agenta/sdk/agents/platform/connections.py | 53 +++++++++----- .../platform/test_write_only_secrets.py | 72 +++++++++++++++++++ 2 files changed, 107 insertions(+), 18 deletions(-) diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index fa5ee59653..08c5d77751 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -13,7 +13,7 @@ import os from dataclasses import dataclass, field, replace -from typing import Any, Dict, Iterable, List, Optional, Sequence, Set +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple import httpx @@ -179,32 +179,49 @@ def _provider_env_var(provider: Optional[str]) -> Optional[str]: return _PROVIDER_ENV_VARS.get(provider.lower()) if provider else None -def _credential_env_var( +def _credential_channels( provider: str, candidate: "_ConnectionCandidate" -) -> Optional[str]: - """The environment variable this candidate's credential would ride. - - Deliberately the variable the harness itself would read, not merely the provider - family's: a Bedrock or Azure candidate authenticates through its own channel, and - reading a family key (say ``OPENAI_API_KEY``) for it would send one service's - credential to another. +) -> List[Tuple[str, ...]]: + """The environment variables this candidate's credential could ride, best first. + + Each entry is one COMPLETE channel: every variable in it must be present for that + channel to authenticate. Deliberately the variables the harness itself would read for + this candidate, never merely the provider family's — a Bedrock or Azure candidate + authenticates through its own channel, and reading a family key (say + ``OPENAI_API_KEY``) for it would send one service's credential to another. The set + mirrors the credential material the plaintext path accepts for the same connection + (``CREDENTIAL_EXTRAS_KEYS``), so a standalone run can supply from the environment + exactly what the vault would have supplied. """ if candidate.deployment == "bedrock": - return "AWS_BEARER_TOKEN_BEDROCK" + return [ + ("AWS_BEARER_TOKEN_BEDROCK",), + ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), + ] + if candidate.deployment in ("vertex_ai", "vertex"): + return [("GOOGLE_APPLICATION_CREDENTIALS",)] if candidate.deployment == "azure": - return "AZURE_OPENAI_API_KEY" - return _provider_env_var(provider) or _provider_env_var(candidate.provider) + return [("AZURE_OPENAI_API_KEY",)] + + env_var = _provider_env_var(provider) or _provider_env_var(candidate.provider) + return [(env_var,)] if env_var else [] def _environment_credential( provider: str, candidate: "_ConnectionCandidate" ) -> Optional[Dict[str, str]]: - """This run's own credential for the candidate, or ``None`` when it has none.""" - env_var = _credential_env_var(provider, candidate) - if not env_var: - return None - value = (os.environ.get(env_var) or "").strip() - return {env_var: value} if value else None + """This run's own credential for the candidate, or ``None`` when it has none. + + A channel counts only when EVERY variable in it is set: half an AWS key pair + authenticates nothing, and passing it on would fail at the provider with a + misleading error instead of here with an actionable one. + """ + for channel in _credential_channels(provider, candidate): + values = {name: (os.environ.get(name) or "").strip() for name in channel} + if all(values.values()): + return values + + return None def _header_name(secret: Dict[str, Any]) -> Optional[str]: diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py index 9ffa9c25a2..67fc6ba65f 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -29,7 +29,10 @@ def _no_ambient_provider_keys(monkeypatch): """ for name in set(PROVIDER_ENV_VARS.values()) | { "AWS_BEARER_TOKEN_BEDROCK", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", "AZURE_OPENAI_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", }: monkeypatch.delenv(name, raising=False) @@ -187,6 +190,75 @@ def test_a_bedrock_connection_never_falls_back_to_the_family_api_key(monkeypatch assert env["AWS_BEARER_TOKEN_BEDROCK"] == "aws-bearer-env" +def _redacted_custom(kind: str, slug: str, extras: dict | None = None) -> dict: + return { + "kind": "custom_provider", + "slug": slug, + "header": {"name": slug}, + "data": { + "kind": kind, + "provider": {"extras": extras or {}}, + "models": [{"slug": "claude-opus-5"}], + "provider_slug": slug, + }, + "write_only": True, + "has_key": True, + } + + +def _claude_model(slug: str) -> ModelRef: + return ModelRef( + provider="anthropic", + model="claude-opus-5", + connection={"mode": "agenta", "slug": slug}, + ) + + +def test_a_bedrock_connection_accepts_an_aws_key_pair_from_the_environment(monkeypatch): + # The same credential material the plaintext path takes from the vault's extras. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "aws-secret-from-env") + redacted = _redacted_custom( + "bedrock", "bedrock-conn", {"aws_region_name": "eu-west-1"} + ) + + resolved = connections._resolve_from_secrets( + secrets=[redacted], model=_claude_model("bedrock-conn"), harness="claude_code" + ) + + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["AWS_ACCESS_KEY_ID"] == "AKIAEXAMPLE" + assert env["AWS_SECRET_ACCESS_KEY"] == "aws-secret-from-env" + + +def test_half_an_aws_key_pair_is_not_a_credential(monkeypatch): + # An access key id with no secret authenticates nothing; failing here names the + # problem, where passing it on would fail at the provider with an auth error. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") + redacted = _redacted_custom("bedrock", "bedrock-conn") + + with pytest.raises(WriteOnlySecretError): + connections._resolve_from_secrets( + secrets=[redacted], + model=_claude_model("bedrock-conn"), + harness="claude_code", + ) + + +def test_a_vertex_connection_accepts_google_application_credentials(monkeypatch): + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/run/secrets/service-account") + redacted = _redacted_custom( + "vertex_ai", "vertex-conn", {"vertex_ai_location": "europe-west1"} + ) + + resolved = connections._resolve_from_secrets( + secrets=[redacted], model=_claude_model("vertex-conn"), harness="claude_code" + ) + + env = {item.binding.name: item.value for item in resolved.credentials} + assert env["GOOGLE_APPLICATION_CREDENTIALS"] == "/run/secrets/service-account" + + def test_plaintext_aws_only_secret_is_not_treated_as_redacted(): plaintext = { "kind": "custom_provider", From 15b2978692b08f898e2f05863bb60a33e8565e97 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 22:49:02 +0200 Subject: [PATCH 11/31] fix(api): require a client secret when an SSO secret is created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SSO branch checked that client_secret was PRESENT, not that it held a value, so a create carrying an explicit null stored an SSO record with no credential — while the webhook and custom-secret branches beside it already checked the value. It now checks the value on the create path only: omission still means "keep the stored one" on update, which is what redacted responses and the edit form depend on. --- api/oss/src/core/secrets/dtos.py | 10 ++- .../tests/pytest/unit/secrets/test_dtos.py | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index 2b1084fb25..3f987ee75d 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -205,9 +205,13 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" ) required_fields = {"client_id", "issuer_url", "scopes"} - if cls.VALUE_REQUIRED: - required_fields.add("client_secret") - if not required_fields.issubset(provider.keys()): + # `client_secret` is checked by VALUE, not by presence: a create carrying an + # explicit null would otherwise store a credential-less SSO record, and a + # value is optional only on the update path (omission means "keep the stored + # one") and in redacted responses. + if not required_fields.issubset(provider.keys()) or ( + cls.VALUE_REQUIRED and provider.get("client_secret") is None + ): raise ValueError( "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" ) diff --git a/api/oss/tests/pytest/unit/secrets/test_dtos.py b/api/oss/tests/pytest/unit/secrets/test_dtos.py index ecef0c90ab..62fd3f1d24 100644 --- a/api/oss/tests/pytest/unit/secrets/test_dtos.py +++ b/api/oss/tests/pytest/unit/secrets/test_dtos.py @@ -344,6 +344,74 @@ def test_create_secret_rejects_an_empty_header(kind): CreateSecretDTO.model_validate(_payload_without_a_header(kind)) +def _sso_payload(client_secret): + return { + "header": {"name": "Okta"}, + "secret": { + "kind": "sso_provider", + "data": { + "provider": { + "client_id": "id", + "client_secret": client_secret, + "issuer_url": "https://issuer.example", + "scopes": ["openid"], + } + }, + }, + } + + +def test_create_sso_secret_rejects_a_null_client_secret(): + # Presence is not a value: an explicit null would store an SSO record with no + # credential. A value may be omitted only on the update path, where omission means + # "keep the stored one". + with pytest.raises(ValidationError, match="SSOProviderSettingsDTO"): + CreateSecretDTO.model_validate(_sso_payload(None)) + + with pytest.raises(ValidationError, match="SSOProviderSettingsDTO"): + CreateSecretDTO.model_validate( + { + "header": {"name": "Okta"}, + "secret": { + "kind": "sso_provider", + "data": { + "provider": { + "client_id": "id", + "issuer_url": "https://issuer.example", + "scopes": ["openid"], + } + }, + }, + } + ) + + +def test_create_sso_secret_accepts_a_real_client_secret(): + secret = CreateSecretDTO.model_validate(_sso_payload("sso-client-secret")) + + assert secret.secret.kind.value == "sso_provider" + + +def test_update_sso_secret_may_omit_the_client_secret(): + # Keep-on-omit: an update without a value keeps the stored one. + update = UpdateSecretDTO.model_validate( + { + "secret": { + "kind": "sso_provider", + "data": { + "provider": { + "client_id": "id", + "issuer_url": "https://issuer.example", + "scopes": ["openid"], + } + }, + } + } + ) + + assert update.secret is not None + + def test_create_secret_allows_an_empty_header_for_a_provider_key(): # The one kind that may arrive unnamed: the service names it after its provider on create. secret = CreateSecretDTO.model_validate( From 0598972adf098839960c1da77ebe5832c4b18d63 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 22:52:26 +0200 Subject: [PATCH 12/31] chore: allow four unit-test credential fixtures that survive in history The fixtures were rewritten to obviously-fake, digit-free strings so the scanner has nothing to find going forward. Four occurrences remain inside commits whose amend could not be replayed, and the scan reads history, not the tree. Each is a unit-test constant handed to a fake DAO; no real credential was ever involved. Fingerprints are anchored to their commit, so they need regenerating if either lane's history is rewritten again. --- .gitleaksignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitleaksignore b/.gitleaksignore index e71f4a670b..decbc8d39e 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -291,3 +291,13 @@ a00f015276504fbf7a4820b26d17eb725c63635b:bench_bulk_insert.py:generic-api-key:30 0ab95c73ed4e993b69081805d1db96dbcc052653:docs/design/agent-workflows/projects/qa/scripts/mcp_qa_server.mjs:generic-api-key:13 0e69cc62f45a348f96239249870d2660cd504402:.github/workflows/16-website-production.yml:generic-api-key:53 f5ebc5469f58a153d2ba72dc1bd0d7baae19c733:web/packages/agenta-entities/tests/unit/secret-persist-redaction.test.ts:generic-api-key:35 + +# Test fixtures for write-only vault secrets (PRs #6164 / #6165). The fixtures themselves +# were rewritten to obviously-fake, digit-free strings; these four occurrences survive in +# commits whose amend could not be replayed. No real credential was ever involved: the +# values are unit-test constants for a fake DAO. Regenerate these lines if either lane's +# history is rewritten again — a fingerprint is anchored to its commit sha. +ea3257c7cc43d635b7bd8a16863d26df8d012c85:api/oss/tests/pytest/unit/secrets/test_write_only.py:generic-api-key:343 +84bb7fe0c9f926c6a60d7bd8577ba64043b655e7:api/oss/tests/pytest/unit/secrets/test_managed_secrets.py:generic-api-key:241 +84bb7fe0c9f926c6a60d7bd8577ba64043b655e7:api/oss/tests/pytest/unit/secrets/test_managed_secrets.py:generic-api-key:248 +84bb7fe0c9f926c6a60d7bd8577ba64043b655e7:api/oss/tests/pytest/unit/secrets/test_managed_secrets.py:generic-api-key:303 From d8b2577e725976ee17175c24d8408fb886de7d97 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:10:08 +0200 Subject: [PATCH 13/31] fix(api): keep the SSO client secret plaintext where it authenticates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redaction was applied inside the one helper every caller used, so the connection test and the edit path read a write-only provider's client secret as empty. A redacted read there does not hide a value from anyone — it tests the provider without its secret, then writes is_valid false and is_active false, taking a working provider out of the login screen. There are now two resolvers, the split the webhooks service already uses for signing secrets: the plain one stays plaintext for the internal callers, and an outward one shapes responses and drops the secret once the record is write-only. Both are documented by what the caller does with the value, not by where it is called from. --- api/ee/src/core/organizations/service.py | 66 +++++++++++----- .../unit/test_write_only_provider_settings.py | 77 +++++++++++++++---- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/api/ee/src/core/organizations/service.py b/api/ee/src/core/organizations/service.py index fcabd4c6b4..0e779f387c 100644 --- a/api/ee/src/core/organizations/service.py +++ b/api/ee/src/core/organizations/service.py @@ -871,9 +871,27 @@ async def delete_provider( await session.commit() return deleted + @staticmethod + def _provider_settings_of(secret) -> dict: + data = secret.data + if hasattr(data, "provider"): + return data.provider.model_dump() + if isinstance(data, dict): + provider = data.get("provider") or {} + if isinstance(provider, dict): + return provider + raise HTTPException(status_code=500, detail="Invalid provider secret format") + async def _get_provider_settings( self, organization_id: str, secret_id: str ) -> dict: + """The provider's settings as this service needs them: PLAINTEXT. + + Internal callers only — testing the connection against the identity provider, and + re-writing the record on edit. Both authenticate or persist, so a redacted client + secret here does not hide a value, it reports a working provider as broken and + deactivates it. Response shaping goes through `_get_outward_provider_settings`. + """ secret = await self._vault_service().get_secret_by_id( secret_id=UUID(secret_id), organization_id=UUID(organization_id), @@ -881,31 +899,41 @@ async def _get_provider_settings( if not secret: raise HTTPException(status_code=404, detail="Provider secret not found") - # This feeds USER-facing provider responses, so a write-only secret loses its - # client_secret here. The login-time reader (the SuperTokens overrides) resolves - # the secret through VaultService directly and keeps plaintext. - secret = redact_secret_response(secret) + return self._provider_settings_of(secret) - data = secret.data - if hasattr(data, "provider"): - return data.provider.model_dump() - if isinstance(data, dict): - provider = data.get("provider") or {} - if isinstance(provider, dict): - if getattr(secret, "write_only", False): - provider = { - key: value - for key, value in provider.items() - if key != "client_secret" - } - return provider - raise HTTPException(status_code=500, detail="Invalid provider secret format") + async def _get_outward_provider_settings( + self, organization_id: str, secret_id: str + ) -> dict: + """The provider's settings as a USER response may carry them. + + Only for response shaping: a write-only record loses its client secret here. The + login-time reader (the SuperTokens overrides) resolves through `VaultService` + directly and keeps plaintext, as the internal resolver above does. + """ + secret = await self._vault_service().get_secret_by_id( + secret_id=UUID(secret_id), + organization_id=UUID(organization_id), + ) + if not secret: + raise HTTPException(status_code=404, detail="Provider secret not found") + + write_only = bool(getattr(secret, "write_only", False)) + settings = self._provider_settings_of(redact_secret_response(secret)) + + if write_only: + # The dict-shaped branch has no typed container for the redaction helper to + # reach into, so the field is dropped here instead. + settings = { + key: value for key, value in settings.items() if key != "client_secret" + } + + return settings async def _to_response( self, provider, organization_id: str ) -> OrganizationProvider: """Convert DBE to response model.""" - settings = await self._get_provider_settings( + settings = await self._get_outward_provider_settings( organization_id, str(provider.secret_id) ) diff --git a/api/ee/tests/pytest/unit/test_write_only_provider_settings.py b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py index 76d59ec925..dad61ec326 100644 --- a/api/ee/tests/pytest/unit/test_write_only_provider_settings.py +++ b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py @@ -1,9 +1,11 @@ -"""EE organization-provider responses respect write-only SSO secrets. - -`_get_provider_settings` feeds the user-facing provider serialization; once the vault -record is write-only it must drop `client_secret` while keeping the non-secret settings. -The login-time reader (SuperTokens overrides) resolves through `VaultService` directly and -is unaffected. +"""EE organization-provider settings: redacted outward, plaintext where it authenticates. + +Two resolvers, and which one a caller uses decides whether SSO keeps working. The outward +one shapes user-facing responses and drops `client_secret` once the vault record is +write-only. The internal one feeds the connection test and the edit path, which +authenticate against the identity provider and persist the record: redacting there does +not hide a value, it reports a working provider as broken and deactivates it. The +login-time reader (SuperTokens overrides) resolves through `VaultService` directly. """ from uuid import uuid4 @@ -46,15 +48,20 @@ def _sso_secret(write_only: bool) -> SecretResponseDTO: ) -@pytest.mark.asyncio -async def test_write_only_sso_secret_drops_client_secret_from_settings(monkeypatch): +def _with_secret(monkeypatch, secret) -> OrganizationProvidersService: monkeypatch.setattr( OrganizationProvidersService, "_vault_service", - staticmethod(lambda: _StubVaultService(_sso_secret(write_only=True))), + staticmethod(lambda: _StubVaultService(secret)), ) + return OrganizationProvidersService() + - settings = await OrganizationProvidersService()._get_provider_settings( +@pytest.mark.asyncio +async def test_write_only_sso_secret_drops_client_secret_from_responses(monkeypatch): + service = _with_secret(monkeypatch, _sso_secret(write_only=True)) + + settings = await service._get_outward_provider_settings( str(ORGANIZATION_ID), str(SECRET_ID) ) @@ -64,15 +71,53 @@ async def test_write_only_sso_secret_drops_client_secret_from_settings(monkeypat @pytest.mark.asyncio -async def test_readable_sso_secret_keeps_todays_settings(monkeypatch): - monkeypatch.setattr( - OrganizationProvidersService, - "_vault_service", - staticmethod(lambda: _StubVaultService(_sso_secret(write_only=False))), +async def test_readable_sso_secret_keeps_todays_responses(monkeypatch): + service = _with_secret(monkeypatch, _sso_secret(write_only=False)) + + settings = await service._get_outward_provider_settings( + str(ORGANIZATION_ID), str(SECRET_ID) ) - settings = await OrganizationProvidersService()._get_provider_settings( + assert settings["client_secret"] == "super-secret-value-123" + + +@pytest.mark.asyncio +async def test_the_internal_resolver_keeps_plaintext_for_a_write_only_secret( + monkeypatch, +): + # What the connection test and the edit path read. Redacting here would test the + # provider with an empty secret and then mark a working provider invalid. + service = _with_secret(monkeypatch, _sso_secret(write_only=True)) + + settings = await service._get_provider_settings( str(ORGANIZATION_ID), str(SECRET_ID) ) assert settings["client_secret"] == "super-secret-value-123" + + +@pytest.mark.asyncio +async def test_testing_a_write_only_provider_uses_the_stored_secret(monkeypatch): + # End to end through `test_provider`: the value handed to the connection check is the + # stored one, and the provider is not deactivated behind a redacted read. + service = _with_secret(monkeypatch, _sso_secret(write_only=True)) + seen: dict = {} + + async def _record(*, issuer_url, client_id, client_secret): + seen.update( + issuer_url=issuer_url, client_id=client_id, client_secret=client_secret + ) + return True + + monkeypatch.setattr(service, "test_oidc_connection", _record) + + settings = await service._get_provider_settings( + str(ORGANIZATION_ID), str(SECRET_ID) + ) + await service.test_oidc_connection( + issuer_url=settings["issuer_url"], + client_id=settings["client_id"], + client_secret=settings.get("client_secret", ""), + ) + + assert seen["client_secret"] == "super-secret-value-123" From 9dcb4028566ab9b31d6039a341203328d3e38380 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:10:18 +0200 Subject: [PATCH 14/31] fix(api): treat a custom secret's format as part of its identity Keep-on-omit compared kind and provider family but not the stored format, so a text-to-json update that omitted its content carried the stored string into the json shape. The validators could not catch it: they run when the payload is built, before the carry-over fills the value in, so what they saw was a value-less shape and what reached the row was a json secret holding a string. A format change is now an identity change and requires an explicit value, and the merged payload is re-validated under the lock before it is persisted, so nothing is stored that a create of the same shape would have refused. Also moves the per-kind primary credential field out of the SDK classifier and onto the API side: it covers kinds the SDK never resolves (SSO providers, webhook signing secrets) and no SDK code reads it. The extras vocabulary stays shared, which is where drift would actually hurt. --- api/oss/src/core/secrets/redaction.py | 28 +++-- api/oss/src/core/secrets/services.py | 52 ++++++++- .../pytest/unit/secrets/test_write_only.py | 105 ++++++++++++++++++ .../sdk/agents/connections/credentials.py | 17 +-- .../connections/test_credentials_parity.py | 11 -- 5 files changed, 179 insertions(+), 34 deletions(-) diff --git a/api/oss/src/core/secrets/redaction.py b/api/oss/src/core/secrets/redaction.py index 25e8fc6dcb..d53579b2e4 100644 --- a/api/oss/src/core/secrets/redaction.py +++ b/api/oss/src/core/secrets/redaction.py @@ -7,21 +7,33 @@ because the workload it runs needs the real key. In-process readers (`VaultService` and below) are untouched: redaction happens strictly at the response boundary. -WHAT counts as credential material is not decided here: the canonical classifier lives in -the SDK (``agenta.sdk.agents.connections.credentials``) and is imported, so the fields the -SDK resolver consumes as credentials and the fields this module strips can never drift. +WHAT counts as credential material inside a connection is not decided here: that +vocabulary lives in the SDK (``agenta.sdk.agents.connections.credentials``) and is +imported, so the extras the SDK resolver consumes as credentials and the extras this +module strips can never drift. The per-kind primary field below is this side's own, +because it covers kinds the SDK never resolves. """ -from typing import Any, Optional +from typing import Any, Dict, Optional, Tuple -from agenta.sdk.agents.connections.credentials import ( - CREDENTIAL_EXTRAS_KEYS, - PRIMARY_CREDENTIAL_FIELDS, -) +from agenta.sdk.agents.connections.credentials import CREDENTIAL_EXTRAS_KEYS from oss.src.core.secrets.dtos import SecretResponseDTO +# The primary value field per secret kind, as (container attribute, field name). Lives +# here rather than in the SDK classifier because it spans kinds the SDK never resolves — +# SSO providers, webhook signing secrets — and no SDK code reads it. The extras +# vocabulary beside it IS shared, and stays imported. +PRIMARY_CREDENTIAL_FIELDS: Dict[str, Tuple[str, str]] = { + "provider_key": ("provider", "key"), + "custom_provider": ("provider", "key"), + "webhook_provider": ("provider", "key"), + "sso_provider": ("provider", "client_secret"), + "custom_secret": ("secret", "content"), +} + + def mask_secret_value(value: str) -> str: """A short, non-reversible display preview like ``sk-****9Qa``. diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 5130ef9cbe..3f30bfabd8 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -2,6 +2,8 @@ from functools import partial from uuid import UUID, uuid4 +from pydantic import ValidationError + from oss.src.utils.env import env from oss.src.utils.helpers import get_slug_from_name_and_id from oss.src.core.secrets.enums import ( @@ -54,6 +56,20 @@ def _provider_family(data: Any) -> Optional[str]: return str(getattr(kind, "value", kind)) +def _secret_format(data: Any) -> Optional[str]: + """A custom secret's stored format (`data.secret.format`); None for other kinds. + + Part of a secret's identity for the same reason the provider family is: the format + decides how the value is validated and read back, so text and json are two different + credentials, not two spellings of one. + """ + container = getattr(data, "secret", None) + fmt = getattr(container, "format", None) if container is not None else None + if fmt is None: + return None + return str(getattr(fmt, "value", fmt)) + + def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> None: """Fill an update payload's omitted value field from the stored record. @@ -85,6 +101,25 @@ def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> _carry_over_saved_extras(stored_data=stored_data, update_data=update_data) +def _revalidate_merged_secret(*, secret: Any) -> None: + """Re-run the payload validators over the update as it will be stored. + + Validation runs at construction, before keep-on-omit fills the value in, so a merged + payload can be a shape no create would have accepted. Re-validating here — inside the + write lock, against the merged result — is what keeps an invalid row from being + committed. + """ + try: + type(secret).model_validate(secret.model_dump()) + except ValidationError as exc: + raise SecretValueRequiredError( + message=( + "the stored value does not fit this update's shape; " + "provide the value explicitly" + ) + ) from exc + + def _require_explicit_value(*, secret: Any) -> None: """Reject a kind/family-changing update that carries no new credential value.""" kind = str(secret.kind.value) @@ -151,10 +186,15 @@ def _resolve_credential_carry_over( if update_secret_dto.secret is None: return - same_identity = stored_secret_dto.kind == ( - update_secret_dto.secret.kind - ) and _provider_family(stored_secret_dto.data) == _provider_family( - update_secret_dto.secret.data + same_identity = ( + stored_secret_dto.kind == update_secret_dto.secret.kind + and _provider_family(stored_secret_dto.data) + == _provider_family(update_secret_dto.secret.data) + # A custom secret's format is identity too: carrying a stored text value into a + # json update would store a string where the shape says object, and the payload + # validators never see it because they ran before the value was filled in. + and _secret_format(stored_secret_dto.data) + == _secret_format(update_secret_dto.secret.data) ) if same_identity: @@ -167,6 +207,10 @@ def _resolve_credential_carry_over( stored_data=stored_secret_dto.data, update_data=update_secret_dto.secret.data, ) + # The payload was validated before the carry-over filled it in, so what the + # validators actually saw was a value-less shape. Re-validate the merged result: + # nothing reaches the row that a create of the same shape would have refused. + _revalidate_merged_secret(secret=update_secret_dto.secret) else: _require_explicit_value(secret=update_secret_dto.secret) diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index de8cd4aa10..e6c66c5323 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -487,6 +487,97 @@ async def test_kind_change_with_omitted_content_is_rejected(service): ) +@pytest.mark.asyncio +async def test_a_format_change_with_omitted_content_is_rejected(service): + # text -> json with no content used to carry the stored STRING into the json shape. + # The payload validators never saw it (they ran before the carry-over filled it in), + # so an invalid row reached the database: a json secret holding a string. + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "Token"}, + secret={ + "kind": "custom_secret", + "data": { + "secret": {"format": "text", "content": "ghp_example_token_xyz"} + }, + }, + write_only=True, + ), + ) + + update = UpdateSecretDTO( + secret={"kind": "custom_secret", "data": {"secret": {"format": "json"}}}, + ) + + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + stored = await service.get_secret_by_id(created.id, project_id=PROJECT_ID) + assert stored.data.secret.format.value == "text" + assert stored.data.secret.content == "ghp_example_token_xyz" + + +@pytest.mark.asyncio +async def test_a_format_change_with_a_new_value_is_allowed(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "Token"}, + secret={ + "kind": "custom_secret", + "data": { + "secret": {"format": "text", "content": "ghp_example_token_xyz"} + }, + }, + write_only=True, + ), + ) + + updated = await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO( + secret={ + "kind": "custom_secret", + "data": { + "secret": {"format": "json", "content": {"token": "ghp-example"}} + }, + }, + ), + ) + + assert updated.data.secret.format.value == "json" + assert updated.data.secret.content == {"token": "ghp-example"} + + +@pytest.mark.asyncio +async def test_an_omitted_content_still_keeps_the_stored_one_within_a_format(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "Token"}, + secret={ + "kind": "custom_secret", + "data": {"secret": {"format": "json", "content": {"token": "abc"}}}, + }, + write_only=True, + ), + ) + + updated = await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO( + secret={"kind": "custom_secret", "data": {"secret": {"format": "json"}}}, + ), + ) + + assert updated.data.secret.content == {"token": "abc"} + + @pytest.mark.asyncio async def test_family_change_does_not_carry_credential_extras(service): created = await service.create_secret( @@ -887,3 +978,17 @@ def test_update_call_sites_build_the_update_path_payload(): "UpdateSecretDTO(secret=...) must be built with UpdateSecretPayloadDTO: " + "; ".join(offenders) ) + + +def test_primary_credential_fields_cover_every_secret_kind(): + # The redaction, the presence report, and the carry-over all key off this map, so a + # kind missing from it silently stops being redacted. + from oss.src.core.secrets.redaction import PRIMARY_CREDENTIAL_FIELDS + + assert set(PRIMARY_CREDENTIAL_FIELDS) == { + "provider_key", + "custom_provider", + "webhook_provider", + "sso_provider", + "custom_secret", + } diff --git a/sdks/python/agenta/sdk/agents/connections/credentials.py b/sdks/python/agenta/sdk/agents/connections/credentials.py index 76094e8b50..a39f93068a 100644 --- a/sdks/python/agenta/sdk/agents/connections/credentials.py +++ b/sdks/python/agenta/sdk/agents/connections/credentials.py @@ -1,12 +1,16 @@ """The canonical classification of credential material inside vault secrets. -One list, consumed by BOTH sides of the write-only contract so they can never drift: +One vocabulary, consumed by BOTH sides of the write-only contract so they can never drift: - the SDK's connection resolver (``platform/connections.py``), which decides which extras are credentials to inject and whether a redacted record still holds usable material; - the API's redaction and update carry-over (``oss/src/core/secrets/redaction.py``), which must strip, refill, and report presence for exactly the same fields. +Scope is deliberately what a CONNECTION carries. The per-kind primary value field lives on +the API side (``oss/src/core/secrets/redaction.py``): it covers secret kinds the SDK never +resolves — SSO providers, webhook signing secrets — and nothing here reads it. + Every key the resolver accepts in a custom provider's ``extras`` must appear in exactly one of the two sets below; a parity test enforces that, so adding an extras key to the resolver without classifying it here fails the build. @@ -14,16 +18,7 @@ from __future__ import annotations -from typing import Dict, FrozenSet, Tuple - -# The primary value field per secret kind, as (container attribute, field name). -PRIMARY_CREDENTIAL_FIELDS: Dict[str, Tuple[str, str]] = { - "provider_key": ("provider", "key"), - "custom_provider": ("provider", "key"), - "webhook_provider": ("provider", "key"), - "sso_provider": ("provider", "client_secret"), - "custom_secret": ("secret", "content"), -} +from typing import Dict, FrozenSet # Extras keys (as stored: the UI's snake_case aliases plus raw env-style names) that hold # credential material. `vertex_ai_credentials`/`GOOGLE_APPLICATION_CREDENTIALS` are diff --git a/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py b/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py index 1365664abe..ca7d4b7686 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py +++ b/sdks/python/oss/tests/pytest/unit/agents/connections/test_credentials_parity.py @@ -8,7 +8,6 @@ from agenta.sdk.agents.connections.credentials import ( CONFIG_EXTRAS_KEYS, CREDENTIAL_EXTRAS_KEYS, - PRIMARY_CREDENTIAL_FIELDS, credential_extras, ) from agenta.sdk.agents.platform.connections import ( @@ -31,16 +30,6 @@ def test_credential_and_config_classifications_are_disjoint(): assert not (CREDENTIAL_EXTRAS_KEYS & CONFIG_EXTRAS_KEYS) -def test_primary_fields_cover_every_secret_kind(): - assert set(PRIMARY_CREDENTIAL_FIELDS) == { - "provider_key", - "custom_provider", - "webhook_provider", - "sso_provider", - "custom_secret", - } - - def test_credential_extras_keeps_only_non_empty_credential_material(): extras = { "api_key": "k", From 71a2857a2afe3df903ddbe7613f2e0dca6648ff1 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:23:14 +0200 Subject: [PATCH 15/31] fix(api,sdk): issue the run credential's grant to the platform runtime, not to whoever asks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing the self-serve grant broke every agent run, and the reason is the shape of the product's path: the playground and the gate post straight to the workflow service with the user's own ApiKey, so the service exchanges THAT credential and the API's in-process mint never runs. Carry-forward alone therefore had nothing to carry, and runs got the redacted shape. The exchange now mints the grant when the caller proves it is the platform runtime and otherwise carries forward what the caller already holds. The proof is a secret only the backend has, sent on the internal hop and compared in constant time, because that route is publicly reachable and the user's token cannot say who is asking. It resolves from AGENTA_SERVICES_INTERNAL_KEY, falling back to AGENTA_AUTH_KEY, which the services container already receives through the same env file as the API — so deployments keep working unchanged, and a dedicated value narrows what one leaked secret can do. The runner keeps carry-forward and is given nothing new; the key never reaches it or a sandbox, and is never logged. --- api/oss/src/apis/fastapi/access/router.py | 64 ++++++++--- api/oss/src/utils/env.py | 12 ++ .../pytest/unit/access/test_grant_exchange.py | 43 ++++++- .../agenta/sdk/middlewares/routing/auth.py | 16 +++ .../unit/test_auth_middleware_credentials.py | 107 ++++++++++++++++++ 5 files changed, 225 insertions(+), 17 deletions(-) create mode 100644 sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py diff --git a/api/oss/src/apis/fastapi/access/router.py b/api/oss/src/apis/fastapi/access/router.py index d775d7bff2..ef7be8d276 100644 --- a/api/oss/src/apis/fastapi/access/router.py +++ b/api/oss/src/apis/fastapi/access/router.py @@ -1,3 +1,4 @@ +from hmac import compare_digest from typing import Any, Dict, List, Optional, Union from uuid import UUID @@ -8,6 +9,7 @@ SECRET_RESOLVE_GRANT, sign_secret_token, ) +from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger from oss.src.utils.caching import get_cache, set_cache from oss.src.utils.context import get_auth_context, get_auth_scope @@ -96,6 +98,47 @@ async def _check_resource_access( return allow_resource +_RUNTIME_KEY_HEADER = "x-agenta-runtime-key" + + +def _is_platform_runtime(request: Request) -> bool: + """Whether this caller proved it is the platform runtime, not a user agent. + + The workflow service exchanges the END USER's credential on the user's behalf, so + nothing about the presented token distinguishes a real run from a browser calling the + same public route. What distinguishes them is a secret only the runtime holds, sent + on this internal hop and compared in constant time. It is never logged, never + returned, and never reaches the runner or a sandbox. + """ + presented = request.headers.get(_RUNTIME_KEY_HEADER) + if not presented: + return False + + expected = env.agenta.services_internal_key + + return bool(expected) and compare_digest(presented, expected) + + +def _run_credential_grants(request: Request, *, action: Optional[str]) -> List[str]: + """The grants the credential this exchange returns may carry. + + Two ways to hold the secret-resolve grant, and no third: the platform runtime asks + for a run credential and proves what it is (the workflow service, on the hop that + starts a run), or the caller already holds the grant on a verified Secret token and + is refreshing it (the runner, every few heartbeats). A session or ApiKey principal + reaching this route directly gets neither, which is what keeps a member who may run a + service from reading write-only values by asking for a credential. + """ + if action == "run_service" and _is_platform_runtime(request): + return [SECRET_RESOLVE_GRANT] + + return [ + grant + for grant in getattr(request.state, "token_grants", ()) or () + if grant == SECRET_RESOLVE_GRANT + ] + + class AccessRouter: def __init__(self) -> None: self.router = APIRouter() @@ -139,20 +182,11 @@ async def check_permissions( # returned credential is uniformly short-lived and renewable — never echoing an # ApiKey/Bearer. Callers (services, the runner) re-check periodically to refresh. # - # This exchange NEVER creates the secret-resolve grant; it only carries forward one - # the caller already holds on a verified Secret token. Minting on `action` alone - # made the grant self-serve: any member who may run a service could ask for it with - # their own session or ApiKey and then spend it on the vault routes, which is the - # write-only guarantee gone. The grant is created in-process where a run actually - # starts (`WorkflowsService._prepare_invoke` / `inspect_workflow`) and then travels - # with the run — the workflow service and the runner both re-exchange the granted - # credential they were handed, so refresh keeps working and nothing else can bootstrap - # a grant it was not given. - carried_grants = [ - grant - for grant in getattr(request.state, "token_grants", ()) or () - if grant == SECRET_RESOLVE_GRANT - ] + # Who may receive a credential that reads write-only secret values: see + # `_run_credential_grants`. Minting on `action` alone made the grant self-serve — + # a member who may run a service could ask for it with their own session or + # ApiKey and spend it on the vault routes, which is the write-only guarantee gone. + grants = _run_credential_grants(request, action=action) secret_token = await sign_secret_token( user_id=user_id, user_email=getattr(request.state, "user_email", None), @@ -160,7 +194,7 @@ async def check_permissions( workspace_id=str(ctx.scope.workspace_id), organization_id=str(ctx.scope.organization_id), organization_name=getattr(request.state, "organization_name", None), - grants=carried_grants or None, + grants=grants or None, ) credentials_header = f"Secret {secret_token}" diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index c64b68e3be..a6397aaf9c 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -603,6 +603,18 @@ class AgentaConfig(BaseModel): auth_key: str = os.getenv("AGENTA_AUTH_KEY") or "replace-me" crypt_key: str = os.getenv("AGENTA_CRYPT_KEY") or "replace-me" + # Shared secret that proves a caller IS the platform runtime (the workflow service), + # as opposed to a browser or an ApiKey holder reaching the same public route. Only a + # caller holding it can be issued a credential that reads write-only secret values. + # Defaults to `auth_key` because the services container already receives it through + # the same env file the API uses, so existing deployments keep working; setting a + # dedicated value narrows what one leaked secret can do. NEVER sent to the runner or + # into a sandbox. + services_internal_key: str = ( + os.getenv("AGENTA_SERVICES_INTERNAL_KEY") + or os.getenv("AGENTA_AUTH_KEY") + or "replace-me" + ) access: AccessConfig = AccessConfig() ai_services: AIServicesConfig = AIServicesConfig() diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index 8d88de0c03..a2a197e132 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -16,6 +16,7 @@ from oss.src.apis.fastapi.access import router as access_router_module from oss.src.apis.fastapi.access.router import AccessRouter from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT +from oss.src.utils.env import env from oss.src.utils.context import ( AuthContext, AuthScope, @@ -26,6 +27,7 @@ SECRET_KEY = "unit-test-secret-key-with-32-bytes" +RUNTIME_KEY = "unit-test-runtime-key-not-a-secret" ORGANIZATION_ID = uuid4() WORKSPACE_ID = uuid4() @@ -36,6 +38,7 @@ @pytest.fixture(name="exchange") def _exchange(monkeypatch): monkeypatch.setattr(auth_module, "_SECRET_KEY", SECRET_KEY) + monkeypatch.setattr(env.agenta, "services_internal_key", RUNTIME_KEY) verdict = {"allow": True} @@ -56,18 +59,23 @@ async def _set_cache(**kwargs): router = AccessRouter() - async def run(action, resource_type="service", carried_grants=()): + async def run(action, resource_type="service", carried_grants=(), runtime_key=None): """Run the exchange as a principal whose credential carries ``carried_grants``. A session or ApiKey principal never has any: `verify_secret_token` is the only path that populates `token_grants`, and only from a verified Secret token. + ``runtime_key`` is what the caller presents as the platform-runtime secret. """ request = Request( { "type": "http", "method": "GET", "path": "/access/permissions/check", - "headers": [], + "headers": ( + [(b"x-agenta-runtime-key", runtime_key.encode())] + if runtime_key is not None + else [] + ), "query_string": b"", "scheme": "http", "server": ("testserver", 80), @@ -133,6 +141,37 @@ async def test_a_granted_caller_keeps_the_grant_through_the_exchange(exchange): assert claims["project_id"] == str(PROJECT_ID) +@pytest.mark.asyncio +async def test_the_platform_runtime_is_issued_the_grant(exchange): + # The path every product run takes: the workflow service exchanges the END USER's + # credential on their behalf, so nothing about the token says "this is a run". The + # runtime proves what it is with a secret only it holds. + run, _ = exchange + + body = _body(await run("run_service", runtime_key=RUNTIME_KEY)) + + claims = _claims(body["credentials"]) + assert claims["grants"] == [SECRET_RESOLVE_GRANT] + + +@pytest.mark.asyncio +async def test_a_wrong_runtime_key_is_not_the_runtime(exchange): + run, _ = exchange + + body = _body(await run("run_service", runtime_key="not-the-key")) + + assert "grants" not in _claims(body["credentials"]) + + +@pytest.mark.asyncio +async def test_the_runtime_key_only_grants_a_run_exchange(exchange): + run, _ = exchange + + body = _body(await run("view_secret", runtime_key=RUNTIME_KEY)) + + assert "grants" not in _claims(body["credentials"]) + + @pytest.mark.asyncio async def test_a_plain_caller_cannot_mint_the_grant_by_asking_for_it(exchange): # The escalation this closes: a member who may run a service could call the exchange diff --git a/sdks/python/agenta/sdk/middlewares/routing/auth.py b/sdks/python/agenta/sdk/middlewares/routing/auth.py index 6d11d89c9a..d11ed240bd 100644 --- a/sdks/python/agenta/sdk/middlewares/routing/auth.py +++ b/sdks/python/agenta/sdk/middlewares/routing/auth.py @@ -21,6 +21,12 @@ AGENTA_RUNTIME_PREFIX = getenv("AGENTA_RUNTIME_PREFIX", "") +# The platform runtime's proof of what it is, for the credential exchange below. +_RUNTIME_KEY_HEADER = "X-Agenta-Runtime-Key" +_RUNTIME_KEY = ( + getenv("AGENTA_SERVICES_INTERNAL_KEY") or getenv("AGENTA_AUTH_KEY") or "" +).strip() + _AUTH_ENABLED = ( getenv("AGENTA_SERVICES_MIDDLEWARE_AUTH_ENABLED") or getenv("AGENTA_SERVICE_MIDDLEWARE_AUTH_ENABLED") @@ -98,6 +104,16 @@ async def get_credentials( authorization = request.headers.get("authorization", None) headers = {"Authorization": authorization} if authorization else None + # This service exchanges the END USER's credential on their behalf, so the token + # it sends says nothing about who is asking. The platform's own secret is what + # says "this is the runtime starting a run", and it is what lets the returned + # credential read write-only secret values. Sent only on this internal hop, never + # logged, never handed to the runner or into a sandbox. A deployment that does not + # set it simply gets a credential without that ability. + runtime_key = _RUNTIME_KEY + if runtime_key: + headers = {**(headers or {}), _RUNTIME_KEY_HEADER: runtime_key} + # COOKIES access_token = request.cookies.get("sAccessToken", None) cookies = {"sAccessToken": access_token} if access_token else None diff --git a/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py new file mode 100644 index 0000000000..8f869719d6 --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py @@ -0,0 +1,107 @@ +"""Unit tests for the routing auth middleware's credential exchange. + +The workflow service exchanges the END USER's credential at `GET /permissions/check`, so +the token it forwards says nothing about who is asking. These tests pin the one thing that +does: the runtime key header that tells the platform a run is starting. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, Dict, Optional + +import pytest + +from agenta.sdk.middlewares.routing import auth as auth_module +from agenta.sdk.utils.cache import TTLLRUCache + + +class _FakeRequest: + def __init__(self) -> None: + self.headers = {"authorization": "ApiKey caller-key"} + self.cookies: Dict[str, str] = {} + self.state = SimpleNamespace(otel={"baggage": {"project_id": "project-1"}}) + self.query_params: Dict[str, str] = {} + + +class _FakeResponse: + def __init__(self, body: Dict[str, Any]) -> None: + self.status_code = 200 + self._body = body + self.headers: Dict[str, str] = {} + + def json(self) -> Dict[str, Any]: + return self._body + + +class _FakeAsyncClient: + """Stands in for httpx.AsyncClient; records the headers each call carried.""" + + call_count = 0 + body: Dict[str, Any] = {} + last_headers: Optional[Dict[str, str]] = None + + async def __aenter__(self) -> "_FakeAsyncClient": + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def get(self, *args: Any, **kwargs: Any) -> _FakeResponse: + type(self).call_count += 1 + type(self).last_headers = kwargs.get("headers") + return _FakeResponse(type(self).body) + + +@pytest.fixture +def platform(monkeypatch): + monkeypatch.setattr(auth_module, "_AUTH_ENABLED", True) + monkeypatch.setattr(auth_module, "_CACHE_ENABLED", True) + monkeypatch.setattr(auth_module, "_cache", TTLLRUCache()) + _FakeAsyncClient.call_count = 0 + _FakeAsyncClient.body = {} + _FakeAsyncClient.last_headers = None + monkeypatch.setattr(auth_module.httpx, "AsyncClient", _FakeAsyncClient) + return _FakeAsyncClient + + +async def _get_credentials() -> Optional[str]: + return await auth_module.get_credentials( + _FakeRequest(), # type: ignore[arg-type] + "http://agenta.test", + ) + + +async def test_deny_body_raises(platform): + platform.body = {"effect": "deny"} + + with pytest.raises(auth_module.DenyException): + await _get_credentials() + + +async def test_the_runtime_key_rides_the_exchange_when_configured( + platform, monkeypatch +): + # What tells the platform that a run is starting, rather than a browser asking for a + # credential: the exchange forwards the END USER's token either way, so this header + # is the only thing that distinguishes them. + monkeypatch.setattr(auth_module, "_RUNTIME_KEY", "runtime-key-for-tests") + platform.body = {"effect": "allow", "credentials": "Secret general-token"} + + await _get_credentials() + + assert platform.last_headers["X-Agenta-Runtime-Key"] == "runtime-key-for-tests" + # The caller's own credential still travels; the key identifies the runtime, it does + # not replace the principal. + assert platform.last_headers["Authorization"] == "ApiKey caller-key" + + +async def test_no_runtime_key_means_no_header(platform, monkeypatch): + # A deployment that configures none simply gets an ungranted credential; it must not + # send an empty header that a comparison might treat as a value. + monkeypatch.setattr(auth_module, "_RUNTIME_KEY", "") + platform.body = {"effect": "allow", "credentials": "Secret general-token"} + + await _get_credentials() + + assert "X-Agenta-Runtime-Key" not in (platform.last_headers or {}) From 037feb0a9fe46d0c02d2eddc4e501e2562649da3 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:25:20 +0200 Subject: [PATCH 16/31] fix(api,sdk): refuse the placeholder as proof of being the platform runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runtime key falls back to AGENTA_AUTH_KEY, whose unconfigured value is the string 'replace-me' committed in every example env file — so a deployment that changed neither would have handed a run credential to anyone who sent it. Both sides now treat the placeholder as no key at all: such a deployment issues no grant, which costs it only the ability to run against write-only secrets (off by default) and never gives that ability to a stranger. The variable is documented in the example env files, and the design note now says who may hold the grant and why the exchange cannot decide it from the requested action. --- api/oss/src/apis/fastapi/access/router.py | 10 +++++++- .../pytest/unit/access/test_grant_exchange.py | 12 ++++++++++ docs/design/write-only-secrets/README.md | 24 +++++++++++++++++++ hosting/docker-compose/ee/env.ee.dev.example | 6 +++++ hosting/docker-compose/ee/env.ee.gh.example | 6 +++++ .../docker-compose/oss/env.oss.dev.example | 6 +++++ hosting/docker-compose/oss/env.oss.gh.example | 6 +++++ .../agenta/sdk/middlewares/routing/auth.py | 4 ++++ 8 files changed, 73 insertions(+), 1 deletion(-) diff --git a/api/oss/src/apis/fastapi/access/router.py b/api/oss/src/apis/fastapi/access/router.py index ef7be8d276..b74c7c84e2 100644 --- a/api/oss/src/apis/fastapi/access/router.py +++ b/api/oss/src/apis/fastapi/access/router.py @@ -99,6 +99,8 @@ async def _check_resource_access( _RUNTIME_KEY_HEADER = "x-agenta-runtime-key" +# The placeholder `env.py` falls back to when nothing is configured. +_UNCONFIGURED_KEY = "replace-me" def _is_platform_runtime(request: Request) -> bool: @@ -115,8 +117,14 @@ def _is_platform_runtime(request: Request) -> bool: return False expected = env.agenta.services_internal_key + # A deployment that configured nothing keeps the well-known placeholder, which anyone + # could send. Treat it as "no runtime configured" rather than as a secret: such a + # deployment issues no grant at all, which costs it only the ability to run against + # write-only secrets — off by default — and never hands the ability to a stranger. + if not expected or expected == _UNCONFIGURED_KEY: + return False - return bool(expected) and compare_digest(presented, expected) + return compare_digest(presented, expected) def _run_credential_grants(request: Request, *, action: Optional[str]) -> List[str]: diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index a2a197e132..96b6c8f7d7 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -172,6 +172,18 @@ async def test_the_runtime_key_only_grants_a_run_exchange(exchange): assert "grants" not in _claims(body["credentials"]) +@pytest.mark.asyncio +async def test_an_unconfigured_deployment_grants_nobody(exchange, monkeypatch): + # The placeholder is in the repo, so anyone could send it. A deployment that + # configured no runtime key must issue no grant rather than accept a known string. + monkeypatch.setattr(env.agenta, "services_internal_key", "replace-me") + run, _ = exchange + + body = _body(await run("run_service", runtime_key="replace-me")) + + assert "grants" not in _claims(body["credentials"]) + + @pytest.mark.asyncio async def test_a_plain_caller_cannot_mint_the_grant_by_asking_for_it(exchange): # The escalation this closes: a member who may run a service could call the exchange diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index da3058cfec..78124808e6 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -144,6 +144,30 @@ read: no session, ApiKey, or list/get call ever returns the value. wants the escape hatch exposed. - Regenerate the Fern client for the new `write_only`, `has_key`, `key_preview` fields. + +## Who may read a value: the grant + +The vault returns plaintext only to a caller whose verified `Secret` token carries the +`secret-resolve` grant. Two callers can hold it, and there is no third: + +- **The platform runtime**, on the hop that starts a run. The workflow service exchanges + the END USER's credential at `/access/permissions/check` on their behalf, so nothing + about the presented token says a run is starting — and that route is reachable by a + browser. The runtime therefore proves what it is with a secret only the backend holds + (`AGENTA_SERVICES_INTERNAL_KEY`, falling back to `AGENTA_AUTH_KEY`), sent as + `X-Agenta-Runtime-Key` on the internal hop and compared in constant time. The + well-known placeholder is refused, so an unconfigured deployment issues no grant rather + than accepting a string anyone could send. +- **A caller refreshing a grant it already holds.** The runner re-exchanges its run + credential every few heartbeats; the exchange carries the grant forward rather than + re-deciding it. The runner is never given the runtime secret, and it never reaches a + sandbox. + +The exchange never mints the grant from the requested `action` alone. It did once, and +that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and +`view_secret`, so any member could ask for a credential and spend it on the vault routes. + + ## Known gap: cache-key tenancy (elsewhere) The platform cache truncates a project id to its last 12 characters, so two projects whose diff --git a/hosting/docker-compose/ee/env.ee.dev.example b/hosting/docker-compose/ee/env.ee.dev.example index ea68f4fdb9..513ec0b6c0 100644 --- a/hosting/docker-compose/ee/env.ee.dev.example +++ b/hosting/docker-compose/ee/env.ee.dev.example @@ -25,6 +25,12 @@ AGENTA_API_URL=http://localhost/api # Agenta - Secrets (REPLACE in production!) # ================================================================== # AGENTA_AUTH_KEY=replace-me +# Proves to the API that a caller IS the platform runtime (the workflow service), so the +# credential it receives may read write-only secret values. The API and the services +# container must hold the SAME value, and a browser must never see it. Falls back to +# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that +# changes neither simply cannot run against write-only secrets. +AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me # ================================================================== # diff --git a/hosting/docker-compose/ee/env.ee.gh.example b/hosting/docker-compose/ee/env.ee.gh.example index 699c33f796..05d6c1f744 100644 --- a/hosting/docker-compose/ee/env.ee.gh.example +++ b/hosting/docker-compose/ee/env.ee.gh.example @@ -29,6 +29,12 @@ AGENTA_API_URL=http://localhost/api # Agenta - Secrets (REPLACE in production!) # ================================================================== # AGENTA_AUTH_KEY=replace-me +# Proves to the API that a caller IS the platform runtime (the workflow service), so the +# credential it receives may read write-only secret values. The API and the services +# container must hold the SAME value, and a browser must never see it. Falls back to +# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that +# changes neither simply cannot run against write-only secrets. +AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me # ================================================================== # diff --git a/hosting/docker-compose/oss/env.oss.dev.example b/hosting/docker-compose/oss/env.oss.dev.example index bf9adc6a57..fb5a547e4b 100644 --- a/hosting/docker-compose/oss/env.oss.dev.example +++ b/hosting/docker-compose/oss/env.oss.dev.example @@ -25,6 +25,12 @@ AGENTA_API_URL=http://localhost/api # Agenta - Secrets (REPLACE in production!) # ================================================================== # AGENTA_AUTH_KEY=replace-me +# Proves to the API that a caller IS the platform runtime (the workflow service), so the +# credential it receives may read write-only secret values. The API and the services +# container must hold the SAME value, and a browser must never see it. Falls back to +# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that +# changes neither simply cannot run against write-only secrets. +AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me # ================================================================== # diff --git a/hosting/docker-compose/oss/env.oss.gh.example b/hosting/docker-compose/oss/env.oss.gh.example index 024ad46e12..7f508e6db3 100644 --- a/hosting/docker-compose/oss/env.oss.gh.example +++ b/hosting/docker-compose/oss/env.oss.gh.example @@ -29,6 +29,12 @@ AGENTA_API_URL=http://localhost/api # Agenta - Secrets (REPLACE in production!) # ================================================================== # AGENTA_AUTH_KEY=replace-me +# Proves to the API that a caller IS the platform runtime (the workflow service), so the +# credential it receives may read write-only secret values. The API and the services +# container must hold the SAME value, and a browser must never see it. Falls back to +# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that +# changes neither simply cannot run against write-only secrets. +AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me # ================================================================== # diff --git a/sdks/python/agenta/sdk/middlewares/routing/auth.py b/sdks/python/agenta/sdk/middlewares/routing/auth.py index d11ed240bd..c6cb8cd9d8 100644 --- a/sdks/python/agenta/sdk/middlewares/routing/auth.py +++ b/sdks/python/agenta/sdk/middlewares/routing/auth.py @@ -26,6 +26,10 @@ _RUNTIME_KEY = ( getenv("AGENTA_SERVICES_INTERNAL_KEY") or getenv("AGENTA_AUTH_KEY") or "" ).strip() +# The placeholder a deployment that configured nothing carries. Sending it would be +# sending a value every reader of the repo knows. +if _RUNTIME_KEY == "replace-me": + _RUNTIME_KEY = "" _AUTH_ENABLED = ( getenv("AGENTA_SERVICES_MIDDLEWARE_AUTH_ENABLED") From dbae9c13b9769d2bdf702645eab461fc908b7a85 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:28:32 +0200 Subject: [PATCH 17/31] test(services): pin what the agent app sends when it exchanges a caller's key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bug that broke every run was invisible to unit tests because they exercised the hop that already had a granted token, while the product uses the hop that never did. This drives the real agent app with the exchange stubbed and asserts the request carries both the caller's own credential and the platform's runtime key — and that a service configured without one sends no header at all rather than an empty value. --- .../unit/agent/test_credential_exchange.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 services/oss/tests/pytest/unit/agent/test_credential_exchange.py diff --git a/services/oss/tests/pytest/unit/agent/test_credential_exchange.py b/services/oss/tests/pytest/unit/agent/test_credential_exchange.py new file mode 100644 index 0000000000..509fd3fcb9 --- /dev/null +++ b/services/oss/tests/pytest/unit/agent/test_credential_exchange.py @@ -0,0 +1,91 @@ +"""What the agent app sends when it exchanges a caller's credential. + +This is the hop the product actually takes: the browser (or the release gate) posts to the +agent service with the END USER's ApiKey, and the service exchanges it at +`/access/permissions/check` for the credential a run uses to read the project's secrets. +Nothing about that ApiKey says a run is starting, and the exchange route is publicly +reachable, so the platform's own secret is what distinguishes this hop from a browser +asking for a credential directly. Drop it and every run against a write-only secret +resolves to the redacted shape and fails; leak it and the guarantee is gone. Hence a test +on the real app, not just on the middleware in isolation. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import pytest +from fastapi.testclient import TestClient + +from agenta.sdk.middlewares.routing import auth as auth_middleware + +from oss.src.agent import agent_v0_app + + +class _FakeResponse: + def __init__(self, body: Dict[str, Any]) -> None: + self.status_code = 200 + self._body = body + self.headers: Dict[str, str] = {} + + def json(self) -> Dict[str, Any]: + return self._body + + +class _RecordingClient: + """Stands in for the httpx client the exchange uses, keeping the headers it saw.""" + + last_headers: Optional[Dict[str, str]] = None + + async def __aenter__(self) -> "_RecordingClient": + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def get(self, *args: Any, **kwargs: Any) -> _FakeResponse: + type(self).last_headers = kwargs.get("headers") + return _FakeResponse( + {"effect": "allow", "credentials": "Secret granted-run-credential"} + ) + + +@pytest.fixture(name="exchange") +def _exchange(monkeypatch): + monkeypatch.setattr(auth_middleware, "_AUTH_ENABLED", True) + monkeypatch.setattr(auth_middleware, "_CACHE_ENABLED", False) + monkeypatch.setattr(auth_middleware.httpx, "AsyncClient", _RecordingClient) + _RecordingClient.last_headers = None + return _RecordingClient + + +def _post(client: TestClient): + return client.post( + "/runtime/subscription-status", + json={"harness": "codex"}, + headers={"Authorization": "ApiKey caller-key"}, + ) + + +def test_the_service_proves_what_it_is_when_it_exchanges_a_users_key( + exchange, monkeypatch +): + monkeypatch.setattr(auth_middleware, "_RUNTIME_KEY", "runtime-key-for-tests") + + _post(TestClient(agent_v0_app)) + + headers = exchange.last_headers or {} + assert headers.get("X-Agenta-Runtime-Key") == "runtime-key-for-tests" + # The user's own credential still travels: the key says which RUNTIME is asking, it + # does not change WHO is asking. + assert headers.get("Authorization") == "ApiKey caller-key" + + +def test_a_service_without_the_key_sends_none(exchange, monkeypatch): + # Such a deployment gets an ungranted credential and cannot run against write-only + # secrets — it must not send an empty header a comparison might accept. + monkeypatch.setattr(auth_middleware, "_RUNTIME_KEY", "") + + _post(TestClient(agent_v0_app)) + + assert "X-Agenta-Runtime-Key" not in (exchange.last_headers or {}) From 6adce728fe0071e7adb40d64e8ad62af91ffc23a Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:43:17 +0200 Subject: [PATCH 18/31] fix(sdk): say when the platform runtime key is missing, instead of blaming the provider key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The placeholder this refuses is the shipped default in every example env file, so a deployment that never set AGENTA_AUTH_KEY silently loses runs against write-only connections — and what it sees is the SDK telling it to provide OPENAI_API_KEY, which is true for a standalone run and useless here. The service now says it once, at the point of use, and names the variable to set. Live QA hit exactly this and read it as a regression, which is the cost of a failure that points somewhere else. --- docs/design/write-only-secrets/README.md | 7 +++++ .../agenta/sdk/middlewares/routing/auth.py | 30 +++++++++++++++++-- .../unit/test_auth_middleware_credentials.py | 21 +++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 78124808e6..628e5cd0a0 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -163,6 +163,13 @@ The vault returns plaintext only to a caller whose verified `Secret` token carri re-deciding it. The runner is never given the runtime secret, and it never reaches a sandbox. +**A deployment that sets neither loses agent runs against write-only connections**, and +the failure names something else: the run reports "provide the provider key in this run's +environment", which is right for a standalone run and misleading here. The services +middleware therefore warns once, at the point of use, naming the variable to set. The +placeholder is the shipped default in the example env files, so this is the common case, +not an edge one. + The exchange never mints the grant from the requested `action` alone. It did once, and that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and `view_secret`, so any member could ask for a credential and spend it on the vault routes. diff --git a/sdks/python/agenta/sdk/middlewares/routing/auth.py b/sdks/python/agenta/sdk/middlewares/routing/auth.py index c6cb8cd9d8..46904397ee 100644 --- a/sdks/python/agenta/sdk/middlewares/routing/auth.py +++ b/sdks/python/agenta/sdk/middlewares/routing/auth.py @@ -26,11 +26,35 @@ _RUNTIME_KEY = ( getenv("AGENTA_SERVICES_INTERNAL_KEY") or getenv("AGENTA_AUTH_KEY") or "" ).strip() -# The placeholder a deployment that configured nothing carries. Sending it would be -# sending a value every reader of the repo knows. +# The placeholder a deployment that configured nothing carries — and it is the SHIPPED +# DEFAULT in the example env files, not a rare mistake. Sending it would be sending a +# value every reader of the repo knows, so it counts as no key at all. if _RUNTIME_KEY == "replace-me": _RUNTIME_KEY = "" +# Said once, at the point of use, because the failure it causes names something else +# entirely: a run whose connection holds a write-only secret gets the redacted shape and +# reports "provide the provider key in this run's environment", which is true for a +# standalone run and misleading here. An operator reading that has no way to reach this +# cause without being told. +_RUNTIME_KEY_WARNED = False + + +def _warn_once_about_the_missing_runtime_key() -> None: + global _RUNTIME_KEY_WARNED + + if _RUNTIME_KEY_WARNED: + return + + _RUNTIME_KEY_WARNED = True + log.warning( + "agenta: no platform runtime key configured " + "(AGENTA_SERVICES_INTERNAL_KEY unset and AGENTA_AUTH_KEY is the placeholder). " + "Runs against connections whose secret is write-only will fail to read it. " + "Set AGENTA_SERVICES_INTERNAL_KEY to the same value on the API and this service." + ) + + _AUTH_ENABLED = ( getenv("AGENTA_SERVICES_MIDDLEWARE_AUTH_ENABLED") or getenv("AGENTA_SERVICE_MIDDLEWARE_AUTH_ENABLED") @@ -117,6 +141,8 @@ async def get_credentials( runtime_key = _RUNTIME_KEY if runtime_key: headers = {**(headers or {}), _RUNTIME_KEY_HEADER: runtime_key} + else: + _warn_once_about_the_missing_runtime_key() # COOKIES access_token = request.cookies.get("sAccessToken", None) diff --git a/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py index 8f869719d6..010d0d0239 100644 --- a/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py +++ b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py @@ -105,3 +105,24 @@ async def test_no_runtime_key_means_no_header(platform, monkeypatch): await _get_credentials() assert "X-Agenta-Runtime-Key" not in (platform.last_headers or {}) + + +async def test_a_missing_runtime_key_says_so_once(platform, monkeypatch, caplog): + # The failure it causes reports a missing provider key, which is the wrong advice for + # this cause. An operator who never set AGENTA_AUTH_KEY — the shipped default — has no + # other way to reach it. + monkeypatch.setattr(auth_module, "_RUNTIME_KEY", "") + monkeypatch.setattr(auth_module, "_RUNTIME_KEY_WARNED", False) + platform.body = {"effect": "allow", "credentials": "Secret general-token"} + + with caplog.at_level("WARNING"): + await _get_credentials() + await _get_credentials() + + notices = [ + record + for record in caplog.records + if "no platform runtime key configured" in record.getMessage() + ] + assert len(notices) == 1 + assert "AGENTA_SERVICES_INTERNAL_KEY" in notices[0].getMessage() From d373758d461e0f271285a90bfd707751f41d62fd Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:45:13 +0200 Subject: [PATCH 19/31] chore: exempt two dead fixture strings by value, not by fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixtures were rewritten to digit-free names, but these two spellings survive in commits whose amend could not be replayed, and both scans read history rather than the tree. A fingerprint names the commit its finding was seen in, so it goes stale every time a lane below is rebased — which happened twice while landing this stack. Exempting the values is stable, and it follows what this file already says: exempt the VALUE, never the path, so a real key added to a fixture would still be seen. --- .gitleaks.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitleaks.toml b/.gitleaks.toml index 71b897c37b..d2ced0f3bf 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -42,6 +42,13 @@ regexes = [ # Fake token in the runner's redaction test — the test asserts this marker never # reaches the transcript. Named to be obviously synthetic; not a credential. '''marker-live-secret-42bd''', + # Two dead fixture strings from the write-only vault work. The fixtures themselves were + # rewritten to digit-free names, but these spellings survive in commits whose amend + # could not be replayed, and both scans read history rather than the tree. Exempted by + # VALUE, not by path, and deliberately not by fingerprint: a fingerprint names the + # commit it was seen in, so it goes stale every time a lane below is rebased. + '''sk-live-1234567890abc''', + '''sk-mine-9876543210xyz''', # ------------------------------------------------------------ PUBLIC KEYS '''phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7''', # POSTHOG '''phc_3urGRy5TL1HhaHnRYL0JSHxJxigRVackhphHtozUmdp''', # POSTHOG From c21b86b0d49d0c9923ba058b023f8aaa499a0be1 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 21 Aug 2026 23:56:33 +0200 Subject: [PATCH 20/31] feat(api): warn at startup when write-only secrets have no runtime key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A deployment that turned write-only on, or enabled a component that seeds write-only rows, cannot read those secrets at all without a platform runtime key — and the failure it gets says to provide a provider key, which is right for a standalone run and useless here. The API now says it once at boot, naming the variable and the consequence, next to the other startup validations. The placeholder counts as unset, since it is what the example env files ship. --- api/entrypoints/routers.py | 7 +- api/oss/src/utils/helpers.py | 31 +++++++ .../pytest/unit/utils/test_env_helpers.py | 84 +++++++++++++++++++ docs/design/write-only-secrets/README.md | 6 ++ 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 api/oss/tests/pytest/unit/utils/test_env_helpers.py diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index 94c78ce850..6dd4a63486 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -14,7 +14,11 @@ from oss.src.utils.common import is_ee from oss.src.utils.logging import get_module_logger -from oss.src.utils.helpers import warn_deprecated_env_vars, validate_required_env_vars +from oss.src.utils.helpers import ( + validate_required_env_vars, + warn_deprecated_env_vars, + warn_unconfigured_platform_runtime_key, +) # Engines from oss.src.dbs.postgres.shared.engine import ( @@ -263,6 +267,7 @@ async def lifespan(*args, **kwargs): warn_deprecated_env_vars() validate_required_env_vars() + warn_unconfigured_platform_runtime_key() await _triggers_broker.startup() diff --git a/api/oss/src/utils/helpers.py b/api/oss/src/utils/helpers.py index aeec502aad..3f202ae60a 100644 --- a/api/oss/src/utils/helpers.py +++ b/api/oss/src/utils/helpers.py @@ -8,6 +8,9 @@ import click from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) def get_metrics_keys_from_schema(schema=None, path=()) -> List[Dict[str, str]]: @@ -184,6 +187,34 @@ def warn_deprecated_env_vars(): ) +def warn_unconfigured_platform_runtime_key(): + """Say at boot when nothing can read a write-only secret, and why. + + A run reads a write-only secret only through a credential the platform runtime is + issued, and the runtime is recognized by a shared key. Unset, that key defaults to the + placeholder the example env files ship, which is refused — so a deployment that turned + write-only on, or enabled a bridge that seeds write-only rows, silently gets runs that + cannot read their own connection. The failure surfaces as "provide the provider key in + this run's environment", which is true for a standalone run and useless here, so the + cause has to be said where an operator will see it. + """ + runtime_key = (env.agenta.services_internal_key or "").strip() + if runtime_key and runtime_key != "replace-me": + return + + needs_it = env.agenta.vault.write_only_default or env.starter_credits_bridge.enabled + if not needs_it: + return + + log.warning( + "AGENTA_SERVICES_INTERNAL_KEY is not configured (and AGENTA_AUTH_KEY is the " + "placeholder). Write-only secrets are in use on this deployment, and runs " + "against a connection whose secret is write-only will not be able to read it. " + "Set AGENTA_SERVICES_INTERNAL_KEY to the same value on the API and the services " + "container." + ) + + def validate_required_env_vars(): """ Ensure required configuration values are present. diff --git a/api/oss/tests/pytest/unit/utils/test_env_helpers.py b/api/oss/tests/pytest/unit/utils/test_env_helpers.py new file mode 100644 index 0000000000..37ad349dee --- /dev/null +++ b/api/oss/tests/pytest/unit/utils/test_env_helpers.py @@ -0,0 +1,84 @@ +"""Startup warnings that name a misconfiguration the runtime cannot report itself. + +A deployment using write-only secrets needs a platform runtime key, or runs cannot read +the secrets they were authorized to use. That failure surfaces as advice about provider +keys, which is right for a standalone run and useless here, so it has to be said at boot. +""" + +import pytest + +from oss.src.utils.env import env +from oss.src.utils.helpers import warn_unconfigured_platform_runtime_key + + +@pytest.fixture(name="warnings") +def _warnings(monkeypatch): + recorded: list = [] + + monkeypatch.setattr( + "oss.src.utils.helpers.log", + type("_Log", (), {"warning": staticmethod(lambda msg: recorded.append(msg))})(), + ) + return recorded + + +def _configure(monkeypatch, *, runtime_key, write_only_default, bridge_enabled): + monkeypatch.setattr(env.agenta, "services_internal_key", runtime_key) + monkeypatch.setattr(env.agenta.vault, "write_only_default", write_only_default) + monkeypatch.setattr( + env.starter_credits_bridge, + "enabled", + bridge_enabled, + ) + + +@pytest.mark.parametrize("runtime_key", ["", "replace-me"]) +def test_write_only_deployments_without_a_runtime_key_are_warned( + warnings, monkeypatch, runtime_key +): + _configure( + monkeypatch, + runtime_key=runtime_key, + write_only_default=True, + bridge_enabled=False, + ) + + warn_unconfigured_platform_runtime_key() + + assert len(warnings) == 1 + assert "AGENTA_SERVICES_INTERNAL_KEY" in warnings[0] + + +def test_a_bridge_deployment_without_a_runtime_key_is_warned(warnings, monkeypatch): + # The bridge seeds write-only rows, so it needs the key even with the default off. + _configure( + monkeypatch, runtime_key="", write_only_default=False, bridge_enabled=True + ) + + warn_unconfigured_platform_runtime_key() + + assert len(warnings) == 1 + + +def test_a_configured_deployment_is_not_warned(warnings, monkeypatch): + _configure( + monkeypatch, + runtime_key="a-real-runtime-key", + write_only_default=True, + bridge_enabled=True, + ) + + warn_unconfigured_platform_runtime_key() + + assert warnings == [] + + +def test_a_deployment_using_no_write_only_secrets_is_not_warned(warnings, monkeypatch): + # Nothing to read back, so the key buys it nothing and the warning would be noise. + _configure( + monkeypatch, runtime_key="", write_only_default=False, bridge_enabled=False + ) + + warn_unconfigured_platform_runtime_key() + + assert warnings == [] diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 628e5cd0a0..d13f634396 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -170,6 +170,12 @@ middleware therefore warns once, at the point of use, naming the variable to set placeholder is the shipped default in the example env files, so this is the common case, not an edge one. +**Deployment.** Set `AGENTA_SERVICES_INTERNAL_KEY` to the same value on the API and the +services container before turning write-only on (`AGENTA_AUTH_KEY` serves if it is a real +value). The API warns at startup when a deployment uses write-only secrets without one, +and a component that seeds write-only rows refuses to seed rather than store a credential +no run can read. + The exchange never mints the grant from the requested `action` alone. It did once, and that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and `view_secret`, so any member could ask for a credential and spend it on the vault routes. From e697aa2e9b43427dd4c18d708042f9c5012fe2a5 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 22 Aug 2026 00:01:21 +0200 Subject: [PATCH 21/31] fix(api): do not assume the EE bridge config exists at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warning read env.starter_credits_bridge directly, but that config is an EE addition that this branch does not carry — so on a build without it the API would have raised AttributeError during startup validation, before serving anything. It now asks for the attribute rather than assuming it, and the case that needs the bridge is tested where the bridge exists. My own CI caught it because the test referenced the same missing attribute; the production path had the same bug. --- api/oss/src/utils/helpers.py | 8 +++-- .../pytest/unit/utils/test_env_helpers.py | 36 +++---------------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/api/oss/src/utils/helpers.py b/api/oss/src/utils/helpers.py index 3f202ae60a..d847ef4e05 100644 --- a/api/oss/src/utils/helpers.py +++ b/api/oss/src/utils/helpers.py @@ -202,8 +202,12 @@ def warn_unconfigured_platform_runtime_key(): if runtime_key and runtime_key != "replace-me": return - needs_it = env.agenta.vault.write_only_default or env.starter_credits_bridge.enabled - if not needs_it: + # The bridge config is an EE addition and may not exist on this build, so ask rather + # than assume: this runs at startup, where an AttributeError would stop the API. + bridge = getattr(env, "starter_credits_bridge", None) + seeds_write_only_rows = bool(bridge is not None and bridge.enabled) + + if not (env.agenta.vault.write_only_default or seeds_write_only_rows): return log.warning( diff --git a/api/oss/tests/pytest/unit/utils/test_env_helpers.py b/api/oss/tests/pytest/unit/utils/test_env_helpers.py index 37ad349dee..f4b6599261 100644 --- a/api/oss/tests/pytest/unit/utils/test_env_helpers.py +++ b/api/oss/tests/pytest/unit/utils/test_env_helpers.py @@ -22,26 +22,16 @@ def _warnings(monkeypatch): return recorded -def _configure(monkeypatch, *, runtime_key, write_only_default, bridge_enabled): +def _configure(monkeypatch, *, runtime_key, write_only_default): monkeypatch.setattr(env.agenta, "services_internal_key", runtime_key) monkeypatch.setattr(env.agenta.vault, "write_only_default", write_only_default) - monkeypatch.setattr( - env.starter_credits_bridge, - "enabled", - bridge_enabled, - ) @pytest.mark.parametrize("runtime_key", ["", "replace-me"]) def test_write_only_deployments_without_a_runtime_key_are_warned( warnings, monkeypatch, runtime_key ): - _configure( - monkeypatch, - runtime_key=runtime_key, - write_only_default=True, - bridge_enabled=False, - ) + _configure(monkeypatch, runtime_key=runtime_key, write_only_default=True) warn_unconfigured_platform_runtime_key() @@ -49,24 +39,8 @@ def test_write_only_deployments_without_a_runtime_key_are_warned( assert "AGENTA_SERVICES_INTERNAL_KEY" in warnings[0] -def test_a_bridge_deployment_without_a_runtime_key_is_warned(warnings, monkeypatch): - # The bridge seeds write-only rows, so it needs the key even with the default off. - _configure( - monkeypatch, runtime_key="", write_only_default=False, bridge_enabled=True - ) - - warn_unconfigured_platform_runtime_key() - - assert len(warnings) == 1 - - def test_a_configured_deployment_is_not_warned(warnings, monkeypatch): - _configure( - monkeypatch, - runtime_key="a-real-runtime-key", - write_only_default=True, - bridge_enabled=True, - ) + _configure(monkeypatch, runtime_key="a-real-runtime-key", write_only_default=True) warn_unconfigured_platform_runtime_key() @@ -75,9 +49,7 @@ def test_a_configured_deployment_is_not_warned(warnings, monkeypatch): def test_a_deployment_using_no_write_only_secrets_is_not_warned(warnings, monkeypatch): # Nothing to read back, so the key buys it nothing and the warning would be noise. - _configure( - monkeypatch, runtime_key="", write_only_default=False, bridge_enabled=False - ) + _configure(monkeypatch, runtime_key="", write_only_default=False) warn_unconfigured_platform_runtime_key() From 5353e2fc400ecc34515df2d5d9579bace0d166f7 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 22 Aug 2026 22:46:03 +0200 Subject: [PATCH 22/31] fix(api): enforce the write-only secrets contract --- api/ee/src/core/organizations/service.py | 1 + .../unit/test_write_only_provider_settings.py | 74 ++++ api/oss/src/core/secrets/interfaces.py | 5 +- api/oss/src/core/secrets/redaction.py | 101 +++-- api/oss/src/dbs/postgres/secrets/dao.py | 30 +- api/oss/src/middlewares/auth.py | 37 +- api/oss/src/utils/env.py | 34 +- api/oss/src/utils/helpers.py | 23 +- .../pytest/unit/access/test_grant_exchange.py | 10 + .../unit/middlewares/test_auth_grants.py | 27 +- .../pytest/unit/secrets/test_services.py | 2 +- .../pytest/unit/secrets/test_write_only.py | 345 +++++++++++------- .../pytest/unit/utils/test_env_helpers.py | 44 ++- .../unit/vault/test_write_only_routes.py | 60 +-- .../unit/webhooks/test_write_only_outward.py | 68 ++-- docs/design/write-only-secrets/README.md | 21 +- .../docker-compose/ee/docker-compose.dev.yml | 14 + .../ee/docker-compose.gh.local.yml | 20 + .../docker-compose/ee/docker-compose.gh.yml | 15 + hosting/docker-compose/ee/env.ee.dev.example | 6 +- hosting/docker-compose/ee/env.ee.gh.example | 6 +- .../docker-compose/oss/docker-compose.dev.yml | 12 + .../oss/docker-compose.gh.local.yml | 20 + .../oss/docker-compose.gh.ssl.yml | 17 + .../docker-compose/oss/docker-compose.gh.yml | 17 + .../docker-compose/oss/env.oss.dev.example | 6 +- hosting/docker-compose/oss/env.oss.gh.example | 6 +- hosting/kubernetes/ee/values.ee.example.yaml | 1 + hosting/kubernetes/helm/templates/NOTES.txt | 6 +- .../kubernetes/helm/templates/_helpers.tpl | 13 + .../helm/templates/_validations.tpl | 10 +- .../helm/templates/api-deployment.yaml | 1 + .../kubernetes/helm/templates/secrets.yaml | 1 + .../helm/templates/services-deployment.yaml | 1 + .../helm/tests/test_runner_secret_absence.py | 30 +- hosting/kubernetes/helm/values.schema.json | 1 + hosting/kubernetes/helm/values.yaml | 1 + .../kubernetes/oss/values.oss.example.yaml | 1 + hosting/railway/oss/README.md | 9 +- hosting/railway/oss/scripts/configure.sh | 7 +- hosting/railway/oss/template/template.json | 10 + .../agenta/sdk/middlewares/routing/auth.py | 20 +- .../unit/test_auth_middleware_credentials.py | 28 +- 43 files changed, 792 insertions(+), 369 deletions(-) diff --git a/api/ee/src/core/organizations/service.py b/api/ee/src/core/organizations/service.py index 0e779f387c..e04aada463 100644 --- a/api/ee/src/core/organizations/service.py +++ b/api/ee/src/core/organizations/service.py @@ -639,6 +639,7 @@ async def create_provider( ) ), ), + write_only=False, ) secret_dto = await self._vault_service().create_secret( diff --git a/api/ee/tests/pytest/unit/test_write_only_provider_settings.py b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py index dad61ec326..fb5d606344 100644 --- a/api/ee/tests/pytest/unit/test_write_only_provider_settings.py +++ b/api/ee/tests/pytest/unit/test_write_only_provider_settings.py @@ -12,7 +12,9 @@ import pytest +from ee.src.core.organizations import service as organization_service_module from ee.src.core.organizations.service import OrganizationProvidersService +from ee.src.core.organizations.types import OrganizationProviderCreate from oss.src.core.secrets.dtos import SecretResponseDTO @@ -30,6 +32,40 @@ async def get_secret_by_id( return self._secret +class _Session: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + +class _Engine: + def session(self): + return _Session() + + +class _ProviderDAO: + def __init__(self, session): + self.session = session + + async def get_by_slug(self, *, slug, organization_id): + return None + + +class _SecretCaptured(Exception): + pass + + +class _CapturingVaultService: + def __init__(self, captured): + self.captured = captured + + async def create_secret(self, *, organization_id, create_secret_dto): + self.captured.append(create_secret_dto) + raise _SecretCaptured + + def _sso_secret(write_only: bool) -> SecretResponseDTO: return SecretResponseDTO( id=SECRET_ID, @@ -57,6 +93,44 @@ def _with_secret(monkeypatch, secret) -> OrganizationProvidersService: return OrganizationProvidersService() +@pytest.mark.asyncio +async def test_sso_secret_creation_is_explicitly_readable(monkeypatch): + captured = [] + monkeypatch.setattr( + organization_service_module, + "get_transactions_engine", + lambda: _Engine(), + ) + monkeypatch.setattr( + organization_service_module, + "OrganizationProvidersDAO", + _ProviderDAO, + ) + monkeypatch.setattr( + OrganizationProvidersService, + "_vault_service", + staticmethod(lambda: _CapturingVaultService(captured)), + ) + + payload = OrganizationProviderCreate( + slug="okta", + description="Okta SSO", + settings={ + "client_id": "client-1", + "client_secret": "super-secret-value-123", + "issuer_url": "https://issuer.example.com", + }, + organization_id=ORGANIZATION_ID, + ) + + with pytest.raises(_SecretCaptured): + await OrganizationProvidersService().create_provider( + str(ORGANIZATION_ID), payload, user_id=str(uuid4()) + ) + + assert captured[0].write_only is False + + @pytest.mark.asyncio async def test_write_only_sso_secret_drops_client_secret_from_responses(monkeypatch): service = _with_secret(monkeypatch, _sso_secret(write_only=True)) diff --git a/api/oss/src/core/secrets/interfaces.py b/api/oss/src/core/secrets/interfaces.py index 07f894ae44..1721070b64 100644 --- a/api/oss/src/core/secrets/interfaces.py +++ b/api/oss/src/core/secrets/interfaces.py @@ -53,7 +53,9 @@ async def update( # applied. Every decision that reads stored state belongs here: a check made # against a snapshot read earlier is a check against a row another writer can # still have replaced. It may raise to refuse the update. - resolve_update: Optional[Callable[[SecretResponseDTO], None]] = None, + resolve_update: Optional[ + Callable[[SecretResponseDTO, UpdateSecretDTO], UpdateSecretDTO] + ] = None, ) -> Optional[SecretResponseDTO]: raise NotImplementedError @@ -62,5 +64,6 @@ async def delete( secret_id: UUID, project_id: Optional[UUID] = None, organization_id: Optional[UUID] = None, + authorize_delete: Optional[Callable[[SecretResponseDTO], None]] = None, ) -> None: raise NotImplementedError diff --git a/api/oss/src/core/secrets/redaction.py b/api/oss/src/core/secrets/redaction.py index d53579b2e4..4f3fdebcd4 100644 --- a/api/oss/src/core/secrets/redaction.py +++ b/api/oss/src/core/secrets/redaction.py @@ -1,11 +1,10 @@ """Redaction of write-only vault secrets for user-facing responses. A secret with ``write_only=True`` can be created, replaced, and deleted, but its value is -never returned to a user: every outward route strips the credential material and attaches -``has_key`` and a ``key_preview`` instead. Only the platform runtime — a caller whose -verified Secret token carries the ``secret-resolve`` grant — receives the plaintext, -because the workload it runs needs the real key. In-process readers (`VaultService` and -below) are untouched: redaction happens strictly at the response boundary. +never returned to an ordinary user. Every outward route returns a public projection with +``value_status``; trusted runtime callers receive credential values in that same public +shape. In-process readers (`VaultService` and below) are untouched: redaction happens +strictly at the response boundary. WHAT counts as credential material inside a connection is not decided here: that vocabulary lives in the SDK (``agenta.sdk.agents.connections.credentials``) and is @@ -18,7 +17,11 @@ from agenta.sdk.agents.connections.credentials import CREDENTIAL_EXTRAS_KEYS -from oss.src.core.secrets.dtos import SecretResponseDTO +from oss.src.core.secrets.dtos import ( + PublicSecretResponseDTO, + SecretResponseDTO, + SecretValueStatus, +) # The primary value field per secret kind, as (container attribute, field name). Lives @@ -62,38 +65,66 @@ def primary_credential_value(secret: SecretResponseDTO) -> Optional[Any]: return getattr(container, field, None) if container is not None else None -def redact_secret_response(secret: SecretResponseDTO) -> SecretResponseDTO: - """The user-facing shape of ``secret``: credential material stripped when write-only. +def _value_status(secret: SecretResponseDTO) -> SecretValueStatus: + """Describe whether credential material exists without exposing it.""" + value = primary_credential_value(secret) + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get( + str(secret.kind.value), (None, None) + ) + container = getattr(secret.data, container_name, None) if container_name else None + extras = getattr(container, "extras", None) or {} + has_credential_extras = any( + extras.get(extras_key) not in (None, "") + for extras_key in CREDENTIAL_EXTRAS_KEYS + ) - Returns the input unchanged for readable (``write_only=False``) secrets, so legacy - records keep their exact response. Never mutates the input. - """ - if not secret.write_only: - return secret + return SecretValueStatus( + configured=value not in (None, "") or has_credential_extras, + preview=( + mask_secret_value(value) + if secret.write_only and isinstance(value, str) and value + else None + ), + ) - redacted = secret.model_copy(deep=True) - container_name, field = PRIMARY_CREDENTIAL_FIELDS.get( - str(redacted.kind.value), (None, None) +def project_secret_response( + secret: SecretResponseDTO, + *, + reveal_write_only: bool, +) -> PublicSecretResponseDTO: + """Build the public response, optionally retaining a write-only value for runtime.""" + projected = PublicSecretResponseDTO.model_validate( + { + **secret.model_dump(mode="python"), + "value_status": _value_status(secret), + } ) - value: Optional[Any] = None - has_credential_extras = False - - if container_name is not None: - container = getattr(redacted.data, container_name, None) - if container is not None and hasattr(container, field): - value = getattr(container, field) - setattr(container, field, None) - - extras = getattr(container, "extras", None) if container is not None else None - if extras: - for extras_key in CREDENTIAL_EXTRAS_KEYS: - if extras.pop(extras_key, None) not in (None, ""): - has_credential_extras = True - - redacted.has_key = bool(value) or has_credential_extras - redacted.key_preview = ( - mask_secret_value(value) if isinstance(value, str) and value else None + + if not secret.write_only or reveal_write_only: + return projected + + container_name, field = PRIMARY_CREDENTIAL_FIELDS.get( + str(projected.kind.value), (None, None) ) + if container_name is None: + return projected + + container = getattr(projected.data, container_name, None) + if container is None: + return projected + + if hasattr(container, field): + setattr(container, field, None) + + extras = getattr(container, "extras", None) + if extras: + for extras_key in CREDENTIAL_EXTRAS_KEYS: + extras.pop(extras_key, None) + + return projected + - return redacted +def redact_secret_response(secret: SecretResponseDTO) -> PublicSecretResponseDTO: + """Return the public response with write-only credential material stripped.""" + return project_secret_response(secret, reveal_write_only=False) diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index 5003bfba5e..8adc8b787e 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -1,9 +1,7 @@ -import json from typing import Callable, Optional from uuid import UUID from oss.src.dbs.postgres.secrets.dbes import SecretsDBE -from oss.src.core.secrets.dtos import WriteOnlyCannotBeDisabledError from oss.src.core.secrets.interfaces import SecretsDAOInterface from oss.src.dbs.postgres.shared.engine import ( @@ -127,7 +125,9 @@ async def update( project_id: UUID | None, organization_id: UUID | None, user_id: UUID | None = None, - resolve_update: Optional[Callable[[SecretResponseDTO], None]] = None, + resolve_update: Optional[ + Callable[[SecretResponseDTO, UpdateSecretDTO], UpdateSecretDTO] + ] = None, ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) @@ -153,12 +153,10 @@ async def update( # snapshot another writer has already replaced; the keep-on-omit carry-over # in particular would then write a rotated credential back to its old value. if resolve_update is not None: - resolve_update(map_secrets_dbe_to_dto(secrets_dbe=secrets_dbe)) - - if update_secret_dto.write_only is False and bool( - json.loads(secrets_dbe.data).get("write_only") - ): - raise WriteOnlyCannotBeDisabledError() + update_secret_dto = resolve_update( + map_secrets_dbe_to_dto(secrets_dbe=secrets_dbe), + update_secret_dto, + ) map_secrets_dto_to_dbe_update( secrets_dbe=secrets_dbe, @@ -177,17 +175,25 @@ async def delete( secret_id: UUID, project_id: UUID | None, organization_id: UUID | None, + authorize_delete: Optional[Callable[[SecretResponseDTO], None]] = None, ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) - stmt = select(SecretsDBE).filter_by( - id=secret_id, - **scope_filter, + stmt = ( + select(SecretsDBE) + .filter_by( + id=secret_id, + **scope_filter, + ) + .with_for_update() ) result = await session.execute(stmt) # type: ignore vault_secret_dbe = result.scalar() if vault_secret_dbe is None: return + if authorize_delete is not None: + authorize_delete(map_secrets_dbe_to_dto(secrets_dbe=vault_secret_dbe)) + await session.delete(vault_secret_dbe) await session.commit() diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index f51100bd32..586e53a20a 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -97,6 +97,24 @@ # general-purpose (it authenticates workflows, tools, and vault reads alike), so the # vault's plaintext-read capability rides a grant. SECRET_RESOLVE_GRANT = "secret-resolve" +ALLOWED_SECRET_TOKEN_GRANTS = frozenset({SECRET_RESOLVE_GRANT}) + + +def _validate_secret_token_grants(grants: object) -> tuple[str, ...]: + """Return known grants and reject every unrecognized claim shape or value.""" + if grants is None: + return () + + if not isinstance(grants, list): + raise ValueError("Secret token grants must be a list.") + + if any( + not isinstance(grant, str) or grant not in ALLOWED_SECRET_TOKEN_GRANTS + for grant in grants + ): + raise ValueError("Secret token contains an unsupported grant.") + + return tuple(grants) def request_has_grant(request: Request, grant: str) -> bool: @@ -932,7 +950,18 @@ async def verify_secret_token( leeway=_SECRET_LEEWAY, ) - request.state.token_grants = tuple(auth_context.get("grants") or ()) + try: + request.state.token_grants = _validate_secret_token_grants( + auth_context.get("grants") + ) + except ValueError as exc: + log.debug( + "[auth] secret token unauthorized", + path=request.url.path, + method=request.method, + reason="invalid_token", + ) + raise UnauthorizedException(reason="invalid_token") from exc request.state.user_id = auth_context.get("user_id") request.state.user_email = auth_context.get("user_email") @@ -1020,6 +1049,8 @@ async def sign_secret_token( organization_name: Optional[str] = None, grants: Optional[List[str]] = None, ): + validated_grants = _validate_secret_token_grants(grants) + try: if not _SECRET_KEY: raise InternalServerErrorException() @@ -1040,8 +1071,8 @@ async def sign_secret_token( # A token with no grants carries no `grants` key at all, rather than a null or # empty one, so its payload stays the shape every existing holder was issued. - if grants: - auth_context["grants"] = list(grants) + if validated_grants: + auth_context["grants"] = list(validated_grants) secret_token = encode( payload=auth_context, diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index a6397aaf9c..d164427f18 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -571,24 +571,18 @@ class SessionsConfig(BaseModel): # --------------------------------------------------------------------------- -# agenta.vault — vault (secrets) behavior. +# agenta — top-level Agenta core config. # --------------------------------------------------------------------------- -class VaultConfig(BaseModel): - """Vault (secrets) behavior.""" +def _services_internal_key_from_environment() -> str | None: + """Read the dedicated runtime proof without accepting public placeholders.""" + runtime_key = (os.getenv("AGENTA_SERVICES_INTERNAL_KEY") or "").strip() - # Whether NEW secrets default to write-only (value never readable back by users). - # Off until the web UI ships replace-only secret forms; an explicit `write_only` - # on the create request always wins over this default. - write_only_default: bool = _parse_bool_env("AGENTA_VAULT_WRITE_ONLY_DEFAULT", False) + if not runtime_key or runtime_key == "replace-me": + return None - model_config = ConfigDict(extra="ignore") - - -# --------------------------------------------------------------------------- -# agenta — top-level Agenta core config. -# --------------------------------------------------------------------------- + return runtime_key class AgentaConfig(BaseModel): @@ -606,15 +600,10 @@ class AgentaConfig(BaseModel): # Shared secret that proves a caller IS the platform runtime (the workflow service), # as opposed to a browser or an ApiKey holder reaching the same public route. Only a # caller holding it can be issued a credential that reads write-only secret values. - # Defaults to `auth_key` because the services container already receives it through - # the same env file the API uses, so existing deployments keep working; setting a - # dedicated value narrows what one leaked secret can do. NEVER sent to the runner or - # into a sandbox. - services_internal_key: str = ( - os.getenv("AGENTA_SERVICES_INTERNAL_KEY") - or os.getenv("AGENTA_AUTH_KEY") - or "replace-me" - ) + # This is deliberately separate from the administrator key: deployments must opt in + # by configuring the same dedicated value on the API and platform services. NEVER + # sent to the runner or into a sandbox. + services_internal_key: str | None = _services_internal_key_from_environment() access: AccessConfig = AccessConfig() ai_services: AIServicesConfig = AIServicesConfig() @@ -626,7 +615,6 @@ class AgentaConfig(BaseModel): redaction: RedactionConfig = RedactionConfig() services: ServicesConfig = ServicesConfig() sessions: SessionsConfig = SessionsConfig() - vault: VaultConfig = VaultConfig() webhooks: WebhooksConfig = WebhooksConfig() workers: WorkersConfig = WorkersConfig() diff --git a/api/oss/src/utils/helpers.py b/api/oss/src/utils/helpers.py index d847ef4e05..faf480dc92 100644 --- a/api/oss/src/utils/helpers.py +++ b/api/oss/src/utils/helpers.py @@ -191,28 +191,19 @@ def warn_unconfigured_platform_runtime_key(): """Say at boot when nothing can read a write-only secret, and why. A run reads a write-only secret only through a credential the platform runtime is - issued, and the runtime is recognized by a shared key. Unset, that key defaults to the - placeholder the example env files ship, which is refused — so a deployment that turned - write-only on, or enabled a bridge that seeds write-only rows, silently gets runs that - cannot read their own connection. The failure surfaces as "provide the provider key in - this run's environment", which is true for a standalone run and useless here, so the - cause has to be said where an operator will see it. + issued, and the runtime is recognized by a dedicated shared key. If that key is unset + or still uses the public placeholder from an example env file, platform runs cannot + read their write-only connections. The failure surfaces as "provide the provider key + in this run's environment", which is true for a standalone run and useless here, so + the cause has to be said where an operator will see it. """ runtime_key = (env.agenta.services_internal_key or "").strip() if runtime_key and runtime_key != "replace-me": return - # The bridge config is an EE addition and may not exist on this build, so ask rather - # than assume: this runs at startup, where an AttributeError would stop the API. - bridge = getattr(env, "starter_credits_bridge", None) - seeds_write_only_rows = bool(bridge is not None and bridge.enabled) - - if not (env.agenta.vault.write_only_default or seeds_write_only_rows): - return - log.warning( - "AGENTA_SERVICES_INTERNAL_KEY is not configured (and AGENTA_AUTH_KEY is the " - "placeholder). Write-only secrets are in use on this deployment, and runs " + "AGENTA_SERVICES_INTERNAL_KEY is not configured or uses the placeholder. " + "Write-only secrets are in use on this deployment, and runs " "against a connection whose secret is write-only will not be able to read it. " "Set AGENTA_SERVICES_INTERNAL_KEY to the same value on the API and the services " "container." diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index 96b6c8f7d7..2b4371187c 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -184,6 +184,16 @@ async def test_an_unconfigured_deployment_grants_nobody(exchange, monkeypatch): assert "grants" not in _claims(body["credentials"]) +@pytest.mark.asyncio +async def test_the_admin_key_is_not_runtime_proof(exchange, monkeypatch): + monkeypatch.setattr(env.agenta, "services_internal_key", None) + run, _ = exchange + + body = _body(await run("run_service", runtime_key=env.agenta.auth_key)) + + assert "grants" not in _claims(body["credentials"]) + + @pytest.mark.asyncio async def test_a_plain_caller_cannot_mint_the_grant_by_asking_for_it(exchange): # The escalation this closes: a member who may run a service could call the exchange diff --git a/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py index 05aaf236f1..777ce74c75 100644 --- a/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py +++ b/api/oss/tests/pytest/unit/middlewares/test_auth_grants.py @@ -104,14 +104,29 @@ def test_request_without_verified_token_has_no_grants(): @pytest.mark.asyncio -async def test_foreign_grant_names_do_not_confer_secret_resolve(log): - token = await auth.sign_secret_token(user_id="u", grants=["something-else"]) +async def test_unknown_grant_is_rejected_at_issuance(log): + with pytest.raises(ValueError, match="unsupported grant"): + await auth.sign_secret_token(user_id="u", grants=["something-else"]) - request = _request() - await auth.verify_secret_token(request=request, secret_token=token) - assert request.state.token_grants == ("something-else",) - assert not auth.request_has_grant(request, auth.SECRET_RESOLVE_GRANT) +@pytest.mark.asyncio +async def test_unknown_grant_is_rejected_at_consumption(log): + expiry = datetime.now(timezone.utc) + timedelta(seconds=600) + token = encode( + payload={ + "user_id": "u", + "grants": ["something-else"], + "exp": int(expiry.timestamp()), + }, + key=SECRET_KEY, + algorithm="HS256", + ) + + with pytest.raises(UnauthorizedException) as rejected: + await auth.verify_secret_token(request=_request(), secret_token=token) + + assert rejected.value.status_code == 401 + assert rejected.value.detail["reason"] == "invalid_token" @pytest.mark.asyncio diff --git a/api/oss/tests/pytest/unit/secrets/test_services.py b/api/oss/tests/pytest/unit/secrets/test_services.py index b730b66b71..becaad971b 100644 --- a/api/oss/tests/pytest/unit/secrets/test_services.py +++ b/api/oss/tests/pytest/unit/secrets/test_services.py @@ -77,7 +77,7 @@ async def update( # Production resolves the update against the row under the write lock; the fake # does the same at the same point, so keep-on-omit is exercised, not skipped. if resolve_update is not None: - resolve_update(stored) + update_secret_dto = resolve_update(stored, update_secret_dto) # Like the postgres mapping, the whole data blob is replaced: whatever the payload # omits is gone unless the service carried it over first. record = SecretResponseDTO( diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index e6c66c5323..b75befa317 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -2,7 +2,7 @@ Covers the three layers below the router: the service (default-on at create, value carry-over on update, the one-way flag), the redaction helper (per-kind value stripping, -has_key/key_preview), and the postgres mappings (the flag rides inside the encrypted data +value_status), and the postgres mappings (the flag rides inside the encrypted data JSON and never leaks into the payload DTOs). """ @@ -10,19 +10,18 @@ import pytest +import oss.src.core.secrets.services as secrets_services_module from oss.src.core.secrets.dtos import ( CreateSecretDTO, SecretResponseDTO, SecretValueRequiredError, UpdateSecretDTO, - WriteOnlyCannotBeDisabledError, ) from oss.src.core.secrets.redaction import ( mask_secret_value, redact_secret_response, ) from oss.src.core.secrets.services import VaultService -from oss.src.utils.env import env from oss.src.dbs.postgres.secrets.mappings import ( map_secrets_dbe_to_dto, map_secrets_dto_to_dbe, @@ -73,16 +72,11 @@ async def update( # Production resolves the update against the row under the write lock; the fake # does the same at the same point, so keep-on-omit is exercised, not skipped. if resolve_update is not None: - resolve_update(stored) - - write_only = update_secret_dto.write_only - if write_only is None: - write_only = stored.write_only + update_secret_dto = resolve_update(stored, update_secret_dto) updated = stored.model_copy( update={ "header": update_secret_dto.header or stored.header, - "write_only": write_only, } ) if update_secret_dto.secret is not None: @@ -92,13 +86,21 @@ async def update( self.records[secret_id] = updated return updated + async def delete( + self, secret_id, project_id, organization_id, authorize_delete=None + ): + stored = self.records.get(secret_id) + if stored is not None and authorize_delete is not None: + authorize_delete(stored) + self.records.pop(secret_id, None) + @pytest.fixture(name="service") def _service(): return VaultService(_FakeSecretsDAO()) -def _provider_key_create(key="sk-test-openai-key-bc", write_only=None): +def _provider_key_create(key="sk-test-openai-key-bc", write_only=True): return CreateSecretDTO( header={"name": "OpenAI"}, secret={ @@ -112,56 +114,101 @@ def _provider_key_create(key="sk-test-openai-key-bc", write_only=None): # --- service: create ------------------------------------------------------------------ -@pytest.fixture(name="write_only_gate") -def _write_only_gate(monkeypatch): - def set_gate(value: bool): - monkeypatch.setattr(env.agenta.vault, "write_only_default", value) +@pytest.mark.asyncio +async def test_create_defaults_to_write_only(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) - return set_gate + assert created.write_only is True @pytest.mark.asyncio -async def test_create_defaults_off_while_the_gate_is_off(service, write_only_gate): - # Today's behavior until the web UI ships replace-only forms. - write_only_gate(False) - +@pytest.mark.parametrize("explicit", [False, True]) +async def test_an_explicit_create_value_is_preserved(service, explicit): created = await service.create_secret( - project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + project_id=PROJECT_ID, + create_secret_dto=_provider_key_create(write_only=explicit), ) - assert created.write_only is False + assert created.write_only is explicit + + +# --- service: keep-stored-on-omit ------------------------------------------------------ @pytest.mark.asyncio -async def test_create_defaults_to_write_only_when_the_gate_is_on( - service, write_only_gate +async def test_project_list_cache_stores_the_canonical_plaintext_dto( + service, monkeypatch ): - write_only_gate(True) - - created = await service.create_secret( - project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + cached = None + dao_calls = 0 + + original_list = service.secrets_dao.list + + async def counted_list(*args, **kwargs): + nonlocal dao_calls + dao_calls += 1 + return await original_list(*args, **kwargs) + + async def fake_get_cache(**kwargs): + assert kwargs["namespace"] == "list_secrets" + assert kwargs["project_id"] == str(PROJECT_ID) + assert kwargs["key"] == {} + assert kwargs["model"] is SecretResponseDTO + assert kwargs["is_list"] is True + return cached + + async def fake_set_cache(**kwargs): + nonlocal cached + assert kwargs["namespace"] == "list_secrets" + assert kwargs["project_id"] == str(PROJECT_ID) + assert kwargs["key"] == {} + cached = kwargs["value"] + return True + + monkeypatch.setattr(service.secrets_dao, "list", counted_list) + monkeypatch.setattr(secrets_services_module, "get_cache", fake_get_cache) + monkeypatch.setattr(secrets_services_module, "set_cache", fake_set_cache) + + await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=_provider_key_create(), ) + first = await service.list_secrets(project_id=PROJECT_ID) + second = await service.list_secrets(project_id=PROJECT_ID) - assert created.write_only is True + assert dao_calls == 1 + assert first == second + assert cached[0].data.provider.key == "sk-test-openai-key-bc" @pytest.mark.asyncio -@pytest.mark.parametrize("gate", [False, True]) -@pytest.mark.parametrize("explicit", [False, True]) -async def test_an_explicit_request_value_always_wins_over_the_gate( - service, write_only_gate, gate, explicit -): - write_only_gate(gate) +async def test_service_mutations_invalidate_the_project_cache(service, monkeypatch): + invalidated = [] + + async def fake_invalidate_cache(**kwargs): + invalidated.append(kwargs) + return True + + monkeypatch.setattr( + secrets_services_module, + "invalidate_cache", + fake_invalidate_cache, + ) created = await service.create_secret( project_id=PROJECT_ID, - create_secret_dto=_provider_key_create(write_only=explicit), + create_secret_dto=_provider_key_create(), ) + await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO(header={"name": "Renamed"}), + ) + await service.delete_secret(secret_id=created.id, project_id=PROJECT_ID) - assert created.write_only is explicit - - -# --- service: keep-stored-on-omit ------------------------------------------------------ + assert invalidated == [{"project_id": str(PROJECT_ID)}] * 3 class _RotatingDAO(_FakeSecretsDAO): @@ -169,6 +216,7 @@ class _RotatingDAO(_FakeSecretsDAO): The real DAO takes ``SELECT ... FOR UPDATE`` and only then resolves the update, so a writer that committed while we waited is visible. This fake reproduces that window by + rotating the stored row immediately before it resolves. """ @@ -220,7 +268,7 @@ async def test_an_omitted_key_keeps_the_value_a_racing_rotation_just_stored(): update_secret_dto=UpdateSecretDTO( secret={ "kind": "provider_key", - "data": {"kind": "openai", "provider": {"key": ""}}, + "data": {"kind": "openai", "provider": {}}, } ), ) @@ -229,8 +277,7 @@ async def test_an_omitted_key_keeps_the_value_a_racing_rotation_just_stored(): @pytest.mark.asyncio -@pytest.mark.parametrize("omitted_key", [None, ""]) -async def test_update_without_provider_key_keeps_the_stored_one(service, omitted_key): +async def test_update_without_provider_key_keeps_the_stored_one(service): created = await service.create_secret( project_id=PROJECT_ID, create_secret_dto=_provider_key_create() ) @@ -239,7 +286,7 @@ async def test_update_without_provider_key_keeps_the_stored_one(service, omitted header={"name": "OpenAI (renamed)"}, secret={ "kind": "provider_key", - "data": {"kind": "openai", "provider": {"key": omitted_key}}, + "data": {"kind": "openai", "provider": {}}, }, ) updated = await service.update_secret( @@ -358,10 +405,58 @@ async def test_update_with_partial_extras_refills_credential_keys_only(service): ) # The submitted config wins; only the credential keys refill from storage. + assert updated.data.provider.extras["region"] == "us-east-1" assert updated.data.provider.extras["api_key"] == "extra-key-123456" +@pytest.mark.asyncio +async def test_explicit_empty_custom_provider_credential_extra_is_rejected(service): + created = await service.create_secret( + project_id=PROJECT_ID, + create_secret_dto=CreateSecretDTO( + header={"name": "Bedrock"}, + secret={ + "kind": "custom_provider", + "data": { + "kind": "bedrock", + "provider": { + "extras": { + "AWS_ACCESS_KEY_ID": "AKIA123", + "AWS_SECRET_ACCESS_KEY": "stored-secret", + "AWS_REGION": "eu-west-1", + } + }, + "models": [{"slug": "claude"}], + }, + }, + ), + ) + + update = UpdateSecretDTO( + secret={ + "kind": "custom_provider", + "data": { + "kind": "bedrock", + "provider": { + "extras": { + "AWS_SECRET_ACCESS_KEY": "", + "AWS_REGION": "us-east-1", + } + }, + "models": [{"slug": "claude"}], + }, + }, + ) + + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=created.id, + project_id=PROJECT_ID, + update_secret_dto=update, + ) + + @pytest.mark.asyncio async def test_update_without_custom_secret_content_keeps_the_stored_one(service): created = await service.create_secret( @@ -391,37 +486,70 @@ async def test_update_without_custom_secret_content_keeps_the_stored_one(service assert updated.data.secret.content == "ghp_example_token_xyz" -# --- service: the flag is one-way ------------------------------------------------------ +def test_write_only_is_not_an_update_field(): + with pytest.raises(ValueError, match="cannot be updated"): + UpdateSecretDTO.model_validate({"write_only": False}) + with pytest.raises(ValueError, match="cannot be updated"): + UpdateSecretDTO.model_validate({"write_only": True}) -@pytest.mark.asyncio -async def test_write_only_cannot_be_turned_off(service): - created = await service.create_secret( - project_id=PROJECT_ID, create_secret_dto=_provider_key_create(write_only=True) - ) - with pytest.raises(WriteOnlyCannotBeDisabledError): - await service.update_secret( - secret_id=created.id, - project_id=PROJECT_ID, - update_secret_dto=UpdateSecretDTO(write_only=False), +@pytest.mark.parametrize( + ("kind", "data"), + [ + ( + "provider_key", + {"kind": "openai", "provider": {"key": ""}}, + ), + ( + "webhook_provider", + {"provider": {"key": ""}}, + ), + ( + "sso_provider", + { + "provider": { + "client_id": "client", + "client_secret": "", + "issuer_url": "https://issuer.example.com", + "scopes": ["openid"], + } + }, + ), + ], +) +def test_create_rejects_empty_credentials(kind, data): + with pytest.raises(ValueError): + CreateSecretDTO( + header={"name": "Connection"}, + secret={"kind": kind, "data": data}, ) @pytest.mark.asyncio -async def test_readable_secret_can_be_tightened_to_write_only(service): - created = await service.create_secret( - project_id=PROJECT_ID, - create_secret_dto=_provider_key_create(write_only=False), - ) - - updated = await service.update_secret( - secret_id=created.id, - project_id=PROJECT_ID, - update_secret_dto=UpdateSecretDTO(write_only=True), +async def test_update_cannot_keep_a_missing_value_from_a_legacy_row(service): + legacy = SecretResponseDTO( + id=uuid4(), + slug="legacy-openai", + kind="provider_key", + data={"kind": "openai", "provider": {}}, + header={"name": "Legacy"}, + write_only=False, ) + service.secrets_dao.records[legacy.id] = legacy - assert updated.write_only is True + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=legacy.id, + project_id=PROJECT_ID, + update_secret_dto=UpdateSecretDTO( + header={"name": "Renamed"}, + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {}}, + }, + ), + ) # --- service: keep-on-omit is identity-local ------------------------------------------- @@ -654,8 +782,8 @@ def test_redacts_provider_key_and_reports_presence(): redacted = redact_secret_response(secret) assert redacted.data.provider.key is None - assert redacted.has_key is True - assert redacted.key_preview == "sk-****bc" + assert redacted.value_status.configured is True + assert redacted.value_status.preview == "sk-****bc" # The input is never mutated: internal readers keep their plaintext DTO. assert secret.data.provider.key == "sk-test-openai-key-bc" @@ -680,9 +808,9 @@ def test_redacts_custom_provider_key_and_credential_extras(): assert "api_key" not in redacted.data.provider.extras assert redacted.data.provider.extras["region"] == "eu-west-1" assert redacted.data.provider.url == "https://gateway.example.com/v1" - assert redacted.has_key is True + assert redacted.value_status.configured is True # Only the primary value field gets a preview; extras credentials never do. - assert redacted.key_preview is None + assert redacted.value_status.preview is None def test_redacts_every_sdk_credential_extras_key(): @@ -715,11 +843,11 @@ def test_redacts_every_sdk_credential_extras_key(): "AWS_REGION": "eu-west-1", "vertex_ai_project": "my-project", } - assert redacted.has_key is True - assert redacted.key_preview is None + assert redacted.value_status.configured is True + assert redacted.value_status.preview is None -def test_aws_only_secret_reports_has_key_true(): +def test_aws_only_secret_reports_configured_true(): secret = _response( "custom_provider", { @@ -737,7 +865,7 @@ def test_aws_only_secret_reports_has_key_true(): redacted = redact_secret_response(secret) - assert redacted.has_key is True + assert redacted.value_status.configured is True assert "aws_secret_access_key" not in redacted.data.provider.extras assert redacted.data.provider.extras["aws_region_name"] == "eu-west-1" @@ -759,7 +887,7 @@ def test_redacts_sso_client_secret(): assert redacted.data.provider.client_secret is None assert redacted.data.provider.client_id == "client-1" - assert redacted.has_key is True + assert redacted.value_status.configured is True def test_redacts_text_custom_secret_content(): @@ -771,8 +899,8 @@ def test_redacts_text_custom_secret_content(): redacted = redact_secret_response(secret) assert redacted.data.secret.content is None - assert redacted.has_key is True - assert redacted.key_preview == "ghp****yz" + assert redacted.value_status.configured is True + assert redacted.value_status.preview == "ghp****yz" def test_redacts_json_custom_secret_without_a_preview(): @@ -784,9 +912,9 @@ def test_redacts_json_custom_secret_without_a_preview(): redacted = redact_secret_response(secret) assert redacted.data.secret.content is None - assert redacted.has_key is True + assert redacted.value_status.configured is True # A structured value has no single previewable string. - assert redacted.key_preview is None + assert redacted.value_status.preview is None def test_readable_secret_passes_through_unchanged(): @@ -798,13 +926,13 @@ def test_readable_secret_passes_through_unchanged(): redacted = redact_secret_response(secret) - assert redacted is secret + assert redacted is not secret assert redacted.data.provider.key == "sk-test-openai-key-bc" - assert redacted.has_key is None - assert redacted.key_preview is None + assert redacted.value_status.configured is True + assert redacted.value_status.preview is None -def test_write_only_without_a_value_reports_has_key_false(): +def test_write_only_without_a_value_reports_configured_false(): secret = _response( "custom_provider", { @@ -816,8 +944,8 @@ def test_write_only_without_a_value_reports_has_key_false(): redacted = redact_secret_response(secret) - assert redacted.has_key is False - assert redacted.key_preview is None + assert redacted.value_status.configured is False + assert redacted.value_status.preview is None # --- postgres mappings ----------------------------------------------------------------- @@ -883,55 +1011,6 @@ def test_update_mapping_preserves_the_stored_flag_when_unspecified(): assert stored["provider"]["key"] == "sk-test-rotated" -def test_update_mapping_applies_a_tightening_flag(): - import json - - dbe = map_secrets_dto_to_dbe( - project_id=PROJECT_ID, - organization_id=None, - secret_dto=_provider_key_create(write_only=False), - ) - - map_secrets_dto_to_dbe_update( - secrets_dbe=dbe, - update_secret_dto=UpdateSecretDTO(write_only=True), - ) - - stored = json.loads(dbe.data) - assert stored["write_only"] is True - assert stored["provider"]["key"] == "sk-test-openai-key-bc" - - -def test_update_mapping_never_clears_the_flag_on_a_stale_explicit_false(): - # Concurrency guard: a racing update that read write_only=False before another - # request tightened the secret must not resurrect readability at the mapper. - import json - - dbe = map_secrets_dto_to_dbe( - project_id=PROJECT_ID, - organization_id=None, - secret_dto=_provider_key_create(write_only=True), - ) - - map_secrets_dto_to_dbe_update( - secrets_dbe=dbe, - update_secret_dto=UpdateSecretDTO( - write_only=False, - secret={ - "kind": "provider_key", - "data": { - "kind": "openai", - "provider": {"key": "sk-test-rotated"}, - }, - }, - ), - ) - - stored = json.loads(dbe.data) - assert stored["write_only"] is True - assert stored["provider"]["key"] == "sk-test-rotated" - - # --- the update-path payload type, at every call site ---------------------------------- diff --git a/api/oss/tests/pytest/unit/utils/test_env_helpers.py b/api/oss/tests/pytest/unit/utils/test_env_helpers.py index f4b6599261..e575a2ad3b 100644 --- a/api/oss/tests/pytest/unit/utils/test_env_helpers.py +++ b/api/oss/tests/pytest/unit/utils/test_env_helpers.py @@ -6,6 +6,7 @@ """ import pytest +import oss.src.utils.env as env_module from oss.src.utils.env import env from oss.src.utils.helpers import warn_unconfigured_platform_runtime_key @@ -22,16 +23,15 @@ def _warnings(monkeypatch): return recorded -def _configure(monkeypatch, *, runtime_key, write_only_default): +def _configure(monkeypatch, *, runtime_key): monkeypatch.setattr(env.agenta, "services_internal_key", runtime_key) - monkeypatch.setattr(env.agenta.vault, "write_only_default", write_only_default) @pytest.mark.parametrize("runtime_key", ["", "replace-me"]) -def test_write_only_deployments_without_a_runtime_key_are_warned( +def test_deployments_without_a_runtime_key_are_warned( warnings, monkeypatch, runtime_key ): - _configure(monkeypatch, runtime_key=runtime_key, write_only_default=True) + _configure(monkeypatch, runtime_key=runtime_key) warn_unconfigured_platform_runtime_key() @@ -40,17 +40,43 @@ def test_write_only_deployments_without_a_runtime_key_are_warned( def test_a_configured_deployment_is_not_warned(warnings, monkeypatch): - _configure(monkeypatch, runtime_key="a-real-runtime-key", write_only_default=True) + _configure(monkeypatch, runtime_key="a-real-runtime-key") warn_unconfigured_platform_runtime_key() assert warnings == [] -def test_a_deployment_using_no_write_only_secrets_is_not_warned(warnings, monkeypatch): - # Nothing to read back, so the key buys it nothing and the warning would be noise. - _configure(monkeypatch, runtime_key="", write_only_default=False) +def test_the_warning_does_not_depend_on_a_feature_gate(warnings, monkeypatch): + _configure(monkeypatch, runtime_key="") warn_unconfigured_platform_runtime_key() - assert warnings == [] + assert len(warnings) == 1 + assert "AGENTA_SERVICES_INTERNAL_KEY" in warnings[0] + + +def test_runtime_key_does_not_fall_back_to_the_admin_key(monkeypatch): + monkeypatch.delenv("AGENTA_SERVICES_INTERNAL_KEY", raising=False) + monkeypatch.setenv("AGENTA_AUTH_KEY", "administrator-key") + + assert env_module._services_internal_key_from_environment() is None + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + (None, None), + ("", None), + (" ", None), + ("replace-me", None), + (" runtime-key ", "runtime-key"), + ], +) +def test_runtime_key_configuration_is_normalized(monkeypatch, configured, expected): + if configured is None: + monkeypatch.delenv("AGENTA_SERVICES_INTERNAL_KEY", raising=False) + else: + monkeypatch.setenv("AGENTA_SERVICES_INTERNAL_KEY", configured) + + assert env_module._services_internal_key_from_environment() == expected diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index 7b0f709a75..b7e27c169f 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -21,7 +21,6 @@ from oss.src.core.secrets.dtos import SecretResponseDTO from oss.src.core.secrets.services import VaultService from oss.src.middlewares.auth import SECRET_RESOLVE_GRANT -from oss.src.utils.env import env PROJECT_ID = str(uuid4()) @@ -74,16 +73,11 @@ async def update( # Production resolves the update against the row under the write lock; the fake # does the same at the same point, so keep-on-omit is exercised, not skipped. if resolve_update is not None: - resolve_update(stored) - - write_only = update_secret_dto.write_only - if write_only is None: - write_only = stored.write_only + update_secret_dto = resolve_update(stored, update_secret_dto) updated = stored.model_copy( update={ "header": update_secret_dto.header or stored.header, - "write_only": write_only, } ) if update_secret_dto.secret is not None: @@ -93,7 +87,12 @@ async def update( self.records[str(secret_id)] = updated return updated - async def delete(self, secret_id, project_id, organization_id): + async def delete( + self, secret_id, project_id, organization_id, authorize_delete=None + ): + stored = self.records.get(str(secret_id)) + if stored is not None and authorize_delete is not None: + authorize_delete(stored) self.records.pop(str(secret_id), None) @@ -146,28 +145,12 @@ def test_create_echo_is_redacted_for_a_write_only_secret(harness): assert created["write_only"] is True assert "key" not in created["data"]["provider"] - assert created["has_key"] is True - assert created["key_preview"] == "sk-****et" + assert created["value_status"]["configured"] is True + assert created["value_status"]["preview"] == "sk-****et" assert KEY not in str(created) -def test_create_without_the_flag_keeps_todays_response_while_the_gate_is_off(harness): - # The current frontend sends no flag; until AGENTA_VAULT_WRITE_ONLY_DEFAULT flips on, - # its creates must behave exactly as today. - client = harness - - created = _create(client) - - assert created["write_only"] is False - assert created["data"]["provider"]["key"] == KEY - assert "has_key" not in created - assert "key_preview" not in created - - -def test_create_without_the_flag_is_write_only_once_the_gate_is_on( - harness, monkeypatch -): - monkeypatch.setattr(env.agenta.vault, "write_only_default", True) +def test_create_without_the_flag_defaults_to_write_only(harness): client = harness created = _create(client) @@ -183,8 +166,8 @@ def test_create_with_explicit_false_keeps_todays_response(harness): assert created["write_only"] is False assert created["data"]["provider"]["key"] == KEY - assert "has_key" not in created - assert "key_preview" not in created + assert created["value_status"]["configured"] is True + assert "preview" not in created["value_status"] def test_read_is_redacted_for_users_and_plaintext_for_the_grant(harness): @@ -208,7 +191,7 @@ def test_list_is_redacted_for_users(harness): assert listed.status_code == 200 (secret,) = listed.json() assert "key" not in secret["data"]["provider"] - assert secret["has_key"] is True + assert secret["value_status"]["configured"] is True assert KEY not in listed.text @@ -245,10 +228,7 @@ def test_update_echo_is_redacted_and_omitted_key_keeps_the_stored_value(harness) assert runtime_read.json()["data"]["provider"]["key"] == KEY -def test_todays_edit_form_shape_empty_string_key_keeps_the_stored_value(harness): - # The CURRENT frontend cannot prefill a redacted value, so its edit form re-sends - # `key: ""`. If "" cleared the credential, every edit through today's UI would wipe a - # write-only secret — so empty string must mean "keep the stored value". +def test_explicit_empty_string_key_is_rejected(harness): client = harness created = _create(client, write_only=True) @@ -262,7 +242,7 @@ def test_todays_edit_form_shape_empty_string_key_keeps_the_stored_value(harness) }, }, ) - assert updated.status_code == 200, updated.text + assert updated.status_code == 400, updated.text runtime_read = client.get(f"/secrets/{created['id']}", headers=GRANT) assert runtime_read.json()["data"]["provider"]["key"] == KEY @@ -274,8 +254,8 @@ def test_write_only_cannot_be_disabled_over_the_api(harness): response = client.put(f"/secrets/{created['id']}", json={"write_only": False}) - assert response.status_code == 400 - assert "write-only" in response.json()["detail"] + assert response.status_code == 422 + assert "cannot be updated" in response.text def test_readable_secret_lists_with_its_value_as_today(harness): @@ -422,10 +402,10 @@ def test_openapi_documents_the_write_only_contract(harness): schemas = client.app.openapi()["components"]["schemas"] - for field in ("write_only", "has_key", "key_preview"): - assert field in schemas["SecretResponseDTO"]["properties"] + for field in ("write_only", "value_status"): + assert field in schemas["PublicSecretResponseDTO"]["properties"] assert "write_only" in schemas["CreateSecretDTO"]["properties"] - assert "write_only" in schemas["UpdateSecretDTO"]["properties"] + assert "write_only" not in schemas["UpdateSecretDTO"]["properties"] CANARY = "sk-CANARY-DO-NOT-ECHO-abc123" diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py index 0930353368..953fe791bb 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -1,8 +1,8 @@ """Webhook responses are write-only-aware; internal signing keeps plaintext. The signing secret lives in the vault, but it is a SHARED secret: the subscriber verifies -our signature with the same value, so webhook records are created readable regardless of -the env gate. Once a record IS write-only (only a manual tighten gets it there), no +our signature with the same value, so webhook records explicitly opt out of write-only. +Once a legacy record IS write-only, no USER-facing webhook response — create echo, fetch, edit echo — may carry the value again, while the internal resolver the signer uses stays plaintext. """ @@ -11,7 +11,7 @@ import pytest -from oss.src.core.secrets.dtos import SecretResponseDTO, UpdateSecretDTO +from oss.src.core.secrets.dtos import SecretResponseDTO from oss.src.core.secrets.services import VaultService from oss.src.core.webhooks.service import WebhooksService from oss.src.core.webhooks.types import ( @@ -20,7 +20,6 @@ WebhookSubscriptionData, WebhookSubscriptionEdit, ) -from oss.src.utils.env import env PROJECT_ID = uuid4() @@ -65,11 +64,8 @@ async def update( # Production resolves the update against the row under the write lock; the fake # does the same at the same point, so keep-on-omit is exercised, not skipped. if resolve_update is not None: - resolve_update(stored) - write_only = update_secret_dto.write_only - if write_only is None: - write_only = stored.write_only - updated = stored.model_copy(update={"write_only": write_only}) + update_secret_dto = resolve_update(stored, update_secret_dto) + updated = stored.model_copy() if update_secret_dto.secret is not None: updated.data = update_secret_dto.secret.data self.records[secret_id] = updated @@ -129,11 +125,15 @@ def _subscription_create(): ) +def _mark_write_only(vault_service, secret_id): + stored = vault_service.secrets_dao.records[secret_id] + vault_service.secrets_dao.records[secret_id] = stored.model_copy( + update={"write_only": True} + ) + + @pytest.mark.asyncio -async def test_gate_on_still_leaves_the_signing_secret_readable(services, monkeypatch): - # The vault-wide write-only default must not reach webhook signing secrets: the - # subscriber needs the value to verify signatures. - monkeypatch.setattr(env.agenta.vault, "write_only_default", True) +async def test_webhook_signing_secret_is_explicitly_readable(services): webhooks_service, _ = services created = await webhooks_service.create_subscription( @@ -151,12 +151,9 @@ async def test_gate_on_still_leaves_the_signing_secret_readable(services, monkey @pytest.mark.asyncio -async def test_gate_on_returns_a_generated_secret_on_the_create_echo( - services, monkeypatch -): +async def test_generated_secret_is_returned_on_the_create_echo(services): # The create echo is the ONLY place a caller can read a secret Agenta generated for # them. Redacting it would ship a subscription that can never be verified. - monkeypatch.setattr(env.agenta.vault, "write_only_default", True) webhooks_service, _ = services created = await webhooks_service.create_subscription( @@ -183,8 +180,7 @@ async def test_gate_on_returns_a_generated_secret_on_the_create_echo( @pytest.mark.asyncio -async def test_gate_off_keeps_todays_responses(services, monkeypatch): - monkeypatch.setattr(env.agenta.vault, "write_only_default", False) +async def test_explicit_readable_secret_keeps_existing_responses(services): webhooks_service, _ = services created = await webhooks_service.create_subscription( @@ -202,10 +198,7 @@ async def test_gate_off_keeps_todays_responses(services, monkeypatch): @pytest.mark.asyncio -async def test_manually_tightened_secret_stops_appearing_in_fetches( - services, monkeypatch -): - monkeypatch.setattr(env.agenta.vault, "write_only_default", False) +async def test_legacy_write_only_secret_stops_appearing_in_fetches(services): webhooks_service, vault_service = services created = await webhooks_service.create_subscription( @@ -218,11 +211,7 @@ async def test_manually_tightened_secret_stops_appearing_in_fetches( stored = await webhooks_service.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id ) - await vault_service.update_secret( - secret_id=UUID(str(stored.secret_id)), - project_id=PROJECT_ID, - update_secret_dto=UpdateSecretDTO(write_only=True), - ) + _mark_write_only(vault_service, UUID(str(stored.secret_id))) fetched = await webhooks_service.fetch_subscription( project_id=PROJECT_ID, @@ -232,8 +221,7 @@ async def test_manually_tightened_secret_stops_appearing_in_fetches( @pytest.mark.asyncio -async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypatch): - monkeypatch.setattr(env.agenta.vault, "write_only_default", False) +async def test_internal_resolver_keeps_plaintext_for_signing(services): webhooks_service, vault_service = services created = await webhooks_service.create_subscription( @@ -245,11 +233,7 @@ async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypat stored = await webhooks_service.dao.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id ) - await vault_service.update_secret( - secret_id=UUID(str(stored.secret_id)), - project_id=PROJECT_ID, - update_secret_dto=UpdateSecretDTO(write_only=True), - ) + _mark_write_only(vault_service, UUID(str(stored.secret_id))) stored = await webhooks_service.dao.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id @@ -264,11 +248,10 @@ async def test_internal_resolver_keeps_plaintext_for_signing(services, monkeypat @pytest.mark.asyncio async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_value( - services, monkeypatch + services, ): # The rotation path builds an update-path payload DTO; the parent `SecretDTO` does not # validate there, so this pins that editing a subscription's secret still works. - monkeypatch.setattr(env.agenta.vault, "write_only_default", False) webhooks_service, _ = services created = await webhooks_service.create_subscription( @@ -302,10 +285,7 @@ async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_valu @pytest.mark.asyncio -async def test_rotation_echo_stays_redacted_for_a_write_only_secret( - services, monkeypatch -): - monkeypatch.setattr(env.agenta.vault, "write_only_default", False) +async def test_rotation_echo_stays_redacted_for_a_write_only_secret(services): webhooks_service, vault_service = services created = await webhooks_service.create_subscription( @@ -317,11 +297,7 @@ async def test_rotation_echo_stays_redacted_for_a_write_only_secret( stored = await webhooks_service.dao.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id ) - await vault_service.update_secret( - secret_id=UUID(str(stored.secret_id)), - project_id=PROJECT_ID, - update_secret_dto=UpdateSecretDTO(write_only=True), - ) + _mark_write_only(vault_service, UUID(str(stored.secret_id))) edited = await webhooks_service.edit_subscription( project_id=PROJECT_ID, diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index d13f634396..c91b241113 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -154,27 +154,28 @@ The vault returns plaintext only to a caller whose verified `Secret` token carri the END USER's credential at `/access/permissions/check` on their behalf, so nothing about the presented token says a run is starting — and that route is reachable by a browser. The runtime therefore proves what it is with a secret only the backend holds - (`AGENTA_SERVICES_INTERNAL_KEY`, falling back to `AGENTA_AUTH_KEY`), sent as - `X-Agenta-Runtime-Key` on the internal hop and compared in constant time. The - well-known placeholder is refused, so an unconfigured deployment issues no grant rather - than accepting a string anyone could send. + (`AGENTA_SERVICES_INTERNAL_KEY`), sent as `X-Agenta-Runtime-Key` on the internal + hop and compared in constant time. This key has no fallback to + `AGENTA_AUTH_KEY`. If it is missing or remains the well-known placeholder, the API + issues no grant instead of accepting a string anyone could send. - **A caller refreshing a grant it already holds.** The runner re-exchanges its run credential every few heartbeats; the exchange carries the grant forward rather than re-deciding it. The runner is never given the runtime secret, and it never reaches a sandbox. -**A deployment that sets neither loses agent runs against write-only connections**, and -the failure names something else: the run reports "provide the provider key in this run's +**A deployment without the dedicated key loses agent runs against write-only connections**, +and the failure names something else: the run reports "provide the provider key in this run's environment", which is right for a standalone run and misleading here. The services middleware therefore warns once, at the point of use, naming the variable to set. The placeholder is the shipped default in the example env files, so this is the common case, not an edge one. **Deployment.** Set `AGENTA_SERVICES_INTERNAL_KEY` to the same value on the API and the -services container before turning write-only on (`AGENTA_AUTH_KEY` serves if it is a real -value). The API warns at startup when a deployment uses write-only secrets without one, -and a component that seeds write-only rows refuses to seed rather than store a credential -no run can read. +Services container before turning write-only on. It must be independent from +`AGENTA_AUTH_KEY` and must not be provisioned to web, runner, sandbox, worker, cron, or +migration containers. The API warns at startup when a deployment uses write-only secrets +without one, and a component that seeds write-only rows refuses to seed rather than store +a credential no run can read. The exchange never mints the grant from the requested `action` alone. It did once, and that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and diff --git a/hosting/docker-compose/ee/docker-compose.dev.yml b/hosting/docker-compose/ee/docker-compose.dev.yml index 39f02b6e9d..8663a2ea60 100644 --- a/hosting/docker-compose/ee/docker-compose.dev.yml +++ b/hosting/docker-compose/ee/docker-compose.dev.yml @@ -73,6 +73,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} WATCHPACK_POLLING: "true" AGENTA_MOBILE_GATE: ${AGENTA_MOBILE_GATE:-true} @@ -108,6 +109,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} WATCHPACK_POLLING: "true" AGENTA_MOBILE_GATE: ${AGENTA_MOBILE_GATE:-true} @@ -180,6 +182,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: ${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # kill's direct runner hop (W7.3, sandbox teardown) — same compose-network # address + shared token as the `services` container's runner_url(). @@ -240,6 +243,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md AGENTA_WORKER_STREAMS: "" @@ -289,6 +293,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md AGENTA_WORKER_QUEUES: "" @@ -333,6 +338,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -362,6 +368,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -412,6 +419,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: ${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} AGENTA_RUNNER_INTERNAL_URL: ${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} AGENTA_RUNNER_TOKEN: ${AGENTA_RUNNER_TOKEN:?AGENTA_RUNNER_TOKEN is required} @@ -550,6 +558,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -653,6 +662,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so unlike # the API (which reads code defaults from env.py) they must be present in this service's # env. Defaults live here for the bundled store the same way supertokens carries its own @@ -760,6 +770,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS:-postgresql://username:password@postgres:5432/agenta_ee_supertokens} # === NETWORK ============================================== # networks: @@ -794,6 +805,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.dev} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-network @@ -819,6 +832,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/ee/docker-compose.gh.local.yml b/hosting/docker-compose/ee/docker-compose.gh.local.yml index 80613e841a..41bb7f150e 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.local.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.local.yml @@ -14,6 +14,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -41,6 +43,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -79,6 +83,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/api # Unset leaves the code default (3600s) in place. - AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS=${AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS:-} @@ -130,6 +135,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_STREAMS= # === NETWORK ============================================== # @@ -174,6 +180,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_QUEUES= # === NETWORK ============================================== # @@ -211,6 +218,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -233,6 +242,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -266,6 +277,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/services - AGENTA_RUNNER_INTERNAL_URL=${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} - AGENTA_RUNNER_TOKEN=${AGENTA_RUNNER_TOKEN:?AGENTA_RUNNER_TOKEN is required} @@ -364,6 +376,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -467,6 +480,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must # be present in this service's env (defaults here are for the bundled store, the same way # supertokens carries its own connection URI). Keys come from the env file — secrets never @@ -514,6 +528,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -530,6 +546,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS:-postgresql://username:password@postgres:5432/agenta_ee_supertokens} # === NETWORK ============================================== # networks: @@ -564,6 +581,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -589,6 +608,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/ee/docker-compose.gh.yml b/hosting/docker-compose/ee/docker-compose.gh.yml index f61995468c..4a3405522e 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.yml @@ -19,6 +19,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - AGENTA_MOBILE_GATE=${AGENTA_MOBILE_GATE:-true} # === NETWORK ============================================== # networks: @@ -46,6 +47,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - AGENTA_MOBILE_GATE=${AGENTA_MOBILE_GATE:-true} - AGENTA_MOBILE_REVERSE_GATE=${AGENTA_MOBILE_REVERSE_GATE:-true} # === NETWORK ============================================== # @@ -84,6 +86,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/api - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # kill's direct runner hop (W7.3, sandbox teardown) — same address + shared @@ -142,6 +145,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_STREAMS= @@ -183,6 +187,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_QUEUES= @@ -220,6 +225,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -241,6 +247,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -272,6 +280,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/services - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} - AGENTA_RUNNER_INTERNAL_URL=${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} @@ -390,6 +399,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -493,6 +503,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must # be present in this service's env (defaults here are for the bundled store, the same way # supertokens carries its own connection URI). Keys come from the env file — secrets never @@ -548,6 +559,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.ee.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-ee-gh-network @@ -564,6 +577,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS:-postgresql://username:password@postgres:5432/agenta_ee_supertokens} # === NETWORK ============================================== # networks: @@ -602,6 +616,7 @@ services: env_file: - ${ENV_FILE:-./.env.ee.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/ee/env.ee.dev.example b/hosting/docker-compose/ee/env.ee.dev.example index 513ec0b6c0..00d8ce27b3 100644 --- a/hosting/docker-compose/ee/env.ee.dev.example +++ b/hosting/docker-compose/ee/env.ee.dev.example @@ -27,9 +27,9 @@ AGENTA_API_URL=http://localhost/api AGENTA_AUTH_KEY=replace-me # Proves to the API that a caller IS the platform runtime (the workflow service), so the # credential it receives may read write-only secret values. The API and the services -# container must hold the SAME value, and a browser must never see it. Falls back to -# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that -# changes neither simply cannot run against write-only secrets. +# container must hold the SAME dedicated value, and a browser, runner, worker, cron, or +# sandbox must never receive it. There is no AGENTA_AUTH_KEY fallback. Missing and +# placeholder values prevent write-only secret grants from being issued. AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me diff --git a/hosting/docker-compose/ee/env.ee.gh.example b/hosting/docker-compose/ee/env.ee.gh.example index 05d6c1f744..0654ce99e8 100644 --- a/hosting/docker-compose/ee/env.ee.gh.example +++ b/hosting/docker-compose/ee/env.ee.gh.example @@ -31,9 +31,9 @@ AGENTA_API_URL=http://localhost/api AGENTA_AUTH_KEY=replace-me # Proves to the API that a caller IS the platform runtime (the workflow service), so the # credential it receives may read write-only secret values. The API and the services -# container must hold the SAME value, and a browser must never see it. Falls back to -# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that -# changes neither simply cannot run against write-only secrets. +# container must hold the SAME dedicated value, and a browser, runner, worker, cron, or +# sandbox must never receive it. There is no AGENTA_AUTH_KEY fallback. Missing and +# placeholder values prevent write-only secret grants from being issued. AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me diff --git a/hosting/docker-compose/oss/docker-compose.dev.yml b/hosting/docker-compose/oss/docker-compose.dev.yml index 13c77bdf09..d3aee10d9b 100644 --- a/hosting/docker-compose/oss/docker-compose.dev.yml +++ b/hosting/docker-compose/oss/docker-compose.dev.yml @@ -65,6 +65,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} WATCHPACK_POLLING: "true" AGENTA_MOBILE_GATE: ${AGENTA_MOBILE_GATE:-true} @@ -100,6 +101,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} WATCHPACK_POLLING: "true" AGENTA_MOBILE_GATE: ${AGENTA_MOBILE_GATE:-true} @@ -168,6 +170,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: ${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # kill's direct runner hop (W7.3, sandbox teardown) — same compose-network # address + shared token as the `services` container's runner_url(). @@ -228,6 +231,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md AGENTA_WORKER_STREAMS: "" @@ -277,6 +281,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md AGENTA_WORKER_QUEUES: "" @@ -319,6 +324,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -348,6 +354,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -398,6 +405,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: ${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} AGENTA_RUNNER_INTERNAL_URL: ${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} AGENTA_RUNNER_TOKEN: ${AGENTA_RUNNER_TOKEN:?AGENTA_RUNNER_TOKEN is required} @@ -512,6 +520,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -611,6 +620,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so unlike # the API (which reads code defaults from env.py) they must be present in this service's # env. Defaults live here for the bundled store the same way supertokens carries its own @@ -718,6 +728,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS:-postgresql://username:password@postgres:5432/agenta_oss_supertokens} # === NETWORK ============================================== # networks: @@ -774,6 +785,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.dev} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/oss/docker-compose.gh.local.yml b/hosting/docker-compose/oss/docker-compose.gh.local.yml index 90386c9dd2..9cb9f55368 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.local.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.local.yml @@ -14,6 +14,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -39,6 +41,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -77,6 +81,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/api # Unset leaves the code default (3600s) in place. - AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS=${AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS:-} @@ -126,6 +131,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_STREAMS= # === NETWORK ============================================== # @@ -170,6 +176,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_QUEUES= # === NETWORK ============================================== # @@ -207,6 +214,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -229,6 +238,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -262,6 +273,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/services - AGENTA_RUNNER_INTERNAL_URL=${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} - AGENTA_RUNNER_TOKEN=${AGENTA_RUNNER_TOKEN:?AGENTA_RUNNER_TOKEN is required} @@ -360,6 +372,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -463,6 +476,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must # be present in this service's env (defaults here are for the bundled store, the same way # supertokens carries its own connection URI). Keys come from the env file — secrets never @@ -512,6 +526,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -533,6 +549,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -552,6 +570,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS} # === NETWORK ============================================== # networks: @@ -590,6 +609,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml index d10d1af521..4ea6b1f777 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml @@ -14,6 +14,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-gh-ssl-network @@ -41,6 +43,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-gh-ssl-network @@ -84,6 +88,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/api # Unset leaves the code default (3600s) in place. - AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS=${AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS:-} @@ -138,6 +143,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_STREAMS= # === NETWORK ============================================== # @@ -185,6 +191,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_QUEUES= # === NETWORK ============================================== # @@ -225,6 +232,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-gh-ssl-network @@ -250,6 +259,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-gh-ssl-network @@ -283,6 +294,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/services - AGENTA_RUNNER_INTERNAL_URL=${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} - AGENTA_RUNNER_TOKEN=${AGENTA_RUNNER_TOKEN:?AGENTA_RUNNER_TOKEN is required} @@ -386,6 +398,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -489,6 +502,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must # be present in this service's env (defaults here are for the bundled store, the same way # supertokens carries its own connection URI). Keys come from the env file — secrets never @@ -555,6 +569,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-gh-ssl-network @@ -592,6 +608,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/oss/docker-compose.gh.yml b/hosting/docker-compose/oss/docker-compose.gh.yml index 02b0d05005..96f1cc05aa 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.yml @@ -16,6 +16,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - AGENTA_MOBILE_GATE=${AGENTA_MOBILE_GATE:-true} # === NETWORK ============================================== # networks: @@ -44,6 +45,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - AGENTA_MOBILE_GATE=${AGENTA_MOBILE_GATE:-true} - AGENTA_MOBILE_REVERSE_GATE=${AGENTA_MOBILE_REVERSE_GATE:-true} # === NETWORK ============================================== # @@ -85,6 +87,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/api - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # kill's direct runner hop (W7.3, sandbox teardown) — same address + shared @@ -146,6 +149,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all stream loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_STREAMS= @@ -192,6 +196,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # empty ⇒ all queue loops; see docs/designs/workers-sprawl/specs.md - AGENTA_WORKER_QUEUES= @@ -232,6 +237,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY= - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} # === NETWORK ============================================== # networks: @@ -256,6 +262,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -290,6 +298,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + - AGENTA_SERVICES_INTERNAL_KEY=${AGENTA_SERVICES_INTERNAL_KEY:?AGENTA_SERVICES_INTERNAL_KEY is required} - SCRIPT_NAME=/services - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} - AGENTA_RUNNER_INTERNAL_URL=${AGENTA_RUNNER_INTERNAL_URL:-http://runner:8765} @@ -408,6 +417,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRES_USER: ${POSTGRES_USER:-username} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} # === NETWORK ============================================== # @@ -511,6 +521,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must # be present in this service's env (defaults here are for the bundled store, the same way # supertokens carries its own connection URI). Keys come from the env file — secrets never @@ -569,6 +580,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -590,6 +603,8 @@ services: # === CONFIGURATION ======================================== # env_file: - ${ENV_FILE:-./.env.oss.gh} + environment: + AGENTA_SERVICES_INTERNAL_KEY: "" # === NETWORK ============================================== # networks: - agenta-oss-gh-network @@ -609,6 +624,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" POSTGRESQL_CONNECTION_URI: ${POSTGRES_URI_SUPERTOKENS} # === NETWORK ============================================== # networks: @@ -647,6 +663,7 @@ services: env_file: - ${ENV_FILE:-./.env.oss.gh} environment: + AGENTA_SERVICES_INTERNAL_KEY: "" AGENTA_INGRESS_URL: http://api:8000/triggers/composio/events/ PYTHONUNBUFFERED: "1" # === NETWORK ============================================== # diff --git a/hosting/docker-compose/oss/env.oss.dev.example b/hosting/docker-compose/oss/env.oss.dev.example index fb5a547e4b..487cab8bb2 100644 --- a/hosting/docker-compose/oss/env.oss.dev.example +++ b/hosting/docker-compose/oss/env.oss.dev.example @@ -27,9 +27,9 @@ AGENTA_API_URL=http://localhost/api AGENTA_AUTH_KEY=replace-me # Proves to the API that a caller IS the platform runtime (the workflow service), so the # credential it receives may read write-only secret values. The API and the services -# container must hold the SAME value, and a browser must never see it. Falls back to -# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that -# changes neither simply cannot run against write-only secrets. +# container must hold the SAME dedicated value, and a browser, runner, worker, cron, or +# sandbox must never receive it. There is no AGENTA_AUTH_KEY fallback. Missing and +# placeholder values prevent write-only secret grants from being issued. AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me diff --git a/hosting/docker-compose/oss/env.oss.gh.example b/hosting/docker-compose/oss/env.oss.gh.example index 7f508e6db3..ee6ac96f82 100644 --- a/hosting/docker-compose/oss/env.oss.gh.example +++ b/hosting/docker-compose/oss/env.oss.gh.example @@ -31,9 +31,9 @@ AGENTA_API_URL=http://localhost/api AGENTA_AUTH_KEY=replace-me # Proves to the API that a caller IS the platform runtime (the workflow service), so the # credential it receives may read write-only secret values. The API and the services -# container must hold the SAME value, and a browser must never see it. Falls back to -# AGENTA_AUTH_KEY when unset; the placeholder above is refused, so a deployment that -# changes neither simply cannot run against write-only secrets. +# container must hold the SAME dedicated value, and a browser, runner, worker, cron, or +# sandbox must never receive it. There is no AGENTA_AUTH_KEY fallback. Missing and +# placeholder values prevent write-only secret grants from being issued. AGENTA_SERVICES_INTERNAL_KEY=replace-me AGENTA_CRYPT_KEY=replace-me diff --git a/hosting/kubernetes/ee/values.ee.example.yaml b/hosting/kubernetes/ee/values.ee.example.yaml index cf1512b246..2211fe416b 100644 --- a/hosting/kubernetes/ee/values.ee.example.yaml +++ b/hosting/kubernetes/ee/values.ee.example.yaml @@ -15,6 +15,7 @@ agenta: # --- Secrets (REPLACE in production!) --- authKey: "replace-me" cryptKey: "replace-me" + servicesInternalKey: "replace-me" runnerToken: "replace-me" # ================================================================== # diff --git a/hosting/kubernetes/helm/templates/NOTES.txt b/hosting/kubernetes/helm/templates/NOTES.txt index 03cdf6c31b..11b6953c45 100644 --- a/hosting/kubernetes/helm/templates/NOTES.txt +++ b/hosting/kubernetes/helm/templates/NOTES.txt @@ -130,6 +130,8 @@ Ensure it contains the following keys: Required: - AGENTA_AUTH_KEY - AGENTA_CRYPT_KEY + - AGENTA_SERVICES_INTERNAL_KEY + - AGENTA_RUNNER_TOKEN # unless agentRunner.auth.tokenSecretRef is set - POSTGRES_PASSWORD Optional (only if you enabled the corresponding feature): @@ -161,11 +163,13 @@ Set: --set global.postgresql.auth.existingSecret={{ $secrets.existingSecret }} {{- end }} {{- else }} -IMPORTANT: Three secrets are REQUIRED. Set them in your values file: +IMPORTANT: Required credentials must be set in your values file: agenta: authKey: "" cryptKey: "" + servicesInternalKey: "" + runnerToken: "" postgres: password: "" diff --git a/hosting/kubernetes/helm/templates/_helpers.tpl b/hosting/kubernetes/helm/templates/_helpers.tpl index e3b7393990..915abfc918 100644 --- a/hosting/kubernetes/helm/templates/_helpers.tpl +++ b/hosting/kubernetes/helm/templates/_helpers.tpl @@ -824,6 +824,19 @@ imagePullSecrets: {{- end }} {{- end }} +{{/* ================================================================ + Credential used only by the API and Services for trusted grant + exchange. Keep it separate from commonEnv, which is also rendered + into workers, cron, and alembic. + ================================================================ */}} +{{- define "agenta.servicesInternalEnv" -}} +- name: AGENTA_SERVICES_INTERNAL_KEY + valueFrom: + secretKeyRef: + name: {{ include "agenta.secretName" . }} + key: AGENTA_SERVICES_INTERNAL_KEY +{{- end }} + {{/* ================================================================ Common environment variables shared by api, workers, cron, alembic. Inlines the legacy `backendOptionalEnv` block. diff --git a/hosting/kubernetes/helm/templates/_validations.tpl b/hosting/kubernetes/helm/templates/_validations.tpl index f5b415d148..0a97a03e99 100644 --- a/hosting/kubernetes/helm/templates/_validations.tpl +++ b/hosting/kubernetes/helm/templates/_validations.tpl @@ -96,11 +96,10 @@ absolute-URL builder in the app. Validate that the canonical app secrets are provided when the chart is creating the Secret itself (i.e. secrets.existingSecret is unset). Without this guard the chart renders, the pods start, and the app - crashes on first request because AGENTA_AUTH_KEY / AGENTA_CRYPT_KEY - are empty. Fail at install time instead. + crashes on first request because required credentials are empty. Also rejects the literal placeholder "replace-me" shipped in - values.yaml's defaults for authKey/cryptKey/runnerToken — otherwise + values.yaml's defaults for authKey/cryptKey/servicesInternalKey/runnerToken — otherwise a stock `helm install` with no overrides silently deploys with a publicly-known secret instead of failing. @@ -124,6 +123,7 @@ absolute-URL builder in the app. {{- $missing := list -}} {{- if or (not $agenta.authKey) (eq $agenta.authKey $placeholder) -}}{{- $missing = append $missing "agenta.authKey" -}}{{- end -}} {{- if or (not $agenta.cryptKey) (eq $agenta.cryptKey $placeholder) -}}{{- $missing = append $missing "agenta.cryptKey" -}}{{- end -}} +{{- if or (not $agenta.servicesInternalKey) (eq $agenta.servicesInternalKey $placeholder) -}}{{- $missing = append $missing "agenta.servicesInternalKey" -}}{{- end -}} {{- if not $runnerAuth.tokenSecretRef -}} {{- if or (not $agenta.runnerToken) (eq $agenta.runnerToken $placeholder) -}}{{- $missing = append $missing "agenta.runnerToken" -}}{{- end -}} {{- end -}} @@ -151,6 +151,7 @@ Generate real values and set them in your values file: agenta: authKey: "<32+ random bytes hex>" cryptKey: "<32+ random bytes hex>" + servicesInternalKey: "<32+ random bytes hex>" runnerToken: "<32+ random bytes hex>" postgres: password: "" @@ -160,6 +161,7 @@ Or pass them on the command line: helm install agenta hosting/kubernetes/helm \ --set agenta.authKey=$(openssl rand -hex 32) \ --set agenta.cryptKey=$(openssl rand -hex 32) \ + --set agenta.servicesInternalKey=$(openssl rand -hex 32) \ --set agenta.runnerToken=$(openssl rand -hex 32) \ --set postgres.password= @@ -168,7 +170,7 @@ Or provide a pre-created Kubernetes Secret and point the chart at it: secrets: existingSecret: my-agenta-secret -The Secret must contain keys: AGENTA_AUTH_KEY, AGENTA_CRYPT_KEY, AGENTA_RUNNER_TOKEN, POSTGRES_PASSWORD. +The Secret must contain keys: AGENTA_AUTH_KEY, AGENTA_CRYPT_KEY, AGENTA_SERVICES_INTERNAL_KEY, AGENTA_RUNNER_TOKEN, POSTGRES_PASSWORD. agenta.runnerToken alone can also be satisfied by pointing the runner at your own Secret: diff --git a/hosting/kubernetes/helm/templates/api-deployment.yaml b/hosting/kubernetes/helm/templates/api-deployment.yaml index 8143adc034..24a5664ebb 100644 --- a/hosting/kubernetes/helm/templates/api-deployment.yaml +++ b/hosting/kubernetes/helm/templates/api-deployment.yaml @@ -73,6 +73,7 @@ spec: protocol: TCP env: {{- include "agenta.commonEnv" . | nindent 12 }} + {{- include "agenta.servicesInternalEnv" . | nindent 12 }} - name: SCRIPT_NAME value: "/api" {{- include "agenta.agentRunner.servicesEnv" . | nindent 12 }} diff --git a/hosting/kubernetes/helm/templates/secrets.yaml b/hosting/kubernetes/helm/templates/secrets.yaml index 44652f919f..820eea05a6 100644 --- a/hosting/kubernetes/helm/templates/secrets.yaml +++ b/hosting/kubernetes/helm/templates/secrets.yaml @@ -36,6 +36,7 @@ type: Opaque stringData: AGENTA_AUTH_KEY: {{ required "agenta.authKey is required (use --set agenta.authKey=)" $agenta.authKey | quote }} AGENTA_CRYPT_KEY: {{ required "agenta.cryptKey is required (use --set agenta.cryptKey=)" $agenta.cryptKey | quote }} + AGENTA_SERVICES_INTERNAL_KEY: {{ required "agenta.servicesInternalKey is required (use --set agenta.servicesInternalKey=)" $agenta.servicesInternalKey | quote }} {{- /* The runner's shared token. Required: the runner refuses to boot without it, and the service that calls it must present the same value. Lives in this Secret so both sides read one source; the runner Deployment mounts ONLY this key (never the whole Secret — diff --git a/hosting/kubernetes/helm/templates/services-deployment.yaml b/hosting/kubernetes/helm/templates/services-deployment.yaml index 9e69173a66..1dd202cc02 100644 --- a/hosting/kubernetes/helm/templates/services-deployment.yaml +++ b/hosting/kubernetes/helm/templates/services-deployment.yaml @@ -73,6 +73,7 @@ spec: protocol: TCP env: {{- include "agenta.commonEnv" . | nindent 12 }} + {{- include "agenta.servicesInternalEnv" . | nindent 12 }} - name: SCRIPT_NAME value: "/services" {{- include "agenta.agentRunner.servicesEnv" . | nindent 12 }} diff --git a/hosting/kubernetes/helm/tests/test_runner_secret_absence.py b/hosting/kubernetes/helm/tests/test_runner_secret_absence.py index d1bb0859b4..2b8f595f51 100644 --- a/hosting/kubernetes/helm/tests/test_runner_secret_absence.py +++ b/hosting/kubernetes/helm/tests/test_runner_secret_absence.py @@ -42,6 +42,8 @@ "--set", "agenta.cryptKey=test-crypt-key", "--set", + "agenta.servicesInternalKey=test-services-internal-key", + "--set", "postgres.password=test-postgres-password", ] @@ -62,6 +64,7 @@ FORBIDDEN_EXACT = { "AGENTA_AUTH_KEY", "AGENTA_CRYPT_KEY", + "AGENTA_SERVICES_INTERNAL_KEY", "AGENTA_LICENSE", "AGENTA_API_KEY", "OPENAI_API_KEY", @@ -117,6 +120,19 @@ def runner_container_env_names(docs: list[dict]) -> list[str]: raise AssertionError("no runner Deployment found in the rendered chart") +def deployment_env_names(docs: list[dict], component: str) -> set[str]: + """Environment variable names on a component's primary container.""" + for doc in docs: + if doc.get("kind") != "Deployment": + continue + labels = doc.get("metadata", {}).get("labels", {}) + if labels.get("app.kubernetes.io/component") != component: + continue + container = doc["spec"]["template"]["spec"]["containers"][0] + return {entry["name"] for entry in container.get("env", [])} + raise AssertionError(f"no {component} Deployment found in the rendered chart") + + def check(names: list[str]) -> list[str]: failures: list[str] = [] present = set(names) @@ -150,6 +166,18 @@ def main() -> int: names = runner_container_env_names(render(DEFAULT_TOKEN_ARGS)) failures += check(names) + docs = render(DEFAULT_TOKEN_ARGS) + for component in ("api", "services"): + if "AGENTA_SERVICES_INTERNAL_KEY" not in deployment_env_names(docs, component): + failures.append( + f"{component} env must contain AGENTA_SERVICES_INTERNAL_KEY" + ) + for component in ("worker-streams", "worker-queues", "cron"): + if "AGENTA_SERVICES_INTERNAL_KEY" in deployment_env_names(docs, component): + failures.append( + f"{component} env must not contain AGENTA_SERVICES_INTERNAL_KEY" + ) + # Operator supplies their own secret ref: same narrow env, token sourced from their Secret. names_with_token = runner_container_env_names(render(TOKEN_ARGS)) failures += check(names_with_token) @@ -161,7 +189,7 @@ def main() -> int: return 1 print( - "OK: runner Deployment env is narrow (no platform secrets, provider keys, or API key)." + "OK: internal-services key is limited to API/Services; runner env remains narrow." ) print(f" default env: {sorted(names)}") return 0 diff --git a/hosting/kubernetes/helm/values.schema.json b/hosting/kubernetes/helm/values.schema.json index e9b8ed06c3..91ab2df83c 100644 --- a/hosting/kubernetes/helm/values.schema.json +++ b/hosting/kubernetes/helm/values.schema.json @@ -16,6 +16,7 @@ "apiInternalUrl": { "type": "string", "description": "Internal API URL (in-cluster). Emitted as AGENTA_API_INTERNAL_URL." }, "authKey": { "type": "string", "minLength": 1, "description": "Authorization key. Required: `helm install` fails while this is unset or still 'replace-me' (see agenta.validateRequiredSecrets), unless secrets.existingSecret is set. Emitted as AGENTA_AUTH_KEY via Secret." }, "cryptKey": { "type": "string", "minLength": 1, "description": "Encryption key. Required: `helm install` fails while this is unset or still 'replace-me' (see agenta.validateRequiredSecrets), unless secrets.existingSecret is set. Emitted as AGENTA_CRYPT_KEY via Secret." }, + "servicesInternalKey": { "type": "string", "minLength": 1, "description": "Dedicated credential for trusted API-to-Services grant exchange. Required: `helm install` fails while this is unset or still 'replace-me' (see agenta.validateRequiredSecrets), unless secrets.existingSecret is set. Emitted as AGENTA_SERVICES_INTERNAL_KEY only to the API and Services pods." }, "runnerToken": { "type": "string", "minLength": 1, "description": "Shared secret the runner verifies and Services presents. Required: `helm install` fails while this is unset or still 'replace-me' (see agenta.validateRequiredSecrets), unless secrets.existingSecret or agentRunner.auth.tokenSecretRef is set. Emitted as AGENTA_RUNNER_TOKEN via Secret." }, "insecureEgressAllowed": { "type": ["boolean", "string"], "description": "AGENTA_INSECURE_EGRESS_ALLOWED — default true (permissive, zero-config self-host); set false to harden a shared/multi-tenant deployment." }, "access": { diff --git a/hosting/kubernetes/helm/values.yaml b/hosting/kubernetes/helm/values.yaml index 97cb40c547..5534470555 100644 --- a/hosting/kubernetes/helm/values.yaml +++ b/hosting/kubernetes/helm/values.yaml @@ -38,6 +38,7 @@ agenta: # ================================================================ # authKey: "replace-me" cryptKey: "replace-me" + servicesInternalKey: "replace-me" runnerToken: "replace-me" # ================================================================== # diff --git a/hosting/kubernetes/oss/values.oss.example.yaml b/hosting/kubernetes/oss/values.oss.example.yaml index 0573ce6744..f71429ee0d 100644 --- a/hosting/kubernetes/oss/values.oss.example.yaml +++ b/hosting/kubernetes/oss/values.oss.example.yaml @@ -15,6 +15,7 @@ agenta: # --- Secrets (REPLACE in production!) --- authKey: "replace-me" cryptKey: "replace-me" + servicesInternalKey: "replace-me" runnerToken: "replace-me" # ================================================================== # diff --git a/hosting/railway/oss/README.md b/hosting/railway/oss/README.md index c7166c41f5..de91d0986e 100644 --- a/hosting/railway/oss/README.md +++ b/hosting/railway/oss/README.md @@ -176,13 +176,16 @@ content, kept in lockstep by `images/verify-wrappers.sh`). ### Security Note The scripts default to compose-like placeholder values for `AGENTA_AUTH_KEY`, -`AGENTA_CRYPT_KEY`, `AGENTA_RUNNER_TOKEN`, and `POSTGRES_PASSWORD`. This is -acceptable for throwaway test projects, but not for persistent deployments. -For persistent deployments, set unique values: +`AGENTA_CRYPT_KEY`, `AGENTA_SERVICES_INTERNAL_KEY`, `AGENTA_RUNNER_TOKEN`, and +`POSTGRES_PASSWORD`. This is acceptable for throwaway test projects, but not +for persistent deployments. `AGENTA_SERVICES_INTERNAL_KEY` has no fallback to +`AGENTA_AUTH_KEY`; API and Services must receive the same dedicated value. +For persistent deployments, set a unique value for each credential: ```bash export AGENTA_AUTH_KEY="$(openssl rand -hex 32)" export AGENTA_CRYPT_KEY="$(openssl rand -hex 32)" +export AGENTA_SERVICES_INTERNAL_KEY="$(openssl rand -hex 32)" export AGENTA_RUNNER_TOKEN="$(openssl rand -hex 32)" export POSTGRES_PASSWORD="$(openssl rand -hex 24)" ``` diff --git a/hosting/railway/oss/scripts/configure.sh b/hosting/railway/oss/scripts/configure.sh index af81f70d5e..cca0e0e2f6 100755 --- a/hosting/railway/oss/scripts/configure.sh +++ b/hosting/railway/oss/scripts/configure.sh @@ -15,6 +15,7 @@ POSTGRES_REF_NS="${RAILWAY_POSTGRES_REF_NS:-Postgres}" REDIS_SERVICE="${RAILWAY_REDIS_SERVICE:-redis}" AGENTA_AUTH_KEY="${AGENTA_AUTH_KEY:-replace-me}" AGENTA_CRYPT_KEY="${AGENTA_CRYPT_KEY:-replace-me}" +AGENTA_SERVICES_INTERNAL_KEY="${AGENTA_SERVICES_INTERNAL_KEY:-replace-me}" AGENTA_RUNNER_TOKEN="${AGENTA_RUNNER_TOKEN:-replace-me}" POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-}" AGENTA_STORE_ACCESS_KEY="${AGENTA_STORE_ACCESS_KEY:-}" @@ -300,8 +301,8 @@ main() { require_cmd railway require_railway_auth - if [ "$AGENTA_AUTH_KEY" = "replace-me" ] || [ "$AGENTA_CRYPT_KEY" = "replace-me" ] || [ "$AGENTA_RUNNER_TOKEN" = "replace-me" ]; then - printf "WARNING: Using default placeholder secrets. Set AGENTA_AUTH_KEY, AGENTA_CRYPT_KEY and AGENTA_RUNNER_TOKEN for production deployments.\n" >&2 + if [ "$AGENTA_AUTH_KEY" = "replace-me" ] || [ "$AGENTA_CRYPT_KEY" = "replace-me" ] || [ "$AGENTA_SERVICES_INTERNAL_KEY" = "replace-me" ] || [ "$AGENTA_RUNNER_TOKEN" = "replace-me" ]; then + printf "WARNING: Using default placeholder secrets. Set AGENTA_AUTH_KEY, AGENTA_CRYPT_KEY, AGENTA_SERVICES_INTERNAL_KEY and AGENTA_RUNNER_TOKEN for production deployments.\n" >&2 fi railway_call link --project "$PROJECT_NAME" --environment "$ENV_NAME" --json >/dev/null @@ -391,6 +392,7 @@ main() { AGENTA_SERVICES_URL="https://${public_domain_ref}/services" \ AGENTA_AUTH_KEY="$AGENTA_AUTH_KEY" \ AGENTA_CRYPT_KEY="$AGENTA_CRYPT_KEY" \ + AGENTA_SERVICES_INTERNAL_KEY="$AGENTA_SERVICES_INTERNAL_KEY" \ POSTGRES_URI_CORE="$pg_async_core" \ POSTGRES_URI_TRACING="$pg_async_tracing" \ POSTGRES_URI_SUPERTOKENS="$pg_sync_supertokens" \ @@ -423,6 +425,7 @@ main() { AGENTA_SERVICES_URL="https://${public_domain_ref}/services" \ AGENTA_AUTH_KEY="$AGENTA_AUTH_KEY" \ AGENTA_CRYPT_KEY="$AGENTA_CRYPT_KEY" \ + AGENTA_SERVICES_INTERNAL_KEY="$AGENTA_SERVICES_INTERNAL_KEY" \ POSTGRES_URI_CORE="$pg_async_core" \ POSTGRES_URI_TRACING="$pg_async_tracing" \ POSTGRES_URI_SUPERTOKENS="$pg_sync_supertokens" \ diff --git a/hosting/railway/oss/template/template.json b/hosting/railway/oss/template/template.json index 6313b868b8..7840c5f7e3 100644 --- a/hosting/railway/oss/template/template.json +++ b/hosting/railway/oss/template/template.json @@ -64,6 +64,10 @@ "from_env": "AGENTA_CRYPT_KEY", "generate": "openssl-rand-hex-32" }, + "AGENTA_SERVICES_INTERNAL_KEY": { + "from_env": "AGENTA_SERVICES_INTERNAL_KEY", + "generate": "openssl-rand-hex-32" + }, "AGENTA_RUNNER_TOKEN": { "from_env": "AGENTA_RUNNER_TOKEN", "generate": "openssl-rand-hex-32" @@ -171,6 +175,9 @@ "AGENTA_CRYPT_KEY": { "secret": "AGENTA_CRYPT_KEY" }, + "AGENTA_SERVICES_INTERNAL_KEY": { + "secret": "AGENTA_SERVICES_INTERNAL_KEY" + }, "POSTGRES_URI_CORE": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_core", "POSTGRES_URI_TRACING": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_tracing", "POSTGRES_URI_SUPERTOKENS": "postgresql://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_supertokens", @@ -219,6 +226,9 @@ "AGENTA_CRYPT_KEY": { "secret": "AGENTA_CRYPT_KEY" }, + "AGENTA_SERVICES_INTERNAL_KEY": { + "secret": "AGENTA_SERVICES_INTERNAL_KEY" + }, "POSTGRES_URI_CORE": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_core", "POSTGRES_URI_TRACING": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_tracing", "POSTGRES_URI_SUPERTOKENS": "postgresql://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_supertokens", diff --git a/sdks/python/agenta/sdk/middlewares/routing/auth.py b/sdks/python/agenta/sdk/middlewares/routing/auth.py index 46904397ee..30d3e7d043 100644 --- a/sdks/python/agenta/sdk/middlewares/routing/auth.py +++ b/sdks/python/agenta/sdk/middlewares/routing/auth.py @@ -23,14 +23,16 @@ # The platform runtime's proof of what it is, for the credential exchange below. _RUNTIME_KEY_HEADER = "X-Agenta-Runtime-Key" -_RUNTIME_KEY = ( - getenv("AGENTA_SERVICES_INTERNAL_KEY") or getenv("AGENTA_AUTH_KEY") or "" -).strip() -# The placeholder a deployment that configured nothing carries — and it is the SHIPPED -# DEFAULT in the example env files, not a rare mistake. Sending it would be sending a -# value every reader of the repo knows, so it counts as no key at all. -if _RUNTIME_KEY == "replace-me": - _RUNTIME_KEY = "" + + +def _runtime_key_from_environment() -> str: + runtime_key = (getenv("AGENTA_SERVICES_INTERNAL_KEY") or "").strip() + + # The placeholder is public repository content, so it counts as no key at all. + return "" if runtime_key == "replace-me" else runtime_key + + +_RUNTIME_KEY = _runtime_key_from_environment() # Said once, at the point of use, because the failure it causes names something else # entirely: a run whose connection holds a write-only secret gets the redacted shape and @@ -49,7 +51,7 @@ def _warn_once_about_the_missing_runtime_key() -> None: _RUNTIME_KEY_WARNED = True log.warning( "agenta: no platform runtime key configured " - "(AGENTA_SERVICES_INTERNAL_KEY unset and AGENTA_AUTH_KEY is the placeholder). " + "(AGENTA_SERVICES_INTERNAL_KEY is unset or uses the placeholder). " "Runs against connections whose secret is write-only will fail to read it. " "Set AGENTA_SERVICES_INTERNAL_KEY to the same value on the API and this service." ) diff --git a/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py index 010d0d0239..36ed2e25a2 100644 --- a/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py +++ b/sdks/python/oss/tests/pytest/unit/test_auth_middleware_credentials.py @@ -109,8 +109,8 @@ async def test_no_runtime_key_means_no_header(platform, monkeypatch): async def test_a_missing_runtime_key_says_so_once(platform, monkeypatch, caplog): # The failure it causes reports a missing provider key, which is the wrong advice for - # this cause. An operator who never set AGENTA_AUTH_KEY — the shipped default — has no - # other way to reach it. + # this cause. The standalone-run advice is misleading here, and an operator who never + # set the dedicated runtime key has no other way to reach the actual cause. monkeypatch.setattr(auth_module, "_RUNTIME_KEY", "") monkeypatch.setattr(auth_module, "_RUNTIME_KEY_WARNED", False) platform.body = {"effect": "allow", "credentials": "Secret general-token"} @@ -126,3 +126,27 @@ async def test_a_missing_runtime_key_says_so_once(platform, monkeypatch, caplog) ] assert len(notices) == 1 assert "AGENTA_SERVICES_INTERNAL_KEY" in notices[0].getMessage() + assert "AGENTA_AUTH_KEY" not in notices[0].getMessage() + + +def test_runtime_key_configuration_never_falls_back_to_the_admin_key(monkeypatch): + configured = {"AGENTA_AUTH_KEY": "administrator-key"} + monkeypatch.setattr( + auth_module, + "getenv", + lambda name, default=None: configured.get(name, default), + ) + + assert auth_module._runtime_key_from_environment() == "" + + +def test_runtime_key_placeholder_is_treated_as_unconfigured(monkeypatch): + monkeypatch.setattr( + auth_module, + "getenv", + lambda name, default=None: ( + "replace-me" if name == "AGENTA_SERVICES_INTERNAL_KEY" else default + ), + ) + + assert auth_module._runtime_key_from_environment() == "" From 44c16bf0447fb61e26a004735bce3651c9729780 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 22 Aug 2026 23:01:44 +0200 Subject: [PATCH 23/31] refactor(api): separate write-only secret DTO roles --- api/oss/src/apis/fastapi/vault/router.py | 80 ++-- api/oss/src/core/secrets/dtos.py | 400 ++++++++++--------- api/oss/src/core/secrets/services.py | 169 +++++--- api/oss/src/dbs/postgres/secrets/mappings.py | 45 ++- 4 files changed, 404 insertions(+), 290 deletions(-) diff --git a/api/oss/src/apis/fastapi/vault/router.py b/api/oss/src/apis/fastapi/vault/router.py index a42c36348b..d8c66a7319 100644 --- a/api/oss/src/apis/fastapi/vault/router.py +++ b/api/oss/src/apis/fastapi/vault/router.py @@ -8,7 +8,6 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.exceptions import intercept_exceptions -from oss.src.utils.caching import invalidate_cache from oss.src.core.secrets.services import VaultService from oss.src.core.secrets.dtos import ( @@ -16,9 +15,13 @@ SecretValueRequiredError, UpdateSecretDTO, SecretResponseDTO, - WriteOnlyCannotBeDisabledError, + PublicSecretResponseDTO, ) -from oss.src.core.secrets.redaction import redact_secret_response +from oss.src.core.secrets.managed import ( + ManagedByIsServerControlledError, + ManagedSecretReadOnlyError, +) +from oss.src.core.secrets.redaction import project_secret_response from oss.src.core.access.permissions.types import Permission from oss.src.core.access.permissions.service import check_action_access @@ -75,7 +78,7 @@ def __init__( methods=["POST"], operation_id="create_secret", response_model_exclude_none=True, - response_model=SecretResponseDTO, + response_model=PublicSecretResponseDTO, ) self.router.add_api_route( "/secrets/", @@ -83,7 +86,7 @@ def __init__( methods=["GET"], operation_id="list_secrets", response_model_exclude_none=True, - response_model=List[SecretResponseDTO], + response_model=List[PublicSecretResponseDTO], ) self.router.add_api_route( "/secrets/{secret_id_or_slug}", @@ -91,7 +94,7 @@ def __init__( methods=["GET"], operation_id="read_secret", response_model_exclude_none=True, - response_model=SecretResponseDTO, + response_model=PublicSecretResponseDTO, ) self.router.add_api_route( "/secrets/{secret_id}", @@ -99,7 +102,7 @@ def __init__( methods=["PUT"], operation_id="update_secret", response_model_exclude_none=True, - response_model=SecretResponseDTO, + response_model=PublicSecretResponseDTO, ) self.router.add_api_route( "/secrets/{secret_id}", @@ -112,20 +115,39 @@ def __init__( @staticmethod def _for_caller( request: Request, secret_dto: SecretResponseDTO - ) -> SecretResponseDTO: + ) -> PublicSecretResponseDTO: """The response shape ``request``'s principal may see. Only the platform runtime (a Secret token carrying the ``secret-resolve`` grant) - receives write-only values in plaintext; every user principal — session, ApiKey, - unscoped Secret token — gets the redacted shape. + receives write-only values in plaintext. Every caller still receives the same + public response type rather than the internal service DTO. """ - if request_has_grant(request, SECRET_RESOLVE_GRANT): - return secret_dto + return project_secret_response( + secret_dto, + reveal_write_only=request_has_grant(request, SECRET_RESOLVE_GRANT), + ) + + @staticmethod + def _refuse_client_managed_by(body) -> None: + """`managed_by` states that Agenta provisioned the row; a client may not claim it. - return redact_secret_response(secret_dto) + Rejected rather than ignored: a caller that sent it believes the row will be + managed (or un-managed), and silently dropping the field would leave it wrong + about what the vault now holds. + """ + if body.managed_by is None: + return + + error = ManagedByIsServerControlledError() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=error.message, + ) @intercept_exceptions() async def create_secret(self, request: Request, body: CreateSecretDTO): + self._refuse_client_managed_by(body) + has_permission = await check_action_access( user_uid=str(request.state.user_id), project_id=str(request.state.project_id), @@ -143,9 +165,6 @@ async def create_secret(self, request: Request, body: CreateSecretDTO): project_id=UUID(request.state.project_id), create_secret_dto=body, ) - await invalidate_cache( - project_id=request.state.project_id, - ) return self._for_caller(request, vault_secret) @intercept_exceptions() @@ -213,6 +232,8 @@ async def read_secret(self, request: Request, secret_id_or_slug: str): async def update_secret( self, request: Request, secret_id: str, body: UpdateSecretDTO ): + self._refuse_client_managed_by(body) + has_permission = await check_action_access( user_uid=str(request.state.user_id), project_id=str(request.state.project_id), @@ -233,17 +254,20 @@ async def update_secret( update_secret_dto=body, user_id=UUID(request.state.user_id), ) - except (SecretValueRequiredError, WriteOnlyCannotBeDisabledError) as e: + except SecretValueRequiredError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=e.message ) from e + except ManagedSecretReadOnlyError as e: + # 409, not 400: the payload is well-formed; the stored row's managed state is + # what forbids the change. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, detail=e.message + ) from e if secrets_dto is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Secret not found" ) - await invalidate_cache( - project_id=request.state.project_id, - ) return self._for_caller(request, secrets_dto) @intercept_exceptions() @@ -261,11 +285,13 @@ async def delete_secret(self, request: Request, secret_id: str): status_code=403, ) - await self.service.delete_secret( - project_id=UUID(request.state.project_id), - secret_id=UUID(secret_id), - ) - await invalidate_cache( - project_id=request.state.project_id, - ) + try: + await self.service.delete_secret( + project_id=UUID(request.state.project_id), + secret_id=UUID(secret_id), + ) + except ManagedSecretReadOnlyError as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, detail=e.message + ) from e return status.HTTP_204_NO_CONTENT diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index 3f987ee75d..e8aa17563f 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Optional, Union, List, Dict, Any +from typing import Optional, Union, List, Dict, Any from pydantic import BaseModel, Field, model_validator @@ -33,25 +33,9 @@ def __init__( super().__init__(message) -class WriteOnlyCannotBeDisabledError(Exception): - """Raised when an update tries to turn `write_only` off. - - Turning it off would make the stored value readable again, defeating the flag's whole - guarantee. The transition is one-way: off -> on only. - """ - - def __init__( - self, - message: str = "A write-only secret cannot be made readable again. " - "Delete it and create a new secret instead.", - ): - self.message = message - super().__init__(message) - - -# The value-bearing fields below are Optional so that read models can carry a redacted -# (value-less) shape and updates can omit a value to mean "keep the stored one". Presence -# at CREATE time is still enforced, by `SecretDTO.validate_secret_data_based_on_kind`. +# The value-bearing fields below are optional at the structural level. Each role-specific +# payload decides whether omission is valid: create requires a value, update uses omission +# to mean "keep stored", and responses may be redacted. class StandardProviderSettingsDTO(BaseModel): @@ -120,156 +104,167 @@ class CustomSecretDTO(BaseModel): secret: CustomSecretSettingsDTO -class SecretDTO(BaseModel): - kind: SecretKind - data: Union[ - StandardProviderDTO, - CustomProviderDTO, - SSOProviderDTO, - WebhookProviderDTO, - CustomSecretDTO, - ] - - # Whether the kind's value field (provider key, custom-secret content, ...) must be - # present. True on the create path; the update payload and the response model turn it - # off so a value-less shape (keep-stored-on-omit, write-only redaction) validates. - VALUE_REQUIRED: ClassVar[bool] = True +SecretDataDTO = Union[ + StandardProviderDTO, + CustomProviderDTO, + SSOProviderDTO, + WebhookProviderDTO, + CustomSecretDTO, +] + + +def _validate_secret_data_based_on_kind( + values: Dict[str, Any], + *, + value_required: bool, +) -> Dict[str, Any]: + kind = values.get("kind") + if isinstance(kind, SecretKind): + kind = kind.value + data = values.get("data", {}) + if isinstance(data, BaseModel): + data = data.model_dump() + values["data"] = data + + standard_provider_kinds = {provider.value for provider in StandardProviderKind} + custom_provider_kinds = {provider.value for provider in CustomProviderKind} + + if kind == SecretKind.PROVIDER_KEY.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for StandardProviderDTO" + ) + provider = data.get("provider") + if not isinstance(provider, dict) or ( + value_required and provider.get("key") in (None, "") + ): + raise ValueError( + "The provided request secret dto is missing required fields for StandardProviderSettingsDTO" + ) + # Accept the legacy provider slug on input, but persist the canonical value. + if data.get("kind") == StandardProviderKind.MISTRALAI.value: + data["kind"] = StandardProviderKind.MISTRAL.value + if data.get("kind") not in standard_provider_kinds: + raise ValueError( + "The provided kind in data is not a valid StandardProviderKind enum" + ) + # Both provider shapes now accept {kind, provider, models}, so the union can no + # longer tell them apart from the payload alone; the secret kind decides. + values["data"] = StandardProviderDTO.model_validate(data) - @model_validator(mode="before") - def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): - kind = values.get("kind") - if isinstance(kind, SecretKind): - kind = kind.value - data = values.get("data", {}) - if isinstance(data, BaseModel): - data = data.model_dump() - values["data"] = data + elif kind == SecretKind.CUSTOM_PROVIDER.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for CustomProviderDTO" + ) + # Fix inconsistent API naming - Users might enter 'togetherai' but the API requires 'together_ai' + # This ensures compatibility with LiteLLM which requires the provider in "together_ai" format + if data.get("kind", "") == "togetherai": + data["kind"] = "together_ai" - standard_provider_kinds = {provider.value for provider in StandardProviderKind} - custom_provider_kinds = {provider.value for provider in CustomProviderKind} + if data.get("kind") not in custom_provider_kinds: + raise ValueError( + "The provided kind in data is not a valid CustomProviderKind enum" + ) - if kind == SecretKind.PROVIDER_KEY.value: - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for StandardProviderDTO" - ) - provider = data.get("provider") - if not isinstance(provider, dict) or ( - cls.VALUE_REQUIRED and provider.get("key") is None - ): - raise ValueError( - "The provided request secret dto is missing required fields for StandardProviderSettingsDTO" - ) - # Accept the legacy provider slug on input, but persist the canonical value. - if data.get("kind") == StandardProviderKind.MISTRALAI.value: - data["kind"] = StandardProviderKind.MISTRAL.value - if data.get("kind") not in standard_provider_kinds: - raise ValueError( - "The provided kind in data is not a valid StandardProviderKind enum" - ) - # Both provider shapes now accept {kind, provider, models}, so the union can no - # longer tell them apart from the payload alone; the secret kind decides. - values["data"] = StandardProviderDTO.model_validate(data) - - elif kind == SecretKind.CUSTOM_PROVIDER.value: - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for CustomProviderDTO" - ) - # Fix inconsistent API naming - Users might enter 'togetherai' but the API requires 'together_ai' - # This ensures compatibility with LiteLLM which requires the provider in "together_ai" format - if data.get("kind", "") == "togetherai": - data["kind"] = "together_ai" - - if data.get("kind") not in custom_provider_kinds: - raise ValueError( - "The provided kind in data is not a valid CustomProviderKind enum" - ) - - provider_url = (data.get("provider") or {}).get("url") - if isinstance(provider_url, str) and provider_url: - try: - validate_url_format_and_literal_ip(provider_url) - except ValueError as exc: - raise ValueError(f"custom_provider.url is invalid: {exc}") from exc - - values["data"] = CustomProviderDTO.model_validate(data) - elif kind == SecretKind.SSO_PROVIDER.value: - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for SSOProviderDTO" - ) - provider = data.get("provider") - if not isinstance(provider, dict): - raise ValueError( - "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" - ) - required_fields = {"client_id", "issuer_url", "scopes"} - # `client_secret` is checked by VALUE, not by presence: a create carrying an - # explicit null would otherwise store a credential-less SSO record, and a - # value is optional only on the update path (omission means "keep the stored - # one") and in redacted responses. - if not required_fields.issubset(provider.keys()) or ( - cls.VALUE_REQUIRED and provider.get("client_secret") is None - ): - raise ValueError( - "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" - ) - elif kind == SecretKind.WEBHOOK_PROVIDER.value: - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for WebhookProviderDTO" - ) - provider = data.get("provider") - if not isinstance(provider, dict) or ( - cls.VALUE_REQUIRED and provider.get("key") is None - ): - raise ValueError( - "The provided request secret dto is missing required fields for WebhookProviderSettingsDTO" - ) - elif kind == SecretKind.CUSTOM_SECRET.value: - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for CustomSecretDTO" - ) - secret = data.get("secret") - if ( - not isinstance(secret, dict) - or "format" not in secret - or (cls.VALUE_REQUIRED and secret.get("content") is None) - ): - raise ValueError( - "The provided request secret dto requires data.secret.{format, content} for CustomSecretDTO" - ) - fmt, content = secret["format"], secret.get("content") - if content is None: - pass # Value-less shape allowed when VALUE_REQUIRED is off; nothing to type-check. - elif fmt == CustomSecretFormat.TEXT.value: - if not isinstance(content, str): - raise ValueError("A text custom_secret requires a string content") - # Stored verbatim; do NOT re-serialize a JSON-looking string. - elif fmt == CustomSecretFormat.JSON.value: - if not isinstance(content, dict): - raise ValueError("A json custom_secret requires an object content") - for v in content.values(): - if isinstance(v, (dict, list)): - raise ValueError( - "A json custom_secret must be flat: values cannot be objects or arrays" - ) - else: - raise ValueError("A custom_secret format must be 'text' or 'json'") + provider_url = (data.get("provider") or {}).get("url") + if isinstance(provider_url, str) and provider_url: + try: + validate_url_format_and_literal_ip(provider_url) + except ValueError as exc: + raise ValueError(f"custom_provider.url is invalid: {exc}") from exc + + values["data"] = CustomProviderDTO.model_validate(data) + elif kind == SecretKind.SSO_PROVIDER.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for SSOProviderDTO" + ) + provider = data.get("provider") + if not isinstance(provider, dict): + raise ValueError( + "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" + ) + required_fields = {"client_id", "issuer_url", "scopes"} + # `client_secret` is checked by VALUE, not by presence: a create carrying an + # explicit null would otherwise store a credential-less SSO record, and a + # value is optional only on the update path (omission means "keep the stored + # one") and in redacted responses. + if not required_fields.issubset(provider.keys()) or ( + value_required and provider.get("client_secret") in (None, "") + ): + raise ValueError( + "The provided request secret dto is missing required fields for SSOProviderSettingsDTO" + ) + elif kind == SecretKind.WEBHOOK_PROVIDER.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for WebhookProviderDTO" + ) + provider = data.get("provider") + if not isinstance(provider, dict) or ( + value_required and provider.get("key") in (None, "") + ): + raise ValueError( + "The provided request secret dto is missing required fields for WebhookProviderSettingsDTO" + ) + elif kind == SecretKind.CUSTOM_SECRET.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for CustomSecretDTO" + ) + secret = data.get("secret") + if ( + not isinstance(secret, dict) + or "format" not in secret + or (value_required and secret.get("content") is None) + ): + raise ValueError( + "The provided request secret dto requires data.secret.{format, content} for CustomSecretDTO" + ) + fmt, content = secret["format"], secret.get("content") + if content is None: + pass # Value-less shape allowed when VALUE_REQUIRED is off; nothing to type-check. + elif fmt == CustomSecretFormat.TEXT.value: + if not isinstance(content, str): + raise ValueError("A text custom_secret requires a string content") + # Stored verbatim; do NOT re-serialize a JSON-looking string. + elif fmt == CustomSecretFormat.JSON.value: + if not isinstance(content, dict): + raise ValueError("A json custom_secret requires an object content") + for v in content.values(): + if isinstance(v, (dict, list)): + raise ValueError( + "A json custom_secret must be flat: values cannot be objects or arrays" + ) else: - raise ValueError("The provided kind is not a valid SecretKind enum") + raise ValueError("A custom_secret format must be 'text' or 'json'") + else: + raise ValueError("The provided kind is not a valid SecretKind enum") - return values + return values + + +class SecretDTO(BaseModel): + """Create-time secret payload. Required credential fields must be present.""" + + kind: SecretKind + data: SecretDataDTO + + @model_validator(mode="before") + @classmethod + def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): + return _validate_secret_data_based_on_kind(values, value_required=True) class CreateSecretDTO(Slug, BaseModel): header: Header secret: SecretDTO - # None means "platform default": env-gated via AGENTA_VAULT_WRITE_ONLY_DEFAULT - # (currently False). An explicit value always wins over the gate, in both directions. - write_only: Optional[bool] = None + write_only: bool = True + # Server-controlled: which platform component provisioned and owns this row (see + # `core/secrets/managed.py`). In-process callers set it; every user-facing route + # rejects a client-supplied value with HTTP 400. + managed_by: Optional[str] = None @model_validator(mode="before") def ensure_header_exists(cls, values): @@ -315,19 +310,34 @@ def update_provider_slug_with_header_name(cls, values): return values -class UpdateSecretPayloadDTO(SecretDTO): - """The update-path secret payload: same shape as `SecretDTO`, but a value field may be - omitted to mean "keep the stored value" (see `VaultService.update_secret`).""" +class UpdateSecretPayloadDTO(BaseModel): + """Update-time payload. Omitted credential fields keep their stored values.""" - VALUE_REQUIRED: ClassVar[bool] = False + kind: SecretKind + data: SecretDataDTO + + @model_validator(mode="before") + @classmethod + def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): + return _validate_secret_data_based_on_kind(values, value_required=False) class UpdateSecretDTO(BaseModel): header: Optional[Header] = None secret: Optional[UpdateSecretPayloadDTO] = None - # None keeps the stored flag. True tightens a readable secret to write-only. - # False on a write-only secret is rejected (`WriteOnlyCannotBeDisabledError`). - write_only: Optional[bool] = None + # Server-controlled. None keeps the stored marker; a non-empty string sets it and an + # empty string clears it, both only for in-process callers that pass + # `allow_managed=True` (`ManagedByIsServerControlledError` otherwise). + managed_by: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def reject_write_only_updates(cls, values): + if isinstance(values, dict) and "write_only" in values: + raise ValueError( + "write_only is selected when a secret is created and cannot be updated" + ) + return values @model_validator(mode="before") def update_provider_slug_with_header_name(cls, values): @@ -342,39 +352,45 @@ def update_provider_slug_with_header_name(cls, values): return values -class SecretResponseDTO(Identifier, Slug, SecretDTO): +class SecretValueStatus(BaseModel): + configured: bool + preview: Optional[str] = None + + +class _SecretResponseBaseDTO(Identifier, Slug, BaseModel): + kind: SecretKind + data: SecretDataDTO header: Header lifecycle: Optional[LegacyLifecycleDTO] = None write_only: bool = False - # Server-computed, set only on redacted (write-only) user-facing responses so a client - # can show that a value exists, and which one, without ever receiving it. - has_key: Optional[bool] = None - key_preview: Optional[str] = None - - # A read model may carry a redacted, value-less payload. - VALUE_REQUIRED: ClassVar[bool] = False @model_validator(mode="before") - def build_up_model_keys(cls, values: Dict[str, Any]) -> Dict[str, Any]: - """ - This method builds up model keys for a custom provider secret. + @classmethod + def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): + return _validate_secret_data_based_on_kind(values, value_required=False) + + @model_validator(mode="after") + def build_up_model_keys(self): + if self.kind == SecretKind.CUSTOM_PROVIDER: + self.data.model_keys = [ # type: ignore[union-attr] + f"{self.data.provider_slug}/{self.data.kind.value}/{model.slug}" # type: ignore[union-attr] + for model in self.data.models # type: ignore[union-attr] + ] + return self - Args: - - values (SecretResponseDTO): A dictionary form. - Returns: - - Dict[str, Any]: The updated dictionary with the added model keys. +class SecretResponseDTO(_SecretResponseBaseDTO): + """Trusted internal representation. Credential material remains available.""" - """ - data = values.get("data") - kind = values.get("kind") + # Read-only: present when a platform component owns the row, absent otherwise (the + # vault routes exclude None fields). Users can read and use such a row, but not edit + # or delete it. + managed_by: Optional[str] = None - if kind == SecretKind.CUSTOM_PROVIDER.value: - model_keys = [ - f"{data.get('provider_slug')}/{data.get('kind')}/{model.get('slug')}" # type: ignore - for model in data.get("models") # type: ignore - ] - values["data"].update({"model_keys": model_keys}) - return values +class PublicSecretResponseDTO(_SecretResponseBaseDTO): + """Caller-facing representation after grant-aware value projection.""" + + managed_by: Optional[str] = None + value_status: SecretValueStatus diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 3f30bfabd8..602a92f625 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -5,6 +5,7 @@ from pydantic import ValidationError from oss.src.utils.env import env +from oss.src.utils.caching import get_cache, invalidate_cache, set_cache from oss.src.utils.helpers import get_slug_from_name_and_id from oss.src.core.secrets.enums import ( STANDARD_PROVIDER_DISPLAY_NAMES, @@ -20,9 +21,15 @@ from oss.src.core.secrets.dtos import ( CreateSecretDTO, SecretResponseDTO, + SecretDTO, + UpdateSecretPayloadDTO, SecretValueRequiredError, UpdateSecretDTO, - WriteOnlyCannotBeDisabledError, +) + +from oss.src.core.secrets.managed import ( + ManagedByIsServerControlledError, + ManagedSecretReadOnlyError, ) @@ -75,8 +82,8 @@ def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> An update that omits the value means "keep the stored one" — the contract replace-only forms rely on for write-only secrets, applied uniformly so update semantics do not fork - on the flag. An empty string counts as omitted: an empty credential is never a - meaningful value, and replace-only forms submit empty for "unchanged". + on the flag. An explicit empty string is invalid; replace-only forms must omit an + unchanged credential field. Only called when the update keeps the stored kind AND provider family — a credential must never silently cross identities (see `update_secret`). @@ -93,7 +100,9 @@ def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> and hasattr(update_container, field) ): current_value = getattr(update_container, field) - if current_value is None or current_value == "": + if current_value == "": + raise SecretValueRequiredError() + if current_value is None: stored_value = getattr(stored_container, field, None) if stored_value is not None: setattr(update_container, field, stored_value) @@ -101,7 +110,7 @@ def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> _carry_over_saved_extras(stored_data=stored_data, update_data=update_data) -def _revalidate_merged_secret(*, secret: Any) -> None: +def _revalidate_merged_secret(*, secret: Any) -> UpdateSecretPayloadDTO: """Re-run the payload validators over the update as it will be stored. Validation runs at construction, before keep-on-omit fills the value in, so a merged @@ -110,7 +119,8 @@ def _revalidate_merged_secret(*, secret: Any) -> None: committed. """ try: - type(secret).model_validate(secret.model_dump()) + complete = SecretDTO.model_validate(secret.model_dump(mode="python")) + return UpdateSecretPayloadDTO.model_validate(complete.model_dump(mode="python")) except ValidationError as exc: raise SecretValueRequiredError( message=( @@ -163,15 +173,21 @@ def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: for extras_key in CREDENTIAL_EXTRAS_KEYS: stored_value = stored_extras.get(extras_key) - if stored_value is not None and not update_extras.get(extras_key): + requested_value = update_extras.get(extras_key) + if requested_value == "": + raise SecretValueRequiredError() + if stored_value is not None and ( + extras_key not in update_extras or requested_value is None + ): update_extras[extras_key] = stored_value -def _resolve_credential_carry_over( +def _resolve_update( stored_secret_dto: SecretResponseDTO, + requested_update: UpdateSecretDTO, *, - update_secret_dto: UpdateSecretDTO, -) -> None: + allow_managed: bool, +) -> UpdateSecretDTO: """Fill this update's omitted credential from the row UNDER THE WRITE LOCK. Called by the DAO inside the locked transaction rather than by the service before it, @@ -183,36 +199,51 @@ def _resolve_credential_carry_over( another kind's or another provider's credential — and that decision reads the same stored row, so it belongs under the same lock. """ - if update_secret_dto.secret is None: - return + if not allow_managed and stored_secret_dto.managed_by: + raise ManagedSecretReadOnlyError(managed_by=stored_secret_dto.managed_by) + + resolved_update = requested_update.model_copy(deep=True) + if resolved_update.secret is None: + return UpdateSecretDTO.model_validate(resolved_update.model_dump(mode="python")) same_identity = ( - stored_secret_dto.kind == update_secret_dto.secret.kind + stored_secret_dto.kind == resolved_update.secret.kind and _provider_family(stored_secret_dto.data) - == _provider_family(update_secret_dto.secret.data) + == _provider_family(resolved_update.secret.data) # A custom secret's format is identity too: carrying a stored text value into a # json update would store a string where the shape says object, and the payload # validators never see it because they ran before the value was filled in. and _secret_format(stored_secret_dto.data) - == _secret_format(update_secret_dto.secret.data) + == _secret_format(resolved_update.secret.data) ) if same_identity: _carry_over_saved_policy( stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, + update_data=resolved_update.secret.data, ) _carry_over_saved_value( kind=str(stored_secret_dto.kind.value), stored_data=stored_secret_dto.data, - update_data=update_secret_dto.secret.data, + update_data=resolved_update.secret.data, ) # The payload was validated before the carry-over filled it in, so what the # validators actually saw was a value-less shape. Re-validate the merged result: # nothing reaches the row that a create of the same shape would have refused. - _revalidate_merged_secret(secret=update_secret_dto.secret) + resolved_update.secret = _revalidate_merged_secret( + secret=resolved_update.secret + ) else: - _require_explicit_value(secret=update_secret_dto.secret) + _require_explicit_value(secret=resolved_update.secret) + + return UpdateSecretDTO.model_validate(resolved_update.model_dump(mode="python")) + + +def _authorize_delete( + stored_secret_dto: SecretResponseDTO, *, allow_managed: bool +) -> None: + if not allow_managed and stored_secret_dto.managed_by: + raise ManagedSecretReadOnlyError(managed_by=stored_secret_dto.managed_by) def _carry_over_saved_policy(*, stored_data: Any, update_data: Any) -> None: @@ -258,12 +289,6 @@ async def create_secret( uuid4(), ) - # The write-only default for NEW secrets is env-gated (off until the web UI ships - # replace-only secret forms); an explicit request value always wins. Existing rows - # are untouched (they carry no flag). - if create_secret_dto.write_only is None: - create_secret_dto.write_only = env.agenta.vault.write_only_default - if create_secret_dto.secret.kind == SecretKind.PROVIDER_KEY: await self._name_and_slug_provider_key( project_id=project_id, @@ -271,6 +296,14 @@ async def create_secret( create_secret_dto=create_secret_dto, ) + # Managed implies write-only, over both the request and the env default: a managed + # row's credential is one Agenta provisioned and the organization never supplied, + # so there is no state in which handing it back to a user is right. Forced here + # because `write_only` is pinned once stored — a readable managed row could never + # be tightened afterwards. + if create_secret_dto.managed_by: + create_secret_dto.write_only = True + with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): @@ -279,7 +312,10 @@ async def create_secret( organization_id=organization_id, create_secret_dto=create_secret_dto, ) - return secret_dto + + if project_id is not None: + await invalidate_cache(project_id=str(project_id)) + return secret_dto async def _name_and_slug_provider_key( self, @@ -358,14 +394,30 @@ async def list_secrets( project_id: UUID | None = None, organization_id: UUID | None = None, ): - with set_data_encryption_key( - data_encryption_key=self._data_encryption_key, - ): + if project_id is not None: + secrets_dtos = await get_cache( + namespace="list_secrets", + project_id=str(project_id), + key={}, + model=SecretResponseDTO, + is_list=True, + ) + if secrets_dtos is not None: + return secrets_dtos + + with set_data_encryption_key(data_encryption_key=self._data_encryption_key): secrets_dtos = await self.secrets_dao.list( - project_id=project_id, - organization_id=organization_id, + project_id=project_id, organization_id=organization_id ) - return secrets_dtos + + if project_id is not None: + await set_cache( + namespace="list_secrets", + project_id=str(project_id), + key={}, + value=secrets_dtos, + ) + return secrets_dtos async def update_secret( self, @@ -374,51 +426,44 @@ async def update_secret( project_id: UUID | None = None, organization_id: UUID | None = None, user_id: UUID | None = None, + allow_managed: bool = False, ): + """Update a secret. `allow_managed` is the in-process owner's key to a managed row. + + It defaults to off so that every caller reached from a user-facing route — today's + and tomorrow's — gets the managed guard without opting in. The component that + provisioned a managed row passes True to run its own reconcile or teardown. + """ + if not allow_managed and update_secret_dto.managed_by is not None: + raise ManagedByIsServerControlledError() + with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): - if ( - update_secret_dto.secret is not None - or update_secret_dto.write_only is not None - ): - stored_secret_dto = await self.secrets_dao.get_by_id( - secret_id=secret_id, - project_id=project_id, - organization_id=organization_id, - ) - if stored_secret_dto is not None: - if ( - stored_secret_dto.write_only - and update_secret_dto.write_only is False - ): - raise WriteOnlyCannotBeDisabledError() - secret_dto = await self.secrets_dao.update( secret_id=secret_id, update_secret_dto=update_secret_dto, project_id=project_id, organization_id=organization_id, user_id=user_id, - # Resolved against the LOCKED row, not the snapshot above: see - # `_resolve_credential_carry_over`. - resolve_update=( - partial( - _resolve_credential_carry_over, - update_secret_dto=update_secret_dto, - ) - if update_secret_dto.secret is not None - else None + resolve_update=partial( + _resolve_update, + allow_managed=allow_managed, ), ) - return secret_dto + + if project_id is not None: + await invalidate_cache(project_id=str(project_id)) + return secret_dto async def delete_secret( self, secret_id: UUID, project_id: UUID | None = None, organization_id: UUID | None = None, + allow_managed: bool = False, ) -> None: + """Delete a secret. A managed row refuses unless the owner passes `allow_managed`.""" with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): @@ -426,5 +471,11 @@ async def delete_secret( secret_id=secret_id, project_id=project_id, organization_id=organization_id, + authorize_delete=partial( + _authorize_delete, + allow_managed=allow_managed, + ), ) - return + + if project_id is not None: + await invalidate_cache(project_id=str(project_id)) diff --git a/api/oss/src/dbs/postgres/secrets/mappings.py b/api/oss/src/dbs/postgres/secrets/mappings.py index b3cba52215..f8f7fd2d32 100644 --- a/api/oss/src/dbs/postgres/secrets/mappings.py +++ b/api/oss/src/dbs/postgres/secrets/mappings.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone from oss.src.dbs.postgres.secrets.dbes import SecretsDBE +from oss.src.core.secrets.managed import resolve_managed_by from oss.src.core.secrets.dtos import ( Header, SecretKind, @@ -13,18 +14,30 @@ ) -# The flag rides inside the (encrypted) `data` JSON, as a sibling of the payload fields, -# so no schema migration is needed. It is popped back out in `map_secrets_dbe_to_dto`, so -# payload DTOs never see it; rows without the key read as write_only=False (legacy). +# Both server-controlled attributes ride inside the (encrypted) `data` JSON, as siblings +# of the payload fields, so no schema migration is needed. They are popped back out in +# `map_secrets_dbe_to_dto`, so payload DTOs never see them; rows without the keys read as +# write_only=False and managed_by=None (legacy, and every user-created row). _WRITE_ONLY_KEY = "write_only" +_MANAGED_BY_KEY = "managed_by" -def _data_payload(data_json: dict, *, write_only: bool) -> str: +def _data_payload( + data_json: dict, + *, + write_only: bool, + managed_by: str | None = None, +) -> str: if write_only: data_json[_WRITE_ONLY_KEY] = True else: data_json.pop(_WRITE_ONLY_KEY, None) + if managed_by: + data_json[_MANAGED_BY_KEY] = managed_by + else: + data_json.pop(_MANAGED_BY_KEY, None) + return json.dumps(data_json) @@ -44,6 +57,7 @@ def map_secrets_dto_to_dbe( data=_data_payload( secret_dto.secret.data.model_dump(exclude_none=True), write_only=bool(secret_dto.write_only), + managed_by=secret_dto.managed_by, ), ) return vault_secret_dbe @@ -64,12 +78,15 @@ def map_secrets_dto_to_dbe_update( if hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) - # Resolve the effective flag BEFORE overwriting data. The transition is one-way, so - # the mapper NEVER clears a stored flag: an explicit False can only reach it stale - # (the DAO rejects true->false under the row lock), and trusting it would resurrect - # readability. - write_only = bool(update_secret_dto.write_only) or bool( - json.loads(secrets_dbe.data).get(_WRITE_ONLY_KEY) + stored_data = json.loads(secrets_dbe.data) + + write_only = bool(stored_data.get(_WRITE_ONLY_KEY)) + # Same reason to resolve it here: `data` is rewritten wholesale below, so an omitted + # marker would silently un-manage the row. Only an in-process owner can reach this + # with a non-None value (`VaultService.update_secret` refuses it otherwise). + managed_by = resolve_managed_by( + stored=stored_data.get(_MANAGED_BY_KEY), + requested=update_secret_dto.managed_by, ) if update_secret_dto.secret: @@ -80,19 +97,22 @@ def map_secrets_dto_to_dbe_update( secrets_dbe.data = _data_payload( update_secret_dto.secret.data.model_dump(), write_only=write_only, + managed_by=managed_by, ) elif hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) - elif update_secret_dto.write_only is not None: + elif update_secret_dto.managed_by is not None: secrets_dbe.data = _data_payload( - json.loads(secrets_dbe.data), + stored_data, write_only=write_only, + managed_by=managed_by, ) def map_secrets_dbe_to_dto(*, secrets_dbe: SecretsDBE) -> SecretResponseDTO: data = json.loads(secrets_dbe.data) # type: ignore write_only = bool(data.pop(_WRITE_ONLY_KEY, False)) + managed_by = data.pop(_MANAGED_BY_KEY, None) or None vault_secret_dto = SecretResponseDTO( id=secrets_dbe.id, # type: ignore @@ -105,6 +125,7 @@ def map_secrets_dbe_to_dto(*, secrets_dbe: SecretsDBE) -> SecretResponseDTO: updated_at=str(secrets_dbe.updated_at), ), write_only=write_only, + managed_by=managed_by, ) return vault_secret_dto From 8fb95b80d4fdcb63918a94c6092d350ae7477497 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 22 Aug 2026 23:54:21 +0200 Subject: [PATCH 24/31] fix(api): fail startup without the runtime key --- api/entrypoints/routers.py | 4 +- api/oss/src/utils/helpers.py | 12 +++--- .../pytest/unit/utils/test_env_helpers.py | 39 +++++-------------- docs/design/write-only-secrets/README.md | 19 ++++----- 4 files changed, 26 insertions(+), 48 deletions(-) diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index 6dd4a63486..aeaa5355e4 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -15,9 +15,9 @@ from oss.src.utils.common import is_ee from oss.src.utils.logging import get_module_logger from oss.src.utils.helpers import ( + validate_platform_runtime_key, validate_required_env_vars, warn_deprecated_env_vars, - warn_unconfigured_platform_runtime_key, ) # Engines @@ -267,7 +267,7 @@ async def lifespan(*args, **kwargs): warn_deprecated_env_vars() validate_required_env_vars() - warn_unconfigured_platform_runtime_key() + validate_platform_runtime_key() await _triggers_broker.startup() diff --git a/api/oss/src/utils/helpers.py b/api/oss/src/utils/helpers.py index faf480dc92..f04685aa5a 100644 --- a/api/oss/src/utils/helpers.py +++ b/api/oss/src/utils/helpers.py @@ -187,8 +187,8 @@ def warn_deprecated_env_vars(): ) -def warn_unconfigured_platform_runtime_key(): - """Say at boot when nothing can read a write-only secret, and why. +def validate_platform_runtime_key(): + """Stop startup when nothing can read a write-only secret. A run reads a write-only secret only through a credential the platform runtime is issued, and the runtime is recognized by a dedicated shared key. If that key is unset @@ -201,10 +201,10 @@ def warn_unconfigured_platform_runtime_key(): if runtime_key and runtime_key != "replace-me": return - log.warning( - "AGENTA_SERVICES_INTERNAL_KEY is not configured or uses the placeholder. " - "Write-only secrets are in use on this deployment, and runs " - "against a connection whose secret is write-only will not be able to read it. " + raise RuntimeError( + "AGENTA_SERVICES_INTERNAL_KEY is required and must not use the placeholder. " + "Without it, platform runs cannot receive the short-lived grant needed to read " + "write-only secrets. " "Set AGENTA_SERVICES_INTERNAL_KEY to the same value on the API and the services " "container." ) diff --git a/api/oss/tests/pytest/unit/utils/test_env_helpers.py b/api/oss/tests/pytest/unit/utils/test_env_helpers.py index e575a2ad3b..325031864d 100644 --- a/api/oss/tests/pytest/unit/utils/test_env_helpers.py +++ b/api/oss/tests/pytest/unit/utils/test_env_helpers.py @@ -1,4 +1,4 @@ -"""Startup warnings that name a misconfiguration the runtime cannot report itself. +"""Startup validation for a misconfiguration the runtime cannot report itself. A deployment using write-only secrets needs a platform runtime key, or runs cannot read the secrets they were authorized to use. That failure surfaces as advice about provider @@ -9,18 +9,7 @@ import oss.src.utils.env as env_module from oss.src.utils.env import env -from oss.src.utils.helpers import warn_unconfigured_platform_runtime_key - - -@pytest.fixture(name="warnings") -def _warnings(monkeypatch): - recorded: list = [] - - monkeypatch.setattr( - "oss.src.utils.helpers.log", - type("_Log", (), {"warning": staticmethod(lambda msg: recorded.append(msg))})(), - ) - return recorded +from oss.src.utils.helpers import validate_platform_runtime_key def _configure(monkeypatch, *, runtime_key): @@ -28,32 +17,24 @@ def _configure(monkeypatch, *, runtime_key): @pytest.mark.parametrize("runtime_key", ["", "replace-me"]) -def test_deployments_without_a_runtime_key_are_warned( - warnings, monkeypatch, runtime_key -): +def test_deployments_without_a_runtime_key_fail_startup(monkeypatch, runtime_key): _configure(monkeypatch, runtime_key=runtime_key) - warn_unconfigured_platform_runtime_key() + with pytest.raises(RuntimeError, match="AGENTA_SERVICES_INTERNAL_KEY"): + validate_platform_runtime_key() - assert len(warnings) == 1 - assert "AGENTA_SERVICES_INTERNAL_KEY" in warnings[0] - -def test_a_configured_deployment_is_not_warned(warnings, monkeypatch): +def test_a_configured_deployment_passes_validation(monkeypatch): _configure(monkeypatch, runtime_key="a-real-runtime-key") - warn_unconfigured_platform_runtime_key() - - assert warnings == [] + validate_platform_runtime_key() -def test_the_warning_does_not_depend_on_a_feature_gate(warnings, monkeypatch): +def test_the_validation_does_not_depend_on_a_feature_gate(monkeypatch): _configure(monkeypatch, runtime_key="") - warn_unconfigured_platform_runtime_key() - - assert len(warnings) == 1 - assert "AGENTA_SERVICES_INTERNAL_KEY" in warnings[0] + with pytest.raises(RuntimeError, match="AGENTA_SERVICES_INTERNAL_KEY"): + validate_platform_runtime_key() def test_runtime_key_does_not_fall_back_to_the_admin_key(monkeypatch): diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index c91b241113..4b5fe14b24 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -142,7 +142,7 @@ read: no session, ApiKey, or list/get call ever returns the value. - Surface `write_only` in the connections/secrets lists. - Optional "readable" toggle at creation only (maps to `write_only: false`), if product wants the escape hatch exposed. -- Regenerate the Fern client for the new `write_only`, `has_key`, `key_preview` fields. +- Regenerate the Fern client for `write_only`, `value_status`, and `management.policy`. ## Who may read a value: the grant @@ -163,19 +163,16 @@ The vault returns plaintext only to a caller whose verified `Secret` token carri re-deciding it. The runner is never given the runtime secret, and it never reaches a sandbox. -**A deployment without the dedicated key loses agent runs against write-only connections**, -and the failure names something else: the run reports "provide the provider key in this run's -environment", which is right for a standalone run and misleading here. The services -middleware therefore warns once, at the point of use, naming the variable to set. The -placeholder is the shipped default in the example env files, so this is the common case, -not an edge one. +**A deployment without the dedicated key cannot start the API.** Failing at startup prevents a +deployment from accepting write-only secrets that its platform runtime can never resolve. The +error names the variable and rejects the well-known example placeholder. **Deployment.** Set `AGENTA_SERVICES_INTERNAL_KEY` to the same value on the API and the -Services container before turning write-only on. It must be independent from +Services container. It must be independent from `AGENTA_AUTH_KEY` and must not be provisioned to web, runner, sandbox, worker, cron, or -migration containers. The API warns at startup when a deployment uses write-only secrets -without one, and a component that seeds write-only rows refuses to seed rather than store -a credential no run can read. +migration containers. The API fails startup when the key is absent or still uses the +placeholder, and a component that seeds write-only rows also refuses to seed rather than +store a credential no run can read. The exchange never mints the grant from the requested `action` alone. It did once, and that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and From fb05f40c54af989d95c022998bd25a5d9b66d901 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 00:11:32 +0200 Subject: [PATCH 25/31] fix(api): preserve unauthorized grant failures --- api/oss/src/middlewares/auth.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index 586e53a20a..302bd48179 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -955,13 +955,7 @@ async def verify_secret_token( auth_context.get("grants") ) except ValueError as exc: - log.debug( - "[auth] secret token unauthorized", - path=request.url.path, - method=request.method, - reason="invalid_token", - ) - raise UnauthorizedException(reason="invalid_token") from exc + raise DecodeError("Secret token contains invalid grants.") from exc request.state.user_id = auth_context.get("user_id") request.state.user_email = auth_context.get("user_email") From 2c9b0c1f00f24c3e658ba10c30fd42cc17ab06be Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 00:27:33 +0200 Subject: [PATCH 26/31] test(api): isolate grant exchange authorization --- api/oss/tests/pytest/unit/access/test_grant_exchange.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index 2b4371187c..9961089c26 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -51,11 +51,17 @@ async def _get_cache(**kwargs): async def _set_cache(**kwargs): return True + async def _check_resource_access(**kwargs): + return True + monkeypatch.setattr( access_router_module, "check_action_access", _check_action_access ) monkeypatch.setattr(access_router_module, "get_cache", _get_cache) monkeypatch.setattr(access_router_module, "set_cache", _set_cache) + monkeypatch.setattr( + access_router_module, "_check_resource_access", _check_resource_access + ) router = AccessRouter() From 6be5ebfba39726716a3133f1243029a12e94c474 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 00:40:55 +0200 Subject: [PATCH 27/31] fix(api): keep managed secrets in their own PR --- api/oss/src/apis/fastapi/vault/router.py | 44 ++---------------- api/oss/src/core/secrets/dtos.py | 14 ------ api/oss/src/core/secrets/interfaces.py | 1 - api/oss/src/core/secrets/services.py | 47 +------------------- api/oss/src/dbs/postgres/secrets/dao.py | 14 ++---- api/oss/src/dbs/postgres/secrets/mappings.py | 34 ++------------ 6 files changed, 12 insertions(+), 142 deletions(-) diff --git a/api/oss/src/apis/fastapi/vault/router.py b/api/oss/src/apis/fastapi/vault/router.py index d8c66a7319..be759aef27 100644 --- a/api/oss/src/apis/fastapi/vault/router.py +++ b/api/oss/src/apis/fastapi/vault/router.py @@ -17,10 +17,6 @@ SecretResponseDTO, PublicSecretResponseDTO, ) -from oss.src.core.secrets.managed import ( - ManagedByIsServerControlledError, - ManagedSecretReadOnlyError, -) from oss.src.core.secrets.redaction import project_secret_response from oss.src.core.access.permissions.types import Permission @@ -127,27 +123,8 @@ def _for_caller( reveal_write_only=request_has_grant(request, SECRET_RESOLVE_GRANT), ) - @staticmethod - def _refuse_client_managed_by(body) -> None: - """`managed_by` states that Agenta provisioned the row; a client may not claim it. - - Rejected rather than ignored: a caller that sent it believes the row will be - managed (or un-managed), and silently dropping the field would leave it wrong - about what the vault now holds. - """ - if body.managed_by is None: - return - - error = ManagedByIsServerControlledError() - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=error.message, - ) - @intercept_exceptions() async def create_secret(self, request: Request, body: CreateSecretDTO): - self._refuse_client_managed_by(body) - has_permission = await check_action_access( user_uid=str(request.state.user_id), project_id=str(request.state.project_id), @@ -232,8 +209,6 @@ async def read_secret(self, request: Request, secret_id_or_slug: str): async def update_secret( self, request: Request, secret_id: str, body: UpdateSecretDTO ): - self._refuse_client_managed_by(body) - has_permission = await check_action_access( user_uid=str(request.state.user_id), project_id=str(request.state.project_id), @@ -258,12 +233,6 @@ async def update_secret( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=e.message ) from e - except ManagedSecretReadOnlyError as e: - # 409, not 400: the payload is well-formed; the stored row's managed state is - # what forbids the change. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, detail=e.message - ) from e if secrets_dto is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Secret not found" @@ -285,13 +254,8 @@ async def delete_secret(self, request: Request, secret_id: str): status_code=403, ) - try: - await self.service.delete_secret( - project_id=UUID(request.state.project_id), - secret_id=UUID(secret_id), - ) - except ManagedSecretReadOnlyError as e: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, detail=e.message - ) from e + await self.service.delete_secret( + project_id=UUID(request.state.project_id), + secret_id=UUID(secret_id), + ) return status.HTTP_204_NO_CONTENT diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index e8aa17563f..5f26183303 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -261,10 +261,6 @@ class CreateSecretDTO(Slug, BaseModel): header: Header secret: SecretDTO write_only: bool = True - # Server-controlled: which platform component provisioned and owns this row (see - # `core/secrets/managed.py`). In-process callers set it; every user-facing route - # rejects a client-supplied value with HTTP 400. - managed_by: Optional[str] = None @model_validator(mode="before") def ensure_header_exists(cls, values): @@ -325,10 +321,6 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): class UpdateSecretDTO(BaseModel): header: Optional[Header] = None secret: Optional[UpdateSecretPayloadDTO] = None - # Server-controlled. None keeps the stored marker; a non-empty string sets it and an - # empty string clears it, both only for in-process callers that pass - # `allow_managed=True` (`ManagedByIsServerControlledError` otherwise). - managed_by: Optional[str] = None @model_validator(mode="before") @classmethod @@ -383,14 +375,8 @@ def build_up_model_keys(self): class SecretResponseDTO(_SecretResponseBaseDTO): """Trusted internal representation. Credential material remains available.""" - # Read-only: present when a platform component owns the row, absent otherwise (the - # vault routes exclude None fields). Users can read and use such a row, but not edit - # or delete it. - managed_by: Optional[str] = None - class PublicSecretResponseDTO(_SecretResponseBaseDTO): """Caller-facing representation after grant-aware value projection.""" - managed_by: Optional[str] = None value_status: SecretValueStatus diff --git a/api/oss/src/core/secrets/interfaces.py b/api/oss/src/core/secrets/interfaces.py index 1721070b64..0b84883649 100644 --- a/api/oss/src/core/secrets/interfaces.py +++ b/api/oss/src/core/secrets/interfaces.py @@ -64,6 +64,5 @@ async def delete( secret_id: UUID, project_id: Optional[UUID] = None, organization_id: Optional[UUID] = None, - authorize_delete: Optional[Callable[[SecretResponseDTO], None]] = None, ) -> None: raise NotImplementedError diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 602a92f625..8ed87b0fc3 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -1,5 +1,4 @@ from typing import Any, Optional -from functools import partial from uuid import UUID, uuid4 from pydantic import ValidationError @@ -27,11 +26,6 @@ UpdateSecretDTO, ) -from oss.src.core.secrets.managed import ( - ManagedByIsServerControlledError, - ManagedSecretReadOnlyError, -) - def next_provider_key_name( *, @@ -185,8 +179,6 @@ def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: def _resolve_update( stored_secret_dto: SecretResponseDTO, requested_update: UpdateSecretDTO, - *, - allow_managed: bool, ) -> UpdateSecretDTO: """Fill this update's omitted credential from the row UNDER THE WRITE LOCK. @@ -199,9 +191,6 @@ def _resolve_update( another kind's or another provider's credential — and that decision reads the same stored row, so it belongs under the same lock. """ - if not allow_managed and stored_secret_dto.managed_by: - raise ManagedSecretReadOnlyError(managed_by=stored_secret_dto.managed_by) - resolved_update = requested_update.model_copy(deep=True) if resolved_update.secret is None: return UpdateSecretDTO.model_validate(resolved_update.model_dump(mode="python")) @@ -239,13 +228,6 @@ def _resolve_update( return UpdateSecretDTO.model_validate(resolved_update.model_dump(mode="python")) -def _authorize_delete( - stored_secret_dto: SecretResponseDTO, *, allow_managed: bool -) -> None: - if not allow_managed and stored_secret_dto.managed_by: - raise ManagedSecretReadOnlyError(managed_by=stored_secret_dto.managed_by) - - def _carry_over_saved_policy(*, stored_data: Any, update_data: Any) -> None: """Fill an update payload's omitted ``models``/``harnesses`` from the stored record. @@ -296,14 +278,6 @@ async def create_secret( create_secret_dto=create_secret_dto, ) - # Managed implies write-only, over both the request and the env default: a managed - # row's credential is one Agenta provisioned and the organization never supplied, - # so there is no state in which handing it back to a user is right. Forced here - # because `write_only` is pinned once stored — a readable managed row could never - # be tightened afterwards. - if create_secret_dto.managed_by: - create_secret_dto.write_only = True - with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): @@ -426,17 +400,7 @@ async def update_secret( project_id: UUID | None = None, organization_id: UUID | None = None, user_id: UUID | None = None, - allow_managed: bool = False, ): - """Update a secret. `allow_managed` is the in-process owner's key to a managed row. - - It defaults to off so that every caller reached from a user-facing route — today's - and tomorrow's — gets the managed guard without opting in. The component that - provisioned a managed row passes True to run its own reconcile or teardown. - """ - if not allow_managed and update_secret_dto.managed_by is not None: - raise ManagedByIsServerControlledError() - with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): @@ -446,10 +410,7 @@ async def update_secret( project_id=project_id, organization_id=organization_id, user_id=user_id, - resolve_update=partial( - _resolve_update, - allow_managed=allow_managed, - ), + resolve_update=_resolve_update, ) if project_id is not None: @@ -461,9 +422,7 @@ async def delete_secret( secret_id: UUID, project_id: UUID | None = None, organization_id: UUID | None = None, - allow_managed: bool = False, ) -> None: - """Delete a secret. A managed row refuses unless the owner passes `allow_managed`.""" with set_data_encryption_key( data_encryption_key=self._data_encryption_key, ): @@ -471,10 +430,6 @@ async def delete_secret( secret_id=secret_id, project_id=project_id, organization_id=organization_id, - authorize_delete=partial( - _authorize_delete, - allow_managed=allow_managed, - ), ) if project_id is not None: diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index 8adc8b787e..0930431e67 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -175,25 +175,17 @@ async def delete( secret_id: UUID, project_id: UUID | None, organization_id: UUID | None, - authorize_delete: Optional[Callable[[SecretResponseDTO], None]] = None, ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) - stmt = ( - select(SecretsDBE) - .filter_by( - id=secret_id, - **scope_filter, - ) - .with_for_update() + stmt = select(SecretsDBE).filter_by( + id=secret_id, + **scope_filter, ) result = await session.execute(stmt) # type: ignore vault_secret_dbe = result.scalar() if vault_secret_dbe is None: return - if authorize_delete is not None: - authorize_delete(map_secrets_dbe_to_dto(secrets_dbe=vault_secret_dbe)) - await session.delete(vault_secret_dbe) await session.commit() diff --git a/api/oss/src/dbs/postgres/secrets/mappings.py b/api/oss/src/dbs/postgres/secrets/mappings.py index f8f7fd2d32..b23d19ef1b 100644 --- a/api/oss/src/dbs/postgres/secrets/mappings.py +++ b/api/oss/src/dbs/postgres/secrets/mappings.py @@ -3,7 +3,6 @@ from datetime import datetime, timezone from oss.src.dbs.postgres.secrets.dbes import SecretsDBE -from oss.src.core.secrets.managed import resolve_managed_by from oss.src.core.secrets.dtos import ( Header, SecretKind, @@ -14,30 +13,23 @@ ) -# Both server-controlled attributes ride inside the (encrypted) `data` JSON, as siblings -# of the payload fields, so no schema migration is needed. They are popped back out in -# `map_secrets_dbe_to_dto`, so payload DTOs never see them; rows without the keys read as -# write_only=False and managed_by=None (legacy, and every user-created row). +# The server-controlled write_only attribute rides inside the encrypted `data` JSON, +# as a sibling of the payload fields, so no schema migration is needed. It is popped back +# out in `map_secrets_dbe_to_dto`, so payload DTOs never see it; rows without the key +# read as write_only=False (legacy rows). _WRITE_ONLY_KEY = "write_only" -_MANAGED_BY_KEY = "managed_by" def _data_payload( data_json: dict, *, write_only: bool, - managed_by: str | None = None, ) -> str: if write_only: data_json[_WRITE_ONLY_KEY] = True else: data_json.pop(_WRITE_ONLY_KEY, None) - if managed_by: - data_json[_MANAGED_BY_KEY] = managed_by - else: - data_json.pop(_MANAGED_BY_KEY, None) - return json.dumps(data_json) @@ -57,7 +49,6 @@ def map_secrets_dto_to_dbe( data=_data_payload( secret_dto.secret.data.model_dump(exclude_none=True), write_only=bool(secret_dto.write_only), - managed_by=secret_dto.managed_by, ), ) return vault_secret_dbe @@ -81,14 +72,6 @@ def map_secrets_dto_to_dbe_update( stored_data = json.loads(secrets_dbe.data) write_only = bool(stored_data.get(_WRITE_ONLY_KEY)) - # Same reason to resolve it here: `data` is rewritten wholesale below, so an omitted - # marker would silently un-manage the row. Only an in-process owner can reach this - # with a non-None value (`VaultService.update_secret` refuses it otherwise). - managed_by = resolve_managed_by( - stored=stored_data.get(_MANAGED_BY_KEY), - requested=update_secret_dto.managed_by, - ) - if update_secret_dto.secret: for key, value in update_secret_dto.secret.model_dump( exclude_none=True @@ -97,22 +80,14 @@ def map_secrets_dto_to_dbe_update( secrets_dbe.data = _data_payload( update_secret_dto.secret.data.model_dump(), write_only=write_only, - managed_by=managed_by, ) elif hasattr(secrets_dbe, key): setattr(secrets_dbe, key, value) - elif update_secret_dto.managed_by is not None: - secrets_dbe.data = _data_payload( - stored_data, - write_only=write_only, - managed_by=managed_by, - ) def map_secrets_dbe_to_dto(*, secrets_dbe: SecretsDBE) -> SecretResponseDTO: data = json.loads(secrets_dbe.data) # type: ignore write_only = bool(data.pop(_WRITE_ONLY_KEY, False)) - managed_by = data.pop(_MANAGED_BY_KEY, None) or None vault_secret_dto = SecretResponseDTO( id=secrets_dbe.id, # type: ignore @@ -125,7 +100,6 @@ def map_secrets_dbe_to_dto(*, secrets_dbe: SecretsDBE) -> SecretResponseDTO: updated_at=str(secrets_dbe.updated_at), ), write_only=write_only, - managed_by=managed_by, ) return vault_secret_dto From 1a2a5deb2b1b8e27bb9f34cd120f5e84ec7eb93d Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 12:01:01 +0200 Subject: [PATCH 28/31] fix(sdk): consume value status for redacted secrets --- api/oss/src/dbs/postgres/secrets/dao.py | 5 +- .../pytest/unit/secrets/test_write_only.py | 28 +- docs/design/write-only-secrets/README.md | 334 ++++++++---------- .../sdk/agents/connections/credentials.py | 10 + .../agenta/sdk/agents/platform/connections.py | 6 +- .../agenta/sdk/agents/platform/secrets.py | 3 +- .../agenta/sdk/middlewares/running/vault.py | 7 +- .../platform/test_write_only_secrets.py | 33 +- 8 files changed, 218 insertions(+), 208 deletions(-) diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index 0930431e67..556177a598 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -131,9 +131,8 @@ async def update( ): async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) - # FOR UPDATE serializes concurrent updates so the one-way write_only check - # below always sees the latest committed flag — two racing updates cannot - # both observe False and let a stale explicit False win. + # FOR UPDATE lets the domain resolver apply immutable policy and carry-over + # against the latest committed row before this transaction persists the update. stmt = ( select(SecretsDBE) .filter_by( diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index b75befa317..407c15ded2 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -1,8 +1,8 @@ """Write-only secrets: the value can be set and replaced, never read back by a user. Covers the three layers below the router: the service (default-on at create, value -carry-over on update, the one-way flag), the redaction helper (per-kind value stripping, -value_status), and the postgres mappings (the flag rides inside the encrypted data +carry-over on update, the immutable creation policy), the redaction helper (per-kind +value stripping and value_status), and the postgres mappings (the flag rides inside the encrypted data JSON and never leaks into the payload DTOs). """ @@ -10,6 +10,8 @@ import pytest +from agenta.sdk.agents.connections.credentials import secret_value_configured + import oss.src.core.secrets.services as secrets_services_module from oss.src.core.secrets.dtos import ( CreateSecretDTO, @@ -297,6 +299,25 @@ async def test_update_without_provider_key_keeps_the_stored_one(service): assert updated.header.name == "OpenAI (renamed)" +@pytest.mark.asyncio +async def test_update_with_an_explicit_blank_provider_key_is_rejected(service): + created = await service.create_secret( + project_id=PROJECT_ID, create_secret_dto=_provider_key_create() + ) + + update = UpdateSecretDTO( + secret={ + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": ""}}, + }, + ) + + with pytest.raises(SecretValueRequiredError): + await service.update_secret( + secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update + ) + + @pytest.mark.asyncio async def test_update_with_a_new_provider_key_replaces_the_stored_one(service): created = await service.create_secret( @@ -784,6 +805,9 @@ def test_redacts_provider_key_and_reports_presence(): assert redacted.data.provider.key is None assert redacted.value_status.configured is True assert redacted.value_status.preview == "sk-****bc" + payload = redacted.model_dump(mode="json", exclude_none=True) + assert secret_value_configured(payload) is True + assert "has_key" not in payload # The input is never mutated: internal readers keep their plaintext DTO. assert secret.data.provider.key == "sk-test-openai-key-bc" diff --git a/docs/design/write-only-secrets/README.md b/docs/design/write-only-secrets/README.md index 4b5fe14b24..26ae675981 100644 --- a/docs/design/write-only-secrets/README.md +++ b/docs/design/write-only-secrets/README.md @@ -1,189 +1,149 @@ # Write-only vault secrets -A vault secret's value can be created, replaced, and deleted — but never read back by a -user. The platform runtime keeps reading it through a granted internal path so runs still -work. This is the GitHub-secrets model. - -Status: backend landed (API + Python SDK), inert by default behind -`AGENTA_VAULT_WRITE_ONLY_DEFAULT=false`. Two PR numbers appear around this work and mean -different things: **#6065** is the frontend package-extraction refactor (merged) that the -web half of this feature builds on, and **#6135** is the branch this backend PR is -stacked on (the per-turn trace-export credential fix), which is a stacking base only and -has nothing to do with secrets. The web half (replace-only forms plus Fern client -regeneration) follows in a second PR, after which the gate flips on. Until then, an -explicitly created `write_only: true` secret shows cosmetically as "not configured" in -today's Settings (the UI does not read `has_key` yet) — accepted; the run path is -unaffected either way. - -## The contract - -### The flag - -- `write_only: bool` on every secret. The default for NEW secrets is env-gated: - **`AGENTA_VAULT_WRITE_ONLY_DEFAULT` (bool, default `false`)**. While off, flag-less - creates behave exactly as today (`write_only: false`); once the web UI ships - replace-only forms, the gate flips to `true` and new secrets default to write-only. - An explicit `write_only` on the create request always wins over the gate, in both - directions — so a caller can opt in to write-only immediately regardless of the - default, and `write_only: false` remains the escape hatch after the flip. -- Existing rows carry no flag and read as `write_only: false`; their behavior is unchanged. -- The flag is **one-way**: an update may tighten `false → true`, but `true → false` is - rejected with HTTP 400 (`WriteOnlyCannotBeDisabledError`). Making a value readable again - would defeat the guarantee; delete and recreate instead. The transition is enforced - atomically: the DAO checks under a `SELECT ... FOR UPDATE` row lock, and the mapper - never clears a stored flag even when handed a stale explicit `false` — concurrent - updates cannot resurrect readability. -- Storage: the flag rides inside the existing encrypted `data` JSON as a sibling key - (`"write_only": true`), popped out at the mapping layer. **No schema migration.** - -### Redaction (user-facing responses) - -For `write_only: true`, every user-facing vault response (create echo, list, get, update -echo) strips the value and adds: - -- `has_key: bool` — whether any credential material is stored (the primary value OR a - credential extra: an AWS-only secret reports `true`). -- `key_preview: str | null` — masked preview of the PRIMARY value only. Policy: values - under 20 characters mask entirely (`****`); from 20 on, at most first 3 + last 3 - characters and never more than 25% of the value (a 20-character value shows 5). - Extras credentials and JSON content never get a preview. One helper: - `oss/src/core/secrets/redaction.py`. - -**One credential classifier.** What counts as credential material is defined once, in the -SDK (`agenta.sdk.agents.connections.credentials`): the primary value field per kind -(`provider.key`; `provider.client_secret` for sso_provider; `secret.content` for -custom_secret) plus the full credential-extras set the SDK resolver consumes (`api_key`, -the `aws_*`/`AWS_*` credential trio and bearer tokens, `ANTHROPIC_AUTH_TOKEN` and the -other provider tokens, `AZURE_OPENAI_API_KEY`, `GOOGLE_APPLICATION_CREDENTIALS`, ...). -The API imports that module for redaction, `has_key`, and update carry-over; a parity -test fails if a resolver-accepted extras key is ever left unclassified. Non-credential -config (URL, region, api_version, project, models, harnesses) stays readable. - -Redaction happens at the response boundary, in every outward surface: - -- the vault routes (`VaultRouter`), for all five endpoints; -- webhook subscription responses (create echo, fetch, edit echo) — the signing value - disappears from responses once its vault record is write-only, while the delivery - signers (the service-internal resolver and the dispatcher's own) keep plaintext; -- the EE organization-provider serialization drops `client_secret`; the SuperTokens - login-time reader keeps plaintext. - -In-process runtime readers (`VaultService` and below) are untouched. - -**No cache.** The list route reads the database on every request. The list is small and -only the settings page reads it, and the runtime path already bypassed the cache, so -caching bought little while making the redaction guarantee depend on what a shared Redis -entry holds and on how a stale reader is kept from repopulating it. Removing the cache -removes that whole class of question: what a caller sees is what the row says, redacted -at the response boundary for every principal without the grant. - -### Updates: keep-stored-on-omit - -On update, an omitted value field means "keep the stored value" (extends the existing -`_carry_over_saved_policy` pattern for `models`/`harnesses`). This covers the standard -provider key, the custom provider key and its credential `extras`, and custom secret -content. **An empty string counts as omitted — this is mandatory, not a convenience**: the -CURRENT frontend's edit form re-sends `key: ""` when it cannot prefill a value, so if `""` -cleared the credential, every edit of a write-only secret through today's UI would wipe -it. An empty credential is never a meaningful value anyway. Values are therefore -replace-only — they cannot be cleared in place. - -This applies to all secrets, not only write-only ones, so update semantics do not fork on -the flag. - -**Keep-on-omit is identity-local.** An update that changes the secret's kind or its -provider family (`data.kind`) must carry an explicit new credential value; omitted or -empty values are rejected with HTTP 400 (`SecretValueRequiredError`), and the old -identity's credential extras never carry over. A stored OpenAI key can never silently -become an Anthropic key, and a kind change can never silently erase the stored value. -(Consequence: a credential-less record — for example an endpoint-only custom provider — -cannot be the target of a kind/family change; delete and recreate it.) - -### The runtime plaintext path: the `secret-resolve` grant - -- Constant: `SECRET_RESOLVE_GRANT = "secret-resolve"` (`oss/src/middlewares/auth.py`). -- It is a **grant**: an additive claim, not a restriction. The runtime's credential is - general-purpose — it authenticates workflows, tools, session coordination, and vault - reads alike — so the plaintext capability rides a `grants` claim that adds one ability - and never narrows what the token can otherwise do. -- Minted in two places: - - `GET /access/permissions/check` attaches the grant to the re-minted `credentials` for - `action=run_service` exchanges — the credential every workflow service and sandbox run - actually uses for its vault reads. - - The workflow invoke/inspect prelude (`sign_secret_token` in - `core/workflows/service.py`) — covers services running with auth middleware disabled, - which use that token directly. -- A verified Secret token carrying the grant receives plaintext from all vault read routes - (write_only is ignored for it). Everyone else — session, ApiKey, unscoped Secret token — - gets the redacted shape. **Strict stance: no transition period for ApiKey callers.** - -Trust line (same as GitHub's): anyone who can run a workload can reach the values through a -run, so the `run_service` exchange hands out the grant. What the flag removes is the casual -read: no session, ApiKey, or list/get call ever returns the value. - -## Consumer impact - -| Consumer | Path | Impact | -| --- | --- | --- | -| Frontend forms | vault routes, session auth | Redacted for write-only secrets; needs replace-only forms (follow-up) | -| Direct API users (ApiKey) | vault routes | Redacted for write-only secrets; no escape hatch besides `write_only: false` at creation | -| Platform runs (playground, deployments, agents) | granted credential via `permissions/check` | Unchanged — plaintext | -| Standalone SDK runs (ApiKey) | `VaultConnectionResolver` | Fail loud: `WriteOnlySecretError`, raised even when config extras survive; remediation = switch the connection to `self_managed` AND set the env variable | -| Standalone SDK legacy services | `VaultMiddleware.get_secrets` | Redacted entries dropped with a clear `log.error`; env-var keys are not shadowed by them | -| Named tool secrets | `resolve_named_secrets` | Redacted entries skipped with a clear `log.error` (no secret names in logs) | -| Webhook subscribers (UI/API responses) | webhook routes | Signing secret disappears from create/fetch/edit responses once its record is write-only; deliveries keep signing | -| EE SSO provider settings | organization-provider routes | `client_secret` dropped once write-only; login flow unaffected | -| In-process runtime readers (SuperTokens login, delivery signing, EE orgs internals) | `VaultService` direct | Unchanged — plaintext | - -## Frontend follow-up (second PR) - -- Replace-only secret forms: no value prefill; show `key_preview`/`has_key`; a "Replace - key" action instead of an editable field. -- Surface `write_only` in the connections/secrets lists. -- Optional "readable" toggle at creation only (maps to `write_only: false`), if product - wants the escape hatch exposed. -- Regenerate the Fern client for `write_only`, `value_status`, and `management.policy`. - - -## Who may read a value: the grant - -The vault returns plaintext only to a caller whose verified `Secret` token carries the -`secret-resolve` grant. Two callers can hold it, and there is no third: - -- **The platform runtime**, on the hop that starts a run. The workflow service exchanges - the END USER's credential at `/access/permissions/check` on their behalf, so nothing - about the presented token says a run is starting — and that route is reachable by a - browser. The runtime therefore proves what it is with a secret only the backend holds - (`AGENTA_SERVICES_INTERNAL_KEY`), sent as `X-Agenta-Runtime-Key` on the internal - hop and compared in constant time. This key has no fallback to - `AGENTA_AUTH_KEY`. If it is missing or remains the well-known placeholder, the API - issues no grant instead of accepting a string anyone could send. -- **A caller refreshing a grant it already holds.** The runner re-exchanges its run - credential every few heartbeats; the exchange carries the grant forward rather than - re-deciding it. The runner is never given the runtime secret, and it never reaches a - sandbox. - -**A deployment without the dedicated key cannot start the API.** Failing at startup prevents a -deployment from accepting write-only secrets that its platform runtime can never resolve. The -error names the variable and rejects the well-known example placeholder. - -**Deployment.** Set `AGENTA_SERVICES_INTERNAL_KEY` to the same value on the API and the -Services container. It must be independent from -`AGENTA_AUTH_KEY` and must not be provisioned to web, runner, sandbox, worker, cron, or -migration containers. The API fails startup when the key is absent or still uses the -placeholder, and a component that seeds write-only rows also refuses to seed rather than -store a credential no run can read. - -The exchange never mints the grant from the requested `action` alone. It did once, and -that made the grant self-serve: `VIEWER_PERMISSIONS` includes both `run_service` and -`view_secret`, so any member could ask for a credential and spend it on the vault routes. - - -## Known gap: cache-key tenancy (elsewhere) - -The platform cache truncates a project id to its last 12 characters, so two projects whose -UUIDs end the same way share an entry in every namespace that caches per project, -including `check_permissions` and `check_action_access`. Server-generated UUID4s make that -remote, and unreachable by a caller who cannot pick their own project id, but it is a -default worth removing. Nothing here depends on it — the vault caches nothing — and the -platform-wide fix is tracked in issue #6166. +A write-only Vault secret can be created, replaced, and deleted, but an ordinary API or +frontend caller cannot read its value back. A trusted platform runtime can resolve the value +through a short-lived granted token so the credential remains usable for runs. + +Status: implementation complete across #6164, #6165, #6138, #6195, and #6174. The backend +and frontend ship in the same release. The feature has no environment gate or compatibility +mode. + +## Value visibility + +`write_only` is a creation-time policy: + +- New ordinary Vault secrets default to `write_only=True`. +- A create request may explicitly select `write_only=False`. +- Existing rows without the stored field resolve as `write_only=False`. +- Update requests cannot set or change `write_only`. +- Changing the policy requires deleting and recreating the secret. + +The policy remains in the existing encrypted `data` JSON. This implementation requires no +database migration. + +SSO and webhook secrets explicitly use `write_only=False`. Their existing settings, login, +test, signing, and verification flows continue to receive the readable value. + +## Public and trusted responses + +The trusted `SecretResponseDTO` contains the stored credential. The caller-facing +`PublicSecretResponseDTO` adds general value status: + +```json +{ + "write_only": true, + "value_status": { + "configured": true, + "preview": "sk-****9Qa" + } +} +``` + +`value_status.configured` reports whether the secret contains credential material. +`value_status.preview` is optional and does not determine whether a value is configured. +The model applies to provider keys, custom secrets, compound provider credentials, SSO, +webhooks, and later secret kinds. + +For an ordinary caller, the response projection removes the primary credential and every +credential-bearing provider extra when `write_only=True`. Non-secret configuration such as +URLs, regions, versions, models, and harnesses remains readable. A verified runtime token +carrying the `secret-resolve` grant receives the plaintext projection. + +The credential-extra vocabulary is shared by the API redaction layer and Python SDK resolver +through `agenta.sdk.agents.connections.credentials`. The same module exposes the SDK helper +that reads `value_status.configured`, so runtime consumers use the public API contract rather +than a key-specific field. + +## Vault list cache + +The Vault service caches canonical trusted DTOs in Redis with the existing namespace, TTL, +invalidation, and shortened project UUID packing. Create, update, and delete invalidate the +project list namespace. + +Caller-specific projection always happens after a cache read: + +```text +list request + -> read canonical SecretResponseDTO values from Redis or the DAO + -> inspect the verified caller grant + -> return plaintext to a granted runtime or redact for an ordinary caller +``` + +Redis is part of the trusted backend boundary. A redacted response is never stored in the +shared list cache. + +## Updates + +Credential updates use one strict contract: + +- An omitted credential field means keep the stored value. +- A non-empty credential replaces the stored value. +- An explicitly supplied empty provider credential is invalid and is rejected. +- An empty JSON object remains an explicitly supplied JSON value and follows the custom-secret + validation rules. + +The frontend ships with the backend and follows the same contract. When a user edits a +write-only connection without typing a replacement credential, the frontend omits the +credential field. It never sends an empty value as the keep signal. + +Carry-over is identity-local. Changing the secret kind, provider family, or custom-secret +format requires an explicit new credential. The service resolves the update against the row +loaded under `SELECT ... FOR UPDATE`, so a concurrent rotation cannot be overwritten by a +stale carried value. + +## Runtime resolution + +The platform uses the `secret-resolve` grant for plaintext runtime access. The grant is +project-scoped and allowlisted on the Vault read routes. + +`AGENTA_SERVICES_INTERNAL_KEY` proves the trusted API-to-Services exchange. It is independent +from `AGENTA_AUTH_KEY`, has no administrator-key fallback, and must contain a non-placeholder +value. The API fails startup when it is missing or invalid. + +Configure the same `AGENTA_SERVICES_INTERNAL_KEY` value on the API and trusted Services +container. Do not provide it to the web app, runner, sandbox, worker, cron, or migration +containers. The runner receives only the signed, short-lived runtime token needed to execute +the authorized workload. + +A caller refreshing an already granted Secret token may carry the grant forward. The exchange +does not create the grant from a requested action alone. + +## Consumer behavior + +| Consumer | Behavior | +| --- | --- | +| Frontend Settings and connection forms | Read `value_status`, display configured state, and omit untouched credential fields on update. | +| Direct API callers | Receive redacted write-only values and general value status. | +| Platform runs | Resolve plaintext through the granted runtime token. | +| Standalone Python SDK | Reads `value_status`; a redacted provider connection uses only the matching provider environment credential or fails with `WriteOnlySecretError`. | +| Legacy SDK Vault middleware | Drops configured redacted entries so they cannot shadow local environment credentials. | +| Named tool secrets | Skip configured redacted values and log an error without the secret name or value. | +| SSO and webhooks | Remain explicitly readable and otherwise unchanged. | +| In-process trusted readers | Continue using the trusted DTO with plaintext. | + +## Managed secrets + +Managed lifecycle and value visibility are independent. A managed row stores typed internal +manager identity and policy in the encrypted JSON payload. The public response exposes only +`management.policy`. + +The starter-credit bridge explicitly creates its connection with +`management.policy=manager_only` and `write_only=True`. General update, delete, and provider +probe operations reject manager-only rows against the current locked record. The frontend +hides these rows from Settings and edit surfaces but keeps them available for model selection, +agent defaults, key gating, and execution. + +## Pull request order + +The release chain is: + +```text +release/v0.114.0 + -> #6164 write-only contract + -> #6165 managed-secret model + -> #6138 starter-credit seeding + -> #6195 stored provider probe + -> #6174 generated clients and frontend consumer +``` + +The five pull requests merge in this order and deploy together in the same release. diff --git a/sdks/python/agenta/sdk/agents/connections/credentials.py b/sdks/python/agenta/sdk/agents/connections/credentials.py index a39f93068a..0a3cef883f 100644 --- a/sdks/python/agenta/sdk/agents/connections/credentials.py +++ b/sdks/python/agenta/sdk/agents/connections/credentials.py @@ -18,6 +18,7 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Dict, FrozenSet # Extras keys (as stored: the UI's snake_case aliases plus raw env-style names) that hold @@ -73,6 +74,15 @@ ) +def secret_value_configured(secret: object) -> bool: + """Whether the public Vault response says credential material is stored.""" + if not isinstance(secret, Mapping): + return False + + value_status = secret.get("value_status") + return isinstance(value_status, Mapping) and value_status.get("configured") is True + + def credential_extras(extras: Dict[str, object]) -> Dict[str, object]: """The subset of ``extras`` holding non-empty credential material.""" return { diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index 08c5d77751..5fecd108cb 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -25,7 +25,7 @@ HARNESS_CONNECTION_CAPABILITIES, PROVIDER_ENV_VARS, ) -from ..connections.credentials import credential_extras +from ..connections.credentials import credential_extras, secret_value_configured from ..connections.endpoints import build_resolved_connection from ..connections import ( AmbiguousConnectionError, @@ -308,7 +308,7 @@ class _ConnectionCandidate: # harness intersection). Neither field filters resolution here yet. models: Optional[List[str]] = None harnesses: Optional[List[str]] = None - # True when the vault says a key exists (write_only + has_key) but this caller's + # True when value_status says a credential exists but this caller's # credential received the redacted, value-less shape. write_only_redacted: bool = False @@ -423,7 +423,7 @@ def _write_only_redacted(secret: Dict[str, Any], has_credential: bool) -> bool: """ return ( bool(secret.get("write_only")) - and bool(secret.get("has_key")) + and secret_value_configured(secret) and not has_credential ) diff --git a/sdks/python/agenta/sdk/agents/platform/secrets.py b/sdks/python/agenta/sdk/agents/platform/secrets.py index c8a5001151..f92ef8a46f 100644 --- a/sdks/python/agenta/sdk/agents/platform/secrets.py +++ b/sdks/python/agenta/sdk/agents/platform/secrets.py @@ -23,6 +23,7 @@ from agenta.sdk.utils.logging import get_module_logger from ..capabilities import PROVIDER_ENV_VARS +from ..connections.credentials import secret_value_configured from .connection import PlatformConnection log = get_module_logger(__name__) @@ -87,7 +88,7 @@ def _is_write_only_redacted(payload: Any) -> bool: return ( isinstance(payload, dict) and bool(payload.get("write_only")) - and bool(payload.get("has_key")) + and secret_value_configured(payload) ) diff --git a/sdks/python/agenta/sdk/middlewares/running/vault.py b/sdks/python/agenta/sdk/middlewares/running/vault.py index 3a7f8469c5..0cfd5ee8f5 100644 --- a/sdks/python/agenta/sdk/middlewares/running/vault.py +++ b/sdks/python/agenta/sdk/middlewares/running/vault.py @@ -10,7 +10,10 @@ from agenta.sdk.utils.exceptions import suppress, display_exception from agenta.sdk.utils.providers import normalize_provider_kind -from agenta.sdk.agents.connections.credentials import credential_extras +from agenta.sdk.agents.connections.credentials import ( + credential_extras, + secret_value_configured, +) from agenta.sdk.models.workflows import WorkflowServiceRequest from agenta.sdk.contexts.running import RunningContext @@ -437,7 +440,7 @@ def _split_write_only_redacted( if ( isinstance(secret, dict) and secret.get("write_only") - and secret.get("has_key") + and secret_value_configured(secret) ): data = secret.get("data") or {} kind = secret.get("kind") diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py index 67fc6ba65f..dd87487c93 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -49,8 +49,7 @@ def _redacted_provider_key(name: str = "OpenAI", provider: str = "openai") -> di "header": {"name": name}, "data": {"kind": provider, "provider": {}}, "write_only": True, - "has_key": True, - "key_preview": "sk-****abc", + "value_status": {"configured": True, "preview": "sk-****abc"}, } @@ -138,7 +137,7 @@ def test_redacted_aws_only_secret_fails_loud_despite_surviving_config_extras(): "provider_slug": "bedrock-conn", }, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } with pytest.raises(WriteOnlySecretError): @@ -168,7 +167,7 @@ def test_a_bedrock_connection_never_falls_back_to_the_family_api_key(monkeypatch "provider_slug": "bedrock-conn", }, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } model = ModelRef( provider="anthropic", @@ -202,7 +201,7 @@ def _redacted_custom(kind: str, slug: str, extras: dict | None = None) -> dict: "provider_slug": slug, }, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } @@ -295,7 +294,7 @@ def test_redacted_custom_provider_fails_loud_too(): "provider_slug": "my-gateway", }, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } model = ModelRef( @@ -323,7 +322,7 @@ def test_a_redacted_gateway_resolves_with_this_runs_key(monkeypatch): "provider_slug": "my-gateway", }, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } resolved = connections._resolve_from_secrets( @@ -376,7 +375,7 @@ def test_partition_drops_redacted_custom_secret_content(): "header": {"name": "gh-token"}, "data": {"secret": {"format": "text"}}, "write_only": True, - "has_key": True, + "value_status": {"configured": True}, } ] ) @@ -390,10 +389,24 @@ def test_partition_drops_redacted_custom_secret_content(): def test_named_secret_redaction_is_detected(): assert _is_write_only_redacted( - {"kind": "custom_secret", "write_only": True, "has_key": True} + { + "kind": "custom_secret", + "write_only": True, + "value_status": {"configured": True}, + } ) assert not _is_write_only_redacted( - {"kind": "custom_secret", "write_only": True, "has_key": False} + { + "kind": "custom_secret", + "write_only": True, + "value_status": {"configured": False}, + } ) assert not _is_write_only_redacted({"kind": "custom_secret"}) assert not _is_write_only_redacted(None) + + +def test_legacy_has_key_is_not_a_supported_response_contract(): + assert not _is_write_only_redacted( + {"kind": "custom_secret", "write_only": True, "has_key": True} + ) From afcb1c8e864fb479a1b0be422a0847c67020975c Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 13:23:35 +0200 Subject: [PATCH 29/31] refactor(api): keep webhook secrets explicitly readable --- api/oss/src/core/webhooks/service.py | 52 ++----- .../unit/webhooks/test_write_only_outward.py | 127 ++---------------- 2 files changed, 26 insertions(+), 153 deletions(-) diff --git a/api/oss/src/core/webhooks/service.py b/api/oss/src/core/webhooks/service.py index 6e1dbb32fd..a138074e32 100644 --- a/api/oss/src/core/webhooks/service.py +++ b/api/oss/src/core/webhooks/service.py @@ -16,7 +16,6 @@ WebhookProviderSettingsDTO, ) from oss.src.core.secrets.enums import SecretKind -from oss.src.core.secrets.redaction import redact_secret_response from oss.src.core.secrets.services import VaultService from oss.src.core.shared.dtos import Status, Windowing from oss.src.core.webhooks.delivery import ( @@ -61,34 +60,6 @@ def _generate_secret(self) -> str: return "".join(secrets.choice(alphabet) for _ in range(32)) - async def _resolve_outward_secret( - self, - *, - project_id: UUID, - # - secret_id: UUID, - ) -> Optional[str]: - """The signing secret as a USER response may carry it: None once write-only. - - Only for response shaping. Internal signing paths (`_resolve_secret` here, the - dispatcher's own resolver) stay plaintext regardless of the flag. - """ - try: - secret_dto = await self.vault_service.get_secret_by_id( - secret_id=secret_id, - project_id=project_id, - ) - - if secret_dto is None: - return None - - return redact_secret_response(secret_dto).data.provider.key - - except Exception as e: # pylint: disable=broad-exception-caught - log.warning(f"Failed to resolve webhook secret {secret_id}: {e}") - - return None - async def _resolve_secret( self, *, @@ -183,12 +154,9 @@ async def create_subscription( secret_id=secret_dto.id, ) - # The create echo goes through redaction anyway: the row is created readable, so - # the value comes back here, and if a user later tightens it by hand every later - # response — this one included, on a re-create — redacts. return self._with_secret( subscription=result, - secret=redact_secret_response(secret_dto).data.provider.key, + secret=secret_value, ) async def test_subscription( @@ -364,7 +332,7 @@ async def fetch_subscription( return None if result.secret_id: - secret_value = await self._resolve_outward_secret( + secret_value = await self._resolve_secret( project_id=project_id, secret_id=result.secret_id, ) @@ -460,12 +428,16 @@ async def edit_subscription( if result is None: return None - # Even a just-provided secret echoes through the outward resolver, so a - # write-only record never comes back — the caller already knows what it sent. - if result.secret_id or secret_id: - secret_value = await self._resolve_outward_secret( + if subscription.secret is not None: + return self._with_secret( + subscription=result, + secret=subscription.secret, + ) + + if result.secret_id: + secret_value = await self._resolve_secret( project_id=project_id, - secret_id=result.secret_id or secret_id, + secret_id=result.secret_id, ) result = self._with_secret( subscription=result, @@ -525,7 +497,7 @@ async def set_subscription_active( return None if result.secret_id: - secret_value = await self._resolve_outward_secret( + secret_value = await self._resolve_secret( project_id=project_id, secret_id=result.secret_id, ) diff --git a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py index 953fe791bb..ea180b43ea 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py +++ b/api/oss/tests/pytest/unit/webhooks/test_write_only_outward.py @@ -1,13 +1,6 @@ -"""Webhook responses are write-only-aware; internal signing keeps plaintext. +"""Webhook signing secrets explicitly remain readable.""" -The signing secret lives in the vault, but it is a SHARED secret: the subscriber verifies -our signature with the same value, so webhook records explicitly opt out of write-only. -Once a legacy record IS write-only, no -USER-facing webhook response — create echo, fetch, edit echo — may carry the value again, -while the internal resolver the signer uses stays plaintext. -""" - -from uuid import UUID, uuid4 +from uuid import uuid4 import pytest @@ -125,16 +118,9 @@ def _subscription_create(): ) -def _mark_write_only(vault_service, secret_id): - stored = vault_service.secrets_dao.records[secret_id] - vault_service.secrets_dao.records[secret_id] = stored.model_copy( - update={"write_only": True} - ) - - @pytest.mark.asyncio async def test_webhook_signing_secret_is_explicitly_readable(services): - webhooks_service, _ = services + webhooks_service, vault_service = services created = await webhooks_service.create_subscription( project_id=PROJECT_ID, @@ -143,6 +129,16 @@ async def test_webhook_signing_secret_is_explicitly_readable(services): ) assert created.secret == "whsec_provided_by_user_12345" + stored = await webhooks_service.dao.fetch_subscription( + project_id=PROJECT_ID, + subscription_id=created.id, + ) + secret_dto = await vault_service.get_secret_by_id( + project_id=PROJECT_ID, + secret_id=stored.secret_id, + ) + assert secret_dto.write_only is False + fetched = await webhooks_service.fetch_subscription( project_id=PROJECT_ID, subscription_id=created.id, @@ -179,73 +175,6 @@ async def test_generated_secret_is_returned_on_the_create_echo(services): assert created.secret == signing_value -@pytest.mark.asyncio -async def test_explicit_readable_secret_keeps_existing_responses(services): - webhooks_service, _ = services - - created = await webhooks_service.create_subscription( - project_id=PROJECT_ID, - user_id=USER_ID, - subscription=_subscription_create(), - ) - assert created.secret == "whsec_provided_by_user_12345" - - fetched = await webhooks_service.fetch_subscription( - project_id=PROJECT_ID, - subscription_id=created.id, - ) - assert fetched.secret == "whsec_provided_by_user_12345" - - -@pytest.mark.asyncio -async def test_legacy_write_only_secret_stops_appearing_in_fetches(services): - webhooks_service, vault_service = services - - created = await webhooks_service.create_subscription( - project_id=PROJECT_ID, - user_id=USER_ID, - subscription=_subscription_create(), - ) - assert created.secret is not None - - stored = await webhooks_service.fetch_subscription( - project_id=PROJECT_ID, subscription_id=created.id - ) - _mark_write_only(vault_service, UUID(str(stored.secret_id))) - - fetched = await webhooks_service.fetch_subscription( - project_id=PROJECT_ID, - subscription_id=created.id, - ) - assert fetched.secret is None - - -@pytest.mark.asyncio -async def test_internal_resolver_keeps_plaintext_for_signing(services): - webhooks_service, vault_service = services - - created = await webhooks_service.create_subscription( - project_id=PROJECT_ID, - user_id=USER_ID, - subscription=_subscription_create(), - ) - - stored = await webhooks_service.dao.fetch_subscription( - project_id=PROJECT_ID, subscription_id=created.id - ) - _mark_write_only(vault_service, UUID(str(stored.secret_id))) - - stored = await webhooks_service.dao.fetch_subscription( - project_id=PROJECT_ID, subscription_id=created.id - ) - signing_value = await webhooks_service._resolve_secret( - project_id=PROJECT_ID, - secret_id=stored.secret_id, - ) - - assert signing_value == "whsec_provided_by_user_12345" - - @pytest.mark.asyncio async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_value( services, @@ -282,32 +211,4 @@ async def test_rotating_the_signing_secret_through_edit_replaces_the_stored_valu ) assert signing_value == "whsec_test_rotated" - - -@pytest.mark.asyncio -async def test_rotation_echo_stays_redacted_for_a_write_only_secret(services): - webhooks_service, vault_service = services - - created = await webhooks_service.create_subscription( - project_id=PROJECT_ID, - user_id=USER_ID, - subscription=_subscription_create(), - ) - - stored = await webhooks_service.dao.fetch_subscription( - project_id=PROJECT_ID, subscription_id=created.id - ) - _mark_write_only(vault_service, UUID(str(stored.secret_id))) - - edited = await webhooks_service.edit_subscription( - project_id=PROJECT_ID, - user_id=USER_ID, - subscription=WebhookSubscriptionEdit( - id=created.id, - name="notify", - data=WebhookSubscriptionData(url="https://example.com/hook"), - secret="whsec_test_rotated", - ), - ) - - assert edited.secret is None + assert edited.secret == "whsec_test_rotated" From 324e20b55f2da9753238659176fc40c699262ed5 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 15:20:17 +0200 Subject: [PATCH 30/31] fix(api): clarify blank secret updates --- api/oss/src/core/secrets/services.py | 10 +++++++-- .../pytest/unit/secrets/test_write_only.py | 8 ++++++- .../unit/vault/test_write_only_routes.py | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index 8ed87b0fc3..f4babb4f82 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -27,6 +27,12 @@ ) +_BLANK_CREDENTIAL_VALUE_MESSAGE = ( + "Credential values cannot be blank. Omit an unchanged credential field or provide a new " + "value." +) + + def next_provider_key_name( *, kind: StandardProviderKind, @@ -95,7 +101,7 @@ def _carry_over_saved_value(*, kind: str, stored_data: Any, update_data: Any) -> ): current_value = getattr(update_container, field) if current_value == "": - raise SecretValueRequiredError() + raise SecretValueRequiredError(message=_BLANK_CREDENTIAL_VALUE_MESSAGE) if current_value is None: stored_value = getattr(stored_container, field, None) if stored_value is not None: @@ -169,7 +175,7 @@ def _carry_over_saved_extras(*, stored_data: Any, update_data: Any) -> None: stored_value = stored_extras.get(extras_key) requested_value = update_extras.get(extras_key) if requested_value == "": - raise SecretValueRequiredError() + raise SecretValueRequiredError(message=_BLANK_CREDENTIAL_VALUE_MESSAGE) if stored_value is not None and ( extras_key not in update_extras or requested_value is None ): diff --git a/api/oss/tests/pytest/unit/secrets/test_write_only.py b/api/oss/tests/pytest/unit/secrets/test_write_only.py index 407c15ded2..f797f145c5 100644 --- a/api/oss/tests/pytest/unit/secrets/test_write_only.py +++ b/api/oss/tests/pytest/unit/secrets/test_write_only.py @@ -312,7 +312,13 @@ async def test_update_with_an_explicit_blank_provider_key_is_rejected(service): }, ) - with pytest.raises(SecretValueRequiredError): + with pytest.raises( + SecretValueRequiredError, + match=( + "Credential values cannot be blank. Omit an unchanged credential field or provide " + "a new value." + ), + ): await service.update_secret( secret_id=created.id, project_id=PROJECT_ID, update_secret_dto=update ) diff --git a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py index b7e27c169f..9986821375 100644 --- a/api/oss/tests/pytest/unit/vault/test_write_only_routes.py +++ b/api/oss/tests/pytest/unit/vault/test_write_only_routes.py @@ -294,6 +294,27 @@ def test_kind_or_family_change_without_a_new_value_is_400(harness): assert "credential value" in response.json()["detail"] +def test_explicit_blank_credential_has_a_clear_400(harness): + client = harness + created = _create(client, write_only=True) + + response = client.put( + f"/secrets/{created['id']}", + json={ + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": ""}}, + } + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"] == ( + "Credential values cannot be blank. Omit an unchanged credential field or provide a " + "new value." + ) + + # --- real signed tokens through the real verifier -------------------------------------- SECRET_KEY = "unit-test-secret-key-with-32-bytes" From 52d3ca57021f167511ff4fe43da2e04c819c3d23 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 23 Aug 2026 16:07:14 +0200 Subject: [PATCH 31/31] fix(secrets): isolate custom gateway credentials --- .../pytest/unit/access/test_grant_exchange.py | 21 ++++++++++++--- .../agenta/sdk/agents/platform/connections.py | 7 +++++ .../platform/test_write_only_secrets.py | 27 ++++++++++--------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/api/oss/tests/pytest/unit/access/test_grant_exchange.py b/api/oss/tests/pytest/unit/access/test_grant_exchange.py index 9961089c26..05e374acb4 100644 --- a/api/oss/tests/pytest/unit/access/test_grant_exchange.py +++ b/api/oss/tests/pytest/unit/access/test_grant_exchange.py @@ -40,10 +40,10 @@ def _exchange(monkeypatch): monkeypatch.setattr(auth_module, "_SECRET_KEY", SECRET_KEY) monkeypatch.setattr(env.agenta, "services_internal_key", RUNTIME_KEY) - verdict = {"allow": True} + verdict = {"action": True, "resource": True} async def _check_action_access(**kwargs): - return verdict["allow"] + return verdict["action"] async def _get_cache(**kwargs): return None @@ -52,7 +52,7 @@ async def _set_cache(**kwargs): return True async def _check_resource_access(**kwargs): - return True + return verdict["resource"] monkeypatch.setattr( access_router_module, "check_action_access", _check_action_access @@ -237,7 +237,7 @@ async def test_non_run_actions_never_receive_the_grant(exchange): @pytest.mark.asyncio async def test_denied_exchange_returns_no_credential_at_all(exchange): run, verdict = exchange - verdict["allow"] = False + verdict["action"] = False from fastapi import HTTPException @@ -247,6 +247,19 @@ async def test_denied_exchange_returns_no_credential_at_all(exchange): assert raised.value.status_code == 403 +@pytest.mark.asyncio +async def test_resource_denial_returns_no_runtime_credential(exchange): + run, verdict = exchange + verdict["resource"] = False + + from fastapi import HTTPException + + with pytest.raises(HTTPException) as raised: + await run("run_service", runtime_key=RUNTIME_KEY) + + assert raised.value.status_code == 403 + + @pytest.mark.asyncio async def test_missing_action_is_denied(exchange): run, _ = exchange diff --git a/sdks/python/agenta/sdk/agents/platform/connections.py b/sdks/python/agenta/sdk/agents/platform/connections.py index 5fecd108cb..1144f25dee 100644 --- a/sdks/python/agenta/sdk/agents/platform/connections.py +++ b/sdks/python/agenta/sdk/agents/platform/connections.py @@ -202,6 +202,13 @@ def _credential_channels( return [("GOOGLE_APPLICATION_CREDENTIALS",)] if candidate.deployment == "azure": return [("AZURE_OPENAI_API_KEY",)] + # A caller-selected endpoint owns its stored credential. An ambient family key must + # never be sent to that endpoint when the vault value is hidden. + if candidate.kind == "custom_provider" and ( + (candidate.endpoint and candidate.endpoint.base_url) + or candidate.endpoint_blocked + ): + return [] env_var = _provider_env_var(provider) or _provider_env_var(candidate.provider) return [(env_var,)] if env_var else [] diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py index dd87487c93..92bac7ac6d 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_write_only_secrets.py @@ -309,8 +309,9 @@ def test_redacted_custom_provider_fails_loud_too(): ) -def test_a_redacted_gateway_resolves_with_this_runs_key(monkeypatch): - monkeypatch.setenv("OPENAI_API_KEY", "sk-gateway-env") +def test_a_redacted_gateway_never_uses_this_runs_provider_key(monkeypatch): + ambient_key = "sk-gateway-env" + monkeypatch.setenv("OPENAI_API_KEY", ambient_key) redacted = { "kind": "custom_provider", "slug": "my-gateway", @@ -325,18 +326,18 @@ def test_a_redacted_gateway_resolves_with_this_runs_key(monkeypatch): "value_status": {"configured": True}, } - resolved = connections._resolve_from_secrets( - secrets=[redacted], - model=ModelRef( - provider="openai", - model="gpt-5.5", - connection={"mode": "agenta", "slug": "my-gateway"}, - ), - harness="pi_core", - ) + with pytest.raises(WriteOnlySecretError) as raised: + connections._resolve_from_secrets( + secrets=[redacted], + model=ModelRef( + provider="openai", + model="gpt-5.5", + connection={"mode": "agenta", "slug": "my-gateway"}, + ), + harness="pi_core", + ) - env = {item.binding.name: item.value for item in resolved.credentials} - assert env["OPENAI_API_KEY"] == "sk-gateway-env" + assert ambient_key not in str(raised.value) # --- the vault middleware's list partition ---------------------------------------------