From eb83f19c97d6dc79470072e08b58caf79fdecc44 Mon Sep 17 00:00:00 2001 From: howlcipher Date: Thu, 13 Aug 2026 15:06:52 -0400 Subject: [PATCH] fix(deploy): bind Azure lifecycle evidence --- README.md | 3 +- docs/devsecops.md | 6 +- docs/roadmap.md | 12 +- docs/safety.md | 26 +-- .../api_platform/infra/terraform/README.md | 6 +- projects/api_platform/infra/terraform/main.tf | 18 +- .../api_platform/infra/terraform/variables.tf | 11 ++ pyproject.toml | 3 + src/devops_learn/tools/azure_tool.py | 46 ++++- src/devops_learn/tools/docker_tool.py | 12 +- src/devops_learn/tools/terraform_tool.py | 22 ++- src/devops_learn/workflows/deploy_flow.py | 177 ++++++++++++++++-- tests/deployment/test_candidate.py | 20 ++ tests/integration/test_azure_lifecycle.py | 86 +++++++++ tests/tools/test_azure_tool.py | 50 ++++- tests/tools/test_real_terraform_tool.py | 27 +++ tests/workflows/test_deploy_flow.py | 116 ++++++++++++ 17 files changed, 573 insertions(+), 68 deletions(-) create mode 100644 tests/integration/test_azure_lifecycle.py create mode 100644 tests/workflows/test_deploy_flow.py diff --git a/README.md b/README.md index 0fa6d29..766583a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,8 @@ Project -> Analysis -> Requirements -> Questions -> Recommendations -> Architect - **Real Terraform execution** (`devops-learn terraform`) runs actual `terraform fmt`/`init`/`validate`/`plan` against a real, committed Azure configuration, with structured `terraform show -json` parsing feeding a - deterministic risk classifier. No `apply`/`destroy` yet; `plan` requires + deterministic risk classifier. This standalone command has no `apply`/`destroy`; + `plan` requires Azure authentication and fails cleanly, with an explanation, without it. - **Simulated cloud/Kubernetes** execution for safe learning without credentials or cost (`devops-learn analyze` without `--real-tools`; diff --git a/docs/devsecops.md b/docs/devsecops.md index 401adfb..1d8d5fb 100644 --- a/docs/devsecops.md +++ b/docs/devsecops.md @@ -144,6 +144,6 @@ not marked live-verified until an opt-in Azure run succeeds. Image scanning requires a locally built image and Docker availability. The first comparison implementation does not solve every move, rename, or package -upgrade equivalence. Policy overrides, real Terraform plan scanning, Azure -apply, runtime detection, and Kubernetes deployment are intentionally out of -scope. +upgrade equivalence. Policy overrides, full Terraform-plan content scanning, +runtime vulnerability detection, and Kubernetes deployment are intentionally +out of scope. diff --git a/docs/roadmap.md b/docs/roadmap.md index 93863f1..91c7436 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -35,10 +35,10 @@ structured parsing of `terraform show -json` feeding the existing `validation/terraform_plan_analysis.analyze()` risk classifier unchanged. `devops-learn terraform` runs the full vertical slice (fmt -> init -> validate -> plan -> risk analysis) and teaches Terraform state, provider, -and resource-address concepts just-in-time (`docs/terraform-state.md`). No -`apply`; `apply_approved_plan`/`destroy_approved_environment` are not even -declared on `RealTerraformTool` and stay simulated until Milestone 3. `plan` -requires Azure authentication and fails cleanly, with an explanation, on a +and resource-address concepts just-in-time (`docs/terraform-state.md`). This +standalone command has no apply action; the separately approved lifecycle +commands introduced in Milestone 3 own real apply and destroy. `plan` requires +Azure authentication and fails cleanly, with an explanation, on a machine without it — `init`/`validate`/`fmt` need no credentials. ## Milestone 3: first Azure vertical slice (implemented, awaiting live verification) @@ -64,8 +64,8 @@ Conftest policy. The deterministic gate includes structural secret redaction, audit/experience evidence, remediation classification, a report artifact, demo, and PR CI. See `docs/devsecops.md`. -The next dependency is the real Terraform plan plus Azure deployment path, -where this gate must be required before any real apply. +The gate is now bound to each deployment candidate before any real apply. Live +Azure verification remains the final opt-in acceptance step. ## Further out (not yet milestoned) diff --git a/docs/safety.md b/docs/safety.md index 892e785..76993b9 100644 --- a/docs/safety.md +++ b/docs/safety.md @@ -74,20 +74,22 @@ resources, applies Terraform, or modifies remote state. It: ## Real Terraform execution -`devops-learn terraform` is the only command that uses `RealTerraformTool` -(`tools/terraform_tool.py`) — `devops-learn analyze --real-tools` and `devops-learn local` both -still use `SimulatedTerraformTool` for the `terraform` tool name; only `python`/`docker` become -real for those commands. `RealTerraformTool` declares only `fmt`/`init`/`validate`/`plan` — -`apply_approved_plan` and `destroy_approved_environment` are real-only Tool -operations. Apply accepts only a saved plan in `.devops_learn/plans` and +`devops-learn terraform`, `devops-learn deploy`, and `devops-learn destroy` +use `RealTerraformTool` (`tools/terraform_tool.py`). `devops-learn analyze +--real-tools` and `devops-learn local` still use `SimulatedTerraformTool` for +the `terraform` tool name. The standalone `terraform` command exposes only +`fmt`/`init`/`validate`/`plan`; `apply_approved_plan` and +`destroy_approved_environment` are real-only operations reached through the +explicit lifecycle commands. Apply accepts only a saved plan in `.devops_learn/plans` and rechecks its digest, source identity, Terraform configuration digest, and candidate context before invoking Terraform. Destroy requires an explicit target environment and is followed by a read-only Azure resource-group check. Neither operation can fall back to simulation; normal CI does not call either. -Every subprocess call goes through `tools/_subprocess_safety.py::run_safely`, which sets an -explicit timeout and always redacts secret-shaped `KEY=value` text and truncates long output -before it reaches a `ToolResult`. `plan()`'s parsing of `terraform show -json` never extracts +Terraform, Docker, and Azure CLI subprocess calls go through +`tools/_subprocess_safety.py::run_safely`, which sets an explicit timeout and +always redacts secret-shaped `KEY=value` text and truncates long output before +it reaches a `ToolResult`. `plan()`'s parsing of `terraform show -json` never extracts raw resource attribute values (`before`/`after`) — only `{address, action}` per resource change — which is the primary defense against leaking sensitive plan data; free-text redaction is defense in depth on top of that. @@ -113,7 +115,9 @@ do support a dry run that skips approval when the operation is simulated. ## Cost awareness -No cloud action in V1 can create a billable resource; `cloud.estimate_cost` deliberately never -returns a specific dollar figure, since V1 has no real pricing data to draw from. +The real-only Azure lifecycle can create billable learning resources only after +explicit confirmations and tool-level approval. `cloud.estimate_cost` still +deliberately never returns a specific dollar figure, since it has no live +pricing data to draw from. Cost impact in a `Recommendation` (e.g. "lower likely cost than a managed cluster") is always qualitative for the same reason. diff --git a/projects/api_platform/infra/terraform/README.md b/projects/api_platform/infra/terraform/README.md index a081afb..9d3e28c 100644 --- a/projects/api_platform/infra/terraform/README.md +++ b/projects/api_platform/infra/terraform/README.md @@ -1,7 +1,9 @@ # infra/terraform -This is a real, valid Terraform configuration for a minimal Azure footprint: -a resource group, a container registry, and a Log Analytics workspace. +This is a real, valid Terraform configuration for a minimal Azure Container +Apps learning footprint: a resource group, container registry, managed +identity with `AcrPull`, Log Analytics workspace, Container Apps Environment, +and (after the image stage) the FastAPI Container App. ## Files in this directory diff --git a/projects/api_platform/infra/terraform/main.tf b/projects/api_platform/infra/terraform/main.tf index df711d4..3fe90c7 100644 --- a/projects/api_platform/infra/terraform/main.tf +++ b/projects/api_platform/infra/terraform/main.tf @@ -7,19 +7,23 @@ locals { tags = { project = "devops-learn-by-doing" - environment = "learning" + environment = var.environment managed-by = "terraform" purpose = "devops-learning" } + resource_prefix = "${var.project_name}-${var.environment}" + # Azure Container Registry names must be alphanumeric only (no hyphens), # unlike most other Azure resource names -- strip them here rather than # relaxing var.project_name itself. - registry_name = replace(var.project_name, "-", "") + registry_name = "${replace(local.resource_prefix, "-", "")}${substr(md5(data.azurerm_client_config.current.subscription_id), 0, 6)}acr" } +data "azurerm_client_config" "current" {} + resource "azurerm_resource_group" "main" { - name = "${var.project_name}-rg" + name = "${local.resource_prefix}-rg" location = var.location tags = local.tags } @@ -33,7 +37,7 @@ resource "azurerm_container_registry" "main" { } resource "azurerm_user_assigned_identity" "container_app" { - name = "${var.project_name}-app-identity" + name = "${local.resource_prefix}-app-identity" resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location tags = local.tags @@ -46,7 +50,7 @@ resource "azurerm_role_assignment" "container_app_acr_pull" { } resource "azurerm_log_analytics_workspace" "main" { - name = "${var.project_name}-logs" + name = "${local.resource_prefix}-logs" resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location sku = "PerGB2018" @@ -55,7 +59,7 @@ resource "azurerm_log_analytics_workspace" "main" { } resource "azurerm_container_app_environment" "main" { - name = "${var.project_name}-env" + name = "${local.resource_prefix}-env" resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location log_analytics_workspace_id = azurerm_log_analytics_workspace.main.id @@ -64,7 +68,7 @@ resource "azurerm_container_app_environment" "main" { resource "azurerm_container_app" "api" { count = var.deploy_application ? 1 : 0 - name = "${var.project_name}-api" + name = "${local.resource_prefix}-api" resource_group_name = azurerm_resource_group.main.name container_app_environment_id = azurerm_container_app_environment.main.id revision_mode = "Single" diff --git a/projects/api_platform/infra/terraform/variables.tf b/projects/api_platform/infra/terraform/variables.tf index f69bb34..5c1a01a 100644 --- a/projects/api_platform/infra/terraform/variables.tf +++ b/projects/api_platform/infra/terraform/variables.tf @@ -10,6 +10,17 @@ variable "location" { default = "eastus" } +variable "environment" { + description = "Learning environment label included in every resource name and tag." + type = string + default = "learning" + + validation { + condition = can(regex("^[a-z0-9-]{2,20}$", var.environment)) + error_message = "environment must be 2-20 lowercase letters, numbers, or hyphens." + } +} + variable "deploy_application" { description = "Set after the ACR image has been pushed. Bootstrap stays image-independent." type = bool diff --git a/pyproject.toml b/pyproject.toml index d352f88..8de909c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,3 +48,6 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests", "projects/api_platform/tests"] pythonpath = ["projects/api_platform"] +markers = [ + "azure_integration: creates and destroys the opt-in Azure learning environment", +] diff --git a/src/devops_learn/tools/azure_tool.py b/src/devops_learn/tools/azure_tool.py index b979d5c..304da0b 100644 --- a/src/devops_learn/tools/azure_tool.py +++ b/src/devops_learn/tools/azure_tool.py @@ -120,12 +120,36 @@ def execute( ] ok, payload, error = _json(command) safe = _container_app_view(payload) + logs = _subprocess_safety.run_safely( + [ + az, + "containerapp", + "logs", + "show", + "--name", + app, + "--resource-group", + resource_group, + "--tail", + "50", + ], + cwd=None, + timeout=45, + ) return ToolResult( ok, "(real) collected Container App evidence" if ok else "(real, failed) could not collect Container App evidence", - safe if ok else {"error": error}, + ( + { + **safe, + "logs": logs.stdout if logs.returncode == 0 else logs.stderr, + "logs_collected": logs.returncode == 0, + } + if ok + else {"error": error} + ), spec.risk_level, False, approval, @@ -235,17 +259,23 @@ def _verify_environment( _container_app_view(payload) if name == "container_app" else _resource_view(payload) ) expected_region = str(params.get("expected_region", "")).lower() - actual_region = str(observed["resource_group"].get("location", "")).lower() expected_tags = params.get("expected_tags", {}) - actual_tags = observed["resource_group"].get("tags", {}) - valid = (not expected_region or expected_region == actual_region) and all( - actual_tags.get(k) == v for k, v in expected_tags.items() + expected_image = str(params.get("expected_image", "")) + resources_match = all( + (not expected_region or str(resource.get("location", "")).lower() == expected_region) + and all(resource.get("tags", {}).get(k) == v for k, v in expected_tags.items()) + for resource in observed.values() + ) + image_matches = ( + not expected_image + or observed["container_app"].get("image") == expected_image ) + valid = resources_match and image_matches return ToolResult( valid, "(real) DECLARED BY TERRAFORM matches OBSERVED IN AZURE" if valid - else "(real, failed) Azure observation does not match declared region or tags", + else "(real, failed) Azure observation does not match declared region, tags, or image", observed, spec.risk_level, False, @@ -267,12 +297,14 @@ def _container_app_view(payload: dict[str, Any]) -> dict[str, Any]: template = properties.get("template", {}) containers = template.get("containers", []) ingress = properties.get("configuration", {}).get("ingress", {}) + fqdn = ingress.get("fqdn") + endpoint = f"https://{fqdn}" if fqdn and not str(fqdn).startswith("http") else fqdn return { "name": payload.get("name"), "location": payload.get("location"), "tags": payload.get("tags", {}), "image": containers[0].get("image") if containers else None, - "endpoint": ingress.get("fqdn"), + "endpoint": endpoint, "revision": properties.get("latestRevisionName"), "running_status": properties.get("runningStatus"), } diff --git a/src/devops_learn/tools/docker_tool.py b/src/devops_learn/tools/docker_tool.py index 726c987..5908cdf 100644 --- a/src/devops_learn/tools/docker_tool.py +++ b/src/devops_learn/tools/docker_tool.py @@ -8,9 +8,9 @@ from __future__ import annotations import shutil -import subprocess from typing import Any, Mapping +from devops_learn.tools import _subprocess_safety from devops_learn.tools.approval import ApprovalRecord from devops_learn.tools.base import RiskLevel, Tool, ToolOperationSpec, ToolResult @@ -62,13 +62,9 @@ def _docker_command() -> str: return command -def _run(command: list[str], *, check: bool = False) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - capture_output=True, - text=True, - check=check, - ) +def _run(command: list[str]) -> _subprocess_safety.SafeRunResult: + """Run a bounded Docker command without exposing raw CLI output.""" + return _subprocess_safety.run_safely(command, cwd=None, timeout=180) class SimulatedDockerTool(Tool): diff --git a/src/devops_learn/tools/terraform_tool.py b/src/devops_learn/tools/terraform_tool.py index 1011d38..c72226a 100644 --- a/src/devops_learn/tools/terraform_tool.py +++ b/src/devops_learn/tools/terraform_tool.py @@ -244,9 +244,12 @@ def _variable_args(params: Mapping[str, Any]) -> list[str]: variables = params.get("variables", {}) if not isinstance(variables, Mapping): raise ValueError("Terraform variables must be a mapping.") - allowed = {"deploy_application", "app_image"} + allowed = {"deploy_application", "app_image", "location", "environment"} if set(variables) - allowed: - raise ValueError("Only deploy_application and app_image are accepted by this workflow.") + raise ValueError( + "Only deploy_application, app_image, location, and environment are accepted " + "by this workflow." + ) args: list[str] = [] for name in sorted(variables): value = variables[name] @@ -254,6 +257,10 @@ def _variable_args(params: Mapping[str, Any]) -> list[str]: raise ValueError("deploy_application must be boolean.") if name == "app_image" and (not isinstance(value, str) or "@sha256:" not in value): raise ValueError("app_image must be a digest-pinned image reference.") + if name == "location" and (not isinstance(value, str) or not value.strip()): + raise ValueError("location must be a non-empty Azure region name.") + if name == "environment" and (not isinstance(value, str) or not value.strip()): + raise ValueError("environment must be a non-empty deployment environment name.") args.extend(["-var", f"{name}={str(value).lower() if isinstance(value, bool) else value}"]) return args @@ -548,6 +555,16 @@ def _apply( "config_digest": terraform_config_digest(working_dir), "plan_digest": _digest_file(plan_path), } + candidate_identity = params.get("candidate_identity") + if not isinstance(candidate_identity, str) or not candidate_identity: + return ToolResult( + False, + "(real, refused) apply requires the approved deployment candidate identity", + {"reason": "candidate identity was not supplied"}, + spec.risk_level, + False, + approval, + ) if any(not value or metadata.get(key) != value for key, value in expected.items()): return ToolResult( False, @@ -572,6 +589,7 @@ def _apply( "stdout": result.stdout, "stderr": result.stderr, "plan_digest": expected["plan_digest"], + "candidate_identity": candidate_identity, }, spec.risk_level, False, diff --git a/src/devops_learn/workflows/deploy_flow.py b/src/devops_learn/workflows/deploy_flow.py index 5c90bef..9618209 100644 --- a/src/devops_learn/workflows/deploy_flow.py +++ b/src/devops_learn/workflows/deploy_flow.py @@ -49,6 +49,7 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: infra = project / "infra" / "terraform" report_path = project / "artifacts" / "deployment" / "azure-deployment-evidence.json" stages: dict[str, str] = {} + observed: dict[str, Any] | None = None if options.cloud != "azure" or not infra.is_dir(): ui.present( "This real lifecycle currently supports the bundled Azure Terraform project only." @@ -105,7 +106,9 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: source = _source_identity(project) config_digest = terraform_config_digest(infra) - bootstrap_security = _security(platform, ui, project, options, local_tag, "bootstrap") + bootstrap_security = _security( + platform, ui, project, options, local_tag, "bootstrap", session_id + ) if bootstrap_security is None: stages["bootstrap_security"] = "REAL FAIL" return @@ -122,13 +125,22 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: created_at=datetime.now(timezone.utc), ) bootstrap_candidate = _plan_and_gate( - platform, ui, infra, bootstrap, {"deploy_application": False} + platform, + ui, + infra, + bootstrap, + { + "deploy_application": False, + "environment": options.environment, + "location": options.location, + }, + session_id, ) if bootstrap_candidate is None: stages["bootstrap_plan_or_eligibility"] = "REAL FAIL OR NOT APPROVED" return stages["bootstrap_plan_or_eligibility"] = "REAL APPROVED" - if not _apply(platform, infra, bootstrap_candidate): + if not _apply(platform, project, infra, bootstrap_candidate, session_id): stages["bootstrap_apply"] = "REAL FAIL OR APPROVAL DENIED" return stages["bootstrap_apply"] = "REAL PASS" @@ -174,7 +186,7 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: stages["image_digest"] = "REAL PASS" application_security = _security( - platform, ui, project, options, image_digest_ref, "application" + platform, ui, project, options, image_digest_ref, "application", session_id ) if application_security is None: stages["application_security"] = "REAL FAIL" @@ -198,16 +210,32 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: ui, infra, application, - {"deploy_application": True, "app_image": image_digest_ref}, + { + "deploy_application": True, + "app_image": image_digest_ref, + "environment": options.environment, + "location": options.location, + }, + session_id, ) if application_candidate is None: stages["application_plan_or_eligibility"] = "REAL FAIL OR NOT APPROVED" return - if not _apply(platform, infra, application_candidate): + stages["application_plan_or_eligibility"] = "REAL APPROVED" + if not _apply(platform, project, infra, application_candidate, session_id): stages["application_apply"] = "REAL FAIL OR APPROVAL DENIED" return stages["application_apply"] = "REAL PASS" - observed = _verify_azure(platform, outputs.details, options) + application_outputs = platform.tool_service.invoke( + "terraform", "output", {"path": str(infra)} + ) + if not application_outputs.success: + stages["application_outputs"] = "REAL FAIL" + return + stages["application_outputs"] = "REAL PASS" + observed = _verify_azure( + platform, application_outputs.details, options, image_digest_ref + ) if observed is None: stages["azure_verification"] = "REAL FAIL" return @@ -219,10 +247,17 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: "azure", "container_app_evidence", { - "resource_group": outputs.details["resource_group_name"], - "container_app_name": outputs.details["container_app_name"], + "resource_group": application_outputs.details["resource_group_name"], + "container_app_name": application_outputs.details["container_app_name"], }, ) + _audit( + platform, + session_id, + AuditEventType.TROUBLESHOOTING_STARTED, + "Health verification failed; collected Container App revision and log evidence", + {"candidate_identity": application_candidate.identity}, + ) return stages["health_verification"] = "REAL PASS" _audit( @@ -255,9 +290,13 @@ def run_deploy_flow(platform: Platform, ui: Ui, options: DeployOptions) -> None: if stages.get("health_verification") == "REAL PASS" else "stopped", stages=stages, + observed_azure=observed, ) platform.session_service.complete(session) - ui.present(f"Sanitized lifecycle evidence: {report_path}") + if report_path.is_file(): + ui.present(f"Sanitized lifecycle evidence: {report_path}") + else: + ui.present("No deployment candidate was created; no lifecycle evidence was written.") def _preflight(platform: Platform, ui: Ui, options: DeployOptions) -> bool: @@ -295,7 +334,13 @@ def _validation(platform: Platform, project: Path) -> bool: def _security( - platform: Platform, ui: Ui, project: Path, options: DeployOptions, image: str, stage: str + platform: Platform, + ui: Ui, + project: Path, + options: DeployOptions, + image: str, + stage: str, + session_id: int, ) -> tuple[Path, SecurityGateDecision] | None: path = project / "artifacts" / "deployment" / f"security-{stage}.json" report = run_security_scan( @@ -303,6 +348,17 @@ def _security( ) if report is None: return None + _audit( + platform, + session_id, + AuditEventType.SECURITY_GATE_EVALUATED, + f"{stage.title()} security evidence bound to deployment candidate", + { + "report_digest": sha256_file(path), + "decision": report.policy.decision.value, + "image": image, + }, + ) return path, report.policy.decision @@ -312,6 +368,7 @@ def _plan_and_gate( infra: Path, candidate: DeploymentCandidate, variables: dict[str, object], + session_id: int, ) -> DeploymentCandidate | None: if not platform.tool_service.invoke("terraform", "fmt", {"path": str(infra)}).success: return None @@ -345,6 +402,16 @@ def _plan_and_gate( if candidate.security_decision == SecurityGateDecision.REQUIRE_APPROVAL.value: prompt = f"Security approval for candidate {completed.identity[:12]} required. Approve?" security_approval = ui.confirm(prompt, default=False) + if not security_approval: + eligibility = evaluate_deployment_eligibility( + validation_passed=True, + gate=SecurityGateDecision.REQUIRE_APPROVAL, + approval_granted=False, + ) + ui.present( + f"DEPLOYMENT ELIGIBILITY: {eligibility.eligibility.value} ({eligibility.reason})" + ) + return None risk_prompt = ( f"Terraform plan risk is {risk.name}. " f"Approve this exact candidate {completed.identity[:12]}?" @@ -352,18 +419,42 @@ def _plan_and_gate( risk_approval = risk.name not in {"HIGH", "DESTRUCTIVE"} or ui.confirm( risk_prompt, default=False ) + if not risk_approval: + ui.present("DEPLOYMENT ELIGIBILITY: ineligible (Terraform plan risk was not approved.)") + return None + cloud_action_approval = ui.confirm( + "Cloud action approval: apply this exact candidate " + f"{completed.identity[:12]} to Azure?", + default=False, + ) eligibility = evaluate_deployment_eligibility( validation_passed=True, gate=SecurityGateDecision(candidate.security_decision or "block"), - approval_granted=security_approval and risk_approval, + approval_granted=security_approval and risk_approval and cloud_action_approval, + ) + ui.present( + f"DEPLOYMENT ELIGIBILITY: {eligibility.eligibility.value} ({eligibility.reason})" ) - ui.present(f"DEPLOYMENT ELIGIBILITY: {eligibility.eligibility.value} ({eligibility.reason})") if eligibility.eligibility is DeploymentEligibility.ELIGIBLE: approvals: list[str] = [] if candidate.security_decision == SecurityGateDecision.REQUIRE_APPROVAL.value: approvals.append("security") if risk.name in {"HIGH", "DESTRUCTIVE"}: approvals.append("terraform_plan_risk") + approvals.append("cloud_action") + _audit( + platform, + session_id, + AuditEventType.USER_APPROVED_PLAN, + "Human approved exact deployment candidate", + { + "candidate_identity": completed.identity, + "plan_digest": completed.terraform_plan_digest, + "security_report_digest": completed.security_report_digest, + "image_digest": completed.image_digest, + "approvals": approvals, + }, + ) return replace( completed, deployment_eligibility=eligibility.eligibility.value, @@ -372,10 +463,25 @@ def _plan_and_gate( return None -def _apply(platform: Platform, infra: Path, candidate: DeploymentCandidate) -> bool: - if not candidate.is_current() or candidate.terraform_config_digest != terraform_config_digest( - infra +def _apply( + platform: Platform, + project: Path, + infra: Path, + candidate: DeploymentCandidate, + session_id: int, +) -> bool: + if ( + not candidate.is_current() + or candidate.terraform_config_digest != terraform_config_digest(infra) + or candidate.source_revision != _source_identity(project) ): + _audit( + platform, + session_id, + AuditEventType.DEPLOYMENT_FAILED, + "Deployment candidate became stale before apply", + {"candidate_identity": candidate.identity}, + ) return False result = platform.tool_service.invoke( "terraform", @@ -388,11 +494,30 @@ def _apply(platform: Platform, infra: Path, candidate: DeploymentCandidate) -> b "source_revision": candidate.source_revision, }, ) + _audit( + platform, + session_id, + AuditEventType.TOOL_INVOKED, + ( + "Approved Terraform apply completed" + if result.success + else "Approved Terraform apply failed" + ), + { + "candidate_identity": candidate.identity, + "plan_digest": candidate.terraform_plan_digest, + "returncode": result.details.get("returncode"), + "approved_by": result.approval.approved_by if result.approval else None, + }, + ) return result.success def _verify_azure( - platform: Platform, outputs: Mapping[str, Any], options: DeployOptions + platform: Platform, + outputs: Mapping[str, Any], + options: DeployOptions, + expected_image: str, ) -> dict[str, Any] | None: result = platform.tool_service.invoke( "azure", @@ -404,6 +529,7 @@ def _verify_azure( "container_app_name": outputs["container_app_name"], "expected_region": options.location, "expected_tags": {"environment": options.environment, "managed-by": "terraform"}, + "expected_image": expected_image, }, ) return dict(result.details) if result.success else None @@ -440,12 +566,19 @@ def _source_identity(project: Path) -> str: return f"{git_sha}:workspace-{digest.hexdigest()}" -def _audit(platform: Platform, session_id: int, event_type: AuditEventType, summary: str) -> None: +def _audit( + platform: Platform, + session_id: int, + event_type: AuditEventType, + summary: str, + payload: Mapping[str, Any] | None = None, +) -> None: platform.audit_service.record( session_id=session_id, event_type=event_type, occurred_at=datetime.now(timezone.utc), summary=summary, + payload=payload, ) @@ -455,7 +588,13 @@ def run_cleanup_flow(platform: Platform, ui: Ui, options: DeployOptions) -> bool infra = project / "infra" / "terraform" if not infra.is_dir() or not _preflight(platform, ui, options): return False - resource_group = "api-platform-rg" + outputs = platform.tool_service.invoke("terraform", "output", {"path": str(infra)}) + resource_group = str(outputs.details.get("resource_group_name", "")) + if not outputs.success or not resource_group: + ui.present( + "DESTROY REFUSED: Terraform state does not identify a deployed learning resource group." + ) + return False ui.present( "DESTROY TARGET\n\n" f"Resource group: {resource_group}\nEnvironment: {options.environment}\n" diff --git a/tests/deployment/test_candidate.py b/tests/deployment/test_candidate.py index 80da091..c237614 100644 --- a/tests/deployment/test_candidate.py +++ b/tests/deployment/test_candidate.py @@ -1,4 +1,5 @@ from dataclasses import replace +from pathlib import Path import pytest @@ -37,3 +38,22 @@ def test_candidate_requires_evidence_paths_for_digests() -> None: replace(_candidate(), terraform_plan_digest="plan") with pytest.raises(ValueError, match="report path"): replace(_candidate(), security_report_path=None) + + +def test_candidate_detects_changed_saved_evidence(tmp_path: Path) -> None: + report = tmp_path / "security.json" + plan = tmp_path / "approved.tfplan" + report.write_text('{"decision":"allow"}\n') + plan.write_bytes(b"approved-plan") + from devops_learn.deployment.candidate import sha256_file + + candidate = replace( + _candidate(), + security_report_path=str(report), + security_report_digest=sha256_file(report), + terraform_plan_path=str(plan), + terraform_plan_digest=sha256_file(plan), + ) + assert candidate.is_current() + plan.write_bytes(b"changed-plan") + assert not candidate.is_current() diff --git a/tests/integration/test_azure_lifecycle.py b/tests/integration/test_azure_lifecycle.py new file mode 100644 index 0000000..5bef35e --- /dev/null +++ b/tests/integration/test_azure_lifecycle.py @@ -0,0 +1,86 @@ +"""Opt-in acceptance test for the real Azure learning lifecycle. + +This is intentionally skipped in normal CI. Setting both environment variables +is an affirmative authorization to create the low-cost lab and destroy it after +the health-check evidence has been written. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROJECT = REPO_ROOT / "projects" / "api_platform" +ENVIRONMENT = "azure-integration" + + +@pytest.mark.azure_integration +def test_real_azure_lifecycle_creates_verifies_and_cleans_up() -> None: + if os.environ.get("RUN_AZURE_INTEGRATION_TESTS") != "1": + pytest.skip("Set RUN_AZURE_INTEGRATION_TESTS=1 to run Azure integration tests.") + if os.environ.get("AZURE_DEPLOYMENT_APPROVED") != "1": + pytest.fail( + "Set AZURE_DEPLOYMENT_APPROVED=1 to explicitly authorize this real Azure test." + ) + + environment = os.environ.copy() + environment["PYTHONPATH"] = str(REPO_ROOT / "src") + deploy = [ + sys.executable, + "-m", + "devops_learn.cli.main", + "deploy", + str(PROJECT), + "--cloud", + "azure", + "--environment", + ENVIRONMENT, + ] + destroy = [ + sys.executable, + "-m", + "devops_learn.cli.main", + "destroy", + str(PROJECT), + "--environment", + ENVIRONMENT, + ] + approvals = "y\n" * 8 + state_path = PROJECT / "infra" / "terraform" / "terraform.tfstate" + try: + completed = subprocess.run( + deploy, + cwd=REPO_ROOT, + env=environment, + input=approvals, + capture_output=True, + text=True, + timeout=1800, + check=False, + ) + output = completed.stdout + completed.stderr + evidence_path = PROJECT / "artifacts" / "deployment" / "azure-deployment-evidence.json" + assert completed.returncode == 0, output + assert evidence_path.is_file(), output + evidence = json.loads(evidence_path.read_text()) + assert evidence["status"] == "completed", output + assert evidence["stages"]["health_verification"] == "REAL PASS", output + finally: + if state_path.is_file(): + cleanup = subprocess.run( + destroy, + cwd=REPO_ROOT, + env=environment, + input="y\ny\n", + capture_output=True, + text=True, + timeout=900, + check=False, + ) + assert cleanup.returncode == 0, cleanup.stdout + cleanup.stderr diff --git a/tests/tools/test_azure_tool.py b/tests/tools/test_azure_tool.py index 8786aca..0efcb70 100644 --- a/tests/tools/test_azure_tool.py +++ b/tests/tools/test_azure_tool.py @@ -5,7 +5,8 @@ AutoApproveApprovalGate, AutoDenyApprovalGate, ) -from devops_learn.tools.azure_tool import _container_app_view +from devops_learn.tools import azure_tool +from devops_learn.tools.azure_tool import AzureCliTool, _container_app_view from devops_learn.tools.base import RiskLevel, Tool, ToolOperationSpec, ToolResult from devops_learn.tools.service import ToolService @@ -29,7 +30,7 @@ def test_container_app_observation_is_sanitized_to_deployment_facts() -> None: "location": "eastus", "tags": {"environment": "learning"}, "image": "acr.azurecr.io/api@sha256:abc", - "endpoint": "api.example.invalid", + "endpoint": "https://api.example.invalid", "revision": "api--one", "running_status": "Running", } @@ -46,6 +47,51 @@ def test_destroy_operation_requires_toolservice_approval() -> None: assert approved.success +def test_azure_verification_rejects_a_different_deployed_image(monkeypatch) -> None: + payloads = { + "group": {"name": "rg", "location": "eastus", "tags": {"environment": "learning"}}, + "acr": {"name": "acr", "location": "eastus", "tags": {"environment": "learning"}}, + "env": {"name": "env", "location": "eastus", "tags": {"environment": "learning"}}, + "app": { + "name": "app", + "location": "eastus", + "tags": {"environment": "learning"}, + "properties": { + "configuration": {"ingress": {"fqdn": "app.example.invalid"}}, + "template": {"containers": [{"image": "acr/api@sha256:old"}]}, + }, + }, + } + + def fake_json(command: list[str], *, timeout: int = 45): + if command[1:3] == ["group", "show"]: + return True, payloads["group"], "" + if command[1:3] == ["acr", "show"]: + return True, payloads["acr"], "" + if command[1:4] == ["containerapp", "env", "show"]: + return True, payloads["env"], "" + return True, payloads["app"], "" + + monkeypatch.setattr(azure_tool, "_az_command", lambda: "az") + monkeypatch.setattr(azure_tool, "_json", fake_json) + result = AzureCliTool().execute( + "verify_environment", + { + "resource_group": "rg", + "acr_name": "acr", + "container_environment_name": "env", + "container_app_name": "app", + "expected_region": "eastus", + "expected_tags": {"environment": "learning"}, + "expected_image": "acr/api@sha256:new", + }, + dry_run=False, + approval=None, + ) + assert not result.success + assert result.details["container_app"]["endpoint"] == "https://app.example.invalid" + + class _DestroySpy(Tool): @property def name(self) -> str: diff --git a/tests/tools/test_real_terraform_tool.py b/tests/tools/test_real_terraform_tool.py index a429047..94ef99c 100644 --- a/tests/tools/test_real_terraform_tool.py +++ b/tests/tools/test_real_terraform_tool.py @@ -208,6 +208,7 @@ def test_saved_plan_binds_digest_source_config_and_candidate(monkeypatch, tmp_pa "plan_path": planned.details["plan_path"], "source_revision": "source-a", "candidate_context": "candidate-a", + "candidate_identity": "candidate-identity-a", }, dry_run=False, approval=approved, @@ -231,6 +232,7 @@ def test_apply_refuses_changed_source_or_plan(monkeypatch, tmp_path) -> None: "plan_path": planned.details["plan_path"], "source_revision": "source-b", "candidate_context": "candidate-a", + "candidate_identity": "candidate-identity-a", }, dry_run=False, approval=ApprovalRecord(granted=True, approved_by="test"), @@ -244,6 +246,7 @@ def test_apply_refuses_changed_source_or_plan(monkeypatch, tmp_path) -> None: "plan_path": planned.details["plan_path"], "source_revision": "source-a", "candidate_context": "candidate-a", + "candidate_identity": "candidate-identity-a", }, dry_run=False, approval=ApprovalRecord(granted=True, approved_by="test"), @@ -251,6 +254,30 @@ def test_apply_refuses_changed_source_or_plan(monkeypatch, tmp_path) -> None: assert not changed_plan.success +def test_apply_refuses_missing_candidate_identity(monkeypatch, tmp_path) -> None: + _use_fake_terraform(monkeypatch) + tool = RealTerraformTool() + planned = tool.execute( + "plan", + {"path": str(tmp_path), "source_revision": "source-a", "candidate_context": "candidate-a"}, + dry_run=False, + approval=None, + ) + result = tool.execute( + "apply_approved_plan", + { + "path": str(tmp_path), + "plan_path": planned.details["plan_path"], + "source_revision": "source-a", + "candidate_context": "candidate-a", + }, + dry_run=False, + approval=ApprovalRecord(granted=True, approved_by="test"), + ) + assert not result.success + assert "candidate identity" in result.summary + + def test_dry_run_never_invokes_subprocess(monkeypatch, tmp_path) -> None: monkeypatch.setenv("PATH", str(tmp_path)) # empty dir: would fail loudly if invoked result = RealTerraformTool().execute( diff --git a/tests/workflows/test_deploy_flow.py b/tests/workflows/test_deploy_flow.py new file mode 100644 index 0000000..7ce7417 --- /dev/null +++ b/tests/workflows/test_deploy_flow.py @@ -0,0 +1,116 @@ +import os +import sqlite3 +from pathlib import Path + +from devops_learn.bootstrap import build_platform +from devops_learn.deployment.candidate import DeploymentCandidate, sha256_file +from devops_learn.domain.enums import ( + CloudProviderKind, + CostPriority, + EnvironmentKind, + ExecutionMode, + ExplanationDepth, + SecurityGateDecision, +) +from devops_learn.domain.question_models import ClarifyingQuestion +from devops_learn.tools.approval import AutoApproveApprovalGate +from devops_learn.tools.terraform_tool import RealTerraformTool, terraform_config_digest +from devops_learn.workflows.deploy_flow import _plan_and_gate +from devops_learn.workflows.ui import Ui + +FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "fake_terraform" + + +class ScriptedUi(Ui): + def __init__(self, confirmations: list[bool]) -> None: + self.confirmations = confirmations + self.presented: list[str] = [] + + def present(self, text: str) -> None: + self.presented.append(text) + + def ask_choice(self, question: ClarifyingQuestion) -> str: + return question.options[0] if question.options else "yes" + + def confirm(self, prompt: str, *, default: bool = False) -> bool: + return self.confirmations.pop(0) + + +def _platform(conn: sqlite3.Connection): + return build_platform( + conn, tools={"terraform": RealTerraformTool()}, approval_gate=AutoApproveApprovalGate() + ) + + +def _candidate(infra: Path, report: Path, decision: SecurityGateDecision) -> DeploymentCandidate: + return DeploymentCandidate( + source_revision="source-a", + project_path="projects/api_platform", + cloud="azure", + environment="learning", + terraform_config_digest=terraform_config_digest(infra), + security_report_path=str(report), + security_report_digest=sha256_file(report), + security_decision=decision.value, + ) + + +def _session_id(platform, infra: Path) -> int: + session = platform.session_service.start( + project_root=str(infra), + mode=ExecutionMode.COLLABORATIVE, + explanation_depth=ExplanationDepth.NORMAL, + cloud=CloudProviderKind.AZURE, + environment=EnvironmentKind.DEV, + cost_priority=CostPriority.LOWEST_COST, + simulation_mode=False, + ) + assert session.id is not None + return session.id + + +def test_blocked_security_evidence_stops_before_cloud_action( + conn: sqlite3.Connection, monkeypatch, tmp_path: Path +) -> None: + monkeypatch.setenv("PATH", f"{FIXTURES_DIR}{os.pathsep}{os.environ['PATH']}") + report = tmp_path / "security.json" + report.write_text('{"decision":"block"}\n') + platform = _platform(conn) + ui = ScriptedUi([]) + + result = _plan_and_gate( + platform, + ui, + tmp_path, + _candidate(tmp_path, report, SecurityGateDecision.BLOCK), + {"deploy_application": False, "location": "eastus"}, + _session_id(platform, tmp_path), + ) + + assert result is None + assert "SECURITY BLOCK" in "\n".join(ui.presented) + assert ui.confirmations == [] + + +def test_required_security_and_cloud_approval_bind_the_saved_plan( + conn: sqlite3.Connection, monkeypatch, tmp_path: Path +) -> None: + monkeypatch.setenv("PATH", f"{FIXTURES_DIR}{os.pathsep}{os.environ['PATH']}") + report = tmp_path / "security.json" + report.write_text('{"decision":"require_approval"}\n') + platform = _platform(conn) + ui = ScriptedUi([True, True]) + + result = _plan_and_gate( + platform, + ui, + tmp_path, + _candidate(tmp_path, report, SecurityGateDecision.REQUIRE_APPROVAL), + {"deploy_application": False, "location": "eastus"}, + _session_id(platform, tmp_path), + ) + + assert result is not None + assert result.human_approvals == ("security", "cloud_action") + assert result.terraform_plan_path is not None + assert Path(result.terraform_plan_path).is_file()