From a2feb940528ae0dab17a5db91e589e80f914fdda Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Mon, 18 May 2026 21:04:33 +0700 Subject: [PATCH 1/9] chore(harness): add agent quality gates --- .github/pull_request_template.md | 31 +++++++++ .github/workflows/ci.yml | 43 ++++++++++++ README.md | 4 +- docs/implementation-plan.md | 43 ++++++++++++ eslint.config.mjs | 8 +++ package.json | 9 ++- src/commands/.gitkeep | 0 src/config/.gitkeep | 0 src/core/.gitkeep | 0 src/dumpers/.gitkeep | 0 src/encryption/.gitkeep | 0 src/notifications/.gitkeep | 0 src/storage/.gitkeep | 0 tool/check-architecture.mjs | 107 +++++++++++++++++++++++++++++ tool/check-commit-message.mjs | 46 +++++++++++++ tool/check-docs.mjs | 92 +++++++++++++++++++++++++ tool/check-env-example.mjs | 41 +++++++++++ tool/check-project-map.mjs | 49 +++++++++++++ tool/check-security.mjs | 69 +++++++++++++++++++ tool/check-source-hygiene.mjs | 69 +++++++++++++++++++ tool/lints/architecture-rules.json | 44 ++++++++++++ tool/verify.mjs | 47 +++++++++++++ 22 files changed, 700 insertions(+), 2 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 src/commands/.gitkeep create mode 100644 src/config/.gitkeep create mode 100644 src/core/.gitkeep create mode 100644 src/dumpers/.gitkeep create mode 100644 src/encryption/.gitkeep create mode 100644 src/notifications/.gitkeep create mode 100644 src/storage/.gitkeep create mode 100644 tool/check-architecture.mjs create mode 100644 tool/check-commit-message.mjs create mode 100644 tool/check-docs.mjs create mode 100644 tool/check-env-example.mjs create mode 100644 tool/check-project-map.mjs create mode 100644 tool/check-security.mjs create mode 100644 tool/check-source-hygiene.mjs create mode 100644 tool/lints/architecture-rules.json create mode 100644 tool/verify.mjs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..56f3acf --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ +## Summary + +- What changed: +- Why it changed: + +## Backup Risk Review + +- [ ] No backup creation, retention, restore, encryption, or storage behavior changed. +- [ ] If backup behavior changed, restore impact is explained below. +- [ ] If retention behavior changed, deletion risk is explained below. +- [ ] If encryption behavior changed, key handling and recovery impact are explained below. +- [ ] If storage provider behavior changed, bucket/path/credential impact is explained below. + +## Operational Evidence + +- [ ] `pnpm verify` passed locally. +- [ ] New or changed harness rules have clear failure messages. +- [ ] No production secrets or real credentials are included. +- [ ] Docs or exec plans were updated when behavior changed. + +## Restore And Rollback Notes + +Describe how to verify restore safety, roll back the change, or recover from a failed run: + +- + +## Deployment Notes + +Mention required env/config changes, schedule changes, or migration steps: + +- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..737035f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + - main + +permissions: + contents: read + +jobs: + verify: + name: Verify + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Validate latest commit message + run: pnpm harness:commit + + - name: Run verification + run: pnpm verify diff --git a/README.md b/README.md index 76c35c9..27af202 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Reusable multi-project backup runner for database dumps, encrypted external stor Draft planning stage. -Start with the engineering proposal: +Start with the engineering proposal and implementation plan: - [Reusable Multi-Project Backup Runner Proposal](docs/reusable-backup-runner-proposal.md) +- [Implementation Plan](docs/implementation-plan.md) +- [Harness Engineering Proposal](docs/harness-engineering-proposal.md) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 6dd5a2e..a9658f8 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -132,6 +132,49 @@ Acceptance criteria: - Repo has no production logic yet. - README points to proposal and implementation plan. +### Phase 1 Harness Smoke Evidence + +Status: Passed on 18 May 2026. + +Verification executed: + +```bash +pnpm verify +``` + +Result: + +- format, lint, type-check, tests, and build passed; +- architecture harness passed; +- source hygiene harness passed; +- docs harness passed; +- env harness passed; +- project-map harness passed; +- security harness passed; +- commit message harness passed. + +Negative smoke tests executed: + +```bash +pnpm harness:commit -- --message "bad commit message" +``` + +Expected failure confirmed: + +- invalid commit message was rejected with the required `type(scope): message` format and allowed commit types. + +Temporary architecture violation created locally: + +```text +src/core/harness-smoke.ts -> src/storage/harness-smoke.ts +``` + +Expected failure confirmed: + +- architecture harness rejected `src/core/**` importing runtime adapter code from `src/storage/**`. + +The temporary smoke files were removed after the negative test, and the clean gate passed again. The harness is ready to guard Phase 2 product implementation. + ## Phase 2 — Config Foundation Goal: build typed, validated configuration before any backup behavior. diff --git a/eslint.config.mjs b/eslint.config.mjs index 1af5706..caab168 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -6,6 +6,14 @@ export default tseslint.config( ignores: ["dist/**", "node_modules/**", "coverage/**"], }, js.configs.recommended, + { + files: ["**/*.mjs"], + languageOptions: { + globals: { + process: "readonly", + }, + }, + }, { files: ["**/*.ts"], extends: [ diff --git a/package.json b/package.json index 7074c53..80754dd 100644 --- a/package.json +++ b/package.json @@ -16,10 +16,17 @@ "build": "tsc -p tsconfig.build.json", "format": "prettier --write .", "format:check": "prettier --check .", + "harness:architecture": "node tool/check-architecture.mjs", + "harness:commit": "node tool/check-commit-message.mjs", + "harness:docs": "node tool/check-docs.mjs", + "harness:env": "node tool/check-env-example.mjs", + "harness:project-map": "node tool/check-project-map.mjs", + "harness:security": "node tool/check-security.mjs", + "harness:source": "node tool/check-source-hygiene.mjs", "lint": "eslint .", "test": "vitest run", "type-check": "tsc --noEmit", - "verify": "pnpm format:check && pnpm lint && pnpm type-check && pnpm test && pnpm build" + "verify": "node tool/verify.mjs" }, "devDependencies": { "@eslint/js": "^9.39.1", diff --git a/src/commands/.gitkeep b/src/commands/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/config/.gitkeep b/src/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/core/.gitkeep b/src/core/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/dumpers/.gitkeep b/src/dumpers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/encryption/.gitkeep b/src/encryption/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/notifications/.gitkeep b/src/notifications/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/storage/.gitkeep b/src/storage/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tool/check-architecture.mjs b/tool/check-architecture.mjs new file mode 100644 index 0000000..cd5c4c9 --- /dev/null +++ b/tool/check-architecture.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const rulesPath = path.join(root, "tool/lints/architecture-rules.json"); + +const fail = (messages) => { + process.stderr.write( + `Architecture harness failed:\n${messages.join("\n")}\n` + ); + process.exit(1); +}; + +const toPosix = (value) => value.split(path.sep).join("/"); + +const walk = (dir) => { + if (!existsSync(dir)) return []; + + const out = []; + for (const entry of readdirSync(dir)) { + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory()) { + out.push(...walk(fullPath)); + continue; + } + + if (/\.(ts|tsx)$/.test(entry)) { + out.push(fullPath); + } + } + + return out; +}; + +const globToRegExp = (glob) => { + const escaped = glob + .replaceAll(".", "\\.") + .replaceAll("/", "\\/") + .replaceAll("**", "__DOUBLE_STAR__") + .replaceAll("*", "[^/]*") + .replaceAll("__DOUBLE_STAR__", ".*"); + return new RegExp(`^${escaped}$`); +}; + +const matchesGlob = (filePath, glob) => globToRegExp(glob).test(filePath); + +const importPattern = + /(?:import|export)\s+(?:type\s+)?(?:[\s\S]*?\s+from\s+)?["']([^"']+)["']/g; + +const resolveImport = (fromFile, specifier) => { + if (!specifier.startsWith(".")) return null; + + const base = path.resolve(path.dirname(fromFile), specifier); + const candidates = [ + base, + `${base}.ts`, + `${base}.tsx`, + path.join(base, "index.ts"), + path.join(base, "index.tsx"), + ]; + + const resolved = candidates.find((candidate) => existsSync(candidate)); + return resolved ? toPosix(path.relative(root, resolved)) : null; +}; + +if (!existsSync(rulesPath)) { + fail([ + `Missing architecture rules file: ${toPosix(path.relative(root, rulesPath))}`, + ]); +} + +const config = JSON.parse(readFileSync(rulesPath, "utf8")); +const rules = Array.isArray(config.rules) ? config.rules : []; +const violations = []; + +for (const file of walk(path.join(root, "src"))) { + const relativeFile = toPosix(path.relative(root, file)); + const source = readFileSync(file, "utf8"); + const imports = [...source.matchAll(importPattern)].map((match) => match[1]); + + for (const specifier of imports) { + const resolved = resolveImport(file, specifier); + if (!resolved) continue; + + for (const rule of rules) { + if (!matchesGlob(relativeFile, rule.from)) continue; + + for (const disallowed of rule.disallow) { + if (matchesGlob(resolved, disallowed)) { + violations.push( + `- ${rule.name}: ${relativeFile} imports ${resolved} via ${specifier}` + ); + } + } + } + } +} + +if (violations.length > 0) { + fail(violations); +} + +process.stdout.write("Architecture harness passed.\n"); diff --git a/tool/check-commit-message.mjs b/tool/check-commit-message.mjs new file mode 100644 index 0000000..38a18d2 --- /dev/null +++ b/tool/check-commit-message.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; + +const args = process.argv.slice(2); +const messageFlagIndex = args.indexOf("--message"); +const message = + messageFlagIndex >= 0 + ? args[messageFlagIndex + 1] + : execFileSync("git", ["log", "-1", "--pretty=%B"], { + encoding: "utf8", + }); + +const firstLine = message?.trim().split("\n")[0] ?? ""; +const allowedTypes = [ + "feat", + "fix", + "docs", + "test", + "refactor", + "chore", + "ci", + "build", + "perf", + "revert", +]; + +const typeGroup = allowedTypes.join("|"); +const conventionalCommitPattern = new RegExp( + `^(${typeGroup})\\([a-z0-9-]+\\): .{1,100}$` +); + +if (!conventionalCommitPattern.test(firstLine)) { + process.stderr.write( + [ + "Commit message harness failed:", + `- Received: ${firstLine || ""}`, + "- Expected: type(scope): message", + `- Allowed types: ${allowedTypes.join(", ")}`, + "- Example: chore(harness): add strict verification gate", + ].join("\n") + "\n" + ); + process.exit(1); +} + +process.stdout.write("Commit message harness passed.\n"); diff --git a/tool/check-docs.mjs b/tool/check-docs.mjs new file mode 100644 index 0000000..5f7333c --- /dev/null +++ b/tool/check-docs.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const ownedDocRoots = ["README.md", "AGENTS.md", "docs"]; + +const requiredFiles = [ + "README.md", + "AGENTS.md", + "docs/reusable-backup-runner-proposal.md", + "docs/implementation-plan.md", + "docs/harness-engineering-proposal.md", + "docs/engineering/agent-pr-loop.md", + "docs/engineering/architecture.md", + "docs/engineering/guardrails.md", + "docs/engineering/security.md", + "docs/engineering/testing.md", + "docs/exec-plans/README.md", + "docs/exec-plans/_template.md", +]; + +const requiredReadmeLinks = [ + "docs/reusable-backup-runner-proposal.md", + "docs/implementation-plan.md", +]; + +const fail = (messages) => { + process.stderr.write(`Docs harness failed:\n${messages.join("\n")}\n`); + process.exit(1); +}; + +const walkMarkdown = (dir) => { + if (!existsSync(dir)) return []; + + const out = []; + for (const entry of readdirSync(dir)) { + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + if (stat.isDirectory()) { + out.push(...walkMarkdown(fullPath)); + } else if (entry.endsWith(".md")) { + out.push(fullPath); + } + } + return out; +}; + +const problems = []; + +for (const file of requiredFiles) { + if (!existsSync(path.join(root, file))) { + problems.push(`- Missing required doc: ${file}`); + } +} + +const readme = readFileSync(path.join(root, "README.md"), "utf8"); +for (const link of requiredReadmeLinks) { + if (!readme.includes(link)) { + problems.push(`- README.md must link to ${link}`); + } +} + +const localMarkdownLinkPattern = /\[[^\]]+\]\(([^)#][^)]+\.md)(?:#[^)]+)?\)/g; +const markdownFiles = ownedDocRoots.flatMap((entry) => { + const fullPath = path.join(root, entry); + if (!existsSync(fullPath)) return []; + if (statSync(fullPath).isDirectory()) return walkMarkdown(fullPath); + return entry.endsWith(".md") ? [fullPath] : []; +}); + +for (const file of markdownFiles) { + const relativeFile = path.relative(root, file); + const content = readFileSync(file, "utf8"); + + for (const match of content.matchAll(localMarkdownLinkPattern)) { + const link = match[1]; + if (/^[a-z]+:\/\//i.test(link)) continue; + + const target = path.resolve(path.dirname(file), link); + if (!existsSync(target)) { + problems.push(`- Broken local doc link in ${relativeFile}: ${link}`); + } + } +} + +if (problems.length > 0) { + fail(problems); +} + +process.stdout.write("Docs harness passed.\n"); diff --git a/tool/check-env-example.mjs b/tool/check-env-example.mjs new file mode 100644 index 0000000..c5c5234 --- /dev/null +++ b/tool/check-env-example.mjs @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const envExamplePath = path.join(root, ".env.example"); + +const fail = (messages) => { + process.stderr.write(`Env harness failed:\n${messages.join("\n")}\n`); + process.exit(1); +}; + +if (!existsSync(envExamplePath)) { + fail(["- Missing .env.example"]); +} + +const content = readFileSync(envExamplePath, "utf8"); +const problems = []; +const suspiciousValuePattern = + /^\s*[A-Z0-9_]*(SECRET|TOKEN|PASSWORD|PRIVATE_KEY|ACCESS_KEY|IDENTITY)[A-Z0-9_]*\s*=\s*("?)(?!$|change-me|example|placeholder|<|your-|test-|dummy-).{8,}\2\s*$/; + +for (const [index, line] of content.split("\n").entries()) { + if (!line.trim() || line.trim().startsWith("#")) continue; + + if (!/^[A-Z0-9_]+=/.test(line)) { + problems.push(`- .env.example line ${index + 1} is not KEY=value format`); + } + + if (suspiciousValuePattern.test(line)) { + problems.push( + `- .env.example line ${index + 1} looks like it may contain a real secret` + ); + } +} + +if (problems.length > 0) { + fail(problems); +} + +process.stdout.write("Env harness passed.\n"); diff --git a/tool/check-project-map.mjs b/tool/check-project-map.mjs new file mode 100644 index 0000000..42f0c74 --- /dev/null +++ b/tool/check-project-map.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const agentsPath = path.join(root, "AGENTS.md"); + +const expectedPaths = [ + "src/cli.ts", + "src/commands", + "src/config", + "src/core", + "src/dumpers", + "src/encryption", + "src/notifications", + "src/storage", + "test", + "tool", + "docs", +]; + +const fail = (messages) => { + process.stderr.write(`Project-map harness failed:\n${messages.join("\n")}\n`); + process.exit(1); +}; + +if (!existsSync(agentsPath)) { + fail(["- Missing AGENTS.md"]); +} + +const agents = readFileSync(agentsPath, "utf8"); +const problems = []; + +for (const expectedPath of expectedPaths) { + if (!existsSync(path.join(root, expectedPath))) { + problems.push(`- AGENTS.md references missing path: ${expectedPath}`); + } + + if (!agents.includes(expectedPath)) { + problems.push(`- AGENTS.md project map must mention: ${expectedPath}`); + } +} + +if (problems.length > 0) { + fail(problems); +} + +process.stdout.write("Project-map harness passed.\n"); diff --git a/tool/check-security.mjs b/tool/check-security.mjs new file mode 100644 index 0000000..62fa88e --- /dev/null +++ b/tool/check-security.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const ignoredDirs = new Set([".git", "node_modules", "dist", "coverage"]); +const forbiddenFilePatterns = [ + /^\.env$/, + /^\.env\.(?!example$).+/, + /(^|\/)age-identity\.txt$/, + /\.(pem|key|p12|pfx)$/, + /service-account.*\.json$/, + /\.(dump|dump\.gz|dump\.gz\.age|sql|sql\.gz|backup)$/, +]; + +const fail = (messages) => { + process.stderr.write(`Security harness failed:\n${messages.join("\n")}\n`); + process.exit(1); +}; + +const walk = (dir) => { + if (!existsSync(dir)) return []; + + const out = []; + for (const entry of readdirSync(dir)) { + if (ignoredDirs.has(entry)) continue; + + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + if (stat.isDirectory()) { + out.push(...walk(fullPath)); + } else { + out.push(fullPath); + } + } + return out; +}; + +const problems = []; + +for (const file of walk(root)) { + const relativeFile = path.relative(root, file).split(path.sep).join("/"); + if (forbiddenFilePatterns.some((pattern) => pattern.test(relativeFile))) { + problems.push( + `- Forbidden secret/backup artifact file present: ${relativeFile}` + ); + } +} + +try { + execFileSync("pnpm", ["audit", "--audit-level=high"], { + cwd: root, + stdio: "pipe", + }); +} catch (error) { + const output = `${error.stdout?.toString() ?? ""}${error.stderr?.toString() ?? ""}`; + problems.push( + "- pnpm audit found high severity dependency issues or failed to run", + output.trim() ? output.trim() : " No audit output was provided." + ); +} + +if (problems.length > 0) { + fail(problems); +} + +process.stdout.write("Security harness passed.\n"); diff --git a/tool/check-source-hygiene.mjs b/tool/check-source-hygiene.mjs new file mode 100644 index 0000000..1255822 --- /dev/null +++ b/tool/check-source-hygiene.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const ignoredDirs = new Set([".git", "node_modules", "dist", "coverage"]); +const allowedProcessEnvFiles = new Set([]); + +const fail = (messages) => { + process.stderr.write( + `Source hygiene harness failed:\n${messages.join("\n")}\n` + ); + process.exit(1); +}; + +const walk = (dir) => { + if (!existsSync(dir)) return []; + + const out = []; + for (const entry of readdirSync(dir)) { + if (ignoredDirs.has(entry)) continue; + + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + if (stat.isDirectory()) { + out.push(...walk(fullPath)); + } else if (/\.(ts|tsx|js|mjs|cjs)$/.test(entry)) { + out.push(fullPath); + } + } + return out; +}; + +const problems = []; +const pendingWorkMarker = "TO" + "DO"; + +for (const file of walk(root)) { + const relativeFile = path.relative(root, file).split(path.sep).join("/"); + const content = readFileSync(file, "utf8"); + + if (content.includes(pendingWorkMarker)) { + problems.push( + `- ${relativeFile} contains ${pendingWorkMarker}. Use an issue/exec-plan note instead.` + ); + } + + if (/console\.(log|debug|info|warn|error)\s*\(/.test(content)) { + problems.push( + `- ${relativeFile} uses console.*. Prefer explicit CLI/logging helpers.` + ); + } + + if ( + content.includes("process.env") && + !allowedProcessEnvFiles.has(relativeFile) && + !relativeFile.startsWith("tool/") + ) { + problems.push( + `- ${relativeFile} reads process.env outside the config boundary. Add config loader first.` + ); + } +} + +if (problems.length > 0) { + fail(problems); +} + +process.stdout.write("Source hygiene harness passed.\n"); diff --git a/tool/lints/architecture-rules.json b/tool/lints/architecture-rules.json new file mode 100644 index 0000000..d35c37e --- /dev/null +++ b/tool/lints/architecture-rules.json @@ -0,0 +1,44 @@ +{ + "rules": [ + { + "name": "core-must-not-import-commands", + "from": "src/core/**", + "disallow": ["src/commands/**"] + }, + { + "name": "core-must-not-import-adapters", + "from": "src/core/**", + "disallow": [ + "src/dumpers/**", + "src/encryption/**", + "src/notifications/**", + "src/storage/**" + ] + }, + { + "name": "config-must-not-import-commands", + "from": "src/config/**", + "disallow": ["src/commands/**"] + }, + { + "name": "dumpers-must-not-import-storage", + "from": "src/dumpers/**", + "disallow": ["src/storage/**"] + }, + { + "name": "storage-must-not-import-dumpers", + "from": "src/storage/**", + "disallow": ["src/dumpers/**"] + }, + { + "name": "encryption-must-not-import-storage-or-dumpers", + "from": "src/encryption/**", + "disallow": ["src/dumpers/**", "src/storage/**"] + }, + { + "name": "notifications-must-not-import-runtime-adapters", + "from": "src/notifications/**", + "disallow": ["src/dumpers/**", "src/encryption/**", "src/storage/**"] + } + ] +} diff --git a/tool/verify.mjs b/tool/verify.mjs new file mode 100644 index 0000000..4185bd2 --- /dev/null +++ b/tool/verify.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; + +const steps = [ + ["Format check", ["pnpm", "format:check"]], + ["ESLint", ["pnpm", "lint"]], + ["TypeScript", ["pnpm", "type-check"]], + ["Tests", ["pnpm", "test"]], + ["Build", ["pnpm", "build"]], + ["Architecture harness", ["pnpm", "harness:architecture"]], + ["Source hygiene harness", ["pnpm", "harness:source"]], + ["Docs harness", ["pnpm", "harness:docs"]], + ["Env harness", ["pnpm", "harness:env"]], + ["Project-map harness", ["pnpm", "harness:project-map"]], + ["Security harness", ["pnpm", "harness:security"]], + [ + "Commit message harness", + [ + "pnpm", + "harness:commit", + "--", + "--message", + "chore(harness): verify scoped commit message format", + ], + ], +]; + +const run = (command) => + new Promise((resolve) => { + const child = spawn(command[0], command.slice(1), { + shell: false, + stdio: "inherit", + }); + child.on("exit", (code) => resolve(code ?? 1)); + }); + +for (const [title, command] of steps) { + process.stdout.write(`\n==> ${title}\n`); + const code = await run(command); + + if (code !== 0) { + process.exit(code); + } +} + +process.stdout.write("\nVerify passed.\n"); From a06f19191012f28fe7e8159d3f9a21ea0c330bdc Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Mon, 18 May 2026 21:37:31 +0700 Subject: [PATCH 2/9] feat(config): add typed config foundation --- README.md | 38 ++++++++ config/targets.example.yaml | 39 ++++++++ docs/implementation-plan.md | 34 +++++++ package.json | 4 + pnpm-lock.yaml | 40 ++++++-- src/cli.ts | 27 ++++++ src/commands/doctor.ts | 93 +++++++++++++++++++ src/config/env.ts | 107 +++++++++++++++++++++ src/config/loader.ts | 62 +++++++++++++ src/config/redact.ts | 22 +++++ src/config/schema.ts | 127 +++++++++++++++++++++++++ src/config/targets.ts | 36 +++++++ src/config/types.ts | 19 ++++ test/config.test.ts | 170 ++++++++++++++++++++++++++++++++++ tool/check-source-hygiene.mjs | 2 +- 15 files changed, 812 insertions(+), 8 deletions(-) create mode 100644 config/targets.example.yaml create mode 100644 src/commands/doctor.ts create mode 100644 src/config/env.ts create mode 100644 src/config/loader.ts create mode 100644 src/config/redact.ts create mode 100644 src/config/schema.ts create mode 100644 src/config/targets.ts create mode 100644 src/config/types.ts create mode 100644 test/config.test.ts diff --git a/README.md b/README.md index 27af202..de19523 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,44 @@ Reusable multi-project backup runner for database dumps, encrypted external stor Draft planning stage. +## Quickstart + +Install dependencies and run the quality gate: + +```bash +pnpm install +pnpm verify +``` + +The CLI currently exposes a minimal baseline command while the product implementation is still behind the harness: + +```bash +pnpm build +node dist/cli.js --version +``` + +## Commit Format + +Use scoped Conventional Commits: + +```text +type(scope): message +``` + +Example: + +```text +chore(harness): add strict verification gate +``` + +Validate the latest commit locally: + +```bash +pnpm harness:commit +``` + +## Docs + Start with the engineering proposal and implementation plan: - [Reusable Multi-Project Backup Runner Proposal](docs/reusable-backup-runner-proposal.md) diff --git a/config/targets.example.yaml b/config/targets.example.yaml new file mode 100644 index 0000000..5c0fa46 --- /dev/null +++ b/config/targets.example.yaml @@ -0,0 +1,39 @@ +version: 1 + +defaults: + encryption: + type: age + recipientEnv: BACKUP_AGE_RECIPIENT + retention: + keepDaily: 7 + keepWeekly: 4 + keepMonthly: 3 + notifications: + telegram: + enabled: true + botTokenEnv: BACKUP_TELEGRAM_BOT_TOKEN + chatIdEnv: BACKUP_TELEGRAM_CHAT_ID + +targets: + - id: maintana + enabled: false + description: Maintana production PostgreSQL database in Docker. + dumper: + type: postgresDocker + container: maintana-postgres + database: maintana + username: maintana + passwordEnv: MAINTANA_POSTGRES_PASSWORD + format: custom + storage: + type: s3 + endpoint: https://example-account-id.r2.cloudflarestorage.com + region: auto + bucket: maintana-backups + prefix: production/postgres + accessKeyIdEnv: MAINTANA_BACKUP_R2_ACCESS_KEY_ID + secretAccessKeyEnv: MAINTANA_BACKUP_R2_SECRET_ACCESS_KEY + retention: + keepDaily: 14 + keepWeekly: 8 + keepMonthly: 6 diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index a9658f8..ae3a6f9 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -61,6 +61,8 @@ Commit only after the relevant verification command passes. ## Phase 0 — Repo Hygiene And Decision Lock +Status: Done on 18 May 2026. + Goal: make sure we are building the right thing before scaffolding. Tasks: @@ -77,8 +79,21 @@ Acceptance criteria: - Open questions for MVP are answered. - No code implementation starts before the MVP boundary is clear. +Decision lock: + +- First supported production target: PostgreSQL running in Docker. +- First external storage target: S3-compatible object storage, specifically Cloudflare R2. +- Storage configuration remains per target, so one Cloudflare account with multiple buckets and separate Cloudflare accounts per project are both supported. +- Encryption default: `age`; production database backups must be encrypted before external upload. +- First notification channel: Telegram. +- Installation model: standalone runner installed on the target VPS, not an app dependency inside Maintana, Orymu backend, Kevly, or future projects. +- First rollout target: Maintana production. +- Product boundary: Phase 2 starts with typed configuration and `doctor`; no real backup side effects are introduced before the config foundation is validated. + ## Phase 1 — Project Harness +Status: Done on 18 May 2026. + Goal: create a strict engineering harness before implementation. Tasks: @@ -177,6 +192,8 @@ The temporary smoke files were removed after the negative test, and the clean ga ## Phase 2 — Config Foundation +Status: Done on 18 May 2026. + Goal: build typed, validated configuration before any backup behavior. Tasks: @@ -214,6 +231,23 @@ Acceptance criteria: - Secrets are redacted in all printed config/debug output. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +node dist/cli.js doctor --config config/targets.example.yaml +``` + +Result: + +- strict YAML config loading is implemented; +- Zod schema rejects invalid config shape and duplicate target ids; +- env reference resolution reports missing required values for enabled targets; +- disabled targets can validate without production secrets; +- config preview redaction masks secret-shaped keys; +- `doctor` validates config shape without running backup side effects; +- example config validates as a disabled Maintana target. + ## Phase 3 — CLI Foundation Goal: create stable command interfaces with no real backup side effects yet. diff --git a/package.json b/package.json index 80754dd..9c03f6f 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,9 @@ "typescript": "^5.9.3", "typescript-eslint": "^8.48.0", "vitest": "^4.1.0" + }, + "dependencies": { + "yaml": "^2.9.0", + "zod": "^4.4.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9b083a..23189db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,13 @@ settings: importers: .: + dependencies: + yaml: + specifier: ^2.9.0 + version: 2.9.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 devDependencies: "@eslint/js": specifier: ^9.39.1 @@ -27,7 +34,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: ^4.1.0 - version: 4.1.6(@types/node@22.19.19)(vite@8.0.13(@types/node@22.19.19)) + version: 4.1.6(@types/node@22.19.19)(vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0)) packages: "@emnapi/core@1.10.0": @@ -1442,6 +1449,14 @@ packages: } engines: { node: ">=0.10.0" } + yaml@2.9.0: + resolution: + { + integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==, + } + engines: { node: ">= 14.6" } + hasBin: true + yocto-queue@0.1.0: resolution: { @@ -1449,6 +1464,12 @@ packages: } engines: { node: ">=10" } + zod@4.4.3: + resolution: + { + integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==, + } + snapshots: "@emnapi/core@1.10.0": dependencies: @@ -1712,13 +1733,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - "@vitest/mocker@4.1.6(vite@8.0.13(@types/node@22.19.19))": + "@vitest/mocker@4.1.6(vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0))": dependencies: "@vitest/spy": 4.1.6 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.13(@types/node@22.19.19) + vite: 8.0.13(@types/node@22.19.19)(yaml@2.9.0) "@vitest/pretty-format@4.1.6": dependencies: @@ -2168,7 +2189,7 @@ snapshots: dependencies: punycode: 2.3.1 - vite@8.0.13(@types/node@22.19.19): + vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -2178,11 +2199,12 @@ snapshots: optionalDependencies: "@types/node": 22.19.19 fsevents: 2.3.3 + yaml: 2.9.0 - vitest@4.1.6(@types/node@22.19.19)(vite@8.0.13(@types/node@22.19.19)): + vitest@4.1.6(@types/node@22.19.19)(vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0)): dependencies: "@vitest/expect": 4.1.6 - "@vitest/mocker": 4.1.6(vite@8.0.13(@types/node@22.19.19)) + "@vitest/mocker": 4.1.6(vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0)) "@vitest/pretty-format": 4.1.6 "@vitest/runner": 4.1.6 "@vitest/snapshot": 4.1.6 @@ -2199,7 +2221,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.13(@types/node@22.19.19) + vite: 8.0.13(@types/node@22.19.19)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: "@types/node": 22.19.19 @@ -2217,4 +2239,8 @@ snapshots: word-wrap@1.2.5: {} + yaml@2.9.0: {} + yocto-queue@0.1.0: {} + + zod@4.4.3: {} diff --git a/src/cli.ts b/src/cli.ts index 62488e0..8efda14 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,11 +1,38 @@ #!/usr/bin/env node +import { formatDoctorResult, runDoctor } from "./commands/doctor.js"; + export const cliName = "ops-backup-runner"; export const getStartupMessage = (): string => `${cliName}: project harness initialized. Backup commands are not implemented yet.`; +const getFlagValue = (args: string[], flag: string): string | undefined => { + const index = args.indexOf(flag); + if (index < 0) return undefined; + return args[index + 1]; +}; + export const main = (): void => { + const args = process.argv.slice(2); + const command = args[0]; + + if (command === "doctor") { + const configPath = getFlagValue(args, "--config"); + if (configPath === undefined) { + process.stderr.write("Doctor failed: missing --config \n"); + process.exitCode = 2; + return; + } + + const result = runDoctor(configPath); + const formatted = formatDoctorResult(result); + const output = result.ok ? process.stdout : process.stderr; + output.write(`${formatted}\n`); + process.exitCode = result.ok ? 0 : 2; + return; + } + process.stdout.write(`${getStartupMessage()}\n`); }; diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts new file mode 100644 index 0000000..e334631 --- /dev/null +++ b/src/commands/doctor.ts @@ -0,0 +1,93 @@ +import { loadConfigFromFile } from "../config/loader.js"; +import { redactConfigPreview } from "../config/redact.js"; +import { resolveTargetEnvReferences } from "../config/env.js"; +import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; + +export interface DoctorTargetResult { + id: string; + enabled: boolean; + ok: boolean; + status: "ready" | "disabled" | "missing-env"; + issues: string[]; +} + +export type DoctorResult = + | { + ok: true; + config: BackupRunnerConfig; + redactedConfig: BackupRunnerConfig; + targets: DoctorTargetResult[]; + } + | { + ok: false; + message: string; + issues: string[]; + }; + +const inspectTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +): DoctorTargetResult => { + if (!target.enabled) { + return { + id: target.id, + enabled: false, + ok: true, + status: "disabled", + issues: [], + }; + } + + const envResult = resolveTargetEnvReferences(config, target); + + return { + id: target.id, + enabled: true, + ok: envResult.ok, + status: envResult.ok ? "ready" : "missing-env", + issues: envResult.issues.map((issue) => issue.message), + }; +}; + +export const runDoctor = (configPath: string): DoctorResult => { + const loadResult = loadConfigFromFile(configPath); + if (!loadResult.ok) { + return loadResult; + } + + const targets = loadResult.config.targets.map((target) => + inspectTarget(loadResult.config, target) + ); + const failedTargets = targets.filter((target) => !target.ok); + + if (failedTargets.length > 0) { + return { + ok: false, + message: "Config is valid, but required runtime environment is missing.", + issues: failedTargets.flatMap((target) => target.issues), + }; + } + + return { + ok: true, + config: loadResult.config, + redactedConfig: redactConfigPreview( + loadResult.config + ) as BackupRunnerConfig, + targets, + }; +}; + +export const formatDoctorResult = (result: DoctorResult): string => { + if (!result.ok) { + return [ + `Doctor failed: ${result.message}`, + ...result.issues.map((issue) => `- ${issue}`), + ].join("\n"); + } + + return [ + "Doctor passed.", + ...result.targets.map((target) => `- ${target.id}: ${target.status}`), + ].join("\n"); +}; diff --git a/src/config/env.ts b/src/config/env.ts new file mode 100644 index 0000000..7ec748d --- /dev/null +++ b/src/config/env.ts @@ -0,0 +1,107 @@ +import type { BackupRunnerConfig, BackupTarget } from "./types.js"; + +export interface EnvReference { + name: string; + owner: string; + requiredForEnabledTarget: boolean; +} + +export interface EnvResolutionIssue { + envName: string; + owner: string; + message: string; +} + +export interface EnvResolutionResult { + ok: boolean; + issues: EnvResolutionIssue[]; +} + +const addEnvReference = ( + references: EnvReference[], + name: string | undefined, + owner: string, + requiredForEnabledTarget: boolean +): void => { + if (name === undefined) return; + references.push({ name, owner, requiredForEnabledTarget }); +}; + +export const getTargetEnvReferences = ( + config: BackupRunnerConfig, + target: BackupTarget +): EnvReference[] => { + const references: EnvReference[] = []; + const encryption = target.encryption ?? config.defaults?.encryption; + const notifications = target.notifications ?? config.defaults?.notifications; + + addEnvReference( + references, + target.dumper.passwordEnv, + `${target.id}.dumper.passwordEnv`, + false + ); + + addEnvReference( + references, + target.storage.accessKeyIdEnv, + `${target.id}.storage.accessKeyIdEnv`, + true + ); + addEnvReference( + references, + target.storage.secretAccessKeyEnv, + `${target.id}.storage.secretAccessKeyEnv`, + true + ); + + if (encryption?.type === "age") { + addEnvReference( + references, + encryption.recipientEnv, + `${target.id}.encryption.recipientEnv`, + true + ); + } + + if (notifications?.telegram?.enabled === true) { + addEnvReference( + references, + notifications.telegram.botTokenEnv, + `${target.id}.notifications.telegram.botTokenEnv`, + true + ); + addEnvReference( + references, + notifications.telegram.chatIdEnv, + `${target.id}.notifications.telegram.chatIdEnv`, + true + ); + } + + return references; +}; + +export const resolveTargetEnvReferences = ( + config: BackupRunnerConfig, + target: BackupTarget, + env: Record = process.env +): EnvResolutionResult => { + if (!target.enabled) { + return { ok: true, issues: [] }; + } + + const issues = getTargetEnvReferences(config, target) + .filter((reference) => reference.requiredForEnabledTarget) + .filter((reference) => env[reference.name] === undefined) + .map((reference) => ({ + envName: reference.name, + owner: reference.owner, + message: `Missing required environment variable ${reference.name} for ${reference.owner}`, + })); + + return { + ok: issues.length === 0, + issues, + }; +}; diff --git a/src/config/loader.ts b/src/config/loader.ts new file mode 100644 index 0000000..8906140 --- /dev/null +++ b/src/config/loader.ts @@ -0,0 +1,62 @@ +import { existsSync, readFileSync } from "node:fs"; + +import { parse } from "yaml"; +import { ZodError } from "zod"; + +import { backupRunnerConfigSchema } from "./schema.js"; +import type { ConfigLoadResult } from "./types.js"; + +export const loadConfigFromFile = (configPath: string): ConfigLoadResult => { + if (!existsSync(configPath)) { + return { + ok: false, + message: `Config file not found: ${configPath}`, + issues: [`Create the file or pass a valid path with --config.`], + }; + } + + let raw: string; + try { + raw = readFileSync(configPath, "utf8"); + } catch (error) { + return { + ok: false, + message: `Config file could not be read: ${configPath}`, + issues: [error instanceof Error ? error.message : "Unknown read error"], + }; + } + + let parsed: unknown; + try { + parsed = parse(raw); + } catch (error) { + return { + ok: false, + message: `Config file is not valid YAML: ${configPath}`, + issues: [error instanceof Error ? error.message : "Unknown YAML error"], + }; + } + + try { + return { + ok: true, + config: backupRunnerConfigSchema.parse(parsed), + }; + } catch (error) { + if (error instanceof ZodError) { + return { + ok: false, + message: `Config file failed validation: ${configPath}`, + issues: error.issues.map( + (issue) => `${issue.path.join(".") || ""}: ${issue.message}` + ), + }; + } + + return { + ok: false, + message: `Config file failed validation: ${configPath}`, + issues: [error instanceof Error ? error.message : "Unknown parse error"], + }; + } +}; diff --git a/src/config/redact.ts b/src/config/redact.ts new file mode 100644 index 0000000..b063a29 --- /dev/null +++ b/src/config/redact.ts @@ -0,0 +1,22 @@ +const redacted = "[REDACTED]"; + +const sensitiveKeyPattern = + /(password|secret|token|privatekey|accesskey|credential)/iu; + +export const redactConfigPreview = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map((item) => redactConfigPreview(item)); + } + + if (value !== null && typeof value === "object") { + const output: Record = {}; + for (const [key, nestedValue] of Object.entries(value)) { + output[key] = sensitiveKeyPattern.test(key) + ? redacted + : redactConfigPreview(nestedValue); + } + return output; + } + + return value; +}; diff --git a/src/config/schema.ts b/src/config/schema.ts new file mode 100644 index 0000000..32f6c9c --- /dev/null +++ b/src/config/schema.ts @@ -0,0 +1,127 @@ +import { z } from "zod"; + +const targetIdSchema = z + .string() + .min(1) + .regex(/^[a-z0-9][a-z0-9-]*$/u, { + message: "target id must use lowercase letters, numbers, and dashes only", + }); + +const envNameSchema = z + .string() + .min(1) + .regex(/^[A-Z][A-Z0-9_]*$/u, { + message: "env references must use uppercase env var names", + }); + +const retentionSchema = z + .object({ + keepDaily: z.number().int().positive().optional(), + keepWeekly: z.number().int().positive().optional(), + keepMonthly: z.number().int().positive().optional(), + maxAgeDays: z.number().int().positive().optional(), + }) + .strict(); + +const postgresDockerDumperSchema = z + .object({ + type: z.literal("postgresDocker"), + container: z.string().min(1), + database: z.string().min(1), + username: z.string().min(1), + passwordEnv: envNameSchema.optional(), + format: z.literal("custom").default("custom"), + dockerBinary: z.string().min(1).optional(), + }) + .strict(); + +const dumperSchema = z.discriminatedUnion("type", [postgresDockerDumperSchema]); + +const s3StorageSchema = z + .object({ + type: z.literal("s3"), + endpoint: z.url(), + region: z.string().min(1), + bucket: z.string().min(1), + prefix: z.string().min(1).optional(), + accessKeyIdEnv: envNameSchema.optional(), + secretAccessKeyEnv: envNameSchema.optional(), + }) + .strict(); + +const storageSchema = z.discriminatedUnion("type", [s3StorageSchema]); + +const ageEncryptionSchema = z + .object({ + type: z.literal("age"), + recipientEnv: envNameSchema.optional(), + }) + .strict(); + +const noneEncryptionSchema = z + .object({ + type: z.literal("none"), + }) + .strict(); + +const encryptionSchema = z.discriminatedUnion("type", [ + ageEncryptionSchema, + noneEncryptionSchema, +]); + +const telegramNotificationSchema = z + .object({ + enabled: z.boolean().default(false), + botTokenEnv: envNameSchema.optional(), + chatIdEnv: envNameSchema.optional(), + }) + .strict(); + +const notificationPolicySchema = z + .object({ + telegram: telegramNotificationSchema.optional(), + }) + .strict(); + +const targetSchema = z + .object({ + id: targetIdSchema, + enabled: z.boolean().default(true), + description: z.string().min(1).optional(), + dumper: dumperSchema, + storage: storageSchema, + retention: retentionSchema.optional(), + encryption: encryptionSchema.optional(), + notifications: notificationPolicySchema.optional(), + }) + .strict(); + +export const backupRunnerConfigSchema = z + .object({ + version: z.literal(1), + defaults: z + .object({ + encryption: encryptionSchema.optional(), + retention: retentionSchema.optional(), + notifications: notificationPolicySchema.optional(), + }) + .strict() + .optional(), + targets: z + .array(targetSchema) + .min(1) + .superRefine((targets, context) => { + const seen = new Set(); + for (const [index, target] of targets.entries()) { + if (seen.has(target.id)) { + context.addIssue({ + code: "custom", + message: `duplicate target id: ${target.id}`, + path: [index, "id"], + }); + } + seen.add(target.id); + } + }), + }) + .strict(); diff --git a/src/config/targets.ts b/src/config/targets.ts new file mode 100644 index 0000000..8b84bb5 --- /dev/null +++ b/src/config/targets.ts @@ -0,0 +1,36 @@ +import type { BackupRunnerConfig, BackupTarget } from "./types.js"; + +export type TargetSelectionResult = + | { + ok: true; + targets: BackupTarget[]; + } + | { + ok: false; + message: string; + }; + +export const selectTargets = ( + config: BackupRunnerConfig, + targetId: string +): TargetSelectionResult => { + if (targetId === "all") { + return { + ok: true, + targets: config.targets.filter((target) => target.enabled), + }; + } + + const target = config.targets.find((item) => item.id === targetId); + if (target === undefined) { + return { + ok: false, + message: `Unknown target: ${targetId}`, + }; + } + + return { + ok: true, + targets: [target], + }; +}; diff --git a/src/config/types.ts b/src/config/types.ts new file mode 100644 index 0000000..87b0366 --- /dev/null +++ b/src/config/types.ts @@ -0,0 +1,19 @@ +import type { z } from "zod"; + +import type { backupRunnerConfigSchema } from "./schema.js"; + +export type BackupRunnerConfig = z.infer; +export type BackupTarget = BackupRunnerConfig["targets"][number]; +export type BackupTargetId = BackupTarget["id"]; +export type BackupRunnerDefaults = NonNullable; + +export type ConfigLoadResult = + | { + ok: true; + config: BackupRunnerConfig; + } + | { + ok: false; + message: string; + issues: string[]; + }; diff --git a/test/config.test.ts b/test/config.test.ts new file mode 100644 index 0000000..423714a --- /dev/null +++ b/test/config.test.ts @@ -0,0 +1,170 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { runDoctor } from "../src/commands/doctor.js"; +import { resolveTargetEnvReferences } from "../src/config/env.js"; +import { loadConfigFromFile } from "../src/config/loader.js"; +import { redactConfigPreview } from "../src/config/redact.js"; + +const writeConfig = (content: string): string => { + const directory = mkdtempSync(path.join(tmpdir(), "ops-backup-runner-")); + const file = path.join(directory, "targets.yaml"); + writeFileSync(file, content, "utf8"); + return file; +}; + +const validConfig = ` +version: 1 +defaults: + encryption: + type: age + recipientEnv: BACKUP_AGE_RECIPIENT +targets: + - id: maintana + enabled: true + dumper: + type: postgresDocker + container: maintana-postgres + database: maintana + username: maintana + passwordEnv: MAINTANA_POSTGRES_PASSWORD + storage: + type: s3 + endpoint: https://example-account-id.r2.cloudflarestorage.com + region: auto + bucket: maintana-backups + accessKeyIdEnv: MAINTANA_BACKUP_R2_ACCESS_KEY_ID + secretAccessKeyEnv: MAINTANA_BACKUP_R2_SECRET_ACCESS_KEY +`; + +describe("config foundation", () => { + it("loads a valid YAML config", () => { + const result = loadConfigFromFile(writeConfig(validConfig)); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.config.targets[0]?.id).toBe("maintana"); + expect(result.config.targets[0]?.dumper.type).toBe("postgresDocker"); + } + }); + + it("rejects unknown dumper types", () => { + const result = loadConfigFromFile( + writeConfig(validConfig.replace("postgresDocker", "mongodbDocker")) + ); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.issues.join("\n")).toContain("Expected 'postgresDocker'"); + } + }); + + it("rejects duplicate target ids", () => { + const result = loadConfigFromFile( + writeConfig(` +version: 1 +targets: + - id: maintana + dumper: + type: postgresDocker + container: one + database: one + username: one + storage: + type: s3 + endpoint: https://one.example.com + region: auto + bucket: one + - id: maintana + dumper: + type: postgresDocker + container: two + database: two + username: two + storage: + type: s3 + endpoint: https://two.example.com + region: auto + bucket: two +`) + ); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.issues.join("\n")).toContain( + "duplicate target id: maintana" + ); + } + }); + + it("reports missing env references for enabled targets", () => { + const result = loadConfigFromFile(writeConfig(validConfig)); + + expect(result.ok).toBe(true); + if (!result.ok) return; + + const target = result.config.targets[0]; + expect(target).toBeDefined(); + if (target === undefined) return; + + const envResult = resolveTargetEnvReferences(result.config, target, {}); + + expect(envResult.ok).toBe(false); + expect(envResult.issues.map((issue) => issue.envName)).toEqual([ + "MAINTANA_BACKUP_R2_ACCESS_KEY_ID", + "MAINTANA_BACKUP_R2_SECRET_ACCESS_KEY", + "BACKUP_AGE_RECIPIENT", + ]); + }); + + it("does not require env references for disabled targets", () => { + const result = runDoctor( + writeConfig(validConfig.replace("enabled: true", "enabled: false")) + ); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.targets[0]?.status).toBe("disabled"); + } + }); + + it("redacts secret-shaped keys in config previews", () => { + const preview = redactConfigPreview({ + accessKeyIdEnv: "ACCESS_KEY_ID", + secretAccessKeyEnv: "SECRET_ACCESS_KEY", + nested: { + botTokenEnv: "BOT_TOKEN", + safe: "visible", + }, + }); + + expect(preview).toEqual({ + accessKeyIdEnv: "[REDACTED]", + secretAccessKeyEnv: "[REDACTED]", + nested: { + botTokenEnv: "[REDACTED]", + safe: "visible", + }, + }); + }); + + it("passes doctor for the disabled example config without production secrets", () => { + const result = runDoctor("config/targets.example.yaml"); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.targets).toEqual([ + { + id: "maintana", + enabled: false, + ok: true, + status: "disabled", + issues: [], + }, + ]); + } + }); +}); diff --git a/tool/check-source-hygiene.mjs b/tool/check-source-hygiene.mjs index 1255822..2913e15 100644 --- a/tool/check-source-hygiene.mjs +++ b/tool/check-source-hygiene.mjs @@ -5,7 +5,7 @@ import path from "node:path"; const root = process.cwd(); const ignoredDirs = new Set([".git", "node_modules", "dist", "coverage"]); -const allowedProcessEnvFiles = new Set([]); +const allowedProcessEnvFiles = new Set(["src/config/env.ts"]); const fail = (messages) => { process.stderr.write( From 6b8a7d3f40c839a99f2106c1e2306192e6450bfc Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 07:30:00 +0700 Subject: [PATCH 3/9] feat(pipeline): add local backup pipeline --- docs/implementation-plan.md | 41 +++ src/cli.ts | 607 +++++++++++++++++++++++++++++++++++- src/config/env.ts | 40 +-- src/config/schema.ts | 24 +- src/core/artifact.ts | 12 + src/core/backup-job.ts | 65 ++++ src/core/manifest.ts | 28 ++ src/core/ports.ts | 28 ++ src/core/temp-workspace.ts | 19 ++ src/dumpers/fake.ts | 17 + src/storage/local.ts | 84 +++++ test/cli.test.ts | 210 ++++++++++++- test/config.test.ts | 2 +- 13 files changed, 1136 insertions(+), 41 deletions(-) create mode 100644 src/core/artifact.ts create mode 100644 src/core/backup-job.ts create mode 100644 src/core/manifest.ts create mode 100644 src/core/ports.ts create mode 100644 src/core/temp-workspace.ts create mode 100644 src/dumpers/fake.ts create mode 100644 src/storage/local.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index ae3a6f9..0c177ff 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -250,6 +250,8 @@ Result: ## Phase 3 — CLI Foundation +Status: Done on 18 May 2026. + Goal: create stable command interfaces with no real backup side effects yet. Tasks: @@ -283,8 +285,29 @@ Acceptance criteria: - `backup all --dry-run` lists enabled targets without executing dump/upload. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +node dist/cli.js backup all --dry-run --config config/targets.example.yaml +node dist/cli.js backup unknown --dry-run --config config/targets.example.yaml +node dist/cli.js restore --help +``` + +Result: + +- command parser is testable through `runCli(args)`; +- `doctor`, `backup`, `list`, `verify`, `restore`, and `prune` have help output; +- shared `--config` and `--json` handling exists; +- `backup all --dry-run` validates target selection without dump/upload/prune/notification side effects; +- unknown target returns usage exit code `2` with a clear message; +- non-dry-run backup returns runtime exit code `1` until Phase 4 implements the local pipeline; +- placeholder target commands validate target selection and explicitly report that no backup side effects were executed. + ## Phase 4 — Local Backup Pipeline +Status: Done on 18 May 2026. + Goal: prove the core pipeline with fake dumper and local storage. Tasks: @@ -319,6 +342,24 @@ Acceptance criteria: - Temp files are cleaned after success and failure. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- `fake` dumper config is supported for local/dev backup targets; +- `local` storage config is supported for local/dev artifact storage; +- backup pipeline writes `fake dump -> gzip -> local artifact -> manifest`; +- manifest includes target id, created time, artifact key, size, sha256, compression, encryption, and storage metadata; +- `list` reads local manifests; +- `verify --latest` checks local artifact sha256 against the manifest; +- `restore` gunzips the stored artifact into the requested output file; +- temp workspace cleanup is handled in the backup job finalizer; +- real PostgreSQL, S3/R2, age encryption, retention pruning, and notifications remain intentionally outside Phase 4. + ## Phase 5 — PostgreSQL Docker Dumper Goal: back up Dockerized PostgreSQL databases. diff --git a/src/cli.ts b/src/cli.ts index 8efda14..334c794 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,11 +1,38 @@ #!/usr/bin/env node +import { writeFileSync } from "node:fs"; + import { formatDoctorResult, runDoctor } from "./commands/doctor.js"; +import { loadConfigFromFile } from "./config/loader.js"; +import { selectTargets } from "./config/targets.js"; +import type { BackupRunnerConfig, BackupTarget } from "./config/types.js"; +import { + restoreLocalBackupArtifact, + runLocalBackupJob, +} from "./core/backup-job.js"; +import { sha256Hex } from "./core/artifact.js"; +import type { BackupManifest } from "./core/manifest.js"; +import { fakeDumper } from "./dumpers/fake.js"; +import { createLocalStorageAdapter } from "./storage/local.js"; export const cliName = "ops-backup-runner"; +export const exitCodes = { + success: 0, + runtimeFailure: 1, + usage: 2, + verificationFailure: 3, +} as const; + +export type CliExitCode = (typeof exitCodes)[keyof typeof exitCodes]; + +export interface CliResult { + exitCode: CliExitCode; + stdout: string; + stderr: string; +} export const getStartupMessage = (): string => - `${cliName}: project harness initialized. Backup commands are not implemented yet.`; + `${cliName}: project harness initialized. Run --help to see available commands.`; const getFlagValue = (args: string[], flag: string): string | undefined => { const index = args.indexOf(flag); @@ -13,27 +40,573 @@ const getFlagValue = (args: string[], flag: string): string | undefined => { return args[index + 1]; }; -export const main = (): void => { - const args = process.argv.slice(2); - const command = args[0]; +const hasFlag = (args: string[], flag: string): boolean => args.includes(flag); + +const renderGlobalHelp = (): string => + [ + cliName, + "", + "Usage:", + " ops-backup-runner [options]", + "", + "Commands:", + " doctor Validate config and required runtime environment.", + " backup Run local fake backup pipeline or validate selection with --dry-run.", + " list List local backup manifests.", + " verify Verify local backup artifact integrity.", + " restore Restore a local backup artifact to a file.", + " prune Prune backups. Placeholder until retention exists.", + "", + "Common options:", + " --config Path to YAML target config.", + " --json Emit machine-readable JSON.", + " --help Show help.", + ].join("\n"); + +const renderCommandHelp = (command: string): string => { + const helpByCommand: Record = { + doctor: [ + "Usage: ops-backup-runner doctor --config [--json]", + "", + "Validates config shape and required runtime environment.", + ], + backup: [ + "Usage: ops-backup-runner backup --config [--dry-run] [--json]", + "", + "Runs fake dumper -> gzip -> local storage for local targets.", + ], + list: [ + "Usage: ops-backup-runner list --config [--json]", + "", + "Lists local backup manifests for selected targets.", + ], + verify: [ + "Usage: ops-backup-runner verify --config [--latest] [--json]", + "", + "Verifies local artifact sha256. Use --latest to verify only the newest manifest.", + ], + restore: [ + "Usage: ops-backup-runner restore --backup --output --config [--json]", + "", + "Restores a local gzip artifact to the given output path.", + ], + prune: [ + "Usage: ops-backup-runner prune --config [--dry-run] [--json]", + "", + "Validates target selection. Retention pruning is implemented after storage exists.", + ], + }; - if (command === "doctor") { - const configPath = getFlagValue(args, "--config"); - if (configPath === undefined) { - process.stderr.write("Doctor failed: missing --config \n"); - process.exitCode = 2; - return; + return (helpByCommand[command] ?? [renderGlobalHelp()]).join("\n"); +}; + +const renderJson = (value: unknown): string => `${JSON.stringify(value)}\n`; + +const success = (stdout: string): CliResult => ({ + exitCode: exitCodes.success, + stdout: stdout.endsWith("\n") ? stdout : `${stdout}\n`, + stderr: "", +}); + +const failure = (exitCode: CliExitCode, stderr: string): CliResult => ({ + exitCode, + stdout: "", + stderr: stderr.endsWith("\n") ? stderr : `${stderr}\n`, +}); + +const loadConfigForCommand = ( + args: string[] +): + | { + ok: true; + config: BackupRunnerConfig; } + | { + ok: false; + result: CliResult; + } => { + const configPath = getFlagValue(args, "--config"); + if (configPath === undefined) { + return { + ok: false, + result: failure(exitCodes.usage, "Missing required --config ."), + }; + } + + const loadResult = loadConfigFromFile(configPath); + if (!loadResult.ok) { + return { + ok: false, + result: failure( + exitCodes.usage, + [ + `Config failed: ${loadResult.message}`, + ...loadResult.issues.map((issue) => `- ${issue}`), + ].join("\n") + ), + }; + } + + return { + ok: true, + config: loadResult.config, + }; +}; + +const selectTargetsForCommand = ( + args: string[], + config: BackupRunnerConfig, + targetId: string | undefined +): + | { + ok: true; + targets: BackupTarget[]; + } + | { + ok: false; + result: CliResult; + } => { + if (targetId === undefined) { + return { + ok: false, + result: failure(exitCodes.usage, "Missing required target argument."), + }; + } + + const selection = selectTargets(config, targetId); + if (!selection.ok) { + const json = hasFlag(args, "--json"); + return { + ok: false, + result: json + ? failure( + exitCodes.usage, + renderJson({ ok: false, error: selection.message }) + ) + : failure(exitCodes.usage, selection.message), + }; + } + + return selection; +}; + +const getEffectiveEncryptionType = ( + config: BackupRunnerConfig, + target: BackupTarget +): "age" | "none" => + (target.encryption ?? config.defaults?.encryption ?? { type: "none" }).type; + +const getLocalStorageForTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +): + | { + ok: true; + storage: ReturnType; + } + | { + ok: false; + message: string; + } => { + const encryptionType = getEffectiveEncryptionType(config, target); + if (target.storage.type !== "local") { + return { + ok: false, + message: `${target.id} uses ${target.storage.type} storage. Phase 4 only supports local storage.`, + }; + } - const result = runDoctor(configPath); - const formatted = formatDoctorResult(result); - const output = result.ok ? process.stdout : process.stderr; - output.write(`${formatted}\n`); - process.exitCode = result.ok ? 0 : 2; - return; + if (encryptionType !== "none") { + return { + ok: false, + message: `${target.id} uses ${encryptionType} encryption. Phase 4 only supports encryption: none.`, + }; } - process.stdout.write(`${getStartupMessage()}\n`); + return { + ok: true, + storage: createLocalStorageAdapter(target), + }; +}; + +const getLocalBackupTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +): + | { + ok: true; + storage: ReturnType; + } + | { + ok: false; + result: CliResult; + } => { + if (target.dumper.type !== "fake") { + return { + ok: false, + result: failure( + exitCodes.runtimeFailure, + `${target.id} uses ${target.dumper.type} dumper. Phase 4 only supports fake dumper.` + ), + }; + } + + const storage = getLocalStorageForTarget(config, target); + if (!storage.ok) { + return { + ok: false, + result: failure(exitCodes.runtimeFailure, storage.message), + }; + } + + return storage; +}; + +const findManifestByBackupId = ( + manifests: BackupManifest[], + backupId: string +): BackupManifest | undefined => + manifests.find((manifest) => manifest.backupId === backupId); + +const sortManifestsNewestFirst = ( + manifests: BackupManifest[] +): BackupManifest[] => + [...manifests].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); + +const runDoctorCommand = (args: string[]): CliResult => { + const configPath = getFlagValue(args, "--config"); + const json = hasFlag(args, "--json"); + + if (configPath === undefined) { + const message = "Doctor failed: missing --config "; + return json + ? failure(exitCodes.usage, renderJson({ ok: false, error: message })) + : failure(exitCodes.usage, message); + } + + const result = runDoctor(configPath); + if (json) { + return result.ok + ? success( + renderJson({ + ok: true, + targets: result.targets, + config: result.redactedConfig, + }) + ) + : failure( + exitCodes.usage, + renderJson({ + ok: false, + error: result.message, + issues: result.issues, + }) + ); + } + + const formatted = formatDoctorResult(result); + return result.ok ? success(formatted) : failure(exitCodes.usage, formatted); +}; + +const runBackupCommand = (args: string[]): CliResult => { + const targetId = args[1]; + const json = hasFlag(args, "--json"); + const dryRun = hasFlag(args, "--dry-run"); + const configResult = loadConfigForCommand(args); + if (!configResult.ok) return configResult.result; + + const selection = selectTargetsForCommand( + args, + configResult.config, + targetId + ); + if (!selection.ok) return selection.result; + + if (dryRun) { + const targetIds = selection.targets.map((target) => target.id); + if (json) { + return success( + renderJson({ + ok: true, + dryRun: true, + command: "backup", + targets: targetIds, + }) + ); + } + + return success( + [ + "Backup dry run passed.", + ...targetIds.map((target) => `- ${target}`), + "No dump, upload, prune, or notification side effects were executed.", + ].join("\n") + ); + } + + const manifests: BackupManifest[] = []; + for (const target of selection.targets) { + const localTarget = getLocalBackupTarget(configResult.config, target); + if (!localTarget.ok) return localTarget.result; + + const result = runLocalBackupJob(target, fakeDumper, localTarget.storage); + manifests.push(result.manifest); + } + + if (json) { + return success( + renderJson({ + ok: true, + command: "backup", + backups: manifests, + }) + ); + } + + return success( + [ + "Backup completed.", + ...manifests.map( + (manifest) => + `- ${manifest.targetId}: ${manifest.backupId} -> ${manifest.storage.artifactKey}` + ), + ].join("\n") + ); +}; + +const runListCommand = (args: string[]): CliResult => { + const targetId = args[1]; + const json = hasFlag(args, "--json"); + const configResult = loadConfigForCommand(args); + if (!configResult.ok) return configResult.result; + + const selection = selectTargetsForCommand( + args, + configResult.config, + targetId + ); + if (!selection.ok) return selection.result; + + const manifests = selection.targets.flatMap((target) => { + const storage = getLocalStorageForTarget(configResult.config, target); + if (!storage.ok) return []; + return storage.storage.listManifests(target.id); + }); + + if (json) { + return success( + renderJson({ + ok: true, + command: "list", + backups: manifests, + }) + ); + } + + if (manifests.length === 0) { + return success("No backups found."); + } + + return success( + [ + "Backups:", + ...manifests.map( + (manifest) => + `- ${manifest.targetId}: ${manifest.backupId} (${String( + manifest.artifact.sizeBytes + )} bytes)` + ), + ].join("\n") + ); +}; + +const runVerifyCommand = (args: string[]): CliResult => { + const targetId = args[1]; + const json = hasFlag(args, "--json"); + const latestOnly = hasFlag(args, "--latest"); + const configResult = loadConfigForCommand(args); + if (!configResult.ok) return configResult.result; + + const selection = selectTargetsForCommand( + args, + configResult.config, + targetId + ); + if (!selection.ok) return selection.result; + + const results: { + backupId: string; + targetId: string; + ok: boolean; + }[] = []; + + for (const target of selection.targets) { + const storage = getLocalStorageForTarget(configResult.config, target); + if (!storage.ok) return failure(exitCodes.runtimeFailure, storage.message); + + const manifests = latestOnly + ? sortManifestsNewestFirst( + storage.storage.listManifests(target.id) + ).slice(0, 1) + : storage.storage.listManifests(target.id); + + for (const manifest of manifests) { + const artifactBytes = storage.storage.readArtifact(manifest); + results.push({ + backupId: manifest.backupId, + targetId: manifest.targetId, + ok: sha256Hex(artifactBytes) === manifest.artifact.sha256, + }); + } + } + + const failed = results.filter((result) => !result.ok); + if (json) { + const payload = { + ok: failed.length === 0, + command: "verify", + results, + }; + return failed.length === 0 + ? success(renderJson(payload)) + : failure(exitCodes.verificationFailure, renderJson(payload)); + } + + if (results.length === 0) { + return failure( + exitCodes.verificationFailure, + "No backups found to verify." + ); + } + + if (failed.length > 0) { + return failure( + exitCodes.verificationFailure, + [ + "Backup verification failed.", + ...failed.map((result) => `- ${result.targetId}: ${result.backupId}`), + ].join("\n") + ); + } + + return success( + [ + "Backup verification passed.", + ...results.map((result) => `- ${result.targetId}: ${result.backupId}`), + ].join("\n") + ); +}; + +const runRestoreCommand = (args: string[]): CliResult => { + const targetId = args[1]; + const json = hasFlag(args, "--json"); + const backupId = getFlagValue(args, "--backup"); + const outputPath = getFlagValue(args, "--output"); + const configResult = loadConfigForCommand(args); + if (!configResult.ok) return configResult.result; + + if (backupId === undefined) { + return failure(exitCodes.usage, "Missing required --backup ."); + } + if (outputPath === undefined) { + return failure(exitCodes.usage, "Missing required --output ."); + } + + const selection = selectTargetsForCommand( + args, + configResult.config, + targetId + ); + if (!selection.ok) return selection.result; + if (selection.targets.length !== 1) { + return failure(exitCodes.usage, "Restore requires exactly one target."); + } + + const target = selection.targets[0]; + if (target === undefined) { + return failure(exitCodes.usage, "Restore requires exactly one target."); + } + + const storage = getLocalStorageForTarget(configResult.config, target); + if (!storage.ok) return failure(exitCodes.runtimeFailure, storage.message); + + const manifest = findManifestByBackupId( + storage.storage.listManifests(target.id), + backupId + ); + if (manifest === undefined) { + return failure(exitCodes.usage, `Backup not found: ${backupId}`); + } + + const restoredBytes = restoreLocalBackupArtifact( + storage.storage.readArtifact(manifest) + ); + writeFileSync(outputPath, restoredBytes); + + if (json) { + return success( + renderJson({ + ok: true, + command: "restore", + backupId, + outputPath, + }) + ); + } + + return success(`Restored ${backupId} to ${outputPath}.`); +}; + +const runPruneCommand = (args: string[]): CliResult => { + const json = hasFlag(args, "--json"); + const message = + "Prune is not implemented until retention policy execution exists."; + return json + ? success( + renderJson({ + ok: true, + command: "prune", + implemented: false, + message, + }) + ) + : success(`${message}\nNo backup side effects were executed.`); +}; + +export const runCli = (args: string[]): CliResult => { + const command = args[0]; + + if ( + command === undefined || + command === "--help" || + command === "-h" || + command === "help" + ) { + return success(renderGlobalHelp()); + } + + if (hasFlag(args, "--help") || hasFlag(args, "-h")) { + return success(renderCommandHelp(command)); + } + + if (command === "doctor") return runDoctorCommand(args); + if (command === "backup") return runBackupCommand(args); + if (command === "list") return runListCommand(args); + if (command === "verify") return runVerifyCommand(args); + if (command === "restore") return runRestoreCommand(args); + if (command === "prune") return runPruneCommand(args); + + return failure( + exitCodes.usage, + `Unknown command: ${command}\n\n${renderGlobalHelp()}` + ); +}; + +export const main = (): void => { + const result = runCli(process.argv.slice(2)); + if (result.stdout.length > 0) process.stdout.write(result.stdout); + if (result.stderr.length > 0) process.stderr.write(result.stderr); + process.exitCode = result.exitCode; }; if (import.meta.url === `file://${process.argv[1] ?? ""}`) { diff --git a/src/config/env.ts b/src/config/env.ts index 7ec748d..3db2622 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -35,25 +35,29 @@ export const getTargetEnvReferences = ( const encryption = target.encryption ?? config.defaults?.encryption; const notifications = target.notifications ?? config.defaults?.notifications; - addEnvReference( - references, - target.dumper.passwordEnv, - `${target.id}.dumper.passwordEnv`, - false - ); + if (target.dumper.type === "postgresDocker") { + addEnvReference( + references, + target.dumper.passwordEnv, + `${target.id}.dumper.passwordEnv`, + false + ); + } - addEnvReference( - references, - target.storage.accessKeyIdEnv, - `${target.id}.storage.accessKeyIdEnv`, - true - ); - addEnvReference( - references, - target.storage.secretAccessKeyEnv, - `${target.id}.storage.secretAccessKeyEnv`, - true - ); + if (target.storage.type === "s3") { + addEnvReference( + references, + target.storage.accessKeyIdEnv, + `${target.id}.storage.accessKeyIdEnv`, + true + ); + addEnvReference( + references, + target.storage.secretAccessKeyEnv, + `${target.id}.storage.secretAccessKeyEnv`, + true + ); + } if (encryption?.type === "age") { addEnvReference( diff --git a/src/config/schema.ts b/src/config/schema.ts index 32f6c9c..b8ac56c 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -35,7 +35,17 @@ const postgresDockerDumperSchema = z }) .strict(); -const dumperSchema = z.discriminatedUnion("type", [postgresDockerDumperSchema]); +const fakeDumperSchema = z + .object({ + type: z.literal("fake"), + bytes: z.string().min(1).default("ops-backup-runner fake dump\n"), + }) + .strict(); + +const dumperSchema = z.discriminatedUnion("type", [ + postgresDockerDumperSchema, + fakeDumperSchema, +]); const s3StorageSchema = z .object({ @@ -49,7 +59,17 @@ const s3StorageSchema = z }) .strict(); -const storageSchema = z.discriminatedUnion("type", [s3StorageSchema]); +const localStorageSchema = z + .object({ + type: z.literal("local"), + rootPath: z.string().min(1), + }) + .strict(); + +const storageSchema = z.discriminatedUnion("type", [ + s3StorageSchema, + localStorageSchema, +]); const ageEncryptionSchema = z .object({ diff --git a/src/core/artifact.ts b/src/core/artifact.ts new file mode 100644 index 0000000..2d19930 --- /dev/null +++ b/src/core/artifact.ts @@ -0,0 +1,12 @@ +import { createHash } from "node:crypto"; + +export const createBackupId = (targetId: string, date = new Date()): string => { + const timestamp = date + .toISOString() + .replaceAll(":", "-") + .replace(/\.\d{3}Z$/u, "Z"); + return `${targetId}-${timestamp}`; +}; + +export const sha256Hex = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); diff --git a/src/core/backup-job.ts b/src/core/backup-job.ts new file mode 100644 index 0000000..5cf2434 --- /dev/null +++ b/src/core/backup-job.ts @@ -0,0 +1,65 @@ +import { gzipSync, gunzipSync } from "node:zlib"; + +import type { BackupTarget } from "../config/types.js"; +import { createBackupId, sha256Hex } from "./artifact.js"; +import type { BackupManifest } from "./manifest.js"; +import type { Dumper, StorageAdapter } from "./ports.js"; +import { createTempWorkspace } from "./temp-workspace.js"; + +export interface BackupJobResult { + manifest: BackupManifest; + tempWorkspaceCleaned: boolean; +} + +export const runLocalBackupJob = ( + target: BackupTarget, + dumper: Dumper, + storage: StorageAdapter +): BackupJobResult => { + const workspace = createTempWorkspace(); + + try { + const dump = dumper.dump(target); + const compressed = gzipSync(dump.bytes); + const backupId = createBackupId(target.id); + const extension = `${dump.extension}.gz`; + const stored = storage.writeArtifact({ + targetId: target.id, + backupId, + artifactBytes: compressed, + extension, + }); + + const manifest: BackupManifest = { + version: 1, + backupId, + targetId: target.id, + createdAt: new Date().toISOString(), + artifact: { + key: stored.artifactKey, + sizeBytes: stored.sizeBytes, + sha256: sha256Hex(compressed), + compression: "gzip", + encryption: "none", + }, + storage: { + type: "local", + artifactKey: stored.artifactKey, + manifestKey: stored.manifestKey, + }, + }; + + storage.writeManifest(manifest); + workspace.cleanup(); + + return { + manifest, + tempWorkspaceCleaned: true, + }; + } finally { + workspace.cleanup(); + } +}; + +export const restoreLocalBackupArtifact = (artifactBytes: Buffer): Buffer => + gunzipSync(artifactBytes); diff --git a/src/core/manifest.ts b/src/core/manifest.ts new file mode 100644 index 0000000..bd1276c --- /dev/null +++ b/src/core/manifest.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; + +export const backupManifestSchema = z + .object({ + version: z.literal(1), + backupId: z.string().min(1), + targetId: z.string().min(1), + createdAt: z.iso.datetime(), + artifact: z + .object({ + key: z.string().min(1), + sizeBytes: z.number().int().nonnegative(), + sha256: z.string().length(64), + compression: z.literal("gzip"), + encryption: z.literal("none"), + }) + .strict(), + storage: z + .object({ + type: z.literal("local"), + artifactKey: z.string().min(1), + manifestKey: z.string().min(1), + }) + .strict(), + }) + .strict(); + +export type BackupManifest = z.infer; diff --git a/src/core/ports.ts b/src/core/ports.ts new file mode 100644 index 0000000..fe9e931 --- /dev/null +++ b/src/core/ports.ts @@ -0,0 +1,28 @@ +import type { BackupManifest } from "./manifest.js"; + +export interface DumpArtifact { + bytes: Buffer; + extension: string; +} + +export interface Dumper { + dump(target: TTarget): DumpArtifact; +} + +export interface StoredArtifact { + artifactKey: string; + manifestKey: string; + sizeBytes: number; +} + +export interface StorageAdapter { + writeArtifact(params: { + targetId: string; + backupId: string; + artifactBytes: Buffer; + extension: string; + }): StoredArtifact; + writeManifest(manifest: BackupManifest): void; + listManifests(targetId: string): BackupManifest[]; + readArtifact(manifest: BackupManifest): Buffer; +} diff --git a/src/core/temp-workspace.ts b/src/core/temp-workspace.ts new file mode 100644 index 0000000..05199ae --- /dev/null +++ b/src/core/temp-workspace.ts @@ -0,0 +1,19 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +export interface TempWorkspace { + path: string; + cleanup(): void; +} + +export const createTempWorkspace = (): TempWorkspace => { + const workspacePath = mkdtempSync(path.join(tmpdir(), "ops-backup-runner-")); + + return { + path: workspacePath, + cleanup(): void { + rmSync(workspacePath, { recursive: true, force: true }); + }, + }; +}; diff --git a/src/dumpers/fake.ts b/src/dumpers/fake.ts new file mode 100644 index 0000000..64d65d9 --- /dev/null +++ b/src/dumpers/fake.ts @@ -0,0 +1,17 @@ +import type { BackupTarget } from "../config/types.js"; +import type { DumpArtifact, Dumper } from "../core/ports.js"; + +export const fakeDumper: Dumper = { + dump(target: BackupTarget): DumpArtifact { + if (target.dumper.type !== "fake") { + throw new Error( + `Unsupported dumper for local pipeline: ${target.dumper.type}` + ); + } + + return { + bytes: Buffer.from(target.dumper.bytes, "utf8"), + extension: "dump", + }; + }, +}; diff --git a/src/storage/local.ts b/src/storage/local.ts new file mode 100644 index 0000000..64ff8c2 --- /dev/null +++ b/src/storage/local.ts @@ -0,0 +1,84 @@ +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from "node:fs"; +import path from "node:path"; + +import type { BackupTarget } from "../config/types.js"; +import { backupManifestSchema, type BackupManifest } from "../core/manifest.js"; +import type { StorageAdapter, StoredArtifact } from "../core/ports.js"; + +const getLocalRoot = (target: BackupTarget): string => { + if (target.storage.type !== "local") { + throw new Error( + `Unsupported storage for local pipeline: ${target.storage.type}` + ); + } + + return target.storage.rootPath; +}; + +const getTargetRoot = (root: string, targetId: string): string => + path.join(root, targetId); + +export const createLocalStorageAdapter = ( + target: BackupTarget +): StorageAdapter => { + const root = getLocalRoot(target); + + return { + writeArtifact(params): StoredArtifact { + const targetRoot = getTargetRoot(root, params.targetId); + const artifactsRoot = path.join(targetRoot, "artifacts"); + const manifestsRoot = path.join(targetRoot, "manifests"); + mkdirSync(artifactsRoot, { recursive: true }); + mkdirSync(manifestsRoot, { recursive: true }); + + const artifactFile = `${params.backupId}.${params.extension}`; + const artifactPath = path.join(artifactsRoot, artifactFile); + writeFileSync(artifactPath, params.artifactBytes); + + return { + artifactKey: path + .relative(root, artifactPath) + .split(path.sep) + .join("/"), + manifestKey: `${params.targetId}/manifests/${params.backupId}.json`, + sizeBytes: params.artifactBytes.byteLength, + }; + }, + + writeManifest(manifest): void { + const manifestPath = path.join(root, manifest.storage.manifestKey); + mkdirSync(path.dirname(manifestPath), { recursive: true }); + writeFileSync( + manifestPath, + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8" + ); + }, + + listManifests(targetId): BackupManifest[] { + const manifestsRoot = path.join( + getTargetRoot(root, targetId), + "manifests" + ); + if (!existsSync(manifestsRoot)) return []; + + return readdirSync(manifestsRoot) + .filter((file) => file.endsWith(".json")) + .sort() + .map((file) => { + const raw = readFileSync(path.join(manifestsRoot, file), "utf8"); + return backupManifestSchema.parse(JSON.parse(raw)); + }); + }, + + readArtifact(manifest): Buffer { + return readFileSync(path.join(root, manifest.storage.artifactKey)); + }, + }; +}; diff --git a/test/cli.test.ts b/test/cli.test.ts index 81deeed..b0aa724 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,13 +1,217 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; -import { cliName, getStartupMessage } from "../src/cli.js"; +import { cliName, exitCodes, getStartupMessage, runCli } from "../src/cli.js"; + +const writeConfig = (content: string): string => { + const directory = mkdtempSync(path.join(tmpdir(), "ops-backup-runner-cli-")); + const file = path.join(directory, "targets.yaml"); + writeFileSync(file, content, "utf8"); + return file; +}; + +const enabledConfig = ` +version: 1 +targets: + - id: maintana + enabled: true + dumper: + type: postgresDocker + container: maintana-postgres + database: maintana + username: maintana + storage: + type: s3 + endpoint: https://example-account-id.r2.cloudflarestorage.com + region: auto + bucket: maintana-backups + - id: kevly + enabled: false + dumper: + type: postgresDocker + container: kevly-postgres + database: kevly + username: kevly + storage: + type: s3 + endpoint: https://example-account-id.r2.cloudflarestorage.com + region: auto + bucket: kevly-backups +`; + +const localConfig = (storageRoot: string): string => ` +version: 1 +targets: + - id: local-demo + enabled: true + dumper: + type: fake + bytes: local fake dump + storage: + type: local + rootPath: ${storageRoot} + encryption: + type: none +`; describe("cli harness baseline", () => { it("exposes the CLI name", () => { expect(cliName).toBe("ops-backup-runner"); }); - it("does not expose backup behavior yet", () => { - expect(getStartupMessage()).toContain("not implemented yet"); + it("keeps a startup message for direct invocation without args", () => { + expect(getStartupMessage()).toContain("project harness initialized"); + }); + + it("shows global help", () => { + const result = runCli(["--help"]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("Commands:"); + expect(result.stdout).toContain("backup"); + }); + + it("shows command help", () => { + const result = runCli(["backup", "--help"]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("backup "); + }); + + it("fails clearly for unknown commands", () => { + const result = runCli(["missing"]); + + expect(result.exitCode).toBe(exitCodes.usage); + expect(result.stderr).toContain("Unknown command: missing"); + }); + + it("fails clearly for unknown targets", () => { + const result = runCli([ + "backup", + "unknown", + "--dry-run", + "--config", + writeConfig(enabledConfig), + ]); + + expect(result.exitCode).toBe(exitCodes.usage); + expect(result.stderr).toContain("Unknown target: unknown"); + }); + + it("lists enabled targets for backup all dry run", () => { + const result = runCli([ + "backup", + "all", + "--dry-run", + "--config", + writeConfig(enabledConfig), + ]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("Backup dry run passed."); + expect(result.stdout).toContain("- maintana"); + expect(result.stdout).not.toContain("- kevly"); + expect(result.stdout).toContain("No dump, upload, prune, or notification"); + }); + + it("emits JSON for backup dry run", () => { + const result = runCli([ + "backup", + "all", + "--dry-run", + "--json", + "--config", + writeConfig(enabledConfig), + ]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(JSON.parse(result.stdout)).toEqual({ + ok: true, + dryRun: true, + command: "backup", + targets: ["maintana"], + }); + }); + + it("rejects real backup execution for unsupported adapters", () => { + const result = runCli([ + "backup", + "maintana", + "--config", + writeConfig(enabledConfig), + ]); + + expect(result.exitCode).toBe(exitCodes.runtimeFailure); + expect(result.stderr).toContain("Phase 4 only supports fake dumper"); + }); + + it("runs local backup, lists it, verifies it, and restores it", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const outputPath = path.join(storageRoot, "restored.dump"); + const configPath = writeConfig(localConfig(storageRoot)); + + const backupResult = runCli([ + "backup", + "local-demo", + "--config", + configPath, + ]); + + expect(backupResult.exitCode).toBe(exitCodes.success); + expect(backupResult.stdout).toContain("Backup completed."); + + const backupId = /local-demo: (?local-demo-[^ ]+)/.exec( + backupResult.stdout + )?.groups?.["backupId"]; + expect(backupId).toBeDefined(); + if (backupId === undefined) return; + + const listResult = runCli(["list", "local-demo", "--config", configPath]); + + expect(listResult.exitCode).toBe(exitCodes.success); + expect(listResult.stdout).toContain(backupId); + + const verifyResult = runCli([ + "verify", + "local-demo", + "--latest", + "--config", + configPath, + ]); + + expect(verifyResult.exitCode).toBe(exitCodes.success); + expect(verifyResult.stdout).toContain("Backup verification passed."); + + const restoreResult = runCli([ + "restore", + "local-demo", + "--backup", + backupId, + "--output", + outputPath, + "--config", + configPath, + ]); + + expect(restoreResult.exitCode).toBe(exitCodes.success); + expect(readFileSync(outputPath, "utf8")).toBe("local fake dump"); + }); + + it("keeps prune as an explicit no-op until retention execution exists", () => { + const result = runCli([ + "prune", + "maintana", + "--config", + writeConfig(enabledConfig), + ]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("Prune is not implemented"); + expect(result.stdout).toContain("No backup side effects were executed."); }); }); diff --git a/test/config.test.ts b/test/config.test.ts index 423714a..fe39a18 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -58,7 +58,7 @@ describe("config foundation", () => { expect(result.ok).toBe(false); if (!result.ok) { - expect(result.issues.join("\n")).toContain("Expected 'postgresDocker'"); + expect(result.issues.join("\n")).toContain("postgresDocker"); } }); From 7d29786f56255a4d4ea8ca8fea015eac408f6341 Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 07:54:00 +0700 Subject: [PATCH 4/9] feat(dumper): add postgres docker support --- docs/implementation-plan.md | 24 +++++ src/cli.ts | 23 ++-- src/commands/doctor.ts | 12 ++- src/config/env.ts | 3 + src/config/schema.ts | 1 + src/dumpers/postgres-docker.ts | 187 +++++++++++++++++++++++++++++++++ test/cli.test.ts | 4 +- test/postgres-docker.test.ts | 157 +++++++++++++++++++++++++++ 8 files changed, 395 insertions(+), 16 deletions(-) create mode 100644 src/dumpers/postgres-docker.ts create mode 100644 test/postgres-docker.test.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 0c177ff..5187376 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -362,6 +362,8 @@ Result: ## Phase 5 — PostgreSQL Docker Dumper +Status: Done on 18 May 2026. + Goal: back up Dockerized PostgreSQL databases. Tasks: @@ -400,6 +402,28 @@ Acceptance criteria: - No plaintext dump is left behind after backup. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- `postgresDocker` dumper is implemented behind the shared dumper port; +- safe `docker exec ... pg_dump` argument construction is unit tested; +- optional `dockerBinary` and `pgRestoreBinary` are supported in config; +- optional `passwordEnv` is passed to Docker as `--env PGPASSWORD` without putting the secret value in command arguments; +- dump stdout is captured as bytes and passed into the existing gzip/local backup pipeline; +- `pg_restore --list` validates the custom dump from stdin before the artifact is accepted; +- docker failures and invalid dump verification failures produce clear target-specific errors; +- doctor checks validate docker binary availability, container existence, and `pg_dump --version` inside the container for enabled `postgresDocker` targets; +- no plaintext dump file is written by the dumper or backup job. + +Integration note: + +- Disposable Postgres container coverage was not added in this phase because the unit suite uses mocked process runners and must stay stable without Docker availability in CI. A real-container test can be added later behind an explicit integration-test command. + ## Phase 6 — S3/R2 Storage Adapter Goal: support Cloudflare R2 and generic S3-compatible storage. diff --git a/src/cli.ts b/src/cli.ts index 334c794..b98ef8d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,9 @@ import { } from "./core/backup-job.js"; import { sha256Hex } from "./core/artifact.js"; import type { BackupManifest } from "./core/manifest.js"; +import type { Dumper } from "./core/ports.js"; import { fakeDumper } from "./dumpers/fake.js"; +import { createPostgresDockerDumper } from "./dumpers/postgres-docker.js"; import { createLocalStorageAdapter } from "./storage/local.js"; export const cliName = "ops-backup-runner"; @@ -241,16 +243,6 @@ const getLocalBackupTarget = ( ok: false; result: CliResult; } => { - if (target.dumper.type !== "fake") { - return { - ok: false, - result: failure( - exitCodes.runtimeFailure, - `${target.id} uses ${target.dumper.type} dumper. Phase 4 only supports fake dumper.` - ), - }; - } - const storage = getLocalStorageForTarget(config, target); if (!storage.ok) { return { @@ -262,6 +254,11 @@ const getLocalBackupTarget = ( return storage; }; +const getDumperForTarget = (target: BackupTarget): Dumper => { + if (target.dumper.type === "fake") return fakeDumper; + return createPostgresDockerDumper(); +}; + const findManifestByBackupId = ( manifests: BackupManifest[], backupId: string @@ -349,7 +346,11 @@ const runBackupCommand = (args: string[]): CliResult => { const localTarget = getLocalBackupTarget(configResult.config, target); if (!localTarget.ok) return localTarget.result; - const result = runLocalBackupJob(target, fakeDumper, localTarget.storage); + const result = runLocalBackupJob( + target, + getDumperForTarget(target), + localTarget.storage + ); manifests.push(result.manifest); } diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index e334631..2a1dae8 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -2,6 +2,7 @@ import { loadConfigFromFile } from "../config/loader.js"; import { redactConfigPreview } from "../config/redact.js"; import { resolveTargetEnvReferences } from "../config/env.js"; import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; +import { checkPostgresDockerTarget } from "../dumpers/postgres-docker.js"; export interface DoctorTargetResult { id: string; @@ -39,13 +40,18 @@ const inspectTarget = ( } const envResult = resolveTargetEnvReferences(config, target); + const postgresDockerResult = checkPostgresDockerTarget(target); + const issues = [ + ...envResult.issues.map((issue) => issue.message), + ...postgresDockerResult.issues, + ]; return { id: target.id, enabled: true, - ok: envResult.ok, - status: envResult.ok ? "ready" : "missing-env", - issues: envResult.issues.map((issue) => issue.message), + ok: envResult.ok && postgresDockerResult.ok, + status: issues.length === 0 ? "ready" : "missing-env", + issues, }; }; diff --git a/src/config/env.ts b/src/config/env.ts index 3db2622..df82d6d 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -17,6 +17,9 @@ export interface EnvResolutionResult { issues: EnvResolutionIssue[]; } +export const getRuntimeEnv = (): Record => + process.env; + const addEnvReference = ( references: EnvReference[], name: string | undefined, diff --git a/src/config/schema.ts b/src/config/schema.ts index b8ac56c..030b270 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -32,6 +32,7 @@ const postgresDockerDumperSchema = z passwordEnv: envNameSchema.optional(), format: z.literal("custom").default("custom"), dockerBinary: z.string().min(1).optional(), + pgRestoreBinary: z.string().min(1).optional(), }) .strict(); diff --git a/src/dumpers/postgres-docker.ts b/src/dumpers/postgres-docker.ts new file mode 100644 index 0000000..afecdb0 --- /dev/null +++ b/src/dumpers/postgres-docker.ts @@ -0,0 +1,187 @@ +import { spawnSync } from "node:child_process"; + +import { getRuntimeEnv } from "../config/env.js"; +import type { BackupTarget } from "../config/types.js"; +import type { DumpArtifact, Dumper } from "../core/ports.js"; + +export interface ProcessRunResult { + status: number | null; + stdout: Buffer; + stderr: string; + error?: Error; +} + +export interface ProcessRunOptions { + input?: Buffer; + env?: Record; +} + +export type ProcessRunner = ( + command: string, + args: string[], + options?: ProcessRunOptions +) => ProcessRunResult; + +export const defaultProcessRunner: ProcessRunner = (command, args, options) => { + const result = spawnSync(command, args, { + input: options?.input, + env: options?.env, + encoding: "buffer", + }); + + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr.toString("utf8"), + ...(result.error === undefined ? {} : { error: result.error }), + }; +}; + +const getPostgresDockerConfig = (target: BackupTarget) => { + if (target.dumper.type !== "postgresDocker") { + throw new Error(`Unsupported dumper: ${target.dumper.type}`); + } + + return target.dumper; +}; + +export const buildPostgresDockerDumpArgs = (target: BackupTarget): string[] => { + const dumper = getPostgresDockerConfig(target); + const args = ["exec"]; + + if (dumper.passwordEnv !== undefined) { + args.push("--env", "PGPASSWORD"); + } + + args.push( + dumper.container, + "pg_dump", + "-U", + dumper.username, + "-d", + dumper.database, + "--format=custom", + "--no-owner", + "--no-privileges" + ); + + return args; +}; + +export const buildPostgresDockerInspectArgs = ( + target: BackupTarget +): string[] => { + const dumper = getPostgresDockerConfig(target); + return ["inspect", "--type", "container", dumper.container]; +}; + +export const buildPostgresDockerPgDumpCheckArgs = ( + target: BackupTarget +): string[] => { + const dumper = getPostgresDockerConfig(target); + return ["exec", dumper.container, "pg_dump", "--version"]; +}; + +const mergePasswordEnv = ( + target: BackupTarget +): Record => { + const dumper = getPostgresDockerConfig(target); + const runtimeEnv = getRuntimeEnv(); + if (dumper.passwordEnv === undefined) return runtimeEnv; + + return { + ...runtimeEnv, + PGPASSWORD: runtimeEnv[dumper.passwordEnv], + }; +}; + +const assertSuccess = ( + result: ProcessRunResult, + failureMessage: string +): void => { + if (result.status === 0) return; + + const detail = + (result.error?.message ?? result.stderr.trim()) || "unknown error"; + throw new Error(`${failureMessage}: ${detail}`); +}; + +export const createPostgresDockerDumper = ( + runner: ProcessRunner = defaultProcessRunner +): Dumper => ({ + dump(target: BackupTarget): DumpArtifact { + const dumper = getPostgresDockerConfig(target); + const dockerBinary = dumper.dockerBinary ?? "docker"; + const dumpResult = runner( + dockerBinary, + buildPostgresDockerDumpArgs(target), + { + env: mergePasswordEnv(target), + } + ); + + assertSuccess( + dumpResult, + `PostgreSQL Docker dump failed for target ${target.id}` + ); + + const pgRestoreBinary = dumper.pgRestoreBinary ?? "pg_restore"; + const restoreCheck = runner(pgRestoreBinary, ["--list"], { + input: dumpResult.stdout, + }); + + assertSuccess( + restoreCheck, + `PostgreSQL dump verification failed for target ${target.id}` + ); + + return { + bytes: dumpResult.stdout, + extension: "dump", + }; + }, +}); + +export interface PostgresDockerDoctorResult { + ok: boolean; + issues: string[]; +} + +export const checkPostgresDockerTarget = ( + target: BackupTarget, + runner: ProcessRunner = defaultProcessRunner +): PostgresDockerDoctorResult => { + if (target.dumper.type !== "postgresDocker") { + return { ok: true, issues: [] }; + } + + const dockerBinary = target.dumper.dockerBinary ?? "docker"; + const checks = [ + { + label: "docker binary", + result: runner(dockerBinary, ["--version"]), + }, + { + label: "docker container", + result: runner(dockerBinary, buildPostgresDockerInspectArgs(target)), + }, + { + label: "pg_dump in container", + result: runner(dockerBinary, buildPostgresDockerPgDumpCheckArgs(target)), + }, + ]; + + const issues = checks + .filter((check) => check.result.status !== 0) + .map((check) => { + const detail = + (check.result.error?.message ?? check.result.stderr.trim()) || + "unknown error"; + return `${target.id} ${check.label} check failed: ${detail}`; + }); + + return { + ok: issues.length === 0, + issues, + }; +}; diff --git a/test/cli.test.ts b/test/cli.test.ts index b0aa724..7da35c6 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -136,7 +136,7 @@ describe("cli harness baseline", () => { }); }); - it("rejects real backup execution for unsupported adapters", () => { + it("rejects real backup execution for unsupported storage adapters", () => { const result = runCli([ "backup", "maintana", @@ -145,7 +145,7 @@ describe("cli harness baseline", () => { ]); expect(result.exitCode).toBe(exitCodes.runtimeFailure); - expect(result.stderr).toContain("Phase 4 only supports fake dumper"); + expect(result.stderr).toContain("Phase 4 only supports local storage"); }); it("runs local backup, lists it, verifies it, and restores it", () => { diff --git a/test/postgres-docker.test.ts b/test/postgres-docker.test.ts new file mode 100644 index 0000000..70ec858 --- /dev/null +++ b/test/postgres-docker.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vitest"; + +import type { BackupTarget } from "../src/config/types.js"; +import { + buildPostgresDockerDumpArgs, + buildPostgresDockerInspectArgs, + buildPostgresDockerPgDumpCheckArgs, + checkPostgresDockerTarget, + createPostgresDockerDumper, + type ProcessRunner, +} from "../src/dumpers/postgres-docker.js"; + +const postgresTarget: BackupTarget = { + id: "maintana", + enabled: true, + dumper: { + type: "postgresDocker", + container: "maintana-postgres", + database: "maintana", + username: "maintana", + format: "custom", + }, + storage: { + type: "local", + rootPath: "/tmp/backups", + }, + encryption: { + type: "none", + }, +}; + +const okRunner: ProcessRunner = () => ({ + status: 0, + stdout: Buffer.from("ok"), + stderr: "", +}); + +describe("postgres docker dumper", () => { + it("builds safe docker exec pg_dump args", () => { + expect(buildPostgresDockerDumpArgs(postgresTarget)).toEqual([ + "exec", + "maintana-postgres", + "pg_dump", + "-U", + "maintana", + "-d", + "maintana", + "--format=custom", + "--no-owner", + "--no-privileges", + ]); + }); + + it("builds docker doctor check args", () => { + expect(buildPostgresDockerInspectArgs(postgresTarget)).toEqual([ + "inspect", + "--type", + "container", + "maintana-postgres", + ]); + expect(buildPostgresDockerPgDumpCheckArgs(postgresTarget)).toEqual([ + "exec", + "maintana-postgres", + "pg_dump", + "--version", + ]); + }); + + it("runs pg_dump and verifies the custom dump with pg_restore list", () => { + const calls: { + command: string; + args: string[]; + input: Buffer | undefined; + }[] = []; + const runner: ProcessRunner = (command, args, options) => { + calls.push({ command, args, input: options?.input }); + return { + status: 0, + stdout: Buffer.from("custom dump bytes"), + stderr: "", + }; + }; + + const artifact = createPostgresDockerDumper(runner).dump(postgresTarget); + + expect(artifact.bytes.toString("utf8")).toBe("custom dump bytes"); + expect(artifact.extension).toBe("dump"); + expect(calls).toEqual([ + { + command: "docker", + args: buildPostgresDockerDumpArgs(postgresTarget), + input: undefined, + }, + { + command: "pg_restore", + args: ["--list"], + input: Buffer.from("custom dump bytes"), + }, + ]); + }); + + it("returns clear dump failure errors", () => { + const runner: ProcessRunner = () => ({ + status: 1, + stdout: Buffer.alloc(0), + stderr: "database does not exist", + }); + + expect(() => + createPostgresDockerDumper(runner).dump(postgresTarget) + ).toThrow( + "PostgreSQL Docker dump failed for target maintana: database does not exist" + ); + }); + + it("returns clear pg_restore validation errors", () => { + let callCount = 0; + const runner: ProcessRunner = () => { + callCount += 1; + return callCount === 1 + ? { + status: 0, + stdout: Buffer.from("not a custom dump"), + stderr: "", + } + : { + status: 1, + stdout: Buffer.alloc(0), + stderr: "input file does not appear to be a valid archive", + }; + }; + + expect(() => + createPostgresDockerDumper(runner).dump(postgresTarget) + ).toThrow( + "PostgreSQL dump verification failed for target maintana: input file does not appear to be a valid archive" + ); + }); + + it("reports docker doctor failures", () => { + const runner: ProcessRunner = (command, args) => { + if (args.includes("inspect")) { + return { + status: 1, + stdout: Buffer.alloc(0), + stderr: "No such container", + }; + } + return okRunner(command, args); + }; + + expect(checkPostgresDockerTarget(postgresTarget, runner)).toEqual({ + ok: false, + issues: ["maintana docker container check failed: No such container"], + }); + }); +}); From e9146452aa7aea85df2330be60be57566cc1af08 Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 08:21:00 +0700 Subject: [PATCH 5/9] feat(storage): add s3 adapter --- docs/implementation-plan.md | 20 ++ package.json | 1 + pnpm-lock.yaml | 661 +++++++++++++++++++++++++++++++++++- src/core/manifest.ts | 2 +- src/core/ports.ts | 8 + src/storage/s3.ts | 353 +++++++++++++++++++ test/s3-storage.test.ts | 217 ++++++++++++ 7 files changed, 1259 insertions(+), 3 deletions(-) create mode 100644 src/storage/s3.ts create mode 100644 test/s3-storage.test.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 5187376..ecc9eda 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -426,6 +426,8 @@ Integration note: ## Phase 6 — S3/R2 Storage Adapter +Status: Done on 18 May 2026. + Goal: support Cloudflare R2 and generic S3-compatible storage. Tasks: @@ -460,6 +462,24 @@ Acceptance criteria: - `doctor` can validate storage config without printing secrets. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- S3-compatible storage adapter is implemented with AWS SDK v3; +- adapter supports upload, head, list-by-prefix, download, and delete operations; +- artifact upload verifies object size with `HeadObject`; +- manifest upload verifies object size with `HeadObject`; +- object metadata includes target id, backup id, created time, and sha256 where relevant; +- key generation supports per-target prefixes for shared buckets; +- config already supports per-target endpoint, bucket, prefix, and credential env references; +- doctor validates missing S3 credential env references through the existing env resolution path without printing secret values; +- tests use a mocked S3 client and do not require real Cloudflare R2 credentials. + ## Phase 7 — Age Encryption Goal: encrypt backups before external upload. diff --git a/package.json b/package.json index 9c03f6f..be81f2d 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "vitest": "^4.1.0" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1048.0", "yaml": "^2.9.0", "zod": "^4.4.3" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23189db..ab18f37 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,9 @@ settings: importers: .: dependencies: + "@aws-sdk/client-s3": + specifier: ^3.1048.0 + version: 3.1048.0 yaml: specifier: ^2.9.0 version: 2.9.0 @@ -37,6 +40,218 @@ importers: version: 4.1.6(@types/node@22.19.19)(vite@8.0.13(@types/node@22.19.19)(yaml@2.9.0)) packages: + "@aws-crypto/crc32@5.2.0": + resolution: + { + integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==, + } + engines: { node: ">=16.0.0" } + + "@aws-crypto/crc32c@5.2.0": + resolution: + { + integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==, + } + + "@aws-crypto/sha1-browser@5.2.0": + resolution: + { + integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==, + } + + "@aws-crypto/sha256-browser@5.2.0": + resolution: + { + integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==, + } + + "@aws-crypto/sha256-js@5.2.0": + resolution: + { + integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==, + } + engines: { node: ">=16.0.0" } + + "@aws-crypto/supports-web-crypto@5.2.0": + resolution: + { + integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==, + } + + "@aws-crypto/util@5.2.0": + resolution: + { + integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==, + } + + "@aws-sdk/client-s3@3.1048.0": + resolution: + { + integrity: sha512-SrJn5FteqqtcDBgQIvqLKk3Qn/2vSsi5XR03I53EDDR4CbCdLysVSNgUnjVncEECMua9Pz+nxO0/lEx3TP+6mA==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/core@3.974.11": + resolution: + { + integrity: sha512-QpnINq5FZH6EOaDEkmHdT7eUunbvD27pDNQypaWjFyYz7Zl1q3UCMQErBZxpmfGfI7MvI2TlK8KTkgNpv8b1ug==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/crc64-nvme@3.972.8": + resolution: + { + integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-env@3.972.37": + resolution: + { + integrity: sha512-/jpPvEh6f7ntmIzf7dNxoNX6Q8vt8UpesCjbW6mFfk4V1NW6bIy9qxcQ6WbA8As5yQhsZOe+xeNd4xHX8kdY2Q==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-http@3.972.39": + resolution: + { + integrity: sha512-pIgTpisWyWg7X1bUbzSjuUYosYTD0Ghz2M0hkSTmb3a6i3qV3uU+NYJPI/E2XSC0HcsZh5rsLPzeXrkb2DS0Cg==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-ini@3.972.41": + resolution: + { + integrity: sha512-u2tyjaxJJzW8UtW4SM1ZcPMDwO6y+kV+llvou+Adts0FAKyzes5jG4izQN+KX3yE8ZROpS5y1LJ//xL2iSf76w==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-login@3.972.41": + resolution: + { + integrity: sha512-0LBitxXiAiaE5nlFPfpNIww/8FRY/I7WIndWsc9GmNFOM7cE1wNpVNQEGEk9Outg5l8xl+3vybxFyUy4l9q/LQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-node@3.972.42": + resolution: + { + integrity: sha512-D4oon2zbqqsWOJUM99Gm3/ZyJ0IJvTXVN3PyloGb3kQEyI36fjCZheZj422lAgTWWd6TSHgiImLt3RIaLdv3dQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-process@3.972.37": + resolution: + { + integrity: sha512-7nVaHBUaWIddASYfVaA9O4D5ZVjewU3sCol9WqZPGfW0nR+0WqE0xHZnD/U2L33PlOB8KNXGKZ6wOES/QijKzg==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-sso@3.972.41": + resolution: + { + integrity: sha512-IOWAWEHe5LkjSKkkUUX9ciV6Y1scHTsnfEkdt5yyC4Slrc7AGbkLPrpntjqh18ksJAMOaVhoBsO8p2WyTcY2wQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/credential-provider-web-identity@3.972.41": + resolution: + { + integrity: sha512-mbACk9Yypa8nm4iGZLs0PofOXEcTDOUw6wDnsPXNDNSd2WNXs1tSo+6nc/fh0jLYdfVZThhBL98PHW4aXFsG5A==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-bucket-endpoint@3.972.13": + resolution: + { + integrity: sha512-JDaukix+kt5KwF7FzNSkfZHpqiPJajVkKJLJexF6z5B44+CN70BXGiQaCEAiCtKtRZNvC16eF3SY9L0bDJPlbA==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-expect-continue@3.972.12": + resolution: + { + integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-flexible-checksums@3.974.19": + resolution: + { + integrity: sha512-GLciZVIvWM3C+ffuqnUqlAZwRjQdLt+KXiqr9+aRwZyKVyF2J5lrJAzzSqwweNl9hUWBN00BhilWXdMI5DjNcw==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-location-constraint@3.972.10": + resolution: + { + integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-sdk-s3@3.972.40": + resolution: + { + integrity: sha512-vyFY4EsAGySqqd87Z7n4qcCYXJO3QArB8VIJzuupY5XuLHIp579HTZldIUGGABvAOzLptfPb9+lJBJcB+3/cvA==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/middleware-ssec@3.972.10": + resolution: + { + integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/nested-clients@3.997.9": + resolution: + { + integrity: sha512-jPR3rnmRI4hWYyzfmTGBr7NblMp8QYYeflHXba1H6+7CGrWVqWKQzaXFQ4qbExqPRsXN3T3L3JxFhr6aouXUGQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/signature-v4-multi-region@3.996.27": + resolution: + { + integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/token-providers@3.1048.0": + resolution: + { + integrity: sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/types@3.973.8": + resolution: + { + integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/util-locate-window@3.965.5": + resolution: + { + integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==, + } + engines: { node: ">=20.0.0" } + + "@aws-sdk/xml-builder@3.972.24": + resolution: + { + integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==, + } + engines: { node: ">=20.0.0" } + + "@aws/lambda-invoke-store@0.2.4": + resolution: + { + integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==, + } + engines: { node: ">=18.0.0" } + "@emnapi/core@1.10.0": resolution: { @@ -170,6 +385,12 @@ packages: "@emnapi/core": ^1.7.1 "@emnapi/runtime": ^1.7.1 + "@nodable/entities@2.1.0": + resolution: + { + integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==, + } + "@oxc-project/types@0.130.0": resolution: { @@ -322,6 +543,69 @@ packages: integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==, } + "@smithy/core@3.24.3": + resolution: + { + integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==, + } + engines: { node: ">=18.0.0" } + + "@smithy/credential-provider-imds@4.3.3": + resolution: + { + integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==, + } + engines: { node: ">=18.0.0" } + + "@smithy/fetch-http-handler@5.4.3": + resolution: + { + integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==, + } + engines: { node: ">=18.0.0" } + + "@smithy/is-array-buffer@2.2.0": + resolution: + { + integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==, + } + engines: { node: ">=14.0.0" } + + "@smithy/node-http-handler@4.7.3": + resolution: + { + integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==, + } + engines: { node: ">=18.0.0" } + + "@smithy/signature-v4@5.4.3": + resolution: + { + integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==, + } + engines: { node: ">=18.0.0" } + + "@smithy/types@4.14.2": + resolution: + { + integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==, + } + engines: { node: ">=18.0.0" } + + "@smithy/util-buffer-from@2.2.0": + resolution: + { + integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==, + } + engines: { node: ">=14.0.0" } + + "@smithy/util-utf8@2.3.0": + resolution: + { + integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==, + } + engines: { node: ">=14.0.0" } + "@standard-schema/spec@1.1.0": resolution: { @@ -558,6 +842,12 @@ packages: } engines: { node: 18 || 20 || >=22 } + bowser@2.14.1: + resolution: + { + integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==, + } + brace-expansion@1.1.14: resolution: { @@ -769,6 +1059,19 @@ packages: integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==, } + fast-xml-builder@1.2.0: + resolution: + { + integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==, + } + + fast-xml-parser@5.7.3: + resolution: + { + integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==, + } + hasBin: true + fdir@6.5.0: resolution: { @@ -1126,6 +1429,13 @@ packages: } engines: { node: ">=8" } + path-expression-matcher@1.5.0: + resolution: + { + integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==, + } + engines: { node: ">=14.0.0" } + path-key@3.1.1: resolution: { @@ -1250,6 +1560,12 @@ packages: } engines: { node: ">=8" } + strnum@2.3.0: + resolution: + { + integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==, + } + supports-color@7.2.0: resolution: { @@ -1449,6 +1765,13 @@ packages: } engines: { node: ">=0.10.0" } + xml-naming@0.1.0: + resolution: + { + integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==, + } + engines: { node: ">=16.0.0" } + yaml@2.9.0: resolution: { @@ -1471,6 +1794,271 @@ packages: } snapshots: + "@aws-crypto/crc32@5.2.0": + dependencies: + "@aws-crypto/util": 5.2.0 + "@aws-sdk/types": 3.973.8 + tslib: 2.8.1 + + "@aws-crypto/crc32c@5.2.0": + dependencies: + "@aws-crypto/util": 5.2.0 + "@aws-sdk/types": 3.973.8 + tslib: 2.8.1 + + "@aws-crypto/sha1-browser@5.2.0": + dependencies: + "@aws-crypto/supports-web-crypto": 5.2.0 + "@aws-crypto/util": 5.2.0 + "@aws-sdk/types": 3.973.8 + "@aws-sdk/util-locate-window": 3.965.5 + "@smithy/util-utf8": 2.3.0 + tslib: 2.8.1 + + "@aws-crypto/sha256-browser@5.2.0": + dependencies: + "@aws-crypto/sha256-js": 5.2.0 + "@aws-crypto/supports-web-crypto": 5.2.0 + "@aws-crypto/util": 5.2.0 + "@aws-sdk/types": 3.973.8 + "@aws-sdk/util-locate-window": 3.965.5 + "@smithy/util-utf8": 2.3.0 + tslib: 2.8.1 + + "@aws-crypto/sha256-js@5.2.0": + dependencies: + "@aws-crypto/util": 5.2.0 + "@aws-sdk/types": 3.973.8 + tslib: 2.8.1 + + "@aws-crypto/supports-web-crypto@5.2.0": + dependencies: + tslib: 2.8.1 + + "@aws-crypto/util@5.2.0": + dependencies: + "@aws-sdk/types": 3.973.8 + "@smithy/util-utf8": 2.3.0 + tslib: 2.8.1 + + "@aws-sdk/client-s3@3.1048.0": + dependencies: + "@aws-crypto/sha1-browser": 5.2.0 + "@aws-crypto/sha256-browser": 5.2.0 + "@aws-crypto/sha256-js": 5.2.0 + "@aws-sdk/core": 3.974.11 + "@aws-sdk/credential-provider-node": 3.972.42 + "@aws-sdk/middleware-bucket-endpoint": 3.972.13 + "@aws-sdk/middleware-expect-continue": 3.972.12 + "@aws-sdk/middleware-flexible-checksums": 3.974.19 + "@aws-sdk/middleware-location-constraint": 3.972.10 + "@aws-sdk/middleware-sdk-s3": 3.972.40 + "@aws-sdk/middleware-ssec": 3.972.10 + "@aws-sdk/signature-v4-multi-region": 3.996.27 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/fetch-http-handler": 5.4.3 + "@smithy/node-http-handler": 4.7.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/core@3.974.11": + dependencies: + "@aws-sdk/types": 3.973.8 + "@aws-sdk/xml-builder": 3.972.24 + "@aws/lambda-invoke-store": 0.2.4 + "@smithy/core": 3.24.3 + "@smithy/signature-v4": 5.4.3 + "@smithy/types": 4.14.2 + bowser: 2.14.1 + tslib: 2.8.1 + + "@aws-sdk/crc64-nvme@3.972.8": + dependencies: + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-env@3.972.37": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-http@3.972.39": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/fetch-http-handler": 5.4.3 + "@smithy/node-http-handler": 4.7.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-ini@3.972.41": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/credential-provider-env": 3.972.37 + "@aws-sdk/credential-provider-http": 3.972.39 + "@aws-sdk/credential-provider-login": 3.972.41 + "@aws-sdk/credential-provider-process": 3.972.37 + "@aws-sdk/credential-provider-sso": 3.972.41 + "@aws-sdk/credential-provider-web-identity": 3.972.41 + "@aws-sdk/nested-clients": 3.997.9 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/credential-provider-imds": 4.3.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-login@3.972.41": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/nested-clients": 3.997.9 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-node@3.972.42": + dependencies: + "@aws-sdk/credential-provider-env": 3.972.37 + "@aws-sdk/credential-provider-http": 3.972.39 + "@aws-sdk/credential-provider-ini": 3.972.41 + "@aws-sdk/credential-provider-process": 3.972.37 + "@aws-sdk/credential-provider-sso": 3.972.41 + "@aws-sdk/credential-provider-web-identity": 3.972.41 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/credential-provider-imds": 4.3.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-process@3.972.37": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-sso@3.972.41": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/nested-clients": 3.997.9 + "@aws-sdk/token-providers": 3.1048.0 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/credential-provider-web-identity@3.972.41": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/nested-clients": 3.997.9 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-bucket-endpoint@3.972.13": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-expect-continue@3.972.12": + dependencies: + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-flexible-checksums@3.974.19": + dependencies: + "@aws-crypto/crc32": 5.2.0 + "@aws-crypto/crc32c": 5.2.0 + "@aws-crypto/util": 5.2.0 + "@aws-sdk/core": 3.974.11 + "@aws-sdk/crc64-nvme": 3.972.8 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-location-constraint@3.972.10": + dependencies: + "@aws-sdk/types": 3.973.8 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-sdk-s3@3.972.40": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/signature-v4-multi-region": 3.996.27 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/signature-v4": 5.4.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/middleware-ssec@3.972.10": + dependencies: + "@aws-sdk/types": 3.973.8 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/nested-clients@3.997.9": + dependencies: + "@aws-crypto/sha256-browser": 5.2.0 + "@aws-crypto/sha256-js": 5.2.0 + "@aws-sdk/core": 3.974.11 + "@aws-sdk/signature-v4-multi-region": 3.996.27 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/fetch-http-handler": 5.4.3 + "@smithy/node-http-handler": 4.7.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/signature-v4-multi-region@3.996.27": + dependencies: + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/signature-v4": 5.4.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/token-providers@3.1048.0": + dependencies: + "@aws-sdk/core": 3.974.11 + "@aws-sdk/nested-clients": 3.997.9 + "@aws-sdk/types": 3.973.8 + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/types@3.973.8": + dependencies: + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@aws-sdk/util-locate-window@3.965.5": + dependencies: + tslib: 2.8.1 + + "@aws-sdk/xml-builder@3.972.24": + dependencies: + "@nodable/entities": 2.1.0 + "@smithy/types": 4.14.2 + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + + "@aws/lambda-invoke-store@0.2.4": {} + "@emnapi/core@1.10.0": dependencies: "@emnapi/wasi-threads": 1.2.1 @@ -1558,6 +2146,8 @@ snapshots: "@tybys/wasm-util": 0.10.2 optional: true + "@nodable/entities@2.1.0": {} + "@oxc-project/types@0.130.0": {} "@rolldown/binding-android-arm64@1.0.1": @@ -1611,6 +2201,54 @@ snapshots: "@rolldown/pluginutils@1.0.1": {} + "@smithy/core@3.24.3": + dependencies: + "@aws-crypto/crc32": 5.2.0 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@smithy/credential-provider-imds@4.3.3": + dependencies: + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@smithy/fetch-http-handler@5.4.3": + dependencies: + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@smithy/is-array-buffer@2.2.0": + dependencies: + tslib: 2.8.1 + + "@smithy/node-http-handler@4.7.3": + dependencies: + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@smithy/signature-v4@5.4.3": + dependencies: + "@smithy/core": 3.24.3 + "@smithy/types": 4.14.2 + tslib: 2.8.1 + + "@smithy/types@4.14.2": + dependencies: + tslib: 2.8.1 + + "@smithy/util-buffer-from@2.2.0": + dependencies: + "@smithy/is-array-buffer": 2.2.0 + tslib: 2.8.1 + + "@smithy/util-utf8@2.3.0": + dependencies: + "@smithy/util-buffer-from": 2.2.0 + tslib: 2.8.1 + "@standard-schema/spec@1.1.0": {} "@tybys/wasm-util@0.10.2": @@ -1790,6 +2428,8 @@ snapshots: balanced-match@4.0.4: {} + bowser@2.14.1: {} + brace-expansion@1.1.14: dependencies: balanced-match: 1.0.2 @@ -1916,6 +2556,18 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-xml-builder@1.2.0: + dependencies: + path-expression-matcher: 1.5.0 + xml-naming: 0.1.0 + + fast-xml-parser@5.7.3: + dependencies: + "@nodable/entities": 2.1.0 + fast-xml-builder: 1.2.0 + path-expression-matcher: 1.5.0 + strnum: 2.3.0 + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -2083,6 +2735,8 @@ snapshots: path-exists@4.0.0: {} + path-expression-matcher@1.5.0: {} + path-key@3.1.1: {} pathe@2.0.3: {} @@ -2144,6 +2798,8 @@ snapshots: strip-json-comments@3.1.1: {} + strnum@2.3.0: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -2163,8 +2819,7 @@ snapshots: dependencies: typescript: 5.9.3 - tslib@2.8.1: - optional: true + tslib@2.8.1: {} type-check@0.4.0: dependencies: @@ -2239,6 +2894,8 @@ snapshots: word-wrap@1.2.5: {} + xml-naming@0.1.0: {} + yaml@2.9.0: {} yocto-queue@0.1.0: {} diff --git a/src/core/manifest.ts b/src/core/manifest.ts index bd1276c..8156ec7 100644 --- a/src/core/manifest.ts +++ b/src/core/manifest.ts @@ -17,7 +17,7 @@ export const backupManifestSchema = z .strict(), storage: z .object({ - type: z.literal("local"), + type: z.enum(["local", "s3"]), artifactKey: z.string().min(1), manifestKey: z.string().min(1), }) diff --git a/src/core/ports.ts b/src/core/ports.ts index fe9e931..463343e 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -25,4 +25,12 @@ export interface StorageAdapter { writeManifest(manifest: BackupManifest): void; listManifests(targetId: string): BackupManifest[]; readArtifact(manifest: BackupManifest): Buffer; + deleteObject?(key: string): void; + headObject?(key: string): StoredObjectHead; +} + +export interface StoredObjectHead { + key: string; + sizeBytes: number; + metadata: Record; } diff --git a/src/storage/s3.ts b/src/storage/s3.ts new file mode 100644 index 0000000..1d3f66b --- /dev/null +++ b/src/storage/s3.ts @@ -0,0 +1,353 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, + type DeleteObjectCommandInput, + type GetObjectCommandInput, + type HeadObjectCommandInput, + type ListObjectsV2CommandInput, + type PutObjectCommandInput, +} from "@aws-sdk/client-s3"; + +import { getRuntimeEnv } from "../config/env.js"; +import type { BackupTarget } from "../config/types.js"; +import { backupManifestSchema, type BackupManifest } from "../core/manifest.js"; +import type { + StorageAdapter, + StoredArtifact, + StoredObjectHead, +} from "../core/ports.js"; + +export interface S3LikeClient { + send(command: unknown): Promise; +} + +interface ByteArrayTransformBody { + transformToByteArray(): Promise; +} + +export type S3StorageAdapter = Omit< + StorageAdapter, + | "writeArtifact" + | "writeManifest" + | "listManifests" + | "readArtifact" + | "deleteObject" + | "headObject" +> & { + writeArtifact(params: { + targetId: string; + backupId: string; + artifactBytes: Buffer; + extension: string; + }): Promise; + writeManifest(manifest: BackupManifest): Promise; + listManifests(targetId: string): Promise; + readArtifact(manifest: BackupManifest): Promise; + deleteObject(key: string): Promise; + headObject(key: string): Promise; +}; + +const getS3StorageConfig = (target: BackupTarget) => { + if (target.storage.type !== "s3") { + throw new Error( + `Unsupported storage for S3 adapter: ${target.storage.type}` + ); + } + + return target.storage; +}; + +const resolveRequiredEnv = ( + name: string | undefined, + owner: string +): string => { + if (name === undefined) { + throw new Error(`Missing env reference for ${owner}`); + } + + const value = getRuntimeEnv()[name]; + if (value === undefined || value.length === 0) { + throw new Error( + `Missing required environment variable ${name} for ${owner}` + ); + } + + return value; +}; + +const joinKey = (...parts: string[]): string => + parts + .filter((part) => part.length > 0) + .join("/") + .replaceAll(/\/+/gu, "/"); + +const getPrefix = (target: BackupTarget): string => { + const storage = getS3StorageConfig(target); + return storage.prefix ?? ""; +}; + +export const buildS3ArtifactKey = ( + target: BackupTarget, + backupId: string, + extension: string +): string => + joinKey( + getPrefix(target), + target.id, + "artifacts", + `${backupId}.${extension}` + ); + +export const buildS3ManifestKey = ( + target: BackupTarget, + backupId: string +): string => + joinKey(getPrefix(target), target.id, "manifests", `${backupId}.json`); + +export const buildS3ManifestPrefix = (target: BackupTarget): string => + joinKey(getPrefix(target), target.id, "manifests") + "/"; + +export const createS3ClientForTarget = (target: BackupTarget): S3Client => { + const storage = getS3StorageConfig(target); + const accessKeyId = resolveRequiredEnv( + storage.accessKeyIdEnv, + `${target.id}.storage.accessKeyIdEnv` + ); + const secretAccessKey = resolveRequiredEnv( + storage.secretAccessKeyEnv, + `${target.id}.storage.secretAccessKeyEnv` + ); + + return new S3Client({ + endpoint: storage.endpoint, + region: storage.region, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle: true, + }); +}; + +const bodyToBuffer = async (body: unknown): Promise => { + if (Buffer.isBuffer(body)) return body; + if (body instanceof Uint8Array) return Buffer.from(body); + if (typeof body === "string") return Buffer.from(body, "utf8"); + if (isByteArrayTransformBody(body)) { + const byteArray = await body.transformToByteArray(); + return Buffer.from(byteArray); + } + + throw new Error("Unsupported S3 body type"); +}; + +const isByteArrayTransformBody = ( + body: unknown +): body is ByteArrayTransformBody => + body !== null && + typeof body === "object" && + "transformToByteArray" in body && + typeof body.transformToByteArray === "function"; + +const getObjectKeys = (response: unknown): string[] => { + if (response === null || typeof response !== "object") return []; + if (!("Contents" in response) || !Array.isArray(response.Contents)) return []; + + const contents = response.Contents as unknown[]; + + return contents.flatMap((item): string[] => { + if ( + item !== null && + typeof item === "object" && + "Key" in item && + typeof item.Key === "string" + ) { + return [item.Key]; + } + return []; + }); +}; + +const getContentLength = (response: unknown): number => { + if ( + response !== null && + typeof response === "object" && + "ContentLength" in response && + typeof response.ContentLength === "number" + ) { + return response.ContentLength; + } + + return 0; +}; + +const getMetadata = (response: unknown): Record => { + if ( + response !== null && + typeof response === "object" && + "Metadata" in response && + response.Metadata !== null && + typeof response.Metadata === "object" + ) { + return Object.fromEntries( + Object.entries(response.Metadata).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + } + + return {}; +}; + +const readObjectBody = async (response: unknown): Promise => { + if (response !== null && typeof response === "object" && "Body" in response) { + return bodyToBuffer(response.Body); + } + + throw new Error("S3 get object response did not include a body"); +}; + +export const createS3StorageAdapter = ( + target: BackupTarget, + client: S3LikeClient = createS3ClientForTarget(target) +): S3StorageAdapter => { + const storage = getS3StorageConfig(target); + + const send = async (command: unknown): Promise => + client.send(command); + + const headObject = async (key: string): Promise => { + const response = await send( + new HeadObjectCommand({ + Bucket: storage.bucket, + Key: key, + } satisfies HeadObjectCommandInput) + ); + + return { + key, + sizeBytes: getContentLength(response), + metadata: getMetadata(response), + }; + }; + + return { + async writeArtifact(params): Promise { + const artifactKey = buildS3ArtifactKey( + target, + params.backupId, + params.extension + ); + const manifestKey = buildS3ManifestKey(target, params.backupId); + const metadata = { + "target-id": params.targetId, + "backup-id": params.backupId, + }; + + await send( + new PutObjectCommand({ + Bucket: storage.bucket, + Key: artifactKey, + Body: params.artifactBytes, + Metadata: metadata, + } satisfies PutObjectCommandInput) + ); + + const head = await headObject(artifactKey); + if (head.sizeBytes !== params.artifactBytes.byteLength) { + throw new Error( + `S3 upload verification failed for ${artifactKey}: expected ${String( + params.artifactBytes.byteLength + )} bytes, got ${String(head.sizeBytes)} bytes` + ); + } + + return { + artifactKey, + manifestKey, + sizeBytes: params.artifactBytes.byteLength, + }; + }, + + async writeManifest(manifest): Promise { + const body = Buffer.from( + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8" + ); + await send( + new PutObjectCommand({ + Bucket: storage.bucket, + Key: manifest.storage.manifestKey, + Body: body, + Metadata: { + "target-id": manifest.targetId, + "backup-id": manifest.backupId, + "created-at": manifest.createdAt, + sha256: manifest.artifact.sha256, + }, + } satisfies PutObjectCommandInput) + ); + + const head = await headObject(manifest.storage.manifestKey); + if (head.sizeBytes !== body.byteLength) { + throw new Error( + `S3 manifest upload verification failed for ${manifest.storage.manifestKey}` + ); + } + }, + + async listManifests(targetId): Promise { + const response = await send( + new ListObjectsV2Command({ + Bucket: storage.bucket, + Prefix: buildS3ManifestPrefix({ + ...target, + id: targetId, + }), + } satisfies ListObjectsV2CommandInput) + ); + + const manifests: BackupManifest[] = []; + for (const key of getObjectKeys(response)) { + const object = await send( + new GetObjectCommand({ + Bucket: storage.bucket, + Key: key, + } satisfies GetObjectCommandInput) + ); + manifests.push( + backupManifestSchema.parse( + JSON.parse((await readObjectBody(object)).toString("utf8")) + ) + ); + } + + return manifests; + }, + + async readArtifact(manifest): Promise { + const response = await send( + new GetObjectCommand({ + Bucket: storage.bucket, + Key: manifest.storage.artifactKey, + } satisfies GetObjectCommandInput) + ); + return readObjectBody(response); + }, + + async deleteObject(key): Promise { + await send( + new DeleteObjectCommand({ + Bucket: storage.bucket, + Key: key, + } satisfies DeleteObjectCommandInput) + ); + }, + + headObject, + }; +}; diff --git a/test/s3-storage.test.ts b/test/s3-storage.test.ts new file mode 100644 index 0000000..fc48a65 --- /dev/null +++ b/test/s3-storage.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, it } from "vitest"; + +import type { BackupTarget } from "../src/config/types.js"; +import type { BackupManifest } from "../src/core/manifest.js"; +import { + buildS3ArtifactKey, + buildS3ManifestKey, + buildS3ManifestPrefix, + createS3StorageAdapter, + type S3LikeClient, +} from "../src/storage/s3.js"; + +const s3Target = { + id: "maintana", + enabled: true, + dumper: { + type: "fake", + bytes: "dump", + }, + storage: { + type: "s3", + endpoint: "https://example-account-id.r2.cloudflarestorage.com", + region: "auto", + bucket: "maintana-backups", + prefix: "production/postgres", + accessKeyIdEnv: "MAINTANA_BACKUP_R2_ACCESS_KEY_ID", + secretAccessKeyEnv: "MAINTANA_BACKUP_R2_SECRET_ACCESS_KEY", + }, + encryption: { + type: "none", + }, +} satisfies BackupTarget; + +const manifest: BackupManifest = { + version: 1, + backupId: "maintana-2026-05-19T08-21-00Z", + targetId: "maintana", + createdAt: "2026-05-19T01:21:00.000Z", + artifact: { + key: "production/postgres/maintana/artifacts/maintana-2026-05-19T08-21-00Z.dump.gz", + sizeBytes: 12, + sha256: "a".repeat(64), + compression: "gzip", + encryption: "none", + }, + storage: { + type: "s3", + artifactKey: + "production/postgres/maintana/artifacts/maintana-2026-05-19T08-21-00Z.dump.gz", + manifestKey: + "production/postgres/maintana/manifests/maintana-2026-05-19T08-21-00Z.json", + }, +}; + +class MockS3Client implements S3LikeClient { + public readonly commands: unknown[] = []; + private readonly objects = new Map(); + + async send(command: unknown): Promise { + await Promise.resolve(); + this.commands.push(command); + const input = getCommandInput(command); + const name = getCommandName(command); + + if (name === "PutObjectCommand") { + const body = input["Body"]; + if (!Buffer.isBuffer(body)) throw new Error("expected buffer body"); + this.objects.set(String(input["Key"]), body); + return {}; + } + + if (name === "HeadObjectCommand") { + const key = String(input["Key"]); + return { + ContentLength: this.objects.get(key)?.byteLength ?? 0, + Metadata: { + checked: "true", + }, + }; + } + + if (name === "ListObjectsV2Command") { + const prefix = String(input["Prefix"]); + return { + Contents: [...this.objects.keys()] + .filter((key) => key.startsWith(prefix)) + .map((Key) => ({ Key })), + }; + } + + if (name === "GetObjectCommand") { + const key = String(input["Key"]); + return { + Body: this.objects.get(key) ?? Buffer.alloc(0), + }; + } + + if (name === "DeleteObjectCommand") { + this.objects.delete(String(input["Key"])); + return {}; + } + + throw new Error(`Unhandled command ${name}`); + } +} + +const getCommandName = (command: unknown): string => { + if ( + command !== null && + typeof command === "object" && + "constructor" in command && + typeof command.constructor === "function" + ) { + return command.constructor.name; + } + + return "UnknownCommand"; +}; + +const getCommandInput = (command: unknown): Record => { + if (command !== null && typeof command === "object" && "input" in command) { + return command.input as Record; + } + + throw new Error("command did not expose input"); +}; + +describe("s3 storage adapter", () => { + it("builds prefixed artifact and manifest keys", () => { + expect(buildS3ArtifactKey(s3Target, "backup-1", "dump.gz")).toBe( + "production/postgres/maintana/artifacts/backup-1.dump.gz" + ); + expect(buildS3ManifestKey(s3Target, "backup-1")).toBe( + "production/postgres/maintana/manifests/backup-1.json" + ); + expect(buildS3ManifestPrefix(s3Target)).toBe( + "production/postgres/maintana/manifests/" + ); + }); + + it("uploads artifact and verifies it with head object", async () => { + const client = new MockS3Client(); + const adapter = createS3StorageAdapter(s3Target, client); + + const stored = await adapter.writeArtifact({ + targetId: "maintana", + backupId: manifest.backupId, + artifactBytes: Buffer.from("hello"), + extension: "dump.gz", + }); + + expect(stored).toEqual({ + artifactKey: + "production/postgres/maintana/artifacts/maintana-2026-05-19T08-21-00Z.dump.gz", + manifestKey: + "production/postgres/maintana/manifests/maintana-2026-05-19T08-21-00Z.json", + sizeBytes: 5, + }); + expect(client.commands.map(getCommandName)).toEqual([ + "PutObjectCommand", + "HeadObjectCommand", + ]); + }); + + it("writes and lists manifests", async () => { + const client = new MockS3Client(); + const adapter = createS3StorageAdapter(s3Target, client); + + await adapter.writeManifest(manifest); + const manifests = await adapter.listManifests("maintana"); + + expect(manifests).toEqual([manifest]); + expect(client.commands.map(getCommandName)).toEqual([ + "PutObjectCommand", + "HeadObjectCommand", + "ListObjectsV2Command", + "GetObjectCommand", + ]); + }); + + it("downloads and deletes objects", async () => { + const client = new MockS3Client(); + const adapter = createS3StorageAdapter(s3Target, client); + + await adapter.writeArtifact({ + targetId: "maintana", + backupId: manifest.backupId, + artifactBytes: Buffer.from("archive"), + extension: "dump.gz", + }); + + await expect(adapter.readArtifact(manifest)).resolves.toEqual( + Buffer.from("archive") + ); + const deletion = adapter.deleteObject(manifest.storage.artifactKey); + await deletion; + await expect(adapter.readArtifact(manifest)).resolves.toEqual( + Buffer.alloc(0) + ); + }); + + it("supports same bucket with different prefixes", () => { + const kevlyTarget = { + ...s3Target, + id: "kevly", + storage: { + ...s3Target.storage, + bucket: "shared-backups", + prefix: "kevly/prod", + }, + } satisfies BackupTarget; + + expect(buildS3ArtifactKey(kevlyTarget, "backup-1", "dump.gz")).toBe( + "kevly/prod/kevly/artifacts/backup-1.dump.gz" + ); + }); +}); From 3564769fb221ea9187d16e890643a516ebc48909 Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 08:49:00 +0700 Subject: [PATCH 6/9] feat(encryption): add age artifact encryption --- docs/implementation-plan.md | 22 ++++ src/cli.ts | 40 ++++--- src/commands/doctor.ts | 10 +- src/config/env.ts | 16 ++- src/config/schema.ts | 3 + src/core/backup-job.ts | 19 +-- src/core/manifest.ts | 2 +- src/core/ports.ts | 6 + src/encryption/age.ts | 103 +++++++++++++++++ src/encryption/none.ts | 11 ++ src/encryption/policy.ts | 19 +++ test/cli.test.ts | 4 +- test/encryption.test.ts | 222 ++++++++++++++++++++++++++++++++++++ 13 files changed, 452 insertions(+), 25 deletions(-) create mode 100644 src/encryption/age.ts create mode 100644 src/encryption/none.ts create mode 100644 src/encryption/policy.ts create mode 100644 test/encryption.test.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index ecc9eda..24837bb 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -482,6 +482,8 @@ Result: ## Phase 7 — Age Encryption +Status: Done on 18 May 2026. + Goal: encrypt backups before external upload. Tasks: @@ -521,6 +523,26 @@ Acceptance criteria: - `none` encryption is blocked for production external storage. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- encryption is modeled as a core pipeline port; +- `none` encryption is implemented for local/dev use; +- `age` encryption adapter is implemented with a process runner and mocked tests; +- backup pipeline now applies encryption after gzip and before storage; +- restore path decrypts before gunzip; +- age recipient is resolved from env for backup encryption; +- age identity path is resolved from env for restore decryption; +- missing identity fails clearly before restore can proceed; +- external storage with `encryption: none` is blocked unless `allowUnsafeExternal: true` is explicitly set; +- external-storage encryption guard is included in doctor validation; +- tests cover encrypted artifact storage, restore decrypt path, age command arguments, missing identity failure, and unsafe external storage guard. + ## Phase 8 — Retention Engine Goal: remove old backups safely. diff --git a/src/cli.ts b/src/cli.ts index b98ef8d..33df8c1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,6 +15,12 @@ import type { BackupManifest } from "./core/manifest.js"; import type { Dumper } from "./core/ports.js"; import { fakeDumper } from "./dumpers/fake.js"; import { createPostgresDockerDumper } from "./dumpers/postgres-docker.js"; +import { createAgeEncryptionAdapter } from "./encryption/age.js"; +import { noneEncryptionAdapter } from "./encryption/none.js"; +import { + getEffectiveEncryptionConfig, + getExternalStorageEncryptionIssue, +} from "./encryption/policy.js"; import { createLocalStorageAdapter } from "./storage/local.js"; export const cliName = "ops-backup-runner"; @@ -192,12 +198,6 @@ const selectTargetsForCommand = ( return selection; }; -const getEffectiveEncryptionType = ( - config: BackupRunnerConfig, - target: BackupTarget -): "age" | "none" => - (target.encryption ?? config.defaults?.encryption ?? { type: "none" }).type; - const getLocalStorageForTarget = ( config: BackupRunnerConfig, target: BackupTarget @@ -210,18 +210,21 @@ const getLocalStorageForTarget = ( ok: false; message: string; } => { - const encryptionType = getEffectiveEncryptionType(config, target); - if (target.storage.type !== "local") { + const externalStorageEncryptionIssue = getExternalStorageEncryptionIssue( + config, + target + ); + if (externalStorageEncryptionIssue !== undefined) { return { ok: false, - message: `${target.id} uses ${target.storage.type} storage. Phase 4 only supports local storage.`, + message: externalStorageEncryptionIssue, }; } - if (encryptionType !== "none") { + if (target.storage.type !== "local") { return { ok: false, - message: `${target.id} uses ${encryptionType} encryption. Phase 4 only supports encryption: none.`, + message: `${target.id} uses ${target.storage.type} storage. Phase 4 only supports local storage.`, }; } @@ -259,6 +262,15 @@ const getDumperForTarget = (target: BackupTarget): Dumper => { return createPostgresDockerDumper(); }; +const getEncryptionForTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +) => { + const encryption = getEffectiveEncryptionConfig(config, target); + if (encryption.type === "none") return noneEncryptionAdapter; + return createAgeEncryptionAdapter(config, target); +}; + const findManifestByBackupId = ( manifests: BackupManifest[], backupId: string @@ -349,7 +361,8 @@ const runBackupCommand = (args: string[]): CliResult => { const result = runLocalBackupJob( target, getDumperForTarget(target), - localTarget.storage + localTarget.storage, + getEncryptionForTarget(configResult.config, target) ); manifests.push(result.manifest); } @@ -540,7 +553,8 @@ const runRestoreCommand = (args: string[]): CliResult => { } const restoredBytes = restoreLocalBackupArtifact( - storage.storage.readArtifact(manifest) + storage.storage.readArtifact(manifest), + getEncryptionForTarget(configResult.config, target) ); writeFileSync(outputPath, restoredBytes); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 2a1dae8..b5ccf7c 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -3,6 +3,7 @@ import { redactConfigPreview } from "../config/redact.js"; import { resolveTargetEnvReferences } from "../config/env.js"; import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; import { checkPostgresDockerTarget } from "../dumpers/postgres-docker.js"; +import { getExternalStorageEncryptionIssue } from "../encryption/policy.js"; export interface DoctorTargetResult { id: string; @@ -41,15 +42,22 @@ const inspectTarget = ( const envResult = resolveTargetEnvReferences(config, target); const postgresDockerResult = checkPostgresDockerTarget(target); + const externalStorageEncryptionIssue = getExternalStorageEncryptionIssue( + config, + target + ); const issues = [ ...envResult.issues.map((issue) => issue.message), ...postgresDockerResult.issues, + ...(externalStorageEncryptionIssue === undefined + ? [] + : [externalStorageEncryptionIssue]), ]; return { id: target.id, enabled: true, - ok: envResult.ok && postgresDockerResult.ok, + ok: issues.length === 0, status: issues.length === 0 ? "ready" : "missing-env", issues, }; diff --git a/src/config/env.ts b/src/config/env.ts index df82d6d..fe1415b 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -17,8 +17,20 @@ export interface EnvResolutionResult { issues: EnvResolutionIssue[]; } +let runtimeEnv: Record = process.env; + export const getRuntimeEnv = (): Record => - process.env; + runtimeEnv; + +export const setRuntimeEnvForTesting = ( + env: Record +): void => { + runtimeEnv = env; +}; + +export const resetRuntimeEnvForTesting = (): void => { + runtimeEnv = process.env; +}; const addEnvReference = ( references: EnvReference[], @@ -92,7 +104,7 @@ export const getTargetEnvReferences = ( export const resolveTargetEnvReferences = ( config: BackupRunnerConfig, target: BackupTarget, - env: Record = process.env + env: Record = getRuntimeEnv() ): EnvResolutionResult => { if (!target.enabled) { return { ok: true, issues: [] }; diff --git a/src/config/schema.ts b/src/config/schema.ts index 030b270..71a8d6b 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -76,12 +76,15 @@ const ageEncryptionSchema = z .object({ type: z.literal("age"), recipientEnv: envNameSchema.optional(), + identityPathEnv: envNameSchema.optional(), + binary: z.string().min(1).optional(), }) .strict(); const noneEncryptionSchema = z .object({ type: z.literal("none"), + allowUnsafeExternal: z.boolean().optional(), }) .strict(); diff --git a/src/core/backup-job.ts b/src/core/backup-job.ts index 5cf2434..8b74b3a 100644 --- a/src/core/backup-job.ts +++ b/src/core/backup-job.ts @@ -3,8 +3,9 @@ import { gzipSync, gunzipSync } from "node:zlib"; import type { BackupTarget } from "../config/types.js"; import { createBackupId, sha256Hex } from "./artifact.js"; import type { BackupManifest } from "./manifest.js"; -import type { Dumper, StorageAdapter } from "./ports.js"; +import type { Dumper, EncryptionAdapter, StorageAdapter } from "./ports.js"; import { createTempWorkspace } from "./temp-workspace.js"; +import { noneEncryptionAdapter } from "../encryption/none.js"; export interface BackupJobResult { manifest: BackupManifest; @@ -14,19 +15,21 @@ export interface BackupJobResult { export const runLocalBackupJob = ( target: BackupTarget, dumper: Dumper, - storage: StorageAdapter + storage: StorageAdapter, + encryption: EncryptionAdapter = noneEncryptionAdapter ): BackupJobResult => { const workspace = createTempWorkspace(); try { const dump = dumper.dump(target); const compressed = gzipSync(dump.bytes); + const encrypted = encryption.encrypt(compressed); const backupId = createBackupId(target.id); const extension = `${dump.extension}.gz`; const stored = storage.writeArtifact({ targetId: target.id, backupId, - artifactBytes: compressed, + artifactBytes: encrypted, extension, }); @@ -38,9 +41,9 @@ export const runLocalBackupJob = ( artifact: { key: stored.artifactKey, sizeBytes: stored.sizeBytes, - sha256: sha256Hex(compressed), + sha256: sha256Hex(encrypted), compression: "gzip", - encryption: "none", + encryption: encryption.type, }, storage: { type: "local", @@ -61,5 +64,7 @@ export const runLocalBackupJob = ( } }; -export const restoreLocalBackupArtifact = (artifactBytes: Buffer): Buffer => - gunzipSync(artifactBytes); +export const restoreLocalBackupArtifact = ( + artifactBytes: Buffer, + encryption: EncryptionAdapter = noneEncryptionAdapter +): Buffer => gunzipSync(encryption.decrypt(artifactBytes)); diff --git a/src/core/manifest.ts b/src/core/manifest.ts index 8156ec7..da82af6 100644 --- a/src/core/manifest.ts +++ b/src/core/manifest.ts @@ -12,7 +12,7 @@ export const backupManifestSchema = z sizeBytes: z.number().int().nonnegative(), sha256: z.string().length(64), compression: z.literal("gzip"), - encryption: z.literal("none"), + encryption: z.enum(["age", "none"]), }) .strict(), storage: z diff --git a/src/core/ports.ts b/src/core/ports.ts index 463343e..62727c2 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -34,3 +34,9 @@ export interface StoredObjectHead { sizeBytes: number; metadata: Record; } + +export interface EncryptionAdapter { + readonly type: "age" | "none"; + encrypt(bytes: Buffer): Buffer; + decrypt(bytes: Buffer): Buffer; +} diff --git a/src/encryption/age.ts b/src/encryption/age.ts new file mode 100644 index 0000000..5a63952 --- /dev/null +++ b/src/encryption/age.ts @@ -0,0 +1,103 @@ +import { spawnSync } from "node:child_process"; + +import { getRuntimeEnv } from "../config/env.js"; +import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; +import type { EncryptionAdapter } from "../core/ports.js"; +import { getEffectiveEncryptionConfig } from "./policy.js"; + +export interface AgeProcessRunResult { + status: number | null; + stdout: Buffer; + stderr: string; + error?: Error; +} + +export type AgeProcessRunner = ( + command: string, + args: string[], + input: Buffer +) => AgeProcessRunResult; + +export const defaultAgeProcessRunner: AgeProcessRunner = ( + command, + args, + input +) => { + const result = spawnSync(command, args, { + input, + encoding: "buffer", + }); + + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr.toString("utf8"), + ...(result.error === undefined ? {} : { error: result.error }), + }; +}; + +const resolveRequiredEnv = ( + name: string | undefined, + owner: string +): string => { + if (name === undefined) { + throw new Error(`Missing env reference for ${owner}`); + } + + const value = getRuntimeEnv()[name]; + if (value === undefined || value.length === 0) { + throw new Error( + `Missing required environment variable ${name} for ${owner}` + ); + } + + return value; +}; + +const assertSuccess = ( + result: AgeProcessRunResult, + message: string +): Buffer => { + if (result.status === 0) return result.stdout; + + const detail = + (result.error?.message ?? result.stderr.trim()) || "unknown error"; + throw new Error(`${message}: ${detail}`); +}; + +export const createAgeEncryptionAdapter = ( + config: BackupRunnerConfig, + target: BackupTarget, + runner: AgeProcessRunner = defaultAgeProcessRunner +): EncryptionAdapter => { + const encryption = getEffectiveEncryptionConfig(config, target); + if (encryption.type !== "age") { + throw new Error(`Target ${target.id} does not use age encryption`); + } + + const binary = encryption.binary ?? "age"; + + return { + type: "age", + encrypt(bytes): Buffer { + const recipient = resolveRequiredEnv( + encryption.recipientEnv, + `${target.id}.encryption.recipientEnv` + ); + return assertSuccess( + runner(binary, ["--encrypt", "--recipient", recipient], bytes), + `Age encryption failed for target ${target.id}` + ); + }, + decrypt(bytes): Buffer { + const identityPath = resolveRequiredEnv( + encryption.identityPathEnv, + `${target.id}.encryption.identityPathEnv` + ); + return assertSuccess( + runner(binary, ["--decrypt", "--identity", identityPath], bytes), + `Age decryption failed for target ${target.id}` + ); + }, + }; +}; diff --git a/src/encryption/none.ts b/src/encryption/none.ts new file mode 100644 index 0000000..d0f915e --- /dev/null +++ b/src/encryption/none.ts @@ -0,0 +1,11 @@ +import type { EncryptionAdapter } from "../core/ports.js"; + +export const noneEncryptionAdapter: EncryptionAdapter = { + type: "none", + encrypt(bytes): Buffer { + return bytes; + }, + decrypt(bytes): Buffer { + return bytes; + }, +}; diff --git a/src/encryption/policy.ts b/src/encryption/policy.ts new file mode 100644 index 0000000..3828fd7 --- /dev/null +++ b/src/encryption/policy.ts @@ -0,0 +1,19 @@ +import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; + +export const getEffectiveEncryptionConfig = ( + config: BackupRunnerConfig, + target: BackupTarget +) => + target.encryption ?? config.defaults?.encryption ?? { type: "none" as const }; + +export const getExternalStorageEncryptionIssue = ( + config: BackupRunnerConfig, + target: BackupTarget +): string | undefined => { + const encryption = getEffectiveEncryptionConfig(config, target); + if (target.storage.type === "local") return undefined; + if (encryption.type !== "none") return undefined; + if (encryption.allowUnsafeExternal === true) return undefined; + + return `${target.id} uses external ${target.storage.type} storage with encryption: none. Use age encryption or set allowUnsafeExternal only for an explicit unsafe test target.`; +}; diff --git a/test/cli.test.ts b/test/cli.test.ts index 7da35c6..8b0fad9 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -145,7 +145,9 @@ describe("cli harness baseline", () => { ]); expect(result.exitCode).toBe(exitCodes.runtimeFailure); - expect(result.stderr).toContain("Phase 4 only supports local storage"); + expect(result.stderr).toContain( + "external s3 storage with encryption: none" + ); }); it("runs local backup, lists it, verifies it, and restores it", () => { diff --git a/test/encryption.test.ts b/test/encryption.test.ts new file mode 100644 index 0000000..0f2acf7 --- /dev/null +++ b/test/encryption.test.ts @@ -0,0 +1,222 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { runDoctor } from "../src/commands/doctor.js"; +import { + resetRuntimeEnvForTesting, + setRuntimeEnvForTesting, +} from "../src/config/env.js"; +import type { BackupRunnerConfig, BackupTarget } from "../src/config/types.js"; +import { + restoreLocalBackupArtifact, + runLocalBackupJob, +} from "../src/core/backup-job.js"; +import type { EncryptionAdapter } from "../src/core/ports.js"; +import { fakeDumper } from "../src/dumpers/fake.js"; +import { + createAgeEncryptionAdapter, + type AgeProcessRunner, +} from "../src/encryption/age.js"; +import { getExternalStorageEncryptionIssue } from "../src/encryption/policy.js"; +import { createLocalStorageAdapter } from "../src/storage/local.js"; + +const localAgeTarget: BackupTarget = { + id: "local-age", + enabled: true, + dumper: { + type: "fake", + bytes: "encrypted dump", + }, + storage: { + type: "local", + rootPath: mkdtempSync(path.join(tmpdir(), "ops-backup-runner-age-")), + }, + encryption: { + type: "age", + recipientEnv: "BACKUP_AGE_RECIPIENT", + identityPathEnv: "BACKUP_AGE_IDENTITY_PATH", + }, +}; + +const ageConfig: BackupRunnerConfig = { + version: 1, + targets: [localAgeTarget], +}; + +describe("encryption adapters", () => { + afterEach(() => { + resetRuntimeEnvForTesting(); + }); + + it("encrypts artifact bytes after gzip and restores by decrypting before gunzip", () => { + const encryption: EncryptionAdapter = { + type: "age", + encrypt(bytes): Buffer { + return Buffer.concat([Buffer.from("age:"), bytes]); + }, + decrypt(bytes): Buffer { + return bytes.subarray("age:".length); + }, + }; + const storage = createLocalStorageAdapter(localAgeTarget); + + const result = runLocalBackupJob( + localAgeTarget, + fakeDumper, + storage, + encryption + ); + const artifact = storage.readArtifact(result.manifest); + + expect(result.manifest.artifact.encryption).toBe("age"); + expect(artifact.subarray(0, 4).toString("utf8")).toBe("age:"); + expect(gunzipSync(artifact.subarray(4)).toString("utf8")).toBe( + "encrypted dump" + ); + expect( + restoreLocalBackupArtifact(artifact, encryption).toString("utf8") + ).toBe("encrypted dump"); + }); + + it("calls age encrypt and decrypt with env-resolved recipient and identity", () => { + setRuntimeEnvForTesting({ + BACKUP_AGE_RECIPIENT: "age1example", + BACKUP_AGE_IDENTITY_PATH: "/secure/identity.txt", + }); + + const calls: { command: string; args: string[]; input: string }[] = []; + const runner: AgeProcessRunner = (command, args, input) => { + calls.push({ command, args, input: input.toString("utf8") }); + return { + status: 0, + stdout: Buffer.from(`out:${input.toString("utf8")}`), + stderr: "", + }; + }; + + const adapter = createAgeEncryptionAdapter( + ageConfig, + localAgeTarget, + runner + ); + + expect(adapter.encrypt(Buffer.from("plain")).toString("utf8")).toBe( + "out:plain" + ); + expect(adapter.decrypt(Buffer.from("cipher")).toString("utf8")).toBe( + "out:cipher" + ); + expect(calls).toEqual([ + { + command: "age", + args: ["--encrypt", "--recipient", "age1example"], + input: "plain", + }, + { + command: "age", + args: ["--decrypt", "--identity", "/secure/identity.txt"], + input: "cipher", + }, + ]); + }); + + it("fails clearly when restore identity is missing", () => { + setRuntimeEnvForTesting({ + BACKUP_AGE_RECIPIENT: "age1example", + }); + + const adapter = createAgeEncryptionAdapter( + ageConfig, + localAgeTarget, + () => ({ + status: 0, + stdout: Buffer.from("unused"), + stderr: "", + }) + ); + + expect(() => adapter.decrypt(Buffer.from("cipher"))).toThrow( + "Missing required environment variable BACKUP_AGE_IDENTITY_PATH" + ); + }); + + it("blocks none encryption for external storage unless explicitly allowed", () => { + const externalTarget: BackupTarget = { + ...localAgeTarget, + id: "external-unsafe", + storage: { + type: "s3", + endpoint: "https://example-account-id.r2.cloudflarestorage.com", + region: "auto", + bucket: "unsafe-backups", + }, + encryption: { + type: "none", + }, + }; + const config: BackupRunnerConfig = { + version: 1, + targets: [externalTarget], + }; + + expect(getExternalStorageEncryptionIssue(config, externalTarget)).toContain( + "external s3 storage with encryption: none" + ); + expect(runUnsafeExternalDoctorConfig().ok).toBe(false); + }); + + it("allows explicit unsafe external none encryption only when configured", () => { + const externalTarget: BackupTarget = { + ...localAgeTarget, + id: "external-test", + storage: { + type: "s3", + endpoint: "https://example-account-id.r2.cloudflarestorage.com", + region: "auto", + bucket: "unsafe-backups", + }, + encryption: { + type: "none", + allowUnsafeExternal: true, + }, + }; + const config: BackupRunnerConfig = { + version: 1, + targets: [externalTarget], + }; + + expect( + getExternalStorageEncryptionIssue(config, externalTarget) + ).toBeUndefined(); + }); +}); + +const runUnsafeExternalDoctorConfig = () => { + const configPath = path.join( + mkdtempSync(path.join(tmpdir(), "ops-backup-runner-doctor-")), + "targets.yaml" + ); + const yaml = [ + "version: 1", + "targets:", + " - id: external-unsafe", + " enabled: true", + " dumper:", + " type: fake", + " bytes: dump", + " storage:", + " type: s3", + " endpoint: https://example-account-id.r2.cloudflarestorage.com", + " region: auto", + " bucket: unsafe-backups", + " encryption:", + " type: none", + ].join("\n"); + + writeFileSync(configPath, yaml, "utf8"); + return runDoctor(configPath); +}; From 00fdada3862e0ff1aaf61435512aa31f84305914 Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 09:17:00 +0700 Subject: [PATCH 7/9] feat(retention): add prune planner and execution --- docs/implementation-plan.md | 19 ++++ src/cli.ts | 114 +++++++++++++++++++--- src/config/schema.ts | 1 + src/core/ports.ts | 1 + src/core/retention.ts | 157 ++++++++++++++++++++++++++++++ src/storage/local.ts | 26 +++++ test/cli.test.ts | 9 +- test/retention.test.ts | 188 ++++++++++++++++++++++++++++++++++++ 8 files changed, 499 insertions(+), 16 deletions(-) create mode 100644 src/core/retention.ts create mode 100644 test/retention.test.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 24837bb..f91e514 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -545,6 +545,8 @@ Result: ## Phase 8 — Retention Engine +Status: Done on 19 May 2026. + Goal: remove old backups safely. Tasks: @@ -579,6 +581,23 @@ Acceptance criteria: - Prune failure does not mark backup upload as failed if backup already succeeded. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- retention policy now supports daily, weekly, monthly, max-age, and manual keep rules; +- local storage can list object keys and delete specific keys for prune execution; +- retention planner keeps selected manifest-backed backups and plans only expired artifact/manifest pairs for deletion; +- unknown objects are reported in prune output and are never deleted; +- `prune --dry-run` prints intended deletions without deleting files; +- `prune` execution deletes only manifest-backed artifact and manifest pairs; +- unsupported external storage targets fail clearly instead of pruning blindly; +- tests cover planner edge cases, dry-run safety, execution deletion safety, and unsupported storage behavior. + ## Phase 9 — Verification Commands Goal: make restore confidence operationally visible. diff --git a/src/cli.ts b/src/cli.ts index 33df8c1..fde4b5e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,7 @@ import { import { sha256Hex } from "./core/artifact.js"; import type { BackupManifest } from "./core/manifest.js"; import type { Dumper } from "./core/ports.js"; +import { createRetentionPlan, type RetentionPlan } from "./core/retention.js"; import { fakeDumper } from "./dumpers/fake.js"; import { createPostgresDockerDumper } from "./dumpers/postgres-docker.js"; import { createAgeEncryptionAdapter } from "./encryption/age.js"; @@ -282,6 +283,43 @@ const sortManifestsNewestFirst = ( ): BackupManifest[] => [...manifests].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +const getRetentionPolicyForTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +) => target.retention ?? config.defaults?.retention ?? {}; + +const createPrunePlanForTarget = ( + config: BackupRunnerConfig, + target: BackupTarget +): + | { + ok: true; + storage: ReturnType; + plan: RetentionPlan; + } + | { + ok: false; + result: CliResult; + } => { + const storage = getLocalStorageForTarget(config, target); + if (!storage.ok) { + return { + ok: false, + result: failure(exitCodes.runtimeFailure, storage.message), + }; + } + + return { + ok: true, + storage: storage.storage, + plan: createRetentionPlan({ + manifests: storage.storage.listManifests(target.id), + objectKeys: storage.storage.listObjectKeys?.(target.id) ?? [], + policy: getRetentionPolicyForTarget(config, target), + }), + }; +}; + const runDoctorCommand = (args: string[]): CliResult => { const configPath = getFlagValue(args, "--config"); const json = hasFlag(args, "--json"); @@ -573,19 +611,71 @@ const runRestoreCommand = (args: string[]): CliResult => { }; const runPruneCommand = (args: string[]): CliResult => { + const targetId = args[1]; const json = hasFlag(args, "--json"); - const message = - "Prune is not implemented until retention policy execution exists."; - return json - ? success( - renderJson({ - ok: true, - command: "prune", - implemented: false, - message, - }) - ) - : success(`${message}\nNo backup side effects were executed.`); + const dryRun = hasFlag(args, "--dry-run"); + const configResult = loadConfigForCommand(args); + if (!configResult.ok) return configResult.result; + + const selection = selectTargetsForCommand( + args, + configResult.config, + targetId + ); + if (!selection.ok) return selection.result; + + const plans: { targetId: string; plan: RetentionPlan }[] = []; + for (const target of selection.targets) { + const result = createPrunePlanForTarget(configResult.config, target); + if (!result.ok) return result.result; + + plans.push({ targetId: target.id, plan: result.plan }); + + if (!dryRun) { + for (const item of result.plan.delete) { + try { + result.storage.deleteObject?.(item.artifactKey); + result.storage.deleteObject?.(item.manifestKey); + } catch (error) { + const message = + error instanceof Error ? error.message : "unknown prune error"; + return failure( + exitCodes.runtimeFailure, + `Prune failed after planning target ${target.id}: ${message}` + ); + } + } + } + } + + if (json) { + return success( + renderJson({ + ok: true, + command: "prune", + dryRun, + plans, + }) + ); + } + + const lines = [dryRun ? "Prune dry run plan:" : "Prune completed:"]; + for (const { targetId: planTargetId, plan } of plans) { + lines.push(`Target: ${planTargetId}`); + lines.push(` keep: ${String(plan.keep.length)}`); + lines.push(` delete: ${String(plan.delete.length)}`); + for (const item of plan.delete) { + lines.push(` - delete ${item.backupId} (${item.reason})`); + } + if (plan.unknownObjectKeys.length > 0) { + lines.push(" unknown objects not deleted:"); + for (const key of plan.unknownObjectKeys) { + lines.push(` - ${key}`); + } + } + } + + return success(lines.join("\n")); }; export const runCli = (args: string[]): CliResult => { diff --git a/src/config/schema.ts b/src/config/schema.ts index 71a8d6b..e41c49c 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -20,6 +20,7 @@ const retentionSchema = z keepWeekly: z.number().int().positive().optional(), keepMonthly: z.number().int().positive().optional(), maxAgeDays: z.number().int().positive().optional(), + keepManual: z.array(z.string().min(1)).optional(), }) .strict(); diff --git a/src/core/ports.ts b/src/core/ports.ts index 62727c2..fb3719e 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -27,6 +27,7 @@ export interface StorageAdapter { readArtifact(manifest: BackupManifest): Buffer; deleteObject?(key: string): void; headObject?(key: string): StoredObjectHead; + listObjectKeys?(targetId: string): string[]; } export interface StoredObjectHead { diff --git a/src/core/retention.ts b/src/core/retention.ts new file mode 100644 index 0000000..890433d --- /dev/null +++ b/src/core/retention.ts @@ -0,0 +1,157 @@ +import type { BackupManifest } from "./manifest.js"; + +export interface RetentionPolicy { + keepDaily?: number | undefined; + keepWeekly?: number | undefined; + keepMonthly?: number | undefined; + maxAgeDays?: number | undefined; + keepManual?: string[] | undefined; +} + +export interface RetentionPlanItem { + backupId: string; + targetId: string; + createdAt: string; + artifactKey: string; + manifestKey: string; + reason: string; +} + +export interface RetentionPlan { + keep: RetentionPlanItem[]; + delete: RetentionPlanItem[]; + unknownObjectKeys: string[]; +} + +const toPlanItem = ( + manifest: BackupManifest, + reason: string +): RetentionPlanItem => ({ + backupId: manifest.backupId, + targetId: manifest.targetId, + createdAt: manifest.createdAt, + artifactKey: manifest.storage.artifactKey, + manifestKey: manifest.storage.manifestKey, + reason, +}); + +const sortNewestFirst = (manifests: BackupManifest[]): BackupManifest[] => + [...manifests].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); + +const dayKey = (createdAt: string): string => createdAt.slice(0, 10); + +const monthKey = (createdAt: string): string => createdAt.slice(0, 7); + +const weekKey = (createdAt: string): string => { + const date = new Date(createdAt); + const utcDate = new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) + ); + const day = utcDate.getUTCDay() || 7; + utcDate.setUTCDate(utcDate.getUTCDate() + 4 - day); + const yearStart = new Date(Date.UTC(utcDate.getUTCFullYear(), 0, 1)); + const week = Math.ceil( + (Math.floor((utcDate.getTime() - yearStart.getTime()) / 86400000) + 1) / 7 + ); + return `${String(utcDate.getUTCFullYear())}-W${String(week).padStart(2, "0")}`; +}; + +const selectNewestPerGroup = ( + manifests: BackupManifest[], + groupKey: (createdAt: string) => string, + limit: number | undefined +): Set => { + if (limit === undefined || limit <= 0) return new Set(); + + const selected = new Set(); + const seenGroups = new Set(); + for (const manifest of sortNewestFirst(manifests)) { + const key = groupKey(manifest.createdAt); + if (seenGroups.has(key)) continue; + if (seenGroups.size >= limit) break; + seenGroups.add(key); + selected.add(manifest.backupId); + } + return selected; +}; + +const getManifestObjectKeys = (manifest: BackupManifest): string[] => [ + manifest.storage.artifactKey, + manifest.storage.manifestKey, +]; + +const isOlderThanMaxAge = ( + manifest: BackupManifest, + maxAgeDays: number | undefined, + now: Date +): boolean => { + if (maxAgeDays === undefined) return false; + const cutoff = now.getTime() - maxAgeDays * 86400000; + return new Date(manifest.createdAt).getTime() < cutoff; +}; + +export const createRetentionPlan = (params: { + manifests: BackupManifest[]; + objectKeys: string[]; + policy: RetentionPolicy; + now?: Date; +}): RetentionPlan => { + const now = params.now ?? new Date(); + const keepIds = new Set(params.policy.keepManual ?? []); + const reasons = new Map(); + + for (const id of keepIds) { + reasons.set(id, "manual"); + } + + const addKeepSet = (ids: Set, reason: string): void => { + for (const id of ids) { + keepIds.add(id); + const existingReason = reasons.get(id); + reasons.set( + id, + existingReason === undefined ? reason : `${existingReason},${reason}` + ); + } + }; + + addKeepSet( + selectNewestPerGroup(params.manifests, dayKey, params.policy.keepDaily), + "daily" + ); + addKeepSet( + selectNewestPerGroup(params.manifests, weekKey, params.policy.keepWeekly), + "weekly" + ); + addKeepSet( + selectNewestPerGroup(params.manifests, monthKey, params.policy.keepMonthly), + "monthly" + ); + + const knownKeys = new Set(params.manifests.flatMap(getManifestObjectKeys)); + const unknownObjectKeys = params.objectKeys.filter( + (key) => !knownKeys.has(key) + ); + + const keep: RetentionPlanItem[] = []; + const deletable: RetentionPlanItem[] = []; + for (const manifest of sortNewestFirst(params.manifests)) { + if (keepIds.has(manifest.backupId)) { + keep.push(toPlanItem(manifest, reasons.get(manifest.backupId) ?? "kept")); + continue; + } + + if (!isOlderThanMaxAge(manifest, params.policy.maxAgeDays, now)) { + keep.push(toPlanItem(manifest, "within-max-age")); + continue; + } + + deletable.push(toPlanItem(manifest, "expired")); + } + + return { + keep, + delete: deletable, + unknownObjectKeys, + }; +}; diff --git a/src/storage/local.ts b/src/storage/local.ts index 64ff8c2..b009259 100644 --- a/src/storage/local.ts +++ b/src/storage/local.ts @@ -1,4 +1,5 @@ import { + unlinkSync, existsSync, mkdirSync, readFileSync, @@ -24,6 +25,17 @@ const getLocalRoot = (target: BackupTarget): string => { const getTargetRoot = (root: string, targetId: string): string => path.join(root, targetId); +const walkFiles = (directory: string): string[] => { + if (!existsSync(directory)) return []; + + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) return walkFiles(fullPath); + if (entry.isFile()) return [fullPath]; + return []; + }); +}; + export const createLocalStorageAdapter = ( target: BackupTarget ): StorageAdapter => { @@ -80,5 +92,19 @@ export const createLocalStorageAdapter = ( readArtifact(manifest): Buffer { return readFileSync(path.join(root, manifest.storage.artifactKey)); }, + + deleteObject(key): void { + const objectPath = path.join(root, key); + if (existsSync(objectPath)) { + unlinkSync(objectPath); + } + }, + + listObjectKeys(targetId): string[] { + const targetRoot = getTargetRoot(root, targetId); + return walkFiles(targetRoot) + .map((file) => path.relative(root, file).split(path.sep).join("/")) + .sort(); + }, }; }; diff --git a/test/cli.test.ts b/test/cli.test.ts index 8b0fad9..9faed29 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -204,7 +204,7 @@ describe("cli harness baseline", () => { expect(readFileSync(outputPath, "utf8")).toBe("local fake dump"); }); - it("keeps prune as an explicit no-op until retention execution exists", () => { + it("rejects prune for unsupported external storage targets", () => { const result = runCli([ "prune", "maintana", @@ -212,8 +212,9 @@ describe("cli harness baseline", () => { writeConfig(enabledConfig), ]); - expect(result.exitCode).toBe(exitCodes.success); - expect(result.stdout).toContain("Prune is not implemented"); - expect(result.stdout).toContain("No backup side effects were executed."); + expect(result.exitCode).toBe(exitCodes.runtimeFailure); + expect(result.stderr).toContain( + "external s3 storage with encryption: none" + ); }); }); diff --git a/test/retention.test.ts b/test/retention.test.ts new file mode 100644 index 0000000..1a46f8d --- /dev/null +++ b/test/retention.test.ts @@ -0,0 +1,188 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { exitCodes, runCli } from "../src/cli.js"; +import type { BackupManifest } from "../src/core/manifest.js"; +import { createRetentionPlan } from "../src/core/retention.js"; + +const manifest = (backupId: string, createdAt: string): BackupManifest => ({ + version: 1, + backupId, + targetId: "local-demo", + createdAt, + artifact: { + key: `local-demo/artifacts/${backupId}.dump.gz`, + sizeBytes: 10, + sha256: "a".repeat(64), + compression: "gzip", + encryption: "none", + }, + storage: { + type: "local", + artifactKey: `local-demo/artifacts/${backupId}.dump.gz`, + manifestKey: `local-demo/manifests/${backupId}.json`, + }, +}); + +const writeConfig = (storageRoot: string): string => { + const directory = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-prune-") + ); + const file = path.join(directory, "targets.yaml"); + writeFileSync( + file, + [ + "version: 1", + "targets:", + " - id: local-demo", + " enabled: true", + " dumper:", + " type: fake", + " bytes: dump", + " storage:", + " type: local", + ` rootPath: ${storageRoot}`, + " encryption:", + " type: none", + " retention:", + " keepDaily: 1", + " maxAgeDays: 1", + ].join("\n"), + "utf8" + ); + return file; +}; + +const writeStoredBackup = (storageRoot: string, item: BackupManifest): void => { + const artifactPath = path.join(storageRoot, item.storage.artifactKey); + const manifestPath = path.join(storageRoot, item.storage.manifestKey); + mkdirSync(path.dirname(artifactPath), { recursive: true }); + mkdirSync(path.dirname(manifestPath), { recursive: true }); + writeFileSync(artifactPath, "artifact", "utf8"); + writeFileSync(manifestPath, `${JSON.stringify(item, null, 2)}\n`, "utf8"); +}; + +describe("retention planner", () => { + it("keeps newest daily backup and deletes older expired manifest-backed pairs", () => { + const newest = manifest("newest", "2026-05-19T00:00:00.000Z"); + const older = manifest("older", "2026-05-17T00:00:00.000Z"); + + const plan = createRetentionPlan({ + manifests: [older, newest], + objectKeys: [ + newest.storage.artifactKey, + newest.storage.manifestKey, + older.storage.artifactKey, + older.storage.manifestKey, + ], + policy: { + keepDaily: 1, + maxAgeDays: 1, + }, + now: new Date("2026-05-19T12:00:00.000Z"), + }); + + expect(plan.keep.map((item) => item.backupId)).toEqual(["newest"]); + expect(plan.delete.map((item) => item.backupId)).toEqual(["older"]); + }); + + it("keeps manual backups and reports unknown objects without deleting them", () => { + const manual = manifest("manual", "2026-05-10T00:00:00.000Z"); + const expired = manifest("expired", "2026-05-09T00:00:00.000Z"); + + const plan = createRetentionPlan({ + manifests: [manual, expired], + objectKeys: [ + manual.storage.artifactKey, + manual.storage.manifestKey, + expired.storage.artifactKey, + expired.storage.manifestKey, + "local-demo/random/unknown.bin", + ], + policy: { + keepManual: ["manual"], + maxAgeDays: 1, + }, + now: new Date("2026-05-19T12:00:00.000Z"), + }); + + expect(plan.keep.map((item) => item.backupId)).toEqual(["manual"]); + expect(plan.delete.map((item) => item.backupId)).toEqual(["expired"]); + expect(plan.unknownObjectKeys).toEqual(["local-demo/random/unknown.bin"]); + }); + + it("prune dry-run prints deletion plan without deleting files", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const configPath = writeConfig(storageRoot); + const oldBackup = manifest("old", "2026-05-17T00:00:00.000Z"); + const newBackup = manifest("new", new Date().toISOString()); + writeStoredBackup(storageRoot, oldBackup); + writeStoredBackup(storageRoot, newBackup); + const unknownPath = path.join(storageRoot, "local-demo/random/unknown.bin"); + mkdirSync(path.dirname(unknownPath), { recursive: true }); + writeFileSync(unknownPath, "unknown", "utf8"); + + const result = runCli([ + "prune", + "local-demo", + "--dry-run", + "--config", + configPath, + ]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("Prune dry run plan:"); + expect(result.stdout).toContain("delete old"); + expect(result.stdout).toContain("unknown objects not deleted"); + expect( + readFileSync( + path.join(storageRoot, oldBackup.storage.artifactKey), + "utf8" + ) + ).toBe("artifact"); + }); + + it("prune execution deletes only manifest-backed artifact and manifest pairs", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const configPath = writeConfig(storageRoot); + const oldBackup = manifest("old", "2026-05-17T00:00:00.000Z"); + const newBackup = manifest("new", new Date().toISOString()); + writeStoredBackup(storageRoot, oldBackup); + writeStoredBackup(storageRoot, newBackup); + const unknownPath = path.join(storageRoot, "local-demo/random/unknown.bin"); + mkdirSync(path.dirname(unknownPath), { recursive: true }); + writeFileSync(unknownPath, "unknown", "utf8"); + + const result = runCli(["prune", "local-demo", "--config", configPath]); + + expect(result.exitCode).toBe(exitCodes.success); + expect(result.stdout).toContain("Prune completed:"); + expect(() => + readFileSync( + path.join(storageRoot, oldBackup.storage.artifactKey), + "utf8" + ) + ).toThrow(); + expect(() => + readFileSync( + path.join(storageRoot, oldBackup.storage.manifestKey), + "utf8" + ) + ).toThrow(); + expect( + readFileSync( + path.join(storageRoot, newBackup.storage.artifactKey), + "utf8" + ) + ).toBe("artifact"); + expect(readFileSync(unknownPath, "utf8")).toBe("unknown"); + }); +}); From 764d19666293a9c518b36752c4af46da0b11c7d4 Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 09:42:00 +0700 Subject: [PATCH 8/9] feat(verify): validate restorable backup artifacts --- docs/implementation-plan.md | 19 ++++ src/cli.ts | 50 +++++++---- src/core/backup-verification.ts | 132 +++++++++++++++++++++++++++ test/backup-verification.test.ts | 149 +++++++++++++++++++++++++++++++ test/cli.test.ts | 18 ++++ 5 files changed, 353 insertions(+), 15 deletions(-) create mode 100644 src/core/backup-verification.ts create mode 100644 test/backup-verification.test.ts diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index f91e514..b040fa7 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -600,6 +600,8 @@ Result: ## Phase 9 — Verification Commands +Status: Done on 19 May 2026. + Goal: make restore confidence operationally visible. Tasks: @@ -624,6 +626,23 @@ Acceptance criteria: - Corrupted artifact fails checksum or restore-list verification. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- `verify --latest` now fails clearly when no backups exist; +- verification checks artifact checksum against the manifest; +- verification decrypts and decompresses the artifact before declaring success; +- PostgreSQL Docker targets run `pg_restore --list ` against a temporary restored dump; +- temporary restore-list files are cleaned up after verification; +- corrupted artifacts fail verification before restore-list execution; +- invalid PostgreSQL dump content fails restore-list verification; +- tests cover empty targets, checksum mismatch, restore-list success, restore-list failure, and existing backup/list/restore flow. + ## Phase 10 — Notifications Goal: alert operators when backup fails. diff --git a/src/cli.ts b/src/cli.ts index fde4b5e..7ffdbed 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,7 +10,10 @@ import { restoreLocalBackupArtifact, runLocalBackupJob, } from "./core/backup-job.js"; -import { sha256Hex } from "./core/artifact.js"; +import { + verifyBackupArtifact, + type BackupVerificationResult, +} from "./core/backup-verification.js"; import type { BackupManifest } from "./core/manifest.js"; import type { Dumper } from "./core/ports.js"; import { createRetentionPlan, type RetentionPlan } from "./core/retention.js"; @@ -486,11 +489,7 @@ const runVerifyCommand = (args: string[]): CliResult => { ); if (!selection.ok) return selection.result; - const results: { - backupId: string; - targetId: string; - ok: boolean; - }[] = []; + const results: BackupVerificationResult[] = []; for (const target of selection.targets) { const storage = getLocalStorageForTarget(configResult.config, target); @@ -503,23 +502,41 @@ const runVerifyCommand = (args: string[]): CliResult => { : storage.storage.listManifests(target.id); for (const manifest of manifests) { - const artifactBytes = storage.storage.readArtifact(manifest); - results.push({ - backupId: manifest.backupId, - targetId: manifest.targetId, - ok: sha256Hex(artifactBytes) === manifest.artifact.sha256, - }); + try { + results.push( + verifyBackupArtifact({ + target, + manifest, + artifactBytes: storage.storage.readArtifact(manifest), + encryption: getEncryptionForTarget(configResult.config, target), + }) + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + results.push({ + backupId: manifest.backupId, + targetId: manifest.targetId, + ok: false, + checks: { + checksum: false, + restore: false, + }, + issues: [`Backup artifact read failed: ${message}`], + tempWorkspaceCleaned: true, + }); + } } } const failed = results.filter((result) => !result.ok); + const noBackups = results.length === 0; if (json) { const payload = { - ok: failed.length === 0, + ok: !noBackups && failed.length === 0, command: "verify", results, }; - return failed.length === 0 + return !noBackups && failed.length === 0 ? success(renderJson(payload)) : failure(exitCodes.verificationFailure, renderJson(payload)); } @@ -536,7 +553,10 @@ const runVerifyCommand = (args: string[]): CliResult => { exitCodes.verificationFailure, [ "Backup verification failed.", - ...failed.map((result) => `- ${result.targetId}: ${result.backupId}`), + ...failed.flatMap((result) => [ + `- ${result.targetId}: ${result.backupId}`, + ...result.issues.map((issue) => ` ${issue}`), + ]), ].join("\n") ); } diff --git a/src/core/backup-verification.ts b/src/core/backup-verification.ts new file mode 100644 index 0000000..29acabc --- /dev/null +++ b/src/core/backup-verification.ts @@ -0,0 +1,132 @@ +import { writeFileSync } from "node:fs"; +import path from "node:path"; + +import type { BackupTarget } from "../config/types.js"; +import { + defaultProcessRunner, + type ProcessRunner, +} from "../dumpers/postgres-docker.js"; +import { sha256Hex } from "./artifact.js"; +import { restoreLocalBackupArtifact } from "./backup-job.js"; +import type { BackupManifest } from "./manifest.js"; +import type { EncryptionAdapter } from "./ports.js"; +import { createTempWorkspace } from "./temp-workspace.js"; + +export interface BackupVerificationResult { + backupId: string; + targetId: string; + ok: boolean; + checks: { + checksum: boolean; + restore: boolean; + postgresRestoreList?: boolean; + }; + issues: string[]; + tempWorkspaceCleaned: boolean; +} + +const errorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +const runPostgresRestoreList = ( + target: BackupTarget, + restoredBytes: Buffer, + runner: ProcessRunner +): { + ok: boolean; + issue: string | undefined; + tempWorkspaceCleaned: boolean; +} => { + const workspace = createTempWorkspace(); + let ok = false; + let issue: string | undefined; + + try { + const dumpPath = path.join(workspace.path, `${target.id}.dump`); + writeFileSync(dumpPath, restoredBytes); + + const pgRestoreBinary = + target.dumper.type === "postgresDocker" + ? (target.dumper.pgRestoreBinary ?? "pg_restore") + : "pg_restore"; + const result = runner(pgRestoreBinary, ["--list", dumpPath]); + if (result.status === 0) { + ok = true; + } else { + const detail = + (result.error?.message ?? result.stderr.trim()) || "unknown error"; + issue = `PostgreSQL restore-list verification failed for ${target.id}: ${detail}`; + } + } finally { + workspace.cleanup(); + } + + return { + ok, + issue, + tempWorkspaceCleaned: true, + }; +}; + +export const verifyBackupArtifact = (params: { + target: BackupTarget; + manifest: BackupManifest; + artifactBytes: Buffer; + encryption: EncryptionAdapter; + processRunner?: ProcessRunner; +}): BackupVerificationResult => { + const checksumOk = + sha256Hex(params.artifactBytes) === params.manifest.artifact.sha256; + const issues: string[] = []; + let restoreOk = false; + let postgresRestoreListOk: boolean | undefined; + let tempWorkspaceCleaned = true; + + if (!checksumOk) { + issues.push(`Checksum mismatch for backup ${params.manifest.backupId}.`); + } else { + try { + const restoredBytes = restoreLocalBackupArtifact( + params.artifactBytes, + params.encryption + ); + restoreOk = true; + + if (params.target.dumper.type === "postgresDocker") { + const postgresResult = runPostgresRestoreList( + params.target, + restoredBytes, + params.processRunner ?? defaultProcessRunner + ); + postgresRestoreListOk = postgresResult.ok; + tempWorkspaceCleaned = postgresResult.tempWorkspaceCleaned; + if (postgresResult.issue !== undefined) { + issues.push(postgresResult.issue); + } + } + } catch (error) { + issues.push( + `Artifact restore verification failed for ${params.manifest.backupId}: ${errorMessage( + error + )}` + ); + } + } + + const checks = { + checksum: checksumOk, + restore: restoreOk, + ...(postgresRestoreListOk === undefined + ? {} + : { postgresRestoreList: postgresRestoreListOk }), + }; + + return { + backupId: params.manifest.backupId, + targetId: params.manifest.targetId, + ok: issues.length === 0, + checks, + issues, + tempWorkspaceCleaned, + }; +}; diff --git a/test/backup-verification.test.ts b/test/backup-verification.test.ts new file mode 100644 index 0000000..8e60808 --- /dev/null +++ b/test/backup-verification.test.ts @@ -0,0 +1,149 @@ +import { existsSync } from "node:fs"; +import { gzipSync } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import type { BackupTarget } from "../src/config/types.js"; +import { sha256Hex } from "../src/core/artifact.js"; +import { verifyBackupArtifact } from "../src/core/backup-verification.js"; +import type { BackupManifest } from "../src/core/manifest.js"; +import { noneEncryptionAdapter } from "../src/encryption/none.js"; +import type { ProcessRunner } from "../src/dumpers/postgres-docker.js"; + +const postgresTarget: BackupTarget = { + id: "maintana", + enabled: true, + dumper: { + type: "postgresDocker", + container: "maintana-postgres", + database: "maintana", + username: "maintana", + format: "custom", + }, + storage: { + type: "local", + rootPath: "/tmp/backups", + }, + encryption: { + type: "none", + }, +}; + +const artifactBytes = gzipSync(Buffer.from("custom postgres dump")); + +const manifest = (sha256 = sha256Hex(artifactBytes)): BackupManifest => ({ + version: 1, + backupId: "maintana-2026-05-19T09-17-00Z", + targetId: "maintana", + createdAt: "2026-05-19T09:17:00.000Z", + artifact: { + key: "maintana/artifacts/maintana.dump.gz", + sizeBytes: artifactBytes.byteLength, + sha256, + compression: "gzip", + encryption: "none", + }, + storage: { + type: "local", + artifactKey: "maintana/artifacts/maintana.dump.gz", + manifestKey: "maintana/manifests/maintana.json", + }, +}); + +describe("backup artifact verification", () => { + it("runs pg_restore list for valid PostgreSQL custom dumps and cleans temp files", () => { + let dumpPath: string | undefined; + const calls: { command: string; args: string[] }[] = []; + const runner: ProcessRunner = (command, args) => { + calls.push({ command, args }); + dumpPath = args[1]; + expect(dumpPath).toBeDefined(); + expect(existsSync(String(dumpPath))).toBe(true); + return { + status: 0, + stdout: Buffer.from("table list"), + stderr: "", + }; + }; + + const result = verifyBackupArtifact({ + target: postgresTarget, + manifest: manifest(), + artifactBytes, + encryption: noneEncryptionAdapter, + processRunner: runner, + }); + + expect(result.ok).toBe(true); + expect(result.checks).toEqual({ + checksum: true, + restore: true, + postgresRestoreList: true, + }); + expect(result.tempWorkspaceCleaned).toBe(true); + expect(calls).toEqual([ + { + command: "pg_restore", + args: ["--list", String(dumpPath)], + }, + ]); + expect(existsSync(String(dumpPath))).toBe(false); + }); + + it("fails checksum verification without running pg_restore", () => { + const calls: string[] = []; + const runner: ProcessRunner = (command) => { + calls.push(command); + return { + status: 0, + stdout: Buffer.alloc(0), + stderr: "", + }; + }; + + const result = verifyBackupArtifact({ + target: postgresTarget, + manifest: manifest("b".repeat(64)), + artifactBytes, + encryption: noneEncryptionAdapter, + processRunner: runner, + }); + + expect(result.ok).toBe(false); + expect(result.checks).toEqual({ + checksum: false, + restore: false, + }); + expect(result.issues).toContain( + "Checksum mismatch for backup maintana-2026-05-19T09-17-00Z." + ); + expect(calls).toEqual([]); + }); + + it("fails when pg_restore list rejects the restored dump", () => { + const runner: ProcessRunner = () => ({ + status: 1, + stdout: Buffer.alloc(0), + stderr: "input file does not appear to be a valid archive", + }); + + const result = verifyBackupArtifact({ + target: postgresTarget, + manifest: manifest(), + artifactBytes, + encryption: noneEncryptionAdapter, + processRunner: runner, + }); + + expect(result.ok).toBe(false); + expect(result.checks).toEqual({ + checksum: true, + restore: true, + postgresRestoreList: false, + }); + expect(result.issues).toEqual([ + "PostgreSQL restore-list verification failed for maintana: input file does not appear to be a valid archive", + ]); + expect(result.tempWorkspaceCleaned).toBe(true); + }); +}); diff --git a/test/cli.test.ts b/test/cli.test.ts index 9faed29..9e28f62 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -204,6 +204,24 @@ describe("cli harness baseline", () => { expect(readFileSync(outputPath, "utf8")).toBe("local fake dump"); }); + it("fails verify latest clearly when no backups exist", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const configPath = writeConfig(localConfig(storageRoot)); + + const result = runCli([ + "verify", + "local-demo", + "--latest", + "--config", + configPath, + ]); + + expect(result.exitCode).toBe(exitCodes.verificationFailure); + expect(result.stderr).toContain("No backups found to verify."); + }); + it("rejects prune for unsupported external storage targets", () => { const result = runCli([ "prune", From a11dbb5363071a04f187425b4d293e51dd3cd87f Mon Sep 17 00:00:00 2001 From: ahmad fikril Date: Tue, 19 May 2026 10:08:00 +0700 Subject: [PATCH 9/9] feat(notifications): add telegram failure alerts --- config/targets.example.yaml | 3 + docs/implementation-plan.md | 19 ++++++ src/cli.ts | 104 +++++++++++++++++++++++++++--- src/config/env.ts | 30 ++++++++- src/config/schema.ts | 3 + src/core/backup-verification.ts | 5 +- src/core/notifications.ts | 16 +++++ src/core/process-runner.ts | 34 ++++++++++ src/dumpers/postgres-docker.ts | 40 ++---------- src/notifications/message.ts | 47 ++++++++++++++ src/notifications/policy.ts | 14 ++++ src/notifications/telegram.ts | 91 ++++++++++++++++++++++++++ test/cli.test.ts | 103 ++++++++++++++++++++++++++++- test/config.test.ts | 111 ++++++++++++++++++++++++++++++++ test/notifications.test.ts | 75 +++++++++++++++++++++ 15 files changed, 644 insertions(+), 51 deletions(-) create mode 100644 src/core/notifications.ts create mode 100644 src/core/process-runner.ts create mode 100644 src/notifications/message.ts create mode 100644 src/notifications/policy.ts create mode 100644 src/notifications/telegram.ts create mode 100644 test/notifications.test.ts diff --git a/config/targets.example.yaml b/config/targets.example.yaml index 5c0fa46..02fd453 100644 --- a/config/targets.example.yaml +++ b/config/targets.example.yaml @@ -11,6 +11,9 @@ defaults: notifications: telegram: enabled: true + onFailure: true + onSuccess: false + successCadence: weekly botTokenEnv: BACKUP_TELEGRAM_BOT_TOKEN chatIdEnv: BACKUP_TELEGRAM_CHAT_ID diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index b040fa7..f821028 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -645,6 +645,8 @@ Result: ## Phase 10 — Notifications +Status: Done on 19 May 2026. + Goal: alert operators when backup fails. Tasks: @@ -676,6 +678,23 @@ Acceptance criteria: - Notification failure is logged but does not hide the original backup failure. - `pnpm verify` passes. +Verification evidence: + +```bash +pnpm verify +``` + +Result: + +- notification core contracts are defined for backup failure alerts; +- Telegram notification config supports failure notifications by default, optional success notifications, and weekly/monthly success cadence settings; +- `doctor` fails clearly when Telegram notifications are enabled without required env references or runtime env values; +- Telegram sendMessage details are isolated in the notification adapter; +- backup failures trigger Telegram alerts when configured; +- Telegram delivery failures are reported alongside the original backup failure without replacing it; +- failure messages redact token/secret-shaped values before output; +- tests cover config validation, message formatting, Telegram request construction, delivery failure redaction, backup failure alerting, and notification failure handling. + ## Phase 11 — Systemd Install Assets Goal: make server installation repeatable. diff --git a/src/cli.ts b/src/cli.ts index 7ffdbed..e41a8ef 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,6 +3,7 @@ import { writeFileSync } from "node:fs"; import { formatDoctorResult, runDoctor } from "./commands/doctor.js"; +import { getRuntimeEnv } from "./config/env.js"; import { loadConfigFromFile } from "./config/loader.js"; import { selectTargets } from "./config/targets.js"; import type { BackupRunnerConfig, BackupTarget } from "./config/types.js"; @@ -25,6 +26,12 @@ import { getEffectiveEncryptionConfig, getExternalStorageEncryptionIssue, } from "./encryption/policy.js"; +import { getServerName, redactSensitiveText } from "./notifications/message.js"; +import { + getEffectiveNotificationsConfig, + shouldNotifyBackupFailure, +} from "./notifications/policy.js"; +import { createTelegramNotifier } from "./notifications/telegram.js"; import { createLocalStorageAdapter } from "./storage/local.js"; export const cliName = "ops-backup-runner"; @@ -323,6 +330,59 @@ const createPrunePlanForTarget = ( }; }; +const sendBackupFailureNotification = (params: { + config: BackupRunnerConfig; + target: BackupTarget; + stage: string; + error: string; +}): string | undefined => { + if (!shouldNotifyBackupFailure(params.config, params.target)) { + return undefined; + } + + const telegram = getEffectiveNotificationsConfig( + params.config, + params.target + )?.telegram; + if (telegram?.enabled !== true) { + return undefined; + } + + const env = getRuntimeEnv(); + const botToken = + telegram.botTokenEnv === undefined ? undefined : env[telegram.botTokenEnv]; + const chatId = + telegram.chatIdEnv === undefined ? undefined : env[telegram.chatIdEnv]; + + if (botToken === undefined || chatId === undefined) { + return `Telegram failure notification skipped: missing Telegram runtime configuration for ${params.target.id}.`; + } + + const notifier = createTelegramNotifier({ botToken, chatId }); + return notifier.notifyFailure({ + targetId: params.target.id, + stage: params.stage, + occurredAt: new Date(), + error: redactSensitiveText(params.error, [botToken, chatId]), + server: getServerName(), + }).message; +}; + +const formatBackupFailure = (params: { + target: BackupTarget; + stage: string; + error: string; + notificationMessage: string | undefined; +}): string => + [ + `Backup failed for ${params.target.id} during ${params.stage}: ${redactSensitiveText( + params.error + )}`, + ...(params.notificationMessage === undefined + ? [] + : [params.notificationMessage]), + ].join("\n"); + const runDoctorCommand = (args: string[]): CliResult => { const configPath = getFlagValue(args, "--config"); const json = hasFlag(args, "--json"); @@ -396,16 +456,40 @@ const runBackupCommand = (args: string[]): CliResult => { const manifests: BackupManifest[] = []; for (const target of selection.targets) { - const localTarget = getLocalBackupTarget(configResult.config, target); - if (!localTarget.ok) return localTarget.result; - - const result = runLocalBackupJob( - target, - getDumperForTarget(target), - localTarget.storage, - getEncryptionForTarget(configResult.config, target) - ); - manifests.push(result.manifest); + let stage = "prepare"; + try { + stage = "storage"; + const localTarget = getLocalBackupTarget(configResult.config, target); + if (!localTarget.ok) { + return localTarget.result; + } + + stage = "backup"; + const result = runLocalBackupJob( + target, + getDumperForTarget(target), + localTarget.storage, + getEncryptionForTarget(configResult.config, target) + ); + manifests.push(result.manifest); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const notificationMessage = sendBackupFailureNotification({ + config: configResult.config, + target, + stage, + error: message, + }); + return failure( + exitCodes.runtimeFailure, + formatBackupFailure({ + target, + stage, + error: message, + notificationMessage, + }) + ); + } } if (json) { diff --git a/src/config/env.ts b/src/config/env.ts index fe1415b..44a6c38 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -110,6 +110,32 @@ export const resolveTargetEnvReferences = ( return { ok: true, issues: [] }; } + const notifications = target.notifications ?? config.defaults?.notifications; + const telegram = notifications?.telegram; + const notificationConfigIssues: EnvResolutionIssue[] = + telegram?.enabled === true + ? [ + ...(telegram.botTokenEnv === undefined + ? [ + { + envName: "botTokenEnv", + owner: `${target.id}.notifications.telegram.botTokenEnv`, + message: `${target.id}.notifications.telegram.botTokenEnv is required when Telegram notifications are enabled`, + }, + ] + : []), + ...(telegram.chatIdEnv === undefined + ? [ + { + envName: "chatIdEnv", + owner: `${target.id}.notifications.telegram.chatIdEnv`, + message: `${target.id}.notifications.telegram.chatIdEnv is required when Telegram notifications are enabled`, + }, + ] + : []), + ] + : []; + const issues = getTargetEnvReferences(config, target) .filter((reference) => reference.requiredForEnabledTarget) .filter((reference) => env[reference.name] === undefined) @@ -120,7 +146,7 @@ export const resolveTargetEnvReferences = ( })); return { - ok: issues.length === 0, - issues, + ok: notificationConfigIssues.length === 0 && issues.length === 0, + issues: [...notificationConfigIssues, ...issues], }; }; diff --git a/src/config/schema.ts b/src/config/schema.ts index e41c49c..be3894e 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -97,6 +97,9 @@ const encryptionSchema = z.discriminatedUnion("type", [ const telegramNotificationSchema = z .object({ enabled: z.boolean().default(false), + onFailure: z.boolean().default(true), + onSuccess: z.boolean().default(false), + successCadence: z.enum(["daily", "weekly", "monthly"]).optional(), botTokenEnv: envNameSchema.optional(), chatIdEnv: envNameSchema.optional(), }) diff --git a/src/core/backup-verification.ts b/src/core/backup-verification.ts index 29acabc..653c447 100644 --- a/src/core/backup-verification.ts +++ b/src/core/backup-verification.ts @@ -2,10 +2,7 @@ import { writeFileSync } from "node:fs"; import path from "node:path"; import type { BackupTarget } from "../config/types.js"; -import { - defaultProcessRunner, - type ProcessRunner, -} from "../dumpers/postgres-docker.js"; +import { defaultProcessRunner, type ProcessRunner } from "./process-runner.js"; import { sha256Hex } from "./artifact.js"; import { restoreLocalBackupArtifact } from "./backup-job.js"; import type { BackupManifest } from "./manifest.js"; diff --git a/src/core/notifications.ts b/src/core/notifications.ts new file mode 100644 index 0000000..f83669b --- /dev/null +++ b/src/core/notifications.ts @@ -0,0 +1,16 @@ +export interface BackupFailureNotification { + targetId: string; + stage: string; + occurredAt: Date; + error: string; + server: string; +} + +export interface NotificationResult { + ok: boolean; + message: string; +} + +export interface FailureNotifier { + notifyFailure(event: BackupFailureNotification): NotificationResult; +} diff --git a/src/core/process-runner.ts b/src/core/process-runner.ts new file mode 100644 index 0000000..769d3cc --- /dev/null +++ b/src/core/process-runner.ts @@ -0,0 +1,34 @@ +import { spawnSync } from "node:child_process"; + +export interface ProcessRunResult { + status: number | null; + stdout: Buffer; + stderr: string; + error?: Error; +} + +export interface ProcessRunOptions { + input?: Buffer; + env?: Record; +} + +export type ProcessRunner = ( + command: string, + args: string[], + options?: ProcessRunOptions +) => ProcessRunResult; + +export const defaultProcessRunner: ProcessRunner = (command, args, options) => { + const result = spawnSync(command, args, { + input: options?.input, + env: options?.env, + encoding: "buffer", + }); + + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr.toString("utf8"), + ...(result.error === undefined ? {} : { error: result.error }), + }; +}; diff --git a/src/dumpers/postgres-docker.ts b/src/dumpers/postgres-docker.ts index afecdb0..ad9686d 100644 --- a/src/dumpers/postgres-docker.ts +++ b/src/dumpers/postgres-docker.ts @@ -1,41 +1,13 @@ -import { spawnSync } from "node:child_process"; - import { getRuntimeEnv } from "../config/env.js"; import type { BackupTarget } from "../config/types.js"; import type { DumpArtifact, Dumper } from "../core/ports.js"; +import { + defaultProcessRunner, + type ProcessRunner, + type ProcessRunResult, +} from "../core/process-runner.js"; -export interface ProcessRunResult { - status: number | null; - stdout: Buffer; - stderr: string; - error?: Error; -} - -export interface ProcessRunOptions { - input?: Buffer; - env?: Record; -} - -export type ProcessRunner = ( - command: string, - args: string[], - options?: ProcessRunOptions -) => ProcessRunResult; - -export const defaultProcessRunner: ProcessRunner = (command, args, options) => { - const result = spawnSync(command, args, { - input: options?.input, - env: options?.env, - encoding: "buffer", - }); - - return { - status: result.status, - stdout: result.stdout, - stderr: result.stderr.toString("utf8"), - ...(result.error === undefined ? {} : { error: result.error }), - }; -}; +export { defaultProcessRunner, type ProcessRunner, type ProcessRunResult }; const getPostgresDockerConfig = (target: BackupTarget) => { if (target.dumper.type !== "postgresDocker") { diff --git a/src/notifications/message.ts b/src/notifications/message.ts new file mode 100644 index 0000000..69224fc --- /dev/null +++ b/src/notifications/message.ts @@ -0,0 +1,47 @@ +import { hostname } from "node:os"; + +import type { BackupFailureNotification } from "../core/notifications.js"; + +const sensitivePattern = + /(password|token|secret|access[_-]?key|credential)(=|:)\S+/giu; + +const formatJakartaTime = (date: Date): string => + new Intl.DateTimeFormat("en-GB", { + day: "2-digit", + month: "short", + year: "numeric", + hour: "2-digit", + minute: "2-digit", + hour12: false, + timeZone: "Asia/Jakarta", + }) + .format(date) + .replace(",", ""); + +export const redactSensitiveText = ( + text: string, + explicitSecrets: string[] = [] +): string => { + const patternRedacted = text.replace(sensitivePattern, "$1$2[REDACTED]"); + + return explicitSecrets + .filter((secret) => secret.length > 0) + .reduce( + (output, secret) => output.split(secret).join("[REDACTED]"), + patternRedacted + ); +}; + +export const getServerName = (): string => hostname(); + +export const formatBackupFailureMessage = ( + event: BackupFailureNotification +): string => + [ + "[Backup Failed]", + `Target: ${event.targetId}`, + `Stage: ${event.stage}`, + `Time: ${formatJakartaTime(event.occurredAt)} WIB`, + `Error: ${redactSensitiveText(event.error)}`, + `Server: ${event.server}`, + ].join("\n"); diff --git a/src/notifications/policy.ts b/src/notifications/policy.ts new file mode 100644 index 0000000..4104a73 --- /dev/null +++ b/src/notifications/policy.ts @@ -0,0 +1,14 @@ +import type { BackupRunnerConfig, BackupTarget } from "../config/types.js"; + +export const getEffectiveNotificationsConfig = ( + config: BackupRunnerConfig, + target: BackupTarget +) => target.notifications ?? config.defaults?.notifications; + +export const shouldNotifyBackupFailure = ( + config: BackupRunnerConfig, + target: BackupTarget +): boolean => { + const telegram = getEffectiveNotificationsConfig(config, target)?.telegram; + return telegram?.enabled === true && telegram.onFailure; +}; diff --git a/src/notifications/telegram.ts b/src/notifications/telegram.ts new file mode 100644 index 0000000..a489c0d --- /dev/null +++ b/src/notifications/telegram.ts @@ -0,0 +1,91 @@ +import { + defaultProcessRunner, + type ProcessRunner, +} from "../core/process-runner.js"; +import type { + BackupFailureNotification, + FailureNotifier, + NotificationResult, +} from "../core/notifications.js"; +import { formatBackupFailureMessage, redactSensitiveText } from "./message.js"; + +export interface TelegramNotifierConfig { + botToken: string; + chatId: string; + runner?: ProcessRunner; +} + +let defaultTelegramProcessRunner: ProcessRunner = defaultProcessRunner; + +export const setTelegramProcessRunnerForTesting = ( + runner: ProcessRunner +): void => { + defaultTelegramProcessRunner = runner; +}; + +export const resetTelegramProcessRunnerForTesting = (): void => { + defaultTelegramProcessRunner = defaultProcessRunner; +}; + +export const buildTelegramSendMessageArgs = (params: { + botToken: string; + chatId: string; + text: string; +}): string[] => [ + "--fail", + "--silent", + "--show-error", + "--request", + "POST", + `https://api.telegram.org/bot${params.botToken}/sendMessage`, + "--header", + "Content-Type: application/json", + "--data", + JSON.stringify({ + chat_id: params.chatId, + text: params.text, + }), +]; + +export const createTelegramNotifier = ( + config: TelegramNotifierConfig +): FailureNotifier => { + if (config.botToken.length === 0) { + throw new Error("Telegram bot token is required."); + } + if (config.chatId.length === 0) { + throw new Error("Telegram chat id is required."); + } + + const runner = config.runner ?? defaultTelegramProcessRunner; + + return { + notifyFailure(event: BackupFailureNotification): NotificationResult { + const result = runner( + "curl", + buildTelegramSendMessageArgs({ + botToken: config.botToken, + chatId: config.chatId, + text: formatBackupFailureMessage(event), + }) + ); + + if (result.status === 0) { + return { + ok: true, + message: "Telegram failure notification sent.", + }; + } + + const detail = redactSensitiveText( + (result.error?.message ?? result.stderr.trim()) || "unknown error", + [config.botToken] + ); + + return { + ok: false, + message: `Telegram failure notification failed: ${detail}`, + }; + }, + }; +}; diff --git a/test/cli.test.ts b/test/cli.test.ts index 9e28f62..2dac556 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -2,9 +2,18 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { + resetRuntimeEnvForTesting, + setRuntimeEnvForTesting, +} from "../src/config/env.js"; +import type { ProcessRunner } from "../src/core/process-runner.js"; import { cliName, exitCodes, getStartupMessage, runCli } from "../src/cli.js"; +import { + resetTelegramProcessRunnerForTesting, + setTelegramProcessRunnerForTesting, +} from "../src/notifications/telegram.js"; const writeConfig = (content: string): string => { const directory = mkdtempSync(path.join(tmpdir(), "ops-backup-runner-cli-")); @@ -57,7 +66,33 @@ targets: type: none `; +const localFailingNotificationConfig = (storageRoot: string): string => ` +version: 1 +targets: + - id: local-demo + enabled: true + dumper: + type: fake + bytes: local fake dump + storage: + type: local + rootPath: ${storageRoot} + encryption: + type: age + recipientEnv: MISSING_AGE_RECIPIENT + notifications: + telegram: + enabled: true + botTokenEnv: BACKUP_TELEGRAM_BOT_TOKEN + chatIdEnv: BACKUP_TELEGRAM_CHAT_ID +`; + describe("cli harness baseline", () => { + afterEach(() => { + resetRuntimeEnvForTesting(); + resetTelegramProcessRunnerForTesting(); + }); + it("exposes the CLI name", () => { expect(cliName).toBe("ops-backup-runner"); }); @@ -222,6 +257,72 @@ describe("cli harness baseline", () => { expect(result.stderr).toContain("No backups found to verify."); }); + it("sends Telegram failure alerts without hiding the backup failure", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const configPath = writeConfig(localFailingNotificationConfig(storageRoot)); + setRuntimeEnvForTesting({ + BACKUP_TELEGRAM_BOT_TOKEN: "bot-secret", + BACKUP_TELEGRAM_CHAT_ID: "12345", + }); + const calls: { command: string; args: string[] }[] = []; + const runner: ProcessRunner = (command, args) => { + calls.push({ command, args }); + return { + status: 0, + stdout: Buffer.from("{}"), + stderr: "", + }; + }; + setTelegramProcessRunnerForTesting(runner); + + const result = runCli(["backup", "local-demo", "--config", configPath]); + + expect(result.exitCode).toBe(exitCodes.runtimeFailure); + expect(result.stderr).toContain( + "Backup failed for local-demo during backup" + ); + expect(result.stderr).toContain( + "Missing required environment variable MISSING_AGE_RECIPIENT" + ); + expect(result.stderr).toContain("Telegram failure notification sent."); + expect(calls).toHaveLength(1); + expect(calls[0]?.command).toBe("curl"); + expect(calls[0]?.args.join(" ")).toContain("sendMessage"); + }); + + it("reports Telegram failure without hiding the original backup failure", () => { + const storageRoot = mkdtempSync( + path.join(tmpdir(), "ops-backup-runner-storage-") + ); + const configPath = writeConfig(localFailingNotificationConfig(storageRoot)); + setRuntimeEnvForTesting({ + BACKUP_TELEGRAM_BOT_TOKEN: "bot-secret", + BACKUP_TELEGRAM_CHAT_ID: "12345", + }); + const runner: ProcessRunner = () => ({ + status: 1, + stdout: Buffer.alloc(0), + stderr: "telegram denied bot-secret", + }); + setTelegramProcessRunnerForTesting(runner); + + const result = runCli(["backup", "local-demo", "--config", configPath]); + + expect(result.exitCode).toBe(exitCodes.runtimeFailure); + expect(result.stderr).toContain( + "Backup failed for local-demo during backup" + ); + expect(result.stderr).toContain( + "Missing required environment variable MISSING_AGE_RECIPIENT" + ); + expect(result.stderr).toContain( + "Telegram failure notification failed: telegram denied [REDACTED]" + ); + expect(result.stderr).not.toContain("bot-secret"); + }); + it("rejects prune for unsupported external storage targets", () => { const result = runCli([ "prune", diff --git a/test/config.test.ts b/test/config.test.ts index fe39a18..28b7d8e 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -120,6 +120,117 @@ targets: ]); }); + it("requires Telegram env names and values when Telegram notifications are enabled", () => { + const configPath = writeConfig(` +version: 1 +targets: + - id: local-demo + enabled: true + dumper: + type: fake + bytes: dump + storage: + type: local + rootPath: /tmp/backups + notifications: + telegram: + enabled: true +`); + const result = loadConfigFromFile(configPath); + + expect(result.ok).toBe(true); + if (!result.ok) return; + + const target = result.config.targets[0]; + expect(target).toBeDefined(); + if (target === undefined) return; + + const envResult = resolveTargetEnvReferences(result.config, target, {}); + + expect(envResult.ok).toBe(false); + expect(envResult.issues.map((issue) => issue.message)).toEqual([ + "local-demo.notifications.telegram.botTokenEnv is required when Telegram notifications are enabled", + "local-demo.notifications.telegram.chatIdEnv is required when Telegram notifications are enabled", + ]); + + const doctorResult = runDoctor(configPath); + expect(doctorResult.ok).toBe(false); + if (!doctorResult.ok) { + expect(doctorResult.issues).toContain( + "local-demo.notifications.telegram.botTokenEnv is required when Telegram notifications are enabled" + ); + } + }); + + it("reports missing Telegram runtime env values when names are configured", () => { + const result = loadConfigFromFile( + writeConfig(` +version: 1 +targets: + - id: local-demo + enabled: true + dumper: + type: fake + bytes: dump + storage: + type: local + rootPath: /tmp/backups + notifications: + telegram: + enabled: true + botTokenEnv: BACKUP_TELEGRAM_BOT_TOKEN + chatIdEnv: BACKUP_TELEGRAM_CHAT_ID +`) + ); + + expect(result.ok).toBe(true); + if (!result.ok) return; + + const target = result.config.targets[0]; + expect(target).toBeDefined(); + if (target === undefined) return; + + const envResult = resolveTargetEnvReferences(result.config, target, {}); + + expect(envResult.ok).toBe(false); + expect(envResult.issues.map((issue) => issue.envName)).toEqual([ + "BACKUP_TELEGRAM_BOT_TOKEN", + "BACKUP_TELEGRAM_CHAT_ID", + ]); + }); + + it("defaults Telegram failure notification policy on when Telegram is enabled", () => { + const result = loadConfigFromFile( + writeConfig(` +version: 1 +targets: + - id: local-demo + enabled: true + dumper: + type: fake + bytes: dump + storage: + type: local + rootPath: /tmp/backups + notifications: + telegram: + enabled: true + botTokenEnv: BACKUP_TELEGRAM_BOT_TOKEN + chatIdEnv: BACKUP_TELEGRAM_CHAT_ID +`) + ); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.config.targets[0]?.notifications?.telegram?.onFailure).toBe( + true + ); + expect(result.config.targets[0]?.notifications?.telegram?.onSuccess).toBe( + false + ); + } + }); + it("does not require env references for disabled targets", () => { const result = runDoctor( writeConfig(validConfig.replace("enabled: true", "enabled: false")) diff --git a/test/notifications.test.ts b/test/notifications.test.ts new file mode 100644 index 0000000..5f65879 --- /dev/null +++ b/test/notifications.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; + +import { formatBackupFailureMessage } from "../src/notifications/message.js"; +import { + buildTelegramSendMessageArgs, + createTelegramNotifier, +} from "../src/notifications/telegram.js"; +import type { ProcessRunner } from "../src/core/process-runner.js"; + +describe("notifications", () => { + it("formats secret-safe backup failure messages", () => { + expect( + formatBackupFailureMessage({ + targetId: "maintana", + stage: "upload", + occurredAt: new Date("2026-05-18T19:00:00.000Z"), + error: "AccessDenied token=secret-token", + server: "orymu-droplet", + }) + ).toBe( + [ + "[Backup Failed]", + "Target: maintana", + "Stage: upload", + "Time: 19 May 2026 02:00 WIB", + "Error: AccessDenied token=[REDACTED]", + "Server: orymu-droplet", + ].join("\n") + ); + }); + + it("builds Telegram sendMessage requests without leaking details into feature code", () => { + const args = buildTelegramSendMessageArgs({ + botToken: "bot-secret", + chatId: "12345", + text: "hello", + }); + + expect(args).toContain( + "https://api.telegram.org/botbot-secret/sendMessage" + ); + expect(args).toContain( + JSON.stringify({ + chat_id: "12345", + text: "hello", + }) + ); + }); + + it("returns typed Telegram send failures with token redaction", () => { + const runner: ProcessRunner = () => ({ + status: 1, + stdout: Buffer.alloc(0), + stderr: "request failed for bot-secret", + }); + + const result = createTelegramNotifier({ + botToken: "bot-secret", + chatId: "12345", + runner, + }).notifyFailure({ + targetId: "maintana", + stage: "backup", + occurredAt: new Date("2026-05-19T02:00:00.000Z"), + error: "failed", + server: "test-server", + }); + + expect(result).toEqual({ + ok: false, + message: + "Telegram failure notification failed: request failed for [REDACTED]", + }); + }); +});