From e2b6937b4c8c6b8e3c8bb4d425e19679210b13a0 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 13 Sep 2026 12:52:52 -0400 Subject: [PATCH 1/3] feat(backup): export bounded database and saved-config bundles --- .github/workflows/ci.yml | 25 ++- README.md | 4 + cmd/beacon-backup/main.go | 39 ++++ docs/backup-export.md | 89 ++++++++ internal/backup/export.go | 231 ++++++++++++++++++++ internal/backup/export_integration_test.go | 216 +++++++++++++++++++ internal/backup/export_test.go | 232 +++++++++++++++++++++ 7 files changed, 835 insertions(+), 1 deletion(-) create mode 100644 cmd/beacon-backup/main.go create mode 100644 docs/backup-export.md create mode 100644 internal/backup/export.go create mode 100644 internal/backup/export_integration_test.go create mode 100644 internal/backup/export_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdb2a3b..0972374 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,18 @@ on: jobs: build: runs-on: ubuntu-latest + services: + backup-postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: backup-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - uses: actions/checkout@v4 @@ -35,6 +47,18 @@ jobs: - name: Test run: go test ./... + - name: Verify backup command against PostgreSQL 16 + run: | + CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ + -e BEACON_BACKUP_TEST_POSTGRES=1 -e BEACON_BACKUP_TEST_BINARY=/beacon-backup \ + -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ + -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ + postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest @@ -54,4 +78,3 @@ jobs: exit 1 fi rm docs/swagger.json.bak - diff --git a/README.md b/README.md index ef3026d..7fd773f 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. +For a bounded private database and saved-config bundle, see +[backup export](docs/backup-export.md). This is a standalone export tool; the +backup web interface and import workflow are separate follow-ups. + --- ## Stack diff --git a/cmd/beacon-backup/main.go b/cmd/beacon-backup/main.go new file mode 100644 index 0000000..7e593e3 --- /dev/null +++ b/cmd/beacon-backup/main.go @@ -0,0 +1,39 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// beacon-backup creates a private database and saved-config export. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +var version = "dev" + +func main() { + var opts backup.Options + flag.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to include verbatim (may contain secrets)") + flag.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz destination (required; never overwritten)") + flag.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size") + flag.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export duration") + flag.Parse() + if flag.NArg() != 0 { + fmt.Fprintln(os.Stderr, "unexpected positional arguments; connection settings use PG* environment variables") + os.Exit(2) + } + opts.Version = version + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := backup.Export(ctx, opts); err != nil { + fmt.Fprintln(os.Stderr, "backup failed:", err) + os.Exit(1) + } + fmt.Println("Backup complete. Store this bundle privately; it may contain keys and message data.") +} diff --git a/docs/backup-export.md b/docs/backup-export.md new file mode 100644 index 0000000..09afb50 --- /dev/null +++ b/docs/backup-export.md @@ -0,0 +1,89 @@ +# Database and saved-config export + +`beacon-backup` is the export foundation for issue #72. It produces a private, +versioned `.tar.gz` using PostgreSQL's `pg_dump`. It does not yet provide a web +interface, account login, scheduled/remote storage or automatic import. + +Build it with `go build ./cmd/beacon-backup`. Install `pg_dump` in the same runtime +as this command; an installation on the Docker host does not install it inside an +app container. Use a client of the same major version as the source PostgreSQL +server; an older client cannot dump a newer server. + +Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings. +`PGDATABASE` is required and must be a plain database name. Prefer a private +`PGPASSFILE` (0600 on Unix) or an existing libpq service configuration for secrets. +The command does not load `.env`, read `POSTGRES_DSN`, start Beacon, run migrations, +subscribe to MQTT or connect to Redis. Connection settings are not command-line +arguments and client stderr is not printed because it can contain private data. + +For example, after configuring those connection settings: + +```sh +beacon-backup -config /private/config.yaml -output /private/beacon-20260913.tar.gz +``` + +Use an output directory controlled by the operator. Staging directories are +0700 and files are 0600 on Unix; Windows operators must use a directory with +appropriately restricted ACLs. Existing destinations, including symlinks, are +never replaced. A hard link publishes the finished archive atomically, so the +destination filesystem must support hard links. Unsupported filesystems fail +without publishing an output. Normal failure, timeout and handled interruption +remove temporary files; a power loss or SIGKILL can leave a private +`.beacon-backup-*` staging directory for the operator to inspect. + +## Format 1 + +The tar contains exactly three regular files with fixed names: + +- `manifest.json`: format version, creation time, exporter version, dump format, + uncompressed payload sizes/SHA-256 hashes and explicit exclusions. +- `database.sql`: one consistent `pg_dump` snapshot of schema and data, including + Beacon's migration journal. Ownership, ACLs and tablespace placement are omitted + so objects can be restored under the destination operator. +- `config.yaml`: the supplied saved file, byte-for-byte, including comments and + any keys. It is captured before the database snapshot; avoid configuration edits + during export if the files must describe the same deployment state. + +This is sensitive, unencrypted data. Store and transfer it privately. The bundle +does not include deployment environment variables, `.env`, external files such as +`borderFile` inputs or TLS keys, runtime-only changes, PostgreSQL roles/cluster +settings, Redis or service/deployment files. Retain those separately. A config +export is not the sanitized admin-config response and is not a complete server +recovery package by itself. + +Defaults are ten minutes and a 1 GiB uncompressed SQL limit. Saved YAML is capped +at 1 MiB. `-timeout` and `-max-bytes` set finite positive limits (SQL maximum 1 TiB). +Allow disk space for both the uncompressed SQL and compressed bundle, roughly +twice the chosen SQL limit plus overhead. A five-second lock-wait limit prevents +waiting indefinitely behind schema changes. Export failure publishes no backup; +check the client version, privileges, connection settings and available capacity +privately. The command deliberately does not expose raw client diagnostics. + +## Restore verification + +Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Inspect +the fixed members, verify the gzip stream and the manifest's payload sizes and +hashes, and extract into private staging. For a **new, empty disposable database**, +with its own explicit `PGDATABASE` and target-role connection settings: + +```sh +psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql +``` + +Restore with a compatible PostgreSQL version and the required extensions already +available. Reconcile the migration journal and representative records/relationships +before relying on the bundle. Review saved configuration and restore external +secrets/files separately before starting a new Beacon server. Never restore over +live data simply to test an export; overwrite/import requires a separate workflow. + +PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html), +[connection environment](https://www.postgresql.org/docs/16/libpq-envars.html), +[password files](https://www.postgresql.org/docs/16/libpq-pgpass.html). + +CI runs the compiled command and its PostgreSQL round-trip test with a dedicated +PostgreSQL 16 service. To repeat it privately, build the command, set the `PG*` +connection variables for an isolated test server and set +`BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's +absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`. +The test role needs permission to create/drop its two randomly named databases; +the test migrates and restores only those databases and removes them afterward. diff --git a/internal/backup/export.go b/internal/backup/export.go new file mode 100644 index 0000000..149ef03 --- /dev/null +++ b/internal/backup/export.go @@ -0,0 +1,231 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package backup exports a database and saved configuration into a private bundle. +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "time" +) + +const ( + DefaultMaxBytes = int64(1 << 30) + DefaultTimeout = 10 * time.Minute + maxConfigBytes = 1 << 20 +) + +var ErrTooLarge = errors.New("backup size limit exceeded") + +// Options selects local files and finite resource limits. Connection settings +// come from libpq's PG* environment; PGDATABASE must explicitly name the database. +type Options struct { + ConfigPath string + OutputPath string + MaxBytes int64 // Maximum uncompressed database dump size. + Timeout time.Duration + Version string +} + +// Manifest describes format 1; hashes cover the uncompressed archive members. +type Manifest struct { + FormatVersion int `json:"format_version"` + CreatedAt time.Time `json:"created_at"` + ToolVersion string `json:"tool_version"` + DatabaseFormat string `json:"database_format"` + Files []File `json:"files"` + Excluded []string `json:"excluded"` +} + +// File identifies one payload by its fixed archive name, size and SHA-256. +type File struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Export creates a complete .tar.gz without replacing an existing destination. +// pg_dump must be installed on PATH. The destination filesystem must support +// hard links; a link publishes the finished private file without an overwrite race. +// The bundle contains secrets if they are present in the database or saved YAML. +func Export(ctx context.Context, opts Options) error { + return export(ctx, opts, dumpCommand) +} + +func dumpCommand(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "pg_dump", "--format=plain", "--no-owner", + "--no-acl", "--no-tablespaces", "--no-password", "--lock-wait-timeout=5000") +} + +func export(ctx context.Context, opts Options, command func(context.Context) *exec.Cmd) error { + if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { + return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") + } + if os.Getenv("PGDATABASE") == "" { + return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used") + } + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + if err := ctx.Err(); err != nil { + return err + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("output already exists or cannot be inspected") + } + config, err := readConfig(opts.ConfigPath) + if err != nil { + return err + } + parent, err := filepath.Abs(filepath.Dir(opts.OutputPath)) + if err != nil { + return errors.New("invalid output directory") + } + temp, err := os.MkdirTemp(parent, ".beacon-backup-") + if err != nil { + return errors.New("cannot create private backup staging directory") + } + defer os.RemoveAll(temp) + dump, err := os.OpenFile(filepath.Join(temp, "database.sql"), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private database dump") + } + defer dump.Close() + hash := sha256.New() + output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} + cmd := command(ctx) + cmd.Stdout, cmd.Stderr = output, io.Discard + cmd.WaitDelay = time.Second + err = cmd.Run() + if output.err != nil { + return output.err + } + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + // Client diagnostics may contain credentials, hostnames or database contents. + return errors.New("pg_dump failed; check the installed client and private connection settings") + } + size := opts.MaxBytes - output.remaining + if size == 0 { + return errors.New("pg_dump produced an empty dump") + } + if _, err = dump.Seek(0, io.SeekStart); err != nil { + return errors.New("cannot read completed database dump") + } + configHash := sha256.Sum256(config) + manifest := Manifest{ + FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: opts.Version, + DatabaseFormat: "postgresql-plain-sql", + Files: []File{ + {Name: "database.sql", Size: size, SHA256: hex.EncodeToString(hash.Sum(nil))}, + {Name: "config.yaml", Size: int64(len(config)), SHA256: hex.EncodeToString(configHash[:])}, + }, + Excluded: []string{"deployment_environment", "external_config_files", "runtime_only_changes", "roles_ownership_acl_tablespaces", "redis_cache"}, + } + metadata, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return errors.New("cannot encode backup manifest") + } + archivePath := filepath.Join(temp, "bundle.tar.gz") + archive, err := os.OpenFile(archivePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private backup archive") + } + defer archive.Close() + // Input is bounded; reserve room for YAML, tar headers and compression overhead. + archiveOutput := &limitedWriter{ctx: ctx, cancel: cancel, dst: archive, remaining: opts.MaxBytes + opts.MaxBytes/100 + 2*maxConfigBytes} + gz := gzip.NewWriter(archiveOutput) + tw := tar.NewWriter(gz) + for _, member := range []struct { + name string + size int64 + data io.Reader + }{ + {"manifest.json", int64(len(metadata)), bytes.NewReader(metadata)}, + {"database.sql", size, dump}, + {"config.yaml", int64(len(config)), bytes.NewReader(config)}, + } { + if err = tw.WriteHeader(&tar.Header{Name: member.name, Size: member.size, Mode: 0600, ModTime: manifest.CreatedAt}); err == nil { + _, err = io.Copy(tw, member.data) + } + if err != nil { + return errors.New("cannot write complete backup archive") + } + } + if err = tw.Close(); err != nil { + return errors.New("cannot complete backup tar") + } + if err = gz.Close(); err != nil { + return errors.New("cannot complete backup compression") + } + if err = archive.Sync(); err != nil { + return errors.New("cannot sync completed backup") + } + if err = archive.Close(); err != nil { + return errors.New("cannot close completed backup") + } + if err = ctx.Err(); err != nil { + return err + } + if err = os.Link(archivePath, opts.OutputPath); err != nil { + return errors.New("cannot publish backup without overwriting; check destination and hard-link support") + } + return nil +} + +func readConfig(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() > maxConfigBytes { + return nil, errors.New("config must be a readable regular file of at most 1 MiB") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("cannot read saved config") + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return nil, errors.New("cannot read saved config within the 1 MiB limit") + } + return data, nil +} + +type limitedWriter struct { + ctx context.Context + cancel context.CancelFunc + dst io.Writer + remaining int64 + err error +} + +func (w *limitedWriter) Write(p []byte) (n int, err error) { + if w.err == nil { + w.err = w.ctx.Err() + } + if w.err == nil && int64(len(p)) > w.remaining { + w.err = ErrTooLarge + } + if w.err == nil { + n, w.err = w.dst.Write(p) + w.remaining -= int64(n) + if w.err == nil && n != len(p) { + w.err = io.ErrShortWrite + } + } + if w.err != nil { + w.cancel() + } + return n, w.err +} diff --git a/internal/backup/export_integration_test.go b/internal/backup/export_integration_test.go new file mode 100644 index 0000000..5ac58a0 --- /dev/null +++ b/internal/backup/export_integration_test.go @@ -0,0 +1,216 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestExportPostgres runs only on an explicitly selected private test server with +// pg_dump/psql installed. It creates and drops two randomly named test databases. +// BEACON_BACKUP_TEST_BINARY points to the exact compiled command being verified. +func TestExportPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("set BEACON_BACKUP_TEST_POSTGRES=1 and BEACON_BACKUP_TEST_BINARY on a private PostgreSQL test server") + } + binary := os.Getenv("BEACON_BACKUP_TEST_BINARY") + if !filepath.IsAbs(binary) { + t.Fatal("the exact backup command must be an absolute path") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private test PostgreSQL") + } + defer admin.Close(context.Background()) + names := []string{"beacon_backup_test_" + strings.ToLower(rand.Text()), "beacon_backup_test_" + strings.ToLower(rand.Text())} + for _, name := range names { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer func() { + cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + if _, err := admin.Exec(cleanup, "DROP DATABASE "+ident); err != nil { + t.Errorf("cannot remove own test database: %v", err) + } + }() + } + poolFor := func(name string) *pgxpool.Pool { + t.Helper() + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid private test connection settings") + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open own test database") + } + return pool + } + source, target := poolFor(names[0]), poolFor(names[1]) + defer source.Close() + defer target.Close() + if err := db.RunMigrations(ctx, source); err != nil { + t.Fatal(err) + } + _, err = source.Exec(ctx, ` +CREATE TABLE backup_fixture (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, message text, raw bytea, detail jsonb); +INSERT INTO backup_fixture (message, raw, detail) VALUES ('café 雪', decode('00ff10', 'hex'), '{"nested":[1,null,true]}'), ('line one +line two', NULL, '{}'); +CREATE TABLE backup_child (id bigint PRIMARY KEY REFERENCES backup_fixture(id)); +INSERT INTO backup_child VALUES (1); +CREATE VIEW backup_view AS SELECT id, message FROM backup_fixture; +CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM backup_child; +`) + if err != nil { + t.Fatal(err) + } + const check = `SELECT json_build_object( + 'rows', (SELECT json_agg(x ORDER BY id) FROM backup_fixture x), + 'children', (SELECT json_agg(x ORDER BY id) FROM backup_child x), + 'view', (SELECT json_agg(x ORDER BY id) FROM backup_view x), + 'materialized', (SELECT json_agg(x) FROM backup_materialized x), + 'migrations', (SELECT json_agg(x ORDER BY filename) FROM schema_migrations x), + 'columns', (SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public') +)::text` + var before, after string + if err := source.QueryRow(ctx, check).Scan(&before); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := []byte("# saved configuration, including synthetic key\nchannel_keys:\n keys: {}\n") + configPath, output := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "backup.tar.gz") + if err := os.WriteFile(configPath, config, 0600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(ctx, binary, "-config", configPath, "-output", output, "-max-bytes", "16777216", "-timeout", "1m") + command.Env = append(os.Environ(), "PGDATABASE="+names[0]) + if err := command.Run(); err != nil { + t.Fatal("exact backup command failed") + } + original, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + for _, failure := range []string{"existing", "size", "connection", "timeout"} { + t.Run(failure, func(t *testing.T) { + failedOutput := filepath.Join(dir, failure+".tar.gz") + limit, timeout, database := "16777216", "30s", names[0] + if failure == "existing" { + failedOutput = output + } else if failure == "size" { + limit = "64" + } else if failure == "connection" { + database = "beacon_backup_test_missing_" + strings.ToLower(rand.Text()) + } else { + lock, err := source.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer lock.Rollback(context.Background()) + if _, err := lock.Exec(ctx, "LOCK TABLE backup_fixture IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal(err) + } + timeout = "1s" + } + failed := exec.CommandContext(ctx, binary, "-config", configPath, "-output", failedOutput, "-max-bytes", limit, "-timeout", timeout) + failed.Env = append(os.Environ(), "PGDATABASE="+database) + message, err := failed.CombinedOutput() + if err == nil || bytes.Contains(message, []byte(database)) || bytes.Contains(message, []byte("café")) { + t.Fatal("expected a failure without database identity or payload diagnostics") + } + if failure == "existing" { + retained, err := os.ReadFile(output) + if err != nil || !bytes.Equal(retained, original) { + t.Fatal("existing output changed") + } + } else if _, err := os.Lstat(failedOutput); !os.IsNotExist(err) { + t.Fatal("failed export published an output") + } + staging, _ := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if len(staging) != 0 { + t.Fatal("failed export retained private staging") + } + }) + } + f, err := os.Open(output) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr, members := tar.NewReader(gz), map[string][]byte{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + members[h.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest Manifest + if err := json.Unmarshal(members["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], config) { + t.Fatal("unexpected bundle members or saved config") + } + for _, member := range manifest.Files { + hash := sha256.Sum256(members[member.Name]) + if member.SHA256 != hex.EncodeToString(hash[:]) || member.Size != int64(len(members[member.Name])) { + t.Fatal("bundle checksum mismatch") + } + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+names[1]) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("restore into own empty database failed") + } + if err := target.QueryRow(ctx, check).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("schema, journal, data, relationships or materialized view changed after restore") + } + var nextID int + if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 { + t.Fatal("identity sequence did not survive restore") + } + t.Log("exact command restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence") +} diff --git a/internal/backup/export_test.go b/internal/backup/export_test.go new file mode 100644 index 0000000..8b3b70e --- /dev/null +++ b/internal/backup/export_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const testSQL = "CREATE TABLE fixture (id bigint PRIMARY KEY, name text);\nINSERT INTO fixture VALUES (1, 'café');\n" + +// TestDumpProcess supplies an actual subprocess, including failure and cancellation. +func TestDumpProcess(t *testing.T) { + mode := os.Getenv("BEACON_BACKUP_TEST_PROCESS") + if mode == "" { + return + } + switch mode { + case "ok": + _, _ = io.WriteString(os.Stdout, testSQL) + case "fail": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + _, _ = io.WriteString(os.Stderr, "PRIVATE_PASSWORD_CANARY") + os.Exit(17) + case "large": + _, _ = os.Stdout.Write(bytes.Repeat([]byte("x"), 65536)) + case "hang": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + time.Sleep(time.Minute) + case "empty": + default: + os.Exit(2) + } + os.Exit(0) +} + +func helper(t *testing.T, mode string) func(context.Context) *exec.Cmd { + t.Helper() + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + return func(ctx context.Context) *exec.Cmd { + cmd := exec.CommandContext(ctx, exe, "-test.run=^TestDumpProcess$") + cmd.Env = append(os.Environ(), "BEACON_BACKUP_TEST_PROCESS="+mode) + return cmd + } +} + +func setup(t *testing.T) Options { + t.Helper() + t.Setenv("PGDATABASE", "fixture") + dir := t.TempDir() + opts := Options{ConfigPath: filepath.Join(dir, "saved.yaml"), OutputPath: filepath.Join(dir, "backup.tar.gz"), + MaxBytes: int64(len(testSQL)), Timeout: 10 * time.Second, Version: "test-revision"} + if err := os.WriteFile(opts.ConfigPath, []byte("# preserve comments\nchannel_keys: {}\n"), 0600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + paths, err := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if err != nil || len(paths) != 0 { + t.Errorf("private staging was not cleaned: %v %v", paths, err) + } + }) + return opts +} + +func TestExportBundle(t *testing.T) { + opts := setup(t) + if err := export(context.Background(), opts, helper(t, "ok")); err != nil { + t.Fatal(err) + } + f, err := os.Open(opts.OutputPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + info, err := f.Stat() + if err != nil || (runtime.GOOS != "windows" && info.Mode().Perm() != 0600) { + t.Fatalf("backup must be private: %v %v", info, err) + } + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string][]byte{} + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil || header.Mode != 0600 || header.Typeflag != tar.TypeReg { + t.Fatalf("invalid archive member: %v %v", header, err) + } + if _, duplicate := files[header.Name]; duplicate { + t.Fatal("duplicate archive name") + } + files[header.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal("invalid gzip footer:", err) + } + config, _ := os.ReadFile(opts.ConfigPath) + if len(files) != 3 || string(files["database.sql"]) != testSQL || !bytes.Equal(files["config.yaml"], config) { + t.Fatal("payloads differ from the supplied SQL and saved config") + } + var manifest Manifest + if err := json.Unmarshal(files["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if manifest.FormatVersion != 1 || manifest.ToolVersion != opts.Version || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 || len(manifest.Excluded) != 5 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + for _, member := range manifest.Files { + digest := sha256.Sum256(files[member.Name]) + if member.Size != int64(len(files[member.Name])) || member.SHA256 != hex.EncodeToString(digest[:]) { + t.Errorf("manifest mismatch for %s", member.Name) + } + } +} + +func TestExportFailureCleanup(t *testing.T) { + for _, mode := range []string{"fail", "large", "hang", "empty"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + if mode == "hang" { + opts.Timeout = time.Second + } + err := export(context.Background(), opts, helper(t, mode)) + if err == nil || strings.Contains(err.Error(), "PRIVATE_PASSWORD_CANARY") { + t.Fatalf("expected sanitized failure, got %v", err) + } + if mode == "large" && !errors.Is(err, ErrTooLarge) { + t.Fatalf("wrong limit error: %v", err) + } + if mode == "hang" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wrong timeout error: %v", err) + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed export published an output") + } + }) + } +} + +func TestExportNeverReplacesDestination(t *testing.T) { + for _, race := range []bool{false, true} { + t.Run(map[bool]string{false: "existing", true: "created-during-export"}[race], func(t *testing.T) { + opts := setup(t) + putExisting := func() { + if err := os.WriteFile(opts.OutputPath, []byte("retain this backup"), 0600); err != nil { + t.Fatal(err) + } + } + if !race { + putExisting() + } + cmd := helper(t, "ok") + err := export(context.Background(), opts, func(ctx context.Context) *exec.Cmd { + if !race { + t.Fatal("existing output must be rejected before starting pg_dump") + } + putExisting() + return cmd(ctx) + }) + data, readErr := os.ReadFile(opts.OutputPath) + if err == nil || readErr != nil || string(data) != "retain this backup" { + t.Fatalf("existing backup changed: %v %v", err, readErr) + } + }) + } +} + +func TestExportValidation(t *testing.T) { + for _, mode := range []string{"database", "limit", "config", "canceled"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch mode { + case "database": + t.Setenv("PGDATABASE", "") + case "limit": + opts.MaxBytes = 0 + case "config": + if err := os.WriteFile(opts.ConfigPath, make([]byte, maxConfigBytes+1), 0600); err != nil { + t.Fatal(err) + } + case "canceled": + cancel() + } + if err := export(ctx, opts, func(context.Context) *exec.Cmd { t.Fatal("dump ran on invalid input"); return nil }); err == nil { + t.Fatal("invalid export succeeded") + } + }) + } +} + +func TestDumpArgumentsExcludeConnectionSecrets(t *testing.T) { + t.Setenv("PGPASSWORD", "PRIVATE_PASSWORD_CANARY") + t.Setenv("POSTGRES_DSN", "postgres://user:PRIVATE_PASSWORD_CANARY@localhost/db") + cmd := dumpCommand(context.Background()) + if strings.Contains(strings.Join(cmd.Args, " "), "PRIVATE_PASSWORD_CANARY") { + t.Fatal("credentials reached command arguments") + } + args := strings.Join(cmd.Args, " ") + for _, required := range []string{"--no-password", "--lock-wait-timeout=5000", "--no-owner", "--no-acl", "--no-tablespaces", "--format=plain"} { + if !strings.Contains(args, required) { + t.Errorf("missing dump boundary: %s", required) + } + } +} From 57b85f078383d9a8b6582fc03aedeec633784a27 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 13 Sep 2026 13:01:45 -0400 Subject: [PATCH 2/3] test(backup): preserve synthetic channel keys in exports --- internal/backup/export_integration_test.go | 2 +- internal/backup/export_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backup/export_integration_test.go b/internal/backup/export_integration_test.go index 5ac58a0..886a16c 100644 --- a/internal/backup/export_integration_test.go +++ b/internal/backup/export_integration_test.go @@ -101,7 +101,7 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba t.Fatal(err) } dir := t.TempDir() - config := []byte("# saved configuration, including synthetic key\nchannel_keys:\n keys: {}\n") + config := []byte("# saved configuration, including a synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n") configPath, output := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "backup.tar.gz") if err := os.WriteFile(configPath, config, 0600); err != nil { t.Fatal(err) diff --git a/internal/backup/export_test.go b/internal/backup/export_test.go index 8b3b70e..a3481af 100644 --- a/internal/backup/export_test.go +++ b/internal/backup/export_test.go @@ -68,7 +68,7 @@ func setup(t *testing.T) Options { dir := t.TempDir() opts := Options{ConfigPath: filepath.Join(dir, "saved.yaml"), OutputPath: filepath.Join(dir, "backup.tar.gz"), MaxBytes: int64(len(testSQL)), Timeout: 10 * time.Second, Version: "test-revision"} - if err := os.WriteFile(opts.ConfigPath, []byte("# preserve comments\nchannel_keys: {}\n"), 0600); err != nil { + if err := os.WriteFile(opts.ConfigPath, []byte("# preserve comments and this synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n"), 0600); err != nil { t.Fatal(err) } t.Cleanup(func() { From f6921a405d2bdbe4f82df65093f323b360237476 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 13 Sep 2026 13:05:37 -0400 Subject: [PATCH 3/3] fix(backup): check writable file closure before publication --- internal/backup/export.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/internal/backup/export.go b/internal/backup/export.go index 149ef03..63abf68 100644 --- a/internal/backup/export.go +++ b/internal/backup/export.go @@ -68,7 +68,7 @@ func dumpCommand(ctx context.Context) *exec.Cmd { "--no-acl", "--no-tablespaces", "--no-password", "--lock-wait-timeout=5000") } -func export(ctx context.Context, opts Options, command func(context.Context) *exec.Cmd) error { +func export(ctx context.Context, opts Options, command func(context.Context) *exec.Cmd) (exportErr error) { if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") } @@ -100,7 +100,7 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex if err != nil { return errors.New("cannot create private database dump") } - defer dump.Close() + defer closeBackupFile(dump, &exportErr) hash := sha256.New() output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} cmd := command(ctx) @@ -143,7 +143,7 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex if err != nil { return errors.New("cannot create private backup archive") } - defer archive.Close() + defer closeBackupFile(archive, &exportErr) // Input is bounded; reserve room for YAML, tar headers and compression overhead. archiveOutput := &limitedWriter{ctx: ctx, cancel: cancel, dst: archive, remaining: opts.MaxBytes + opts.MaxBytes/100 + 2*maxConfigBytes} gz := gzip.NewWriter(archiveOutput) @@ -164,6 +164,9 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex return errors.New("cannot write complete backup archive") } } + if err = dump.Close(); err != nil { + return errors.New("cannot close completed database dump") + } if err = tw.Close(); err != nil { return errors.New("cannot complete backup tar") } @@ -185,6 +188,14 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex return nil } +// Cleanup also checks close errors. Both writable files are explicitly closed +// before publication; a deferred second close may therefore report ErrClosed. +func closeBackupFile(file *os.File, exportErr *error) { + if err := file.Close(); err != nil && !errors.Is(err, os.ErrClosed) && *exportErr == nil { + *exportErr = errors.New("cannot close private backup file") + } +} + func readConfig(path string) ([]byte, error) { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Size() > maxConfigBytes {