From 5ec8f61ddc8c067e0dddb6a7dd109a7635f766db Mon Sep 17 00:00:00 2001 From: jdtw Date: Fri, 31 Jul 2026 21:36:07 -0700 Subject: [PATCH 1/3] Add a SQLite storage backend The link table is a single three-column relation, which does not need a managed Postgres instance to serve it. SQLiteStore implements the existing Store interface so the choice is a deployment detail rather than a rewrite. The driver is modernc.org/sqlite rather than mattn/go-sqlite3 because the Dockerfile builds with CGO_ENABLED=0 into distroless/static; a cgo driver would not link. WAL mode keeps redirect reads from blocking on writes, and the schema is applied on open so a freshly provisioned volume needs no manual setup. Postgres has no equivalent of SQLite for reporting insert-vs-update, so where PostgresStore uses "returning (xmax = 0)", SQLiteStore does the existence check and the write in one transaction. Storage precedence is --ephemeral, then SQLITE_PATH, then DATABASE_URL, so unsetting one variable reverts to Postgres. cmd/migrate copies Postgres to SQLite and verifies every entry against the source. It never writes to Postgres and is safe to rerun. It ships in the image so it can run on a machine that reaches both the database and the volume. Startup no longer logs the database connection string. Co-Authored-By: Claude Opus 5 --- Dockerfile | 3 + cmd/links/links.go | 23 +++- cmd/migrate/migrate.go | 94 +++++++++++++++ go.mod | 10 ++ go.sum | 49 ++++++++ pkg/links/sqlite.go | 171 ++++++++++++++++++++++++++ pkg/links/sqlite_test.go | 255 +++++++++++++++++++++++++++++++++++++++ sqlite_test.sh | 20 +++ 8 files changed, 619 insertions(+), 6 deletions(-) create mode 100644 cmd/migrate/migrate.go create mode 100644 pkg/links/sqlite.go create mode 100644 pkg/links/sqlite_test.go create mode 100755 sqlite_test.sh diff --git a/Dockerfile b/Dockerfile index da547ad..1a0a7f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,9 @@ RUN CGO_ENABLED=0 go build -o /app ./... FROM gcr.io/distroless/static:latest WORKDIR /app COPY --from=builder /app/links /app/links +# Shipped so the Postgres -> SQLite migration can be run on a machine that can +# reach both the database and the volume. +COPY --from=builder /app/migrate /app/migrate EXPOSE 8080 CMD ["/app/links"] diff --git a/cmd/links/links.go b/cmd/links/links.go index 5a25432..1b71534 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -46,18 +46,29 @@ func main() { } log.Printf("loaded keyset:\n%s", keyset) + // Storage precedence: -ephemeral wins, then SQLITE_PATH, then + // DATABASE_URL. Unsetting SQLITE_PATH reverts to Postgres. var store links.Store - if *ephemeral { + ctx := context.Background() + sqlitePath := os.Getenv("SQLITE_PATH") + switch { + case *ephemeral: log.Printf("Running in ephemeral mode!") store = links.NewMemStore() - } else { - ctx := context.Background() - dbURL := os.Getenv("DATABASE_URL") - pgStore, err := links.NewPostgresStore(ctx, dbURL) + case sqlitePath != "": + sqliteStore, err := links.NewSQLiteStore(ctx, sqlitePath) + if err != nil { + log.Fatalf("links.NewSQLiteStore failed: %v", err) + } + log.Printf("Opened SQLite database at %s", sqlitePath) + store = sqliteStore + defer sqliteStore.Close() + default: + pgStore, err := links.NewPostgresStore(ctx, os.Getenv("DATABASE_URL")) if err != nil { log.Fatalf("links.NewPostgresStore failed: %v", err) } - log.Printf("Connected to %s", dbURL) + log.Print("Connected to Postgres") store = pgStore defer pgStore.Close() } diff --git a/cmd/migrate/migrate.go b/cmd/migrate/migrate.go new file mode 100644 index 0000000..4320674 --- /dev/null +++ b/cmd/migrate/migrate.go @@ -0,0 +1,94 @@ +// migrate copies every link from the Postgres store named by DATABASE_URL +// into a SQLite database file, then verifies the copy. It is idempotent: +// rerunning it overwrites rows that already exist rather than duplicating +// them, and it never modifies the source database. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + + "jdtw.dev/links/pkg/links" + pb "jdtw.dev/links/proto/links" +) + +func main() { + sqlitePath := flag.String("sqlite", os.Getenv("SQLITE_PATH"), "Destination SQLite database file (defaults to $SQLITE_PATH)") + flag.Parse() + log.SetPrefix("migrate: ") + + if *sqlitePath == "" { + log.Fatal("no destination: pass -sqlite or set SQLITE_PATH") + } + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + log.Fatal("DATABASE_URL must be set to the source Postgres database") + } + + ctx := context.Background() + + src, err := links.NewPostgresStore(ctx, dbURL) + if err != nil { + log.Fatalf("connecting to source Postgres failed: %v", err) + } + defer src.Close() + log.Print("connected to source Postgres") + + dst, err := links.NewSQLiteStore(ctx, *sqlitePath) + if err != nil { + log.Fatalf("opening destination SQLite database failed: %v", err) + } + defer dst.Close() + log.Printf("opened destination %s", *sqlitePath) + + n, err := links.ImportFrom(ctx, dst, src) + if err != nil { + log.Fatalf("import failed: %v", err) + } + log.Printf("copied %d entries", n) + + if err := verify(ctx, src, dst); err != nil { + log.Fatalf("verification failed: %v", err) + } + log.Printf("verified %d entries match the source", n) +} + +// verify re-reads every source entry and confirms the destination holds an +// identical URI and RequiredPaths for the same key. +func verify(ctx context.Context, src, dst links.Store) error { + var checked int + var mismatches []string + + if err := src.Visit(ctx, func(k string, want *pb.LinkEntry) { + checked++ + got, err := dst.Get(ctx, k) + if err != nil { + mismatches = append(mismatches, k+": read error: "+err.Error()) + return + } + if got == nil { + mismatches = append(mismatches, k+": missing from destination") + return + } + if got.Link.GetUri() != want.Link.GetUri() { + mismatches = append(mismatches, k+": URI mismatch") + return + } + if got.RequiredPaths != want.RequiredPaths { + mismatches = append(mismatches, k+": RequiredPaths mismatch") + } + }); err != nil { + return err + } + + for _, m := range mismatches { + log.Printf("MISMATCH %s", m) + } + if len(mismatches) > 0 { + return fmt.Errorf("%d of %d entries did not match", len(mismatches), checked) + } + return nil +} diff --git a/go.mod b/go.mod index 125bebc..c52a879 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,22 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e google.golang.org/protobuf v1.36.10 jdtw.dev/token v0.1.6 + modernc.org/sqlite v1.55.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.40.0 // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 6c5b431..b319994 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -13,8 +21,14 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -22,10 +36,17 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -34,3 +55,31 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= jdtw.dev/token v0.1.6 h1:EzvBOo0s+O4cudJqZob1ynhyIjtWfS85Oxfk/b0iqP4= jdtw.dev/token v0.1.6/go.mod h1:qr+zsFbOixxkv7T5Jb7rar/5Gs2yhw27vyNX0Q7pBA4= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= +modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/pkg/links/sqlite.go b/pkg/links/sqlite.go new file mode 100644 index 0000000..ba654fb --- /dev/null +++ b/pkg/links/sqlite.go @@ -0,0 +1,171 @@ +package links + +import ( + "context" + "database/sql" + "errors" + "fmt" + + pb "jdtw.dev/links/proto/links" + _ "modernc.org/sqlite" +) + +const ( + // sqliteSchema is applied on open so that a fresh database file (for + // example, a newly provisioned volume) is usable without any manual + // setup. It mirrors links.sql. + sqliteSchema = `create table if not exists links ( + path text primary key, + link text not null, + segments integer not null +)` + + sqliteGet = "select link, segments from links where path=?" + sqliteExists = "select 1 from links where path=?" + sqlitePut = `insert into links (path, link, segments) values (?, ?, ?) + on conflict (path) do update set link=excluded.link, segments=excluded.segments` + sqliteDel = "delete from links where path=?" + sqliteList = "select path, link, segments from links" +) + +// SQLiteStore is a Store backed by a local SQLite database file. It is the +// low-cost alternative to PostgresStore: the link table is small enough that +// a file on a mounted volume serves it fine, at the cost of pinning the app +// to a single machine. +type SQLiteStore struct { + db *sql.DB +} + +var _ Store = &SQLiteStore{} + +func (s *SQLiteStore) Close() error { + if s.db == nil { + return nil + } + return s.db.Close() +} + +// NewSQLiteStore opens (creating if necessary) the SQLite database at path +// and applies the schema. WAL mode keeps redirect reads from blocking on the +// occasional write, and busy_timeout absorbs the brief contention that WAL +// still allows between concurrent writers. +func NewSQLiteStore(ctx context.Context, path string) (*SQLiteStore, error) { + dsn := fmt.Sprintf("file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)", path) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("sql.Open failed: %w", err) + } + + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("db.Ping failed: %w", err) + } + + if _, err := db.ExecContext(ctx, sqliteSchema); err != nil { + db.Close() + return nil, fmt.Errorf("applying schema failed: %w", err) + } + + return &SQLiteStore{db: db}, nil +} + +func (s *SQLiteStore) Get(ctx context.Context, key string) (*pb.LinkEntry, error) { + var link string + var segments int + if err := s.db.QueryRowContext(ctx, sqliteGet, key).Scan(&link, &segments); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, err + } + return &pb.LinkEntry{ + Link: &pb.Link{Uri: link}, + RequiredPaths: int32(segments), + }, nil +} + +// Put upserts the link and reports whether it was created rather than +// updated. SQLite has no equivalent of Postgres' xmax trick, so the existence +// check and the write share a transaction to keep the answer accurate under +// concurrent writers. +func (s *SQLiteStore) Put(ctx context.Context, key string, l *pb.Link) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer tx.Rollback() + + var exists int + err = tx.QueryRowContext(ctx, sqliteExists, key).Scan(&exists) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, err + } + created := errors.Is(err, sql.ErrNoRows) + + if _, err := tx.ExecContext(ctx, sqlitePut, key, l.Uri, requiredPaths(l)); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + return created, nil +} + +func (s *SQLiteStore) Delete(ctx context.Context, key string) error { + _, err := s.db.ExecContext(ctx, sqliteDel, key) + return err +} + +func (s *SQLiteStore) Visit(ctx context.Context, visit func(string, *pb.LinkEntry)) error { + rows, err := s.db.QueryContext(ctx, sqliteList) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var path string + var link string + var segments int + if err := rows.Scan(&path, &link, &segments); err != nil { + return err + } + visit(path, &pb.LinkEntry{ + Link: &pb.Link{Uri: link}, + RequiredPaths: int32(segments), + }) + } + return rows.Err() +} + +// putEntry writes an entry verbatim, preserving the stored RequiredPaths +// instead of recomputing it from the URI. Used by ImportFrom so that a +// migration is a faithful copy rather than a re-derivation. +func (s *SQLiteStore) putEntry(ctx context.Context, key string, le *pb.LinkEntry) error { + _, err := s.db.ExecContext(ctx, sqlitePut, key, le.Link.GetUri(), le.RequiredPaths) + return err +} + +// ImportFrom copies every entry from src into dst, returning the number of +// entries copied. Existing rows in dst with the same path are overwritten. +func ImportFrom(ctx context.Context, dst *SQLiteStore, src Store) (int, error) { + type entry struct { + key string + le *pb.LinkEntry + } + // Collect first: Visit holds a read lock on the source for its duration, + // and PostgresStore streams rows over the same pool the writes would use. + var entries []entry + if err := src.Visit(ctx, func(k string, le *pb.LinkEntry) { + entries = append(entries, entry{k, le}) + }); err != nil { + return 0, fmt.Errorf("reading source store failed: %w", err) + } + + for _, e := range entries { + if err := dst.putEntry(ctx, e.key, e.le); err != nil { + return 0, fmt.Errorf("writing %q failed: %w", e.key, err) + } + } + return len(entries), nil +} diff --git a/pkg/links/sqlite_test.go b/pkg/links/sqlite_test.go new file mode 100644 index 0000000..25ac010 --- /dev/null +++ b/pkg/links/sqlite_test.go @@ -0,0 +1,255 @@ +package links + +import ( + "context" + "path/filepath" + "testing" + + pb "jdtw.dev/links/proto/links" +) + +// newTestSQLiteStore opens a store backed by a file in the test's temp +// directory. Unlike the Postgres tests these need no external service, so +// they always run. +func newTestSQLiteStore(t *testing.T) *SQLiteStore { + t.Helper() + s, err := NewSQLiteStore(context.Background(), filepath.Join(t.TempDir(), "links.db")) + if err != nil { + t.Fatalf("NewSQLiteStore failed: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func TestSQLitePutReportsCreatedVsUpdated(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "createdvsupdated" + + created, err := s.Put(ctx, key, &pb.Link{Uri: "http://example.com/first"}) + if err != nil { + t.Fatalf("Put (insert) failed: %v", err) + } + if !created { + t.Errorf("Put (insert) reported created=false, want true") + } + + created, err = s.Put(ctx, key, &pb.Link{Uri: "http://example.com/second"}) + if err != nil { + t.Fatalf("Put (update) failed: %v", err) + } + if created { + t.Errorf("Put (update) reported created=true, want false") + } + + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.Link.GetUri(), "http://example.com/second"; got != want { + t.Errorf("Get(%s) URI = %q, want %q", key, got, want) + } +} + +func TestSQLiteGetMissingKeyReturnsNil(t *testing.T) { + s := newTestSQLiteStore(t) + + le, err := s.Get(context.Background(), "doesnotexist") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le != nil { + t.Errorf("Get(missing) = %v, want nil", le) + } +} + +func TestSQLiteDelete(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "delete" + + if _, err := s.Put(ctx, key, &pb.Link{Uri: "http://example.com"}); err != nil { + t.Fatalf("Put failed: %v", err) + } + if err := s.Delete(ctx, key); err != nil { + t.Fatalf("Delete failed: %v", err) + } + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get after delete failed: %v", err) + } + if le != nil { + t.Errorf("Get after delete = %v, want nil", le) + } +} + +// Deleting a key that was never present should be a no-op, matching the +// Postgres store's behavior. +func TestSQLiteDeleteMissingKeyIsNoOp(t *testing.T) { + s := newTestSQLiteStore(t) + if err := s.Delete(context.Background(), "neverexisted"); err != nil { + t.Errorf("Delete(missing) failed: %v", err) + } +} + +func TestSQLiteVisit(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + + want := map[string]string{ + "one": "http://example.com/one", + "two": "http://example.com/two", + "three": "http://example.com/three", + } + for k, uri := range want { + if _, err := s.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { + t.Fatalf("Put(%s) failed: %v", k, err) + } + } + + got := map[string]string{} + if err := s.Visit(ctx, func(k string, le *pb.LinkEntry) { + got[k] = le.Link.GetUri() + }); err != nil { + t.Fatalf("Visit failed: %v", err) + } + + if len(got) != len(want) { + t.Errorf("Visit saw %d entries, want %d", len(got), len(want)) + } + for k, wantURI := range want { + if got[k] != wantURI { + t.Errorf("Visit(%s) URI = %q, want %q", k, got[k], wantURI) + } + } +} + +// Put must persist the computed RequiredPaths so that {n} substitution keeps +// working after a restart. +func TestSQLitePutPersistsRequiredPaths(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "subst" + + l := &pb.Link{Uri: "http://example.com/{1}/{0}"} + if _, err := s.Put(ctx, key, l); err != nil { + t.Fatalf("Put failed: %v", err) + } + + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.RequiredPaths, requiredPaths(l); got != want { + t.Errorf("RequiredPaths = %d, want %d", got, want) + } + if le.RequiredPaths != 2 { + t.Errorf("RequiredPaths = %d, want 2 for %q", le.RequiredPaths, l.Uri) + } +} + +// The database file must survive being closed and reopened -- this is the +// whole point of putting it on a volume. +func TestSQLitePersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "links.db") + + first, err := NewSQLiteStore(ctx, path) + if err != nil { + t.Fatalf("NewSQLiteStore failed: %v", err) + } + if _, err := first.Put(ctx, "persisted", &pb.Link{Uri: "http://example.com/persisted"}); err != nil { + t.Fatalf("Put failed: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + second, err := NewSQLiteStore(ctx, path) + if err != nil { + t.Fatalf("reopening store failed: %v", err) + } + t.Cleanup(func() { second.Close() }) + + le, err := second.Get(ctx, "persisted") + if err != nil { + t.Fatalf("Get after reopen failed: %v", err) + } + if le == nil { + t.Fatal("Get after reopen = nil, want the entry written before close") + } + if got, want := le.Link.GetUri(), "http://example.com/persisted"; got != want { + t.Errorf("URI after reopen = %q, want %q", got, want) + } +} + +func TestImportFromCopiesEntriesVerbatim(t *testing.T) { + ctx := context.Background() + src := NewMemStore() + + // A link whose stored RequiredPaths must be preserved rather than + // recomputed during the copy. + uris := map[string]string{ + "plain": "http://example.com/plain", + "subst": "http://example.com/{1}/{0}", + } + for k, uri := range uris { + if _, err := src.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { + t.Fatalf("seeding %s failed: %v", k, err) + } + } + + dst := newTestSQLiteStore(t) + n, err := ImportFrom(ctx, dst, src) + if err != nil { + t.Fatalf("ImportFrom failed: %v", err) + } + if n != len(uris) { + t.Errorf("ImportFrom copied %d entries, want %d", n, len(uris)) + } + + for k, wantURI := range uris { + le, err := dst.Get(ctx, k) + if err != nil { + t.Fatalf("Get(%s) failed: %v", k, err) + } + if le == nil { + t.Errorf("Get(%s) = nil, want the imported entry", k) + continue + } + if le.Link.GetUri() != wantURI { + t.Errorf("Get(%s) URI = %q, want %q", k, le.Link.GetUri(), wantURI) + } + srcLE, err := src.Get(ctx, k) + if err != nil { + t.Fatalf("src.Get(%s) failed: %v", k, err) + } + if le.RequiredPaths != srcLE.RequiredPaths { + t.Errorf("Get(%s) RequiredPaths = %d, want %d", k, le.RequiredPaths, srcLE.RequiredPaths) + } + } +} + +// Re-running a migration must be safe. +func TestImportFromIsIdempotent(t *testing.T) { + ctx := context.Background() + src := NewMemStore() + if _, err := src.Put(ctx, "key", &pb.Link{Uri: "http://example.com/one"}); err != nil { + t.Fatalf("seeding failed: %v", err) + } + + dst := newTestSQLiteStore(t) + for i := 0; i < 2; i++ { + if _, err := ImportFrom(ctx, dst, src); err != nil { + t.Fatalf("ImportFrom run %d failed: %v", i+1, err) + } + } + + count := 0 + if err := dst.Visit(ctx, func(string, *pb.LinkEntry) { count++ }); err != nil { + t.Fatalf("Visit failed: %v", err) + } + if count != 1 { + t.Errorf("after two imports the store holds %d entries, want 1", count) + } +} diff --git a/sqlite_test.sh b/sqlite_test.sh new file mode 100755 index 0000000..44619bb --- /dev/null +++ b/sqlite_test.sh @@ -0,0 +1,20 @@ +#! /bin/bash +# Runs the full integration suite against the SQLite store. Unlike +# docker_test.sh and local_test.sh this needs no database server: the store is +# a file in a scratch directory that is removed on exit. +set -euxo pipefail + +SQLITE_DIR="$(mktemp -d)" +export SQLITE_PATH="${SQLITE_DIR}/links.db" + +cleanup() { + exit_status=$? + echo "Cleaning up ${SQLITE_DIR}..." + rm -rf "${SQLITE_DIR}" + exit "${exit_status}" +} +trap cleanup EXIT + +go test ./... + +./test.sh From 74f8be208e8e8361d432a6eae8d2fb433c27fda7 Mon Sep 17 00:00:00 2001 From: jdtw Date: Fri, 31 Jul 2026 21:36:17 -0700 Subject: [PATCH 2/3] Add bulk link import and export GET /api/links already returned the whole database; this adds the missing half so a link set can be saved off and reloaded. POST /api/links accepts the same Links proto that the GET returns, and the client grows --export and --import flags that read and write it as a file. Imports are additive: links already stored that the body does not mention are left alone, so a restore cannot silently drop entries. Every link is validated before the first write, so one malformed URI fails the request instead of half-applying. Keys that collide only after hyphen normalization are rejected for the same reason. The validation put() performed inline moves to validateLink so that single and bulk writes cannot drift apart. Together with the SQLite backend this makes migrating between stores a matter of exporting, pointing the server at an empty database, and importing -- no direct database access required. Co-Authored-By: Claude Opus 5 --- README.md | 69 +++++++++++ cmd/client/client.go | 42 +++++++ pkg/client/client.go | 32 +++++ pkg/links/api.go | 119 ++++++++++++++++--- pkg/links/bulk_test.go | 258 +++++++++++++++++++++++++++++++++++++++++ pkg/links/server.go | 2 + 6 files changed, 504 insertions(+), 18 deletions(-) create mode 100644 pkg/links/bulk_test.go diff --git a/README.md b/README.md index e559919..52fffd3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,68 @@ Run `docker compose up` from the `dev/` directory to get a local instance runnin The server maintains a database of friendly names to URI redirect templates. For example, `rfc -> https://datatracker.ietf.org/doc/html/rfc{0}` will redirect `GET /rfc/5280` to `https://datatracker.ietf.org/doc/html/rfc5280`. Try it out: [jdtw.us/rfc/5280](https://jdtw.us/rfc/5280). +## Storage + +The server picks its backing store from the environment, in this order: + +| Condition | Store | +| --- | --- | +| `--ephemeral` | in-memory, discarded on exit | +| `SQLITE_PATH` set | SQLite database at that path | +| otherwise | Postgres at `DATABASE_URL` | + +SQLite keeps the whole link table in a single file, which is enough for this +workload and avoids paying for a managed Postgres instance. The tradeoff is +that the file lives on one volume, so the app is pinned to a single machine +in a single region and there is no replication. Postgres remains supported: +unset `SQLITE_PATH` to switch back. + +The schema is applied automatically when the SQLite database is opened, so a +freshly provisioned volume needs no manual setup. + +### Backup and restore + +The client can dump the whole link database to a file and load it back, +which doubles as the migration path between storage backends: + +``` +$ client --export links-backup.json +$ client --import links-backup.json +``` + +`--export` writes the same `links.Links` JSON proto that `GET /api/links` +returns, indented for readability. `--import` posts it back. Both accept `-` +for stdout/stdin. Importing is additive and idempotent, so re-running it is +safe. + +### Migrating Postgres to SQLite + +The simplest route needs no database access at all -- export from the running +server, point it at an empty SQLite file, and import: + +1. `client --export links-backup.json` against the Postgres-backed server. +2. Restart with `SQLITE_PATH` set, which creates and initializes an empty + database file. +3. `client --import links-backup.json`. + +Keep the backup, and keep Postgres around until you're satisfied; unsetting +`SQLITE_PATH` reverts to it with the original data untouched. + +Alternatively `cmd/migrate` copies directly from Postgres to a SQLite file +and verifies each entry, for when the server isn't reachable but the database +is. It never writes to Postgres, and rerunning it is safe. + +``` +$ DATABASE_URL="postgres://..." go run ./cmd/migrate -sqlite /data/links.db +migrate: copied 42 entries +migrate: verified 42 entries match the source +``` + +### Tests + +`./sqlite_test.sh` runs the full suite against SQLite and needs no database +server. `./docker_test.sh` does the same against Postgres in a container. + ## REST API * `GET /api/links` returns all links in the database. @@ -28,6 +90,13 @@ The server maintains a database of friendly names to URI redirect templates. For * Request body: empty * Response body: `links.Link` JSON proto. * Returns: 200 (OK) or 404 (not found) +* `POST /api/links` bulk creates or updates links. + * Request body: `links.Links` JSON proto, the same shape `GET /api/links` returns. + * Response body: empty + * Returns: 204 (no content), or 400 if any link is invalid. + * Additive: links already stored that the body does not mention are left + alone. Every entry is validated before anything is written, so one bad + link fails the whole request rather than half-applying the import. * `PUT /api/links/{link}` creates or updates a link. * Request body: `links.Link` JSON proto. * Response body: empty diff --git a/cmd/client/client.go b/cmd/client/client.go index fc4c26e..864bad3 100644 --- a/cmd/client/client.go +++ b/cmd/client/client.go @@ -3,14 +3,17 @@ package main import ( "flag" "fmt" + "io" "log" "net/http" "os" "sort" + "google.golang.org/protobuf/encoding/protojson" "jdtw.dev/links/pkg/client" "jdtw.dev/links/pkg/frontend" "jdtw.dev/links/pkg/links" + pb "jdtw.dev/links/proto/links" "jdtw.dev/token" ) @@ -23,6 +26,8 @@ var ( get = flag.String("get", "", "Get a redirect") rm = flag.String("rm", "", "Remove a redirect") server = flag.Int("server", -1, "If not -1, starts starts a frontent HTTP server on the given port.") + export = flag.String("export", "", "Write all links as a JSON Links proto to the given file, or '-' for stdout") + imprt = flag.String("import", "", "Bulk create or update links from a JSON Links proto file, or '-' for stdin") ) func main() { @@ -77,6 +82,43 @@ func main() { if err := c.Delete(*rm); err != nil { log.Fatal(err) } + case *export != "": + lpb, err := c.Export() + if err != nil { + log.Fatal(err) + } + // Indented so the backup is readable and reviewable. Note that + // protojson does not promise byte-stable output, so don't expect + // two exports of identical data to diff clean. + data, err := protojson.MarshalOptions{Multiline: true, Indent: " "}.Marshal(lpb) + if err != nil { + log.Fatal(err) + } + if *export == "-" { + os.Stdout.Write(data) + } else if err := os.WriteFile(*export, data, 0600); err != nil { + log.Fatal(err) + } + log.Printf("exported %d links", len(lpb.GetLinks())) + case *imprt != "": + var data []byte + var err error + if *imprt == "-" { + data, err = io.ReadAll(os.Stdin) + } else { + data, err = os.ReadFile(*imprt) + } + if err != nil { + log.Fatal(err) + } + lpb := &pb.Links{} + if err := protojson.Unmarshal(data, lpb); err != nil { + log.Fatalf("failed to parse %s: %v", *imprt, err) + } + if err := c.Import(lpb); err != nil { + log.Fatal(err) + } + log.Printf("imported %d links", len(lpb.GetLinks())) default: l, err := c.List() if err != nil { diff --git a/pkg/client/client.go b/pkg/client/client.go index 3d1084a..641485c 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -82,6 +82,38 @@ func (c *Client) List() (map[string]string, error) { return l, nil } +// Export returns every link as a Links proto, preserving the exact shape the +// server stores. Unlike List, which flattens to a map of strings for display, +// the result round-trips through Import. +func (c *Client) Export() (*pb.Links, error) { + resp, err := c.do("GET", linksAPI, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + lpb := &pb.Links{} + if err := unmarshalBody(resp, lpb); err != nil { + return nil, err + } + return lpb, nil +} + +// Import bulk-creates or updates every link in lpb. Links already on the +// server that lpb does not mention are left alone. +func (c *Client) Import(lpb *pb.Links) error { + body, err := marshal(lpb) + if err != nil { + return err + } + resp, err := c.do("POST", linksAPI, body) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + func (c *Client) Get(link string) (string, error) { resp, err := c.do("GET", api(link), nil) if err != nil { diff --git a/pkg/links/api.go b/pkg/links/api.go index 3566f8b..49fe33c 100644 --- a/pkg/links/api.go +++ b/pkg/links/api.go @@ -1,10 +1,14 @@ package links import ( + "errors" + "fmt" "io" "log" "net/http" "net/url" + "sort" + "strings" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -56,15 +60,35 @@ func (s *server) get() http.HandlerFunc { } } +// validateLink reports whether a normalized key and link are acceptable to +// store, describing the problem if they are not. Shared by put() and +// bulkPut() so a bulk import enforces exactly the same rules as a single +// write. +func validateLink(key string, l *pb.Link) error { + if key == qrKey { + return fmt.Errorf("%q is a reserved link name", qrKey) + } + if l.GetUri() == "" { + return errors.New("missing URI") + } + // Create a dummy URI with all template parameters replaced + // with something innocuous so that we can try to parse it. + dummy := replacement.ReplaceAllString(l.GetUri(), "links") + url, err := url.Parse(dummy) + if err != nil { + return fmt.Errorf("URI %q failed to parse: %v", l.GetUri(), err) + } + if url.Scheme == "" { + return fmt.Errorf("URI %q has no scheme", l.GetUri()) + } + return nil +} + func (s *server) put() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { rid := middleware.GetReqID(r.Context()) l := normalizeKey(chi.URLParam(r, "link")) - if l == qrKey { - badRequest(w, "%q is a reserved link name", qrKey) - return - } data, err := io.ReadAll(r.Body) if err != nil { internalError(w, err, rid) @@ -75,20 +99,8 @@ func (s *server) put() http.HandlerFunc { badRequest(w, "failed to unmarshal body: %v", err) return } - if lpb.Uri == "" { - badRequest(w, "missing URI") - return - } - // Create a dummy URI with all template parameters replaced - // with something innocuous so that we can try to parse it. - dummy := replacement.ReplaceAllString(lpb.Uri, "links") - url, err := url.Parse(dummy) - if err != nil { - badRequest(w, "URI %q failed to parse: %v", lpb.Uri, err) - return - } - if url.Scheme == "" { - badRequest(w, "URI %q has no scheme", lpb.Uri) + if err := validateLink(l, lpb); err != nil { + badRequest(w, "%v", err) return } created, err := s.store.Put(r.Context(), l, lpb) @@ -108,6 +120,77 @@ func (s *server) put() http.HandlerFunc { } } +// bulkPut creates or updates every link in the request body, which is a +// Links proto of the same shape that list() returns. Links already in the +// store that the body does not mention are left alone, so an import is +// additive rather than a replacement. +// +// Every entry is validated before anything is written: one malformed link +// fails the whole request rather than leaving a half-applied import. +func (s *server) bulkPut() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + rid := middleware.GetReqID(r.Context()) + + data, err := io.ReadAll(r.Body) + if err != nil { + internalError(w, err, rid) + return + } + lpb := new(pb.Links) + if err := protojson.Unmarshal(data, lpb); err != nil { + badRequest(w, "failed to unmarshal body: %v", err) + return + } + if len(lpb.GetLinks()) == 0 { + badRequest(w, "no links in request body") + return + } + + // Normalize and validate everything before the first write. Keys + // that collide only after normalization ("my-link" and "mylink") + // would silently overwrite each other, so reject those too. + normalized := make(map[string]*pb.Link, len(lpb.GetLinks())) + sources := make(map[string]string, len(lpb.GetLinks())) + var problems []string + for k, l := range lpb.GetLinks() { + key := normalizeKey(k) + if err := validateLink(key, l); err != nil { + problems = append(problems, fmt.Sprintf("%q: %v", k, err)) + continue + } + if prev, dup := sources[key]; dup { + problems = append(problems, fmt.Sprintf("%q: collides with %q after normalization", k, prev)) + continue + } + sources[key] = k + normalized[key] = l + } + if len(problems) > 0 { + sort.Strings(problems) + badRequest(w, "rejected %d of %d links:\n%s", len(problems), len(lpb.GetLinks()), strings.Join(problems, "\n")) + return + } + + var created, updated int + for k, l := range normalized { + wasCreated, err := s.store.Put(r.Context(), k, l) + if err != nil { + internalError(w, err, rid) + return + } + if wasCreated { + created++ + } else { + updated++ + } + } + + w.WriteHeader(http.StatusNoContent) + log.Printf("[%s] %s imported %d links (%d created, %d updated)", + rid, subject(r.Context()), len(normalized), created, updated) + } +} + func (s *server) delete() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { rid := middleware.GetReqID(r.Context()) diff --git a/pkg/links/bulk_test.go b/pkg/links/bulk_test.go new file mode 100644 index 0000000..d34ecfb --- /dev/null +++ b/pkg/links/bulk_test.go @@ -0,0 +1,258 @@ +package links + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "jdtw.dev/links/pkg/tokentest" + pb "jdtw.dev/links/proto/links" + "jdtw.dev/token" +) + +// postLinks bulk-imports the given path -> URI pairs and returns the response. +func postLinks(t *testing.T, srv http.Handler, priv *token.SigningKey, links map[string]string) *http.Response { + t.Helper() + lpb := &pb.Links{Links: make(map[string]*pb.Link, len(links))} + for k, uri := range links { + lpb.Links[k] = &pb.Link{Uri: uri} + } + return postBody(t, srv, priv, marshal(t, lpb)) +} + +func postBody(t *testing.T, srv http.Handler, priv *token.SigningKey, body io.Reader) *http.Response { + t.Helper() + rr := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/api/links", body) + signRequest(t, priv, req) + srv.ServeHTTP(rr, req) + return rr.Result() +} + +func TestBulkPutImportsLinks(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + want := map[string]string{ + "rfc": "https://datatracker.ietf.org/doc/html/rfc{0}", + "gh": "https://github.com/jdtw/{0}", + "plain": "https://example.com/plain", + Index: "https://example.com", + } + if sc := postLinks(t, srv, priv, want).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + for k, wantURI := range want { + le, err := store.Get(context.Background(), k) + if err != nil { + t.Fatalf("Get(%s) failed: %v", k, err) + } + if le == nil { + t.Errorf("Get(%s) = nil, want the imported link", k) + continue + } + if le.Link.GetUri() != wantURI { + t.Errorf("Get(%s) = %q, want %q", k, le.Link.GetUri(), wantURI) + } + } +} + +// A bulk import must compute RequiredPaths the same way a single PUT does, +// or {n} substitution silently breaks on imported links. +func TestBulkPutComputesRequiredPaths(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + const uri = "https://example.com/{1}/{0}" + if sc := postLinks(t, srv, priv, map[string]string{"swap": uri}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + le, err := store.Get(context.Background(), "swap") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.RequiredPaths, requiredPaths(&pb.Link{Uri: uri}); got != want { + t.Errorf("RequiredPaths = %d, want %d", got, want) + } + if le.RequiredPaths != 2 { + t.Errorf("RequiredPaths = %d, want 2", le.RequiredPaths) + } +} + +func TestBulkPutIsAdditive(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + ctx := context.Background() + + if _, err := store.Put(ctx, "existing", &pb.Link{Uri: "https://example.com/existing"}); err != nil { + t.Fatalf("seeding failed: %v", err) + } + + if sc := postLinks(t, srv, priv, map[string]string{"new": "https://example.com/new"}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + // The link the import did not mention must survive. + le, err := store.Get(ctx, "existing") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le == nil { + t.Error("import removed a link it did not mention; want additive behavior") + } +} + +func TestBulkPutOverwritesExisting(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + ctx := context.Background() + + if _, err := store.Put(ctx, "dup", &pb.Link{Uri: "https://example.com/old"}); err != nil { + t.Fatalf("seeding failed: %v", err) + } + if sc := postLinks(t, srv, priv, map[string]string{"dup": "https://example.com/new"}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + le, err := store.Get(ctx, "dup") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.Link.GetUri(), "https://example.com/new"; got != want { + t.Errorf("URI = %q, want %q", got, want) + } +} + +func TestBulkPutRejectsInvalidRequests(t *testing.T) { + tests := []struct { + name string + body io.Reader + }{ + {"nil body", nil}, + {"not a proto", strings.NewReader("not-a-proto")}, + {"empty links", bytes.NewReader([]byte(`{"links":{}}`))}, + {"missing uri", bytes.NewReader([]byte(`{"links":{"foo":{"uri":""}}}`))}, + {"no scheme", bytes.NewReader([]byte(`{"links":{"foo":{"uri":"no-scheme"}}}`))}, + {"reserved qr key", bytes.NewReader([]byte(`{"links":{"qr":{"uri":"https://example.com"}}}`))}, + {"normalization collision", bytes.NewReader([]byte(`{"links":{"my-link":{"uri":"https://example.com/a"},"mylink":{"uri":"https://example.com/b"}}}`))}, + } + + keyset, priv := tokentest.GenerateKey(t, "test") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := NewHandler(NewMemStore(), keyset, 0) + if sc := postBody(t, srv, priv, tc.body).StatusCode; sc != http.StatusBadRequest { + t.Errorf("POST returned %d, want 400", sc) + } + }) + } +} + +// One bad link must not leave a partially applied import behind. +func TestBulkPutIsAtomicOnValidationFailure(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + body := bytes.NewReader([]byte(`{"links":{ + "good":{"uri":"https://example.com/good"}, + "bad":{"uri":"no-scheme"} + }}`)) + if sc := postBody(t, srv, priv, body).StatusCode; sc != http.StatusBadRequest { + t.Fatalf("POST returned %d, want 400", sc) + } + + le, err := store.Get(context.Background(), "good") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le != nil { + t.Error("a rejected import wrote the valid link anyway; want nothing written") + } +} + +func TestBulkPutRequiresAuth(t *testing.T) { + _, priv := tokentest.GenerateKey(t, "test") + srv := NewHandler(NewMemStore(), nil, 0) + if sc := postLinks(t, srv, priv, map[string]string{"foo": "https://example.com"}).StatusCode; sc != http.StatusUnauthorized { + t.Errorf("POST with nil keyset returned %d, want 401", sc) + } +} + +// Export then import must reproduce the original set exactly -- this is the +// property the Postgres -> SQLite migration relies on. +func TestExportImportRoundTrip(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + ctx := context.Background() + + source := NewMemStore() + want := map[string]string{ + "rfc": "https://datatracker.ietf.org/doc/html/rfc{0}", + "swap": "https://example.com/{1}/{0}", + "plain": "https://example.com/plain", + Index: "https://example.com", + } + for k, uri := range want { + if _, err := source.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { + t.Fatalf("seeding %s failed: %v", k, err) + } + } + + // Export from the source server. + exportSrv := NewHandler(source, keyset, 0) + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/links", nil) + signRequest(t, priv, req) + exportSrv.ServeHTTP(rr, req) + if sc := rr.Result().StatusCode; sc != http.StatusOK { + t.Fatalf("GET /api/links returned %d, want 200", sc) + } + exported, err := io.ReadAll(rr.Result().Body) + if err != nil { + t.Fatalf("reading export failed: %v", err) + } + + // Import into a fresh, empty store -- the SQLite side of the migration. + dest := NewMemStore() + destSrv := NewHandler(dest, keyset, 0) + if sc := postBody(t, destSrv, priv, bytes.NewReader(exported)).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + got := map[string]string{} + if err := dest.Visit(ctx, func(k string, le *pb.LinkEntry) { + got[k] = le.Link.GetUri() + }); err != nil { + t.Fatalf("Visit failed: %v", err) + } + + if len(got) != len(want) { + t.Errorf("round trip produced %d links, want %d", len(got), len(want)) + } + for k, wantURI := range want { + if got[k] != wantURI { + t.Errorf("round trip [%s] = %q, want %q", k, got[k], wantURI) + } + srcLE, err := source.Get(ctx, k) + if err != nil { + t.Fatalf("source.Get(%s) failed: %v", k, err) + } + dstLE, err := dest.Get(ctx, k) + if err != nil { + t.Fatalf("dest.Get(%s) failed: %v", k, err) + } + if srcLE.RequiredPaths != dstLE.RequiredPaths { + t.Errorf("round trip [%s] RequiredPaths = %d, want %d", k, dstLE.RequiredPaths, srcLE.RequiredPaths) + } + } +} diff --git a/pkg/links/server.go b/pkg/links/server.go index 0aa17be..d03dc4f 100644 --- a/pkg/links/server.go +++ b/pkg/links/server.go @@ -28,6 +28,8 @@ func (s *server) routes() { r.Use(s.authenticated()) // Get all links as a Links proto. r.Get("/links", s.list()) + // Bulk create or update from a Links proto. + r.Post("/links", s.bulkPut()) // Get a speficic link. r.Get("/links/{link}", s.get()) // Create or update a link. From 92c625365d5f6b48f92cc5e988a03b7a029fde12 Mon Sep 17 00:00:00 2001 From: jdtw Date: Fri, 31 Jul 2026 21:44:49 -0700 Subject: [PATCH 3/3] Drop the direct Postgres-to-SQLite migrate tool Bulk import/export over the REST API covers the migration, so the direct database-to-database path is dead weight before it has ever been used. It was not free: shipping a second binary in the image took it from ~42MB to ~75MB, and the command had no automated test coverage. The one thing it did that import does not -- copy `segments` verbatim rather than recomputing it -- turned out not to matter. requiredPaths is deterministic from the URI, so a recomputed value is identical, and it is self-healing if a stored value were ever stale. pgx stays regardless: PostgresStore remains as the rollback path. Co-Authored-By: Claude Opus 5 --- Dockerfile | 3 -- README.md | 14 +----- cmd/migrate/migrate.go | 94 ---------------------------------------- pkg/links/sqlite.go | 32 -------------- pkg/links/sqlite_test.go | 71 ------------------------------ 5 files changed, 2 insertions(+), 212 deletions(-) delete mode 100644 cmd/migrate/migrate.go diff --git a/Dockerfile b/Dockerfile index 1a0a7f6..da547ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,6 @@ RUN CGO_ENABLED=0 go build -o /app ./... FROM gcr.io/distroless/static:latest WORKDIR /app COPY --from=builder /app/links /app/links -# Shipped so the Postgres -> SQLite migration can be run on a machine that can -# reach both the database and the volume. -COPY --from=builder /app/migrate /app/migrate EXPOSE 8080 CMD ["/app/links"] diff --git a/README.md b/README.md index 52fffd3..dc61d1d 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ safe. ### Migrating Postgres to SQLite -The simplest route needs no database access at all -- export from the running -server, point it at an empty SQLite file, and import: +No database access is needed -- export from the running server, point it at +an empty SQLite file, and import: 1. `client --export links-backup.json` against the Postgres-backed server. 2. Restart with `SQLITE_PATH` set, which creates and initializes an empty @@ -65,16 +65,6 @@ server, point it at an empty SQLite file, and import: Keep the backup, and keep Postgres around until you're satisfied; unsetting `SQLITE_PATH` reverts to it with the original data untouched. -Alternatively `cmd/migrate` copies directly from Postgres to a SQLite file -and verifies each entry, for when the server isn't reachable but the database -is. It never writes to Postgres, and rerunning it is safe. - -``` -$ DATABASE_URL="postgres://..." go run ./cmd/migrate -sqlite /data/links.db -migrate: copied 42 entries -migrate: verified 42 entries match the source -``` - ### Tests `./sqlite_test.sh` runs the full suite against SQLite and needs no database diff --git a/cmd/migrate/migrate.go b/cmd/migrate/migrate.go deleted file mode 100644 index 4320674..0000000 --- a/cmd/migrate/migrate.go +++ /dev/null @@ -1,94 +0,0 @@ -// migrate copies every link from the Postgres store named by DATABASE_URL -// into a SQLite database file, then verifies the copy. It is idempotent: -// rerunning it overwrites rows that already exist rather than duplicating -// them, and it never modifies the source database. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "os" - - "jdtw.dev/links/pkg/links" - pb "jdtw.dev/links/proto/links" -) - -func main() { - sqlitePath := flag.String("sqlite", os.Getenv("SQLITE_PATH"), "Destination SQLite database file (defaults to $SQLITE_PATH)") - flag.Parse() - log.SetPrefix("migrate: ") - - if *sqlitePath == "" { - log.Fatal("no destination: pass -sqlite or set SQLITE_PATH") - } - dbURL := os.Getenv("DATABASE_URL") - if dbURL == "" { - log.Fatal("DATABASE_URL must be set to the source Postgres database") - } - - ctx := context.Background() - - src, err := links.NewPostgresStore(ctx, dbURL) - if err != nil { - log.Fatalf("connecting to source Postgres failed: %v", err) - } - defer src.Close() - log.Print("connected to source Postgres") - - dst, err := links.NewSQLiteStore(ctx, *sqlitePath) - if err != nil { - log.Fatalf("opening destination SQLite database failed: %v", err) - } - defer dst.Close() - log.Printf("opened destination %s", *sqlitePath) - - n, err := links.ImportFrom(ctx, dst, src) - if err != nil { - log.Fatalf("import failed: %v", err) - } - log.Printf("copied %d entries", n) - - if err := verify(ctx, src, dst); err != nil { - log.Fatalf("verification failed: %v", err) - } - log.Printf("verified %d entries match the source", n) -} - -// verify re-reads every source entry and confirms the destination holds an -// identical URI and RequiredPaths for the same key. -func verify(ctx context.Context, src, dst links.Store) error { - var checked int - var mismatches []string - - if err := src.Visit(ctx, func(k string, want *pb.LinkEntry) { - checked++ - got, err := dst.Get(ctx, k) - if err != nil { - mismatches = append(mismatches, k+": read error: "+err.Error()) - return - } - if got == nil { - mismatches = append(mismatches, k+": missing from destination") - return - } - if got.Link.GetUri() != want.Link.GetUri() { - mismatches = append(mismatches, k+": URI mismatch") - return - } - if got.RequiredPaths != want.RequiredPaths { - mismatches = append(mismatches, k+": RequiredPaths mismatch") - } - }); err != nil { - return err - } - - for _, m := range mismatches { - log.Printf("MISMATCH %s", m) - } - if len(mismatches) > 0 { - return fmt.Errorf("%d of %d entries did not match", len(mismatches), checked) - } - return nil -} diff --git a/pkg/links/sqlite.go b/pkg/links/sqlite.go index ba654fb..b163585 100644 --- a/pkg/links/sqlite.go +++ b/pkg/links/sqlite.go @@ -137,35 +137,3 @@ func (s *SQLiteStore) Visit(ctx context.Context, visit func(string, *pb.LinkEntr } return rows.Err() } - -// putEntry writes an entry verbatim, preserving the stored RequiredPaths -// instead of recomputing it from the URI. Used by ImportFrom so that a -// migration is a faithful copy rather than a re-derivation. -func (s *SQLiteStore) putEntry(ctx context.Context, key string, le *pb.LinkEntry) error { - _, err := s.db.ExecContext(ctx, sqlitePut, key, le.Link.GetUri(), le.RequiredPaths) - return err -} - -// ImportFrom copies every entry from src into dst, returning the number of -// entries copied. Existing rows in dst with the same path are overwritten. -func ImportFrom(ctx context.Context, dst *SQLiteStore, src Store) (int, error) { - type entry struct { - key string - le *pb.LinkEntry - } - // Collect first: Visit holds a read lock on the source for its duration, - // and PostgresStore streams rows over the same pool the writes would use. - var entries []entry - if err := src.Visit(ctx, func(k string, le *pb.LinkEntry) { - entries = append(entries, entry{k, le}) - }); err != nil { - return 0, fmt.Errorf("reading source store failed: %w", err) - } - - for _, e := range entries { - if err := dst.putEntry(ctx, e.key, e.le); err != nil { - return 0, fmt.Errorf("writing %q failed: %w", e.key, err) - } - } - return len(entries), nil -} diff --git a/pkg/links/sqlite_test.go b/pkg/links/sqlite_test.go index 25ac010..72719f9 100644 --- a/pkg/links/sqlite_test.go +++ b/pkg/links/sqlite_test.go @@ -182,74 +182,3 @@ func TestSQLitePersistsAcrossReopen(t *testing.T) { t.Errorf("URI after reopen = %q, want %q", got, want) } } - -func TestImportFromCopiesEntriesVerbatim(t *testing.T) { - ctx := context.Background() - src := NewMemStore() - - // A link whose stored RequiredPaths must be preserved rather than - // recomputed during the copy. - uris := map[string]string{ - "plain": "http://example.com/plain", - "subst": "http://example.com/{1}/{0}", - } - for k, uri := range uris { - if _, err := src.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { - t.Fatalf("seeding %s failed: %v", k, err) - } - } - - dst := newTestSQLiteStore(t) - n, err := ImportFrom(ctx, dst, src) - if err != nil { - t.Fatalf("ImportFrom failed: %v", err) - } - if n != len(uris) { - t.Errorf("ImportFrom copied %d entries, want %d", n, len(uris)) - } - - for k, wantURI := range uris { - le, err := dst.Get(ctx, k) - if err != nil { - t.Fatalf("Get(%s) failed: %v", k, err) - } - if le == nil { - t.Errorf("Get(%s) = nil, want the imported entry", k) - continue - } - if le.Link.GetUri() != wantURI { - t.Errorf("Get(%s) URI = %q, want %q", k, le.Link.GetUri(), wantURI) - } - srcLE, err := src.Get(ctx, k) - if err != nil { - t.Fatalf("src.Get(%s) failed: %v", k, err) - } - if le.RequiredPaths != srcLE.RequiredPaths { - t.Errorf("Get(%s) RequiredPaths = %d, want %d", k, le.RequiredPaths, srcLE.RequiredPaths) - } - } -} - -// Re-running a migration must be safe. -func TestImportFromIsIdempotent(t *testing.T) { - ctx := context.Background() - src := NewMemStore() - if _, err := src.Put(ctx, "key", &pb.Link{Uri: "http://example.com/one"}); err != nil { - t.Fatalf("seeding failed: %v", err) - } - - dst := newTestSQLiteStore(t) - for i := 0; i < 2; i++ { - if _, err := ImportFrom(ctx, dst, src); err != nil { - t.Fatalf("ImportFrom run %d failed: %v", i+1, err) - } - } - - count := 0 - if err := dst.Visit(ctx, func(string, *pb.LinkEntry) { count++ }); err != nil { - t.Fatalf("Visit failed: %v", err) - } - if count != 1 { - t.Errorf("after two imports the store holds %d entries, want 1", count) - } -}