Add a SQLite storage backend, with bulk import/export

README.md

@@ -18,6 +18,58 @@ Run `docker compose up` from the `dev/` directory to get a local instance runnin
 
 The server maintains a database of friendly names to URI redirect templates. For example, `rfc -> https://datatracker.ietf.org/doc/html/rfc{0}` will redirect `GET /rfc/5280` to `https://datatracker.ietf.org/doc/html/rfc5280`. Try it out: [jdtw.us/rfc/5280](https://jdtw.us/rfc/5280).
 
+## Storage
+
+The server picks its backing store from the environment, in this order:
+
+| Condition | Store |
+| --- | --- |
+| `--ephemeral` | in-memory, discarded on exit |
+| `SQLITE_PATH` set | SQLite database at that path |
+| otherwise | Postgres at `DATABASE_URL` |
+
+SQLite keeps the whole link table in a single file, which is enough for this
+workload and avoids paying for a managed Postgres instance. The tradeoff is
+that the file lives on one volume, so the app is pinned to a single machine
+in a single region and there is no replication. Postgres remains supported:
+unset `SQLITE_PATH` to switch back.
+
+The schema is applied automatically when the SQLite database is opened, so a
+freshly provisioned volume needs no manual setup.
+
+### Backup and restore
+
+The client can dump the whole link database to a file and load it back,
+which doubles as the migration path between storage backends:
+
+```
+$ client --export links-backup.json
+$ client --import links-backup.json
+```
+
+`--export` writes the same `links.Links` JSON proto that `GET /api/links`
+returns, indented for readability. `--import` posts it back. Both accept `-`
+for stdout/stdin. Importing is additive and idempotent, so re-running it is
+safe.
+
+### Migrating Postgres to SQLite
+
+No database access is needed -- export from the running server, point it at
+an empty SQLite file, and import:
+
+1. `client --export links-backup.json` against the Postgres-backed server.
+2. Restart with `SQLITE_PATH` set, which creates and initializes an empty
+   database file.
+3. `client --import links-backup.json`.
+
+Keep the backup, and keep Postgres around until you're satisfied; unsetting
+`SQLITE_PATH` reverts to it with the original data untouched.
+
+### Tests
+
+`./sqlite_test.sh` runs the full suite against SQLite and needs no database
+server. `./docker_test.sh` does the same against Postgres in a container.
+
 ## REST API
 
 * `GET /api/links` returns all links in the database.
@@ -28,6 +80,13 @@ The server maintains a database of friendly names to URI redirect templates. For
   * Request body: empty
   * Response body: `links.Link` JSON proto.
   * Returns: 200 (OK) or 404 (not found)
+* `POST /api/links` bulk creates or updates links.
+  * Request body: `links.Links` JSON proto, the same shape `GET /api/links` returns.
+  * Response body: empty
+  * Returns: 204 (no content), or 400 if any link is invalid.
+  * Additive: links already stored that the body does not mention are left
+    alone. Every entry is validated before anything is written, so one bad
+    link fails the whole request rather than half-applying the import.
 * `PUT /api/links/{link}` creates or updates a link.
   * Request body: `links.Link` JSON proto.
   * Response body: empty

cmd/client/client.go

@@ -3,14 +3,17 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
 	"sort"
 
+	"google.golang.org/protobuf/encoding/protojson"
 	"jdtw.dev/links/pkg/client"
 	"jdtw.dev/links/pkg/frontend"
 	"jdtw.dev/links/pkg/links"
+	pb "jdtw.dev/links/proto/links"
 	"jdtw.dev/token"
 )
 
@@ -23,6 +26,8 @@ var (
 	get    = flag.String("get", "", "Get a redirect")
 	rm     = flag.String("rm", "", "Remove a redirect")
 	server = flag.Int("server", -1, "If not -1, starts starts a frontent HTTP server on the given port.")
+	export = flag.String("export", "", "Write all links as a JSON Links proto to the given file, or '-' for stdout")
+	imprt  = flag.String("import", "", "Bulk create or update links from a JSON Links proto file, or '-' for stdin")
 )
 
 func main() {
@@ -77,6 +82,43 @@ func main() {
 		if err := c.Delete(*rm); err != nil {
 			log.Fatal(err)
 		}
+	case *export != "":
+		lpb, err := c.Export()
+		if err != nil {
+			log.Fatal(err)
+		}
+		// Indented so the backup is readable and reviewable. Note that
+		// protojson does not promise byte-stable output, so don't expect
+		// two exports of identical data to diff clean.
+		data, err := protojson.MarshalOptions{Multiline: true, Indent: "  "}.Marshal(lpb)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if *export == "-" {
+			os.Stdout.Write(data)
+		} else if err := os.WriteFile(*export, data, 0600); err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("exported %d links", len(lpb.GetLinks()))
+	case *imprt != "":
+		var data []byte
+		var err error
+		if *imprt == "-" {
+			data, err = io.ReadAll(os.Stdin)
+		} else {
+			data, err = os.ReadFile(*imprt)
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		lpb := &pb.Links{}
+		if err := protojson.Unmarshal(data, lpb); err != nil {
+			log.Fatalf("failed to parse %s: %v", *imprt, err)
+		}
+		if err := c.Import(lpb); err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("imported %d links", len(lpb.GetLinks()))
 	default:
 		l, err := c.List()
 		if err != nil {

cmd/links/links.go

@@ -46,18 +46,29 @@ func main() {
 	}
 	log.Printf("loaded keyset:\n%s", keyset)
 
+	// Storage precedence: -ephemeral wins, then SQLITE_PATH, then
+	// DATABASE_URL. Unsetting SQLITE_PATH reverts to Postgres.
 	var store links.Store
-	if *ephemeral {
+	ctx := context.Background()
+	sqlitePath := os.Getenv("SQLITE_PATH")
+	switch {
+	case *ephemeral:
 		log.Printf("Running in ephemeral mode!")
 		store = links.NewMemStore()
-	} else {
-		ctx := context.Background()
-		dbURL := os.Getenv("DATABASE_URL")
-		pgStore, err := links.NewPostgresStore(ctx, dbURL)
+	case sqlitePath != "":
+		sqliteStore, err := links.NewSQLiteStore(ctx, sqlitePath)
+		if err != nil {
+			log.Fatalf("links.NewSQLiteStore failed: %v", err)
+		}
+		log.Printf("Opened SQLite database at %s", sqlitePath)
+		store = sqliteStore
+		defer sqliteStore.Close()
+	default:
+		pgStore, err := links.NewPostgresStore(ctx, os.Getenv("DATABASE_URL"))
 		if err != nil {
 			log.Fatalf("links.NewPostgresStore failed: %v", err)
 		}
-		log.Printf("Connected to %s", dbURL)
+		log.Print("Connected to Postgres")
 		store = pgStore
 		defer pgStore.Close()
 	}

go.mod

@@ -8,12 +8,22 @@ require (
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	google.golang.org/protobuf v1.36.10
 	jdtw.dev/token v0.1.6
+	modernc.org/sqlite v1.55.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/text v0.40.0 // indirect
+	modernc.org/libc v1.74.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -1,10 +1,18 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
 github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -13,19 +21,32 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
 github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
 golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
 golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -34,3 +55,31 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 jdtw.dev/token v0.1.6 h1:EzvBOo0s+O4cudJqZob1ynhyIjtWfS85Oxfk/b0iqP4=
 jdtw.dev/token v0.1.6/go.mod h1:qr+zsFbOixxkv7T5Jb7rar/5Gs2yhw27vyNX0Q7pBA4=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
+modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM=
+modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

pkg/client/client.go

@@ -82,6 +82,38 @@ func (c *Client) List() (map[string]string, error) {
 	return l, nil
 }
 
+// Export returns every link as a Links proto, preserving the exact shape the
+// server stores. Unlike List, which flattens to a map of strings for display,
+// the result round-trips through Import.
+func (c *Client) Export() (*pb.Links, error) {
+	resp, err := c.do("GET", linksAPI, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	lpb := &pb.Links{}
+	if err := unmarshalBody(resp, lpb); err != nil {
+		return nil, err
+	}
+	return lpb, nil
+}
+
+// Import bulk-creates or updates every link in lpb. Links already on the
+// server that lpb does not mention are left alone.
+func (c *Client) Import(lpb *pb.Links) error {
+	body, err := marshal(lpb)
+	if err != nil {
+		return err
+	}
+	resp, err := c.do("POST", linksAPI, body)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
 func (c *Client) Get(link string) (string, error) {
 	resp, err := c.do("GET", api(link), nil)
 	if err != nil {