diff --git a/README.md b/README.md index e559919..dc61d1d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,58 @@ Run `docker compose up` from the `dev/` directory to get a local instance runnin The server maintains a database of friendly names to URI redirect templates. For example, `rfc -> https://datatracker.ietf.org/doc/html/rfc{0}` will redirect `GET /rfc/5280` to `https://datatracker.ietf.org/doc/html/rfc5280`. Try it out: [jdtw.us/rfc/5280](https://jdtw.us/rfc/5280). +## Storage + +The server picks its backing store from the environment, in this order: + +| Condition | Store | +| --- | --- | +| `--ephemeral` | in-memory, discarded on exit | +| `SQLITE_PATH` set | SQLite database at that path | +| otherwise | Postgres at `DATABASE_URL` | + +SQLite keeps the whole link table in a single file, which is enough for this +workload and avoids paying for a managed Postgres instance. The tradeoff is +that the file lives on one volume, so the app is pinned to a single machine +in a single region and there is no replication. Postgres remains supported: +unset `SQLITE_PATH` to switch back. + +The schema is applied automatically when the SQLite database is opened, so a +freshly provisioned volume needs no manual setup. + +### Backup and restore + +The client can dump the whole link database to a file and load it back, +which doubles as the migration path between storage backends: + +``` +$ client --export links-backup.json +$ client --import links-backup.json +``` + +`--export` writes the same `links.Links` JSON proto that `GET /api/links` +returns, indented for readability. `--import` posts it back. Both accept `-` +for stdout/stdin. Importing is additive and idempotent, so re-running it is +safe. + +### Migrating Postgres to SQLite + +No database access is needed -- export from the running server, point it at +an empty SQLite file, and import: + +1. `client --export links-backup.json` against the Postgres-backed server. +2. Restart with `SQLITE_PATH` set, which creates and initializes an empty + database file. +3. `client --import links-backup.json`. + +Keep the backup, and keep Postgres around until you're satisfied; unsetting +`SQLITE_PATH` reverts to it with the original data untouched. + +### Tests + +`./sqlite_test.sh` runs the full suite against SQLite and needs no database +server. `./docker_test.sh` does the same against Postgres in a container. + ## REST API * `GET /api/links` returns all links in the database. @@ -28,6 +80,13 @@ The server maintains a database of friendly names to URI redirect templates. For * Request body: empty * Response body: `links.Link` JSON proto. * Returns: 200 (OK) or 404 (not found) +* `POST /api/links` bulk creates or updates links. + * Request body: `links.Links` JSON proto, the same shape `GET /api/links` returns. + * Response body: empty + * Returns: 204 (no content), or 400 if any link is invalid. + * Additive: links already stored that the body does not mention are left + alone. Every entry is validated before anything is written, so one bad + link fails the whole request rather than half-applying the import. * `PUT /api/links/{link}` creates or updates a link. * Request body: `links.Link` JSON proto. * Response body: empty diff --git a/cmd/client/client.go b/cmd/client/client.go index fc4c26e..864bad3 100644 --- a/cmd/client/client.go +++ b/cmd/client/client.go @@ -3,14 +3,17 @@ package main import ( "flag" "fmt" + "io" "log" "net/http" "os" "sort" + "google.golang.org/protobuf/encoding/protojson" "jdtw.dev/links/pkg/client" "jdtw.dev/links/pkg/frontend" "jdtw.dev/links/pkg/links" + pb "jdtw.dev/links/proto/links" "jdtw.dev/token" ) @@ -23,6 +26,8 @@ var ( get = flag.String("get", "", "Get a redirect") rm = flag.String("rm", "", "Remove a redirect") server = flag.Int("server", -1, "If not -1, starts starts a frontent HTTP server on the given port.") + export = flag.String("export", "", "Write all links as a JSON Links proto to the given file, or '-' for stdout") + imprt = flag.String("import", "", "Bulk create or update links from a JSON Links proto file, or '-' for stdin") ) func main() { @@ -77,6 +82,43 @@ func main() { if err := c.Delete(*rm); err != nil { log.Fatal(err) } + case *export != "": + lpb, err := c.Export() + if err != nil { + log.Fatal(err) + } + // Indented so the backup is readable and reviewable. Note that + // protojson does not promise byte-stable output, so don't expect + // two exports of identical data to diff clean. + data, err := protojson.MarshalOptions{Multiline: true, Indent: " "}.Marshal(lpb) + if err != nil { + log.Fatal(err) + } + if *export == "-" { + os.Stdout.Write(data) + } else if err := os.WriteFile(*export, data, 0600); err != nil { + log.Fatal(err) + } + log.Printf("exported %d links", len(lpb.GetLinks())) + case *imprt != "": + var data []byte + var err error + if *imprt == "-" { + data, err = io.ReadAll(os.Stdin) + } else { + data, err = os.ReadFile(*imprt) + } + if err != nil { + log.Fatal(err) + } + lpb := &pb.Links{} + if err := protojson.Unmarshal(data, lpb); err != nil { + log.Fatalf("failed to parse %s: %v", *imprt, err) + } + if err := c.Import(lpb); err != nil { + log.Fatal(err) + } + log.Printf("imported %d links", len(lpb.GetLinks())) default: l, err := c.List() if err != nil { diff --git a/cmd/links/links.go b/cmd/links/links.go index 5a25432..1b71534 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -46,18 +46,29 @@ func main() { } log.Printf("loaded keyset:\n%s", keyset) + // Storage precedence: -ephemeral wins, then SQLITE_PATH, then + // DATABASE_URL. Unsetting SQLITE_PATH reverts to Postgres. var store links.Store - if *ephemeral { + ctx := context.Background() + sqlitePath := os.Getenv("SQLITE_PATH") + switch { + case *ephemeral: log.Printf("Running in ephemeral mode!") store = links.NewMemStore() - } else { - ctx := context.Background() - dbURL := os.Getenv("DATABASE_URL") - pgStore, err := links.NewPostgresStore(ctx, dbURL) + case sqlitePath != "": + sqliteStore, err := links.NewSQLiteStore(ctx, sqlitePath) + if err != nil { + log.Fatalf("links.NewSQLiteStore failed: %v", err) + } + log.Printf("Opened SQLite database at %s", sqlitePath) + store = sqliteStore + defer sqliteStore.Close() + default: + pgStore, err := links.NewPostgresStore(ctx, os.Getenv("DATABASE_URL")) if err != nil { log.Fatalf("links.NewPostgresStore failed: %v", err) } - log.Printf("Connected to %s", dbURL) + log.Print("Connected to Postgres") store = pgStore defer pgStore.Close() } diff --git a/go.mod b/go.mod index 125bebc..c52a879 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,22 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e google.golang.org/protobuf v1.36.10 jdtw.dev/token v0.1.6 + modernc.org/sqlite v1.55.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.40.0 // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 6c5b431..b319994 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -13,8 +21,14 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -22,10 +36,17 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -34,3 +55,31 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= jdtw.dev/token v0.1.6 h1:EzvBOo0s+O4cudJqZob1ynhyIjtWfS85Oxfk/b0iqP4= jdtw.dev/token v0.1.6/go.mod h1:qr+zsFbOixxkv7T5Jb7rar/5Gs2yhw27vyNX0Q7pBA4= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= +modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/pkg/client/client.go b/pkg/client/client.go index 3d1084a..641485c 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -82,6 +82,38 @@ func (c *Client) List() (map[string]string, error) { return l, nil } +// Export returns every link as a Links proto, preserving the exact shape the +// server stores. Unlike List, which flattens to a map of strings for display, +// the result round-trips through Import. +func (c *Client) Export() (*pb.Links, error) { + resp, err := c.do("GET", linksAPI, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + lpb := &pb.Links{} + if err := unmarshalBody(resp, lpb); err != nil { + return nil, err + } + return lpb, nil +} + +// Import bulk-creates or updates every link in lpb. Links already on the +// server that lpb does not mention are left alone. +func (c *Client) Import(lpb *pb.Links) error { + body, err := marshal(lpb) + if err != nil { + return err + } + resp, err := c.do("POST", linksAPI, body) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + func (c *Client) Get(link string) (string, error) { resp, err := c.do("GET", api(link), nil) if err != nil { diff --git a/pkg/links/api.go b/pkg/links/api.go index 3566f8b..49fe33c 100644 --- a/pkg/links/api.go +++ b/pkg/links/api.go @@ -1,10 +1,14 @@ package links import ( + "errors" + "fmt" "io" "log" "net/http" "net/url" + "sort" + "strings" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -56,15 +60,35 @@ func (s *server) get() http.HandlerFunc { } } +// validateLink reports whether a normalized key and link are acceptable to +// store, describing the problem if they are not. Shared by put() and +// bulkPut() so a bulk import enforces exactly the same rules as a single +// write. +func validateLink(key string, l *pb.Link) error { + if key == qrKey { + return fmt.Errorf("%q is a reserved link name", qrKey) + } + if l.GetUri() == "" { + return errors.New("missing URI") + } + // Create a dummy URI with all template parameters replaced + // with something innocuous so that we can try to parse it. + dummy := replacement.ReplaceAllString(l.GetUri(), "links") + url, err := url.Parse(dummy) + if err != nil { + return fmt.Errorf("URI %q failed to parse: %v", l.GetUri(), err) + } + if url.Scheme == "" { + return fmt.Errorf("URI %q has no scheme", l.GetUri()) + } + return nil +} + func (s *server) put() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { rid := middleware.GetReqID(r.Context()) l := normalizeKey(chi.URLParam(r, "link")) - if l == qrKey { - badRequest(w, "%q is a reserved link name", qrKey) - return - } data, err := io.ReadAll(r.Body) if err != nil { internalError(w, err, rid) @@ -75,20 +99,8 @@ func (s *server) put() http.HandlerFunc { badRequest(w, "failed to unmarshal body: %v", err) return } - if lpb.Uri == "" { - badRequest(w, "missing URI") - return - } - // Create a dummy URI with all template parameters replaced - // with something innocuous so that we can try to parse it. - dummy := replacement.ReplaceAllString(lpb.Uri, "links") - url, err := url.Parse(dummy) - if err != nil { - badRequest(w, "URI %q failed to parse: %v", lpb.Uri, err) - return - } - if url.Scheme == "" { - badRequest(w, "URI %q has no scheme", lpb.Uri) + if err := validateLink(l, lpb); err != nil { + badRequest(w, "%v", err) return } created, err := s.store.Put(r.Context(), l, lpb) @@ -108,6 +120,77 @@ func (s *server) put() http.HandlerFunc { } } +// bulkPut creates or updates every link in the request body, which is a +// Links proto of the same shape that list() returns. Links already in the +// store that the body does not mention are left alone, so an import is +// additive rather than a replacement. +// +// Every entry is validated before anything is written: one malformed link +// fails the whole request rather than leaving a half-applied import. +func (s *server) bulkPut() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + rid := middleware.GetReqID(r.Context()) + + data, err := io.ReadAll(r.Body) + if err != nil { + internalError(w, err, rid) + return + } + lpb := new(pb.Links) + if err := protojson.Unmarshal(data, lpb); err != nil { + badRequest(w, "failed to unmarshal body: %v", err) + return + } + if len(lpb.GetLinks()) == 0 { + badRequest(w, "no links in request body") + return + } + + // Normalize and validate everything before the first write. Keys + // that collide only after normalization ("my-link" and "mylink") + // would silently overwrite each other, so reject those too. + normalized := make(map[string]*pb.Link, len(lpb.GetLinks())) + sources := make(map[string]string, len(lpb.GetLinks())) + var problems []string + for k, l := range lpb.GetLinks() { + key := normalizeKey(k) + if err := validateLink(key, l); err != nil { + problems = append(problems, fmt.Sprintf("%q: %v", k, err)) + continue + } + if prev, dup := sources[key]; dup { + problems = append(problems, fmt.Sprintf("%q: collides with %q after normalization", k, prev)) + continue + } + sources[key] = k + normalized[key] = l + } + if len(problems) > 0 { + sort.Strings(problems) + badRequest(w, "rejected %d of %d links:\n%s", len(problems), len(lpb.GetLinks()), strings.Join(problems, "\n")) + return + } + + var created, updated int + for k, l := range normalized { + wasCreated, err := s.store.Put(r.Context(), k, l) + if err != nil { + internalError(w, err, rid) + return + } + if wasCreated { + created++ + } else { + updated++ + } + } + + w.WriteHeader(http.StatusNoContent) + log.Printf("[%s] %s imported %d links (%d created, %d updated)", + rid, subject(r.Context()), len(normalized), created, updated) + } +} + func (s *server) delete() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { rid := middleware.GetReqID(r.Context()) diff --git a/pkg/links/bulk_test.go b/pkg/links/bulk_test.go new file mode 100644 index 0000000..d34ecfb --- /dev/null +++ b/pkg/links/bulk_test.go @@ -0,0 +1,258 @@ +package links + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "jdtw.dev/links/pkg/tokentest" + pb "jdtw.dev/links/proto/links" + "jdtw.dev/token" +) + +// postLinks bulk-imports the given path -> URI pairs and returns the response. +func postLinks(t *testing.T, srv http.Handler, priv *token.SigningKey, links map[string]string) *http.Response { + t.Helper() + lpb := &pb.Links{Links: make(map[string]*pb.Link, len(links))} + for k, uri := range links { + lpb.Links[k] = &pb.Link{Uri: uri} + } + return postBody(t, srv, priv, marshal(t, lpb)) +} + +func postBody(t *testing.T, srv http.Handler, priv *token.SigningKey, body io.Reader) *http.Response { + t.Helper() + rr := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/api/links", body) + signRequest(t, priv, req) + srv.ServeHTTP(rr, req) + return rr.Result() +} + +func TestBulkPutImportsLinks(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + want := map[string]string{ + "rfc": "https://datatracker.ietf.org/doc/html/rfc{0}", + "gh": "https://github.com/jdtw/{0}", + "plain": "https://example.com/plain", + Index: "https://example.com", + } + if sc := postLinks(t, srv, priv, want).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + for k, wantURI := range want { + le, err := store.Get(context.Background(), k) + if err != nil { + t.Fatalf("Get(%s) failed: %v", k, err) + } + if le == nil { + t.Errorf("Get(%s) = nil, want the imported link", k) + continue + } + if le.Link.GetUri() != wantURI { + t.Errorf("Get(%s) = %q, want %q", k, le.Link.GetUri(), wantURI) + } + } +} + +// A bulk import must compute RequiredPaths the same way a single PUT does, +// or {n} substitution silently breaks on imported links. +func TestBulkPutComputesRequiredPaths(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + const uri = "https://example.com/{1}/{0}" + if sc := postLinks(t, srv, priv, map[string]string{"swap": uri}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + le, err := store.Get(context.Background(), "swap") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.RequiredPaths, requiredPaths(&pb.Link{Uri: uri}); got != want { + t.Errorf("RequiredPaths = %d, want %d", got, want) + } + if le.RequiredPaths != 2 { + t.Errorf("RequiredPaths = %d, want 2", le.RequiredPaths) + } +} + +func TestBulkPutIsAdditive(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + ctx := context.Background() + + if _, err := store.Put(ctx, "existing", &pb.Link{Uri: "https://example.com/existing"}); err != nil { + t.Fatalf("seeding failed: %v", err) + } + + if sc := postLinks(t, srv, priv, map[string]string{"new": "https://example.com/new"}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + // The link the import did not mention must survive. + le, err := store.Get(ctx, "existing") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le == nil { + t.Error("import removed a link it did not mention; want additive behavior") + } +} + +func TestBulkPutOverwritesExisting(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + ctx := context.Background() + + if _, err := store.Put(ctx, "dup", &pb.Link{Uri: "https://example.com/old"}); err != nil { + t.Fatalf("seeding failed: %v", err) + } + if sc := postLinks(t, srv, priv, map[string]string{"dup": "https://example.com/new"}).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + le, err := store.Get(ctx, "dup") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.Link.GetUri(), "https://example.com/new"; got != want { + t.Errorf("URI = %q, want %q", got, want) + } +} + +func TestBulkPutRejectsInvalidRequests(t *testing.T) { + tests := []struct { + name string + body io.Reader + }{ + {"nil body", nil}, + {"not a proto", strings.NewReader("not-a-proto")}, + {"empty links", bytes.NewReader([]byte(`{"links":{}}`))}, + {"missing uri", bytes.NewReader([]byte(`{"links":{"foo":{"uri":""}}}`))}, + {"no scheme", bytes.NewReader([]byte(`{"links":{"foo":{"uri":"no-scheme"}}}`))}, + {"reserved qr key", bytes.NewReader([]byte(`{"links":{"qr":{"uri":"https://example.com"}}}`))}, + {"normalization collision", bytes.NewReader([]byte(`{"links":{"my-link":{"uri":"https://example.com/a"},"mylink":{"uri":"https://example.com/b"}}}`))}, + } + + keyset, priv := tokentest.GenerateKey(t, "test") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := NewHandler(NewMemStore(), keyset, 0) + if sc := postBody(t, srv, priv, tc.body).StatusCode; sc != http.StatusBadRequest { + t.Errorf("POST returned %d, want 400", sc) + } + }) + } +} + +// One bad link must not leave a partially applied import behind. +func TestBulkPutIsAtomicOnValidationFailure(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + store := NewMemStore() + srv := NewHandler(store, keyset, 0) + + body := bytes.NewReader([]byte(`{"links":{ + "good":{"uri":"https://example.com/good"}, + "bad":{"uri":"no-scheme"} + }}`)) + if sc := postBody(t, srv, priv, body).StatusCode; sc != http.StatusBadRequest { + t.Fatalf("POST returned %d, want 400", sc) + } + + le, err := store.Get(context.Background(), "good") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le != nil { + t.Error("a rejected import wrote the valid link anyway; want nothing written") + } +} + +func TestBulkPutRequiresAuth(t *testing.T) { + _, priv := tokentest.GenerateKey(t, "test") + srv := NewHandler(NewMemStore(), nil, 0) + if sc := postLinks(t, srv, priv, map[string]string{"foo": "https://example.com"}).StatusCode; sc != http.StatusUnauthorized { + t.Errorf("POST with nil keyset returned %d, want 401", sc) + } +} + +// Export then import must reproduce the original set exactly -- this is the +// property the Postgres -> SQLite migration relies on. +func TestExportImportRoundTrip(t *testing.T) { + keyset, priv := tokentest.GenerateKey(t, "test") + ctx := context.Background() + + source := NewMemStore() + want := map[string]string{ + "rfc": "https://datatracker.ietf.org/doc/html/rfc{0}", + "swap": "https://example.com/{1}/{0}", + "plain": "https://example.com/plain", + Index: "https://example.com", + } + for k, uri := range want { + if _, err := source.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { + t.Fatalf("seeding %s failed: %v", k, err) + } + } + + // Export from the source server. + exportSrv := NewHandler(source, keyset, 0) + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/links", nil) + signRequest(t, priv, req) + exportSrv.ServeHTTP(rr, req) + if sc := rr.Result().StatusCode; sc != http.StatusOK { + t.Fatalf("GET /api/links returned %d, want 200", sc) + } + exported, err := io.ReadAll(rr.Result().Body) + if err != nil { + t.Fatalf("reading export failed: %v", err) + } + + // Import into a fresh, empty store -- the SQLite side of the migration. + dest := NewMemStore() + destSrv := NewHandler(dest, keyset, 0) + if sc := postBody(t, destSrv, priv, bytes.NewReader(exported)).StatusCode; sc != http.StatusNoContent { + t.Fatalf("POST returned %d, want 204", sc) + } + + got := map[string]string{} + if err := dest.Visit(ctx, func(k string, le *pb.LinkEntry) { + got[k] = le.Link.GetUri() + }); err != nil { + t.Fatalf("Visit failed: %v", err) + } + + if len(got) != len(want) { + t.Errorf("round trip produced %d links, want %d", len(got), len(want)) + } + for k, wantURI := range want { + if got[k] != wantURI { + t.Errorf("round trip [%s] = %q, want %q", k, got[k], wantURI) + } + srcLE, err := source.Get(ctx, k) + if err != nil { + t.Fatalf("source.Get(%s) failed: %v", k, err) + } + dstLE, err := dest.Get(ctx, k) + if err != nil { + t.Fatalf("dest.Get(%s) failed: %v", k, err) + } + if srcLE.RequiredPaths != dstLE.RequiredPaths { + t.Errorf("round trip [%s] RequiredPaths = %d, want %d", k, dstLE.RequiredPaths, srcLE.RequiredPaths) + } + } +} diff --git a/pkg/links/server.go b/pkg/links/server.go index 0aa17be..d03dc4f 100644 --- a/pkg/links/server.go +++ b/pkg/links/server.go @@ -28,6 +28,8 @@ func (s *server) routes() { r.Use(s.authenticated()) // Get all links as a Links proto. r.Get("/links", s.list()) + // Bulk create or update from a Links proto. + r.Post("/links", s.bulkPut()) // Get a speficic link. r.Get("/links/{link}", s.get()) // Create or update a link. diff --git a/pkg/links/sqlite.go b/pkg/links/sqlite.go new file mode 100644 index 0000000..b163585 --- /dev/null +++ b/pkg/links/sqlite.go @@ -0,0 +1,139 @@ +package links + +import ( + "context" + "database/sql" + "errors" + "fmt" + + pb "jdtw.dev/links/proto/links" + _ "modernc.org/sqlite" +) + +const ( + // sqliteSchema is applied on open so that a fresh database file (for + // example, a newly provisioned volume) is usable without any manual + // setup. It mirrors links.sql. + sqliteSchema = `create table if not exists links ( + path text primary key, + link text not null, + segments integer not null +)` + + sqliteGet = "select link, segments from links where path=?" + sqliteExists = "select 1 from links where path=?" + sqlitePut = `insert into links (path, link, segments) values (?, ?, ?) + on conflict (path) do update set link=excluded.link, segments=excluded.segments` + sqliteDel = "delete from links where path=?" + sqliteList = "select path, link, segments from links" +) + +// SQLiteStore is a Store backed by a local SQLite database file. It is the +// low-cost alternative to PostgresStore: the link table is small enough that +// a file on a mounted volume serves it fine, at the cost of pinning the app +// to a single machine. +type SQLiteStore struct { + db *sql.DB +} + +var _ Store = &SQLiteStore{} + +func (s *SQLiteStore) Close() error { + if s.db == nil { + return nil + } + return s.db.Close() +} + +// NewSQLiteStore opens (creating if necessary) the SQLite database at path +// and applies the schema. WAL mode keeps redirect reads from blocking on the +// occasional write, and busy_timeout absorbs the brief contention that WAL +// still allows between concurrent writers. +func NewSQLiteStore(ctx context.Context, path string) (*SQLiteStore, error) { + dsn := fmt.Sprintf("file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)", path) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("sql.Open failed: %w", err) + } + + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("db.Ping failed: %w", err) + } + + if _, err := db.ExecContext(ctx, sqliteSchema); err != nil { + db.Close() + return nil, fmt.Errorf("applying schema failed: %w", err) + } + + return &SQLiteStore{db: db}, nil +} + +func (s *SQLiteStore) Get(ctx context.Context, key string) (*pb.LinkEntry, error) { + var link string + var segments int + if err := s.db.QueryRowContext(ctx, sqliteGet, key).Scan(&link, &segments); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, err + } + return &pb.LinkEntry{ + Link: &pb.Link{Uri: link}, + RequiredPaths: int32(segments), + }, nil +} + +// Put upserts the link and reports whether it was created rather than +// updated. SQLite has no equivalent of Postgres' xmax trick, so the existence +// check and the write share a transaction to keep the answer accurate under +// concurrent writers. +func (s *SQLiteStore) Put(ctx context.Context, key string, l *pb.Link) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer tx.Rollback() + + var exists int + err = tx.QueryRowContext(ctx, sqliteExists, key).Scan(&exists) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, err + } + created := errors.Is(err, sql.ErrNoRows) + + if _, err := tx.ExecContext(ctx, sqlitePut, key, l.Uri, requiredPaths(l)); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + return created, nil +} + +func (s *SQLiteStore) Delete(ctx context.Context, key string) error { + _, err := s.db.ExecContext(ctx, sqliteDel, key) + return err +} + +func (s *SQLiteStore) Visit(ctx context.Context, visit func(string, *pb.LinkEntry)) error { + rows, err := s.db.QueryContext(ctx, sqliteList) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var path string + var link string + var segments int + if err := rows.Scan(&path, &link, &segments); err != nil { + return err + } + visit(path, &pb.LinkEntry{ + Link: &pb.Link{Uri: link}, + RequiredPaths: int32(segments), + }) + } + return rows.Err() +} diff --git a/pkg/links/sqlite_test.go b/pkg/links/sqlite_test.go new file mode 100644 index 0000000..72719f9 --- /dev/null +++ b/pkg/links/sqlite_test.go @@ -0,0 +1,184 @@ +package links + +import ( + "context" + "path/filepath" + "testing" + + pb "jdtw.dev/links/proto/links" +) + +// newTestSQLiteStore opens a store backed by a file in the test's temp +// directory. Unlike the Postgres tests these need no external service, so +// they always run. +func newTestSQLiteStore(t *testing.T) *SQLiteStore { + t.Helper() + s, err := NewSQLiteStore(context.Background(), filepath.Join(t.TempDir(), "links.db")) + if err != nil { + t.Fatalf("NewSQLiteStore failed: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func TestSQLitePutReportsCreatedVsUpdated(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "createdvsupdated" + + created, err := s.Put(ctx, key, &pb.Link{Uri: "http://example.com/first"}) + if err != nil { + t.Fatalf("Put (insert) failed: %v", err) + } + if !created { + t.Errorf("Put (insert) reported created=false, want true") + } + + created, err = s.Put(ctx, key, &pb.Link{Uri: "http://example.com/second"}) + if err != nil { + t.Fatalf("Put (update) failed: %v", err) + } + if created { + t.Errorf("Put (update) reported created=true, want false") + } + + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.Link.GetUri(), "http://example.com/second"; got != want { + t.Errorf("Get(%s) URI = %q, want %q", key, got, want) + } +} + +func TestSQLiteGetMissingKeyReturnsNil(t *testing.T) { + s := newTestSQLiteStore(t) + + le, err := s.Get(context.Background(), "doesnotexist") + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if le != nil { + t.Errorf("Get(missing) = %v, want nil", le) + } +} + +func TestSQLiteDelete(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "delete" + + if _, err := s.Put(ctx, key, &pb.Link{Uri: "http://example.com"}); err != nil { + t.Fatalf("Put failed: %v", err) + } + if err := s.Delete(ctx, key); err != nil { + t.Fatalf("Delete failed: %v", err) + } + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get after delete failed: %v", err) + } + if le != nil { + t.Errorf("Get after delete = %v, want nil", le) + } +} + +// Deleting a key that was never present should be a no-op, matching the +// Postgres store's behavior. +func TestSQLiteDeleteMissingKeyIsNoOp(t *testing.T) { + s := newTestSQLiteStore(t) + if err := s.Delete(context.Background(), "neverexisted"); err != nil { + t.Errorf("Delete(missing) failed: %v", err) + } +} + +func TestSQLiteVisit(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + + want := map[string]string{ + "one": "http://example.com/one", + "two": "http://example.com/two", + "three": "http://example.com/three", + } + for k, uri := range want { + if _, err := s.Put(ctx, k, &pb.Link{Uri: uri}); err != nil { + t.Fatalf("Put(%s) failed: %v", k, err) + } + } + + got := map[string]string{} + if err := s.Visit(ctx, func(k string, le *pb.LinkEntry) { + got[k] = le.Link.GetUri() + }); err != nil { + t.Fatalf("Visit failed: %v", err) + } + + if len(got) != len(want) { + t.Errorf("Visit saw %d entries, want %d", len(got), len(want)) + } + for k, wantURI := range want { + if got[k] != wantURI { + t.Errorf("Visit(%s) URI = %q, want %q", k, got[k], wantURI) + } + } +} + +// Put must persist the computed RequiredPaths so that {n} substitution keeps +// working after a restart. +func TestSQLitePutPersistsRequiredPaths(t *testing.T) { + s := newTestSQLiteStore(t) + ctx := context.Background() + const key = "subst" + + l := &pb.Link{Uri: "http://example.com/{1}/{0}"} + if _, err := s.Put(ctx, key, l); err != nil { + t.Fatalf("Put failed: %v", err) + } + + le, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got, want := le.RequiredPaths, requiredPaths(l); got != want { + t.Errorf("RequiredPaths = %d, want %d", got, want) + } + if le.RequiredPaths != 2 { + t.Errorf("RequiredPaths = %d, want 2 for %q", le.RequiredPaths, l.Uri) + } +} + +// The database file must survive being closed and reopened -- this is the +// whole point of putting it on a volume. +func TestSQLitePersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "links.db") + + first, err := NewSQLiteStore(ctx, path) + if err != nil { + t.Fatalf("NewSQLiteStore failed: %v", err) + } + if _, err := first.Put(ctx, "persisted", &pb.Link{Uri: "http://example.com/persisted"}); err != nil { + t.Fatalf("Put failed: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + second, err := NewSQLiteStore(ctx, path) + if err != nil { + t.Fatalf("reopening store failed: %v", err) + } + t.Cleanup(func() { second.Close() }) + + le, err := second.Get(ctx, "persisted") + if err != nil { + t.Fatalf("Get after reopen failed: %v", err) + } + if le == nil { + t.Fatal("Get after reopen = nil, want the entry written before close") + } + if got, want := le.Link.GetUri(), "http://example.com/persisted"; got != want { + t.Errorf("URI after reopen = %q, want %q", got, want) + } +} diff --git a/sqlite_test.sh b/sqlite_test.sh new file mode 100755 index 0000000..44619bb --- /dev/null +++ b/sqlite_test.sh @@ -0,0 +1,20 @@ +#! /bin/bash +# Runs the full integration suite against the SQLite store. Unlike +# docker_test.sh and local_test.sh this needs no database server: the store is +# a file in a scratch directory that is removed on exit. +set -euxo pipefail + +SQLITE_DIR="$(mktemp -d)" +export SQLITE_PATH="${SQLITE_DIR}/links.db" + +cleanup() { + exit_status=$? + echo "Cleaning up ${SQLITE_DIR}..." + rm -rf "${SQLITE_DIR}" + exit "${exit_status}" +} +trap cleanup EXIT + +go test ./... + +./test.sh