From b63f4f531265d695d9d9cf9d2efbede36a25057d Mon Sep 17 00:00:00 2001 From: Amen <2668333+amenophis1er@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:31:27 -0500 Subject: [PATCH] feat: restore keeps document ids (--preserve-ids, on by default for an empty target) Restore POSTs each document with its backup id when the target has none of the restored (non-singleton) documents, or with --clean, so links and systems that name a document by id survive a restore. --preserve-ids forces it into a populated target (overwriting same-id documents); --preserve-ids=false opts out. An unlistable or unreadable target is never assumed empty. A refused write now reports the server's reason instead of always naming content:write. --- README.md | 4 + cmd/restore.go | 85 ++++++++++++++++++++- cmd/restore_test.go | 175 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 257 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 19c5694..3a3c948 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,10 @@ trokky restore --input backup.zip --dry-run # Restore with options trokky restore --input backup.zip --clean --overwrite trokky restore --input backup.zip --collections posts --with-dependencies + +# Keep the backup's document ids in an instance that already has content +# (the default when the target is empty or with --clean; overwrites same-id documents) +trokky restore --input backup.zip --preserve-ids ``` ## Project Scaffolding diff --git a/cmd/restore.go b/cmd/restore.go index e2c0ecd..02775f3 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -33,6 +33,14 @@ Documents are restored first, media is uploaded second, and references are rewritten last. If the token cannot write documents the restore aborts before anything is uploaded, so a half-restored instance is never left behind. +Document ids are kept from the backup when the target has none of the restored +documents (or with --clean), so links and other systems that name a document by +id keep working. Into an instance that already has documents, new ids are minted +and references between documents are rewritten; --preserve-ids keeps the backup's +ids there too, overwriting any document with the same id (on a server older than +Trokky 3.6, an overwritten document's slug may come back with a numeric suffix). +Media files always get new ids, and references to them are rewritten. + Example: trokky restore --input backup.zip trokky restore --input backup.zip --collections posts,pages @@ -54,6 +62,8 @@ Example: dryRun, _ := cmd.Flags().GetBool("dry-run") clean, _ := cmd.Flags().GetBool("clean") overwrite, _ := cmd.Flags().GetBool("overwrite") + preserveIDs, _ := cmd.Flags().GetBool("preserve-ids") + preserveIDsSet := cmd.Flags().Changed("preserve-ids") // Open zip zr, err := zip.OpenReader(input) @@ -259,6 +269,38 @@ Example: fmt.Fprintf(out, " [DRY RUN] Would restore %d media file(s)\n", len(manifest.MediaIndex)) } + // Document ids. Kept as they are in the backup, anything that refers to a document by id + // from outside the CMS (a link, a redirect map, another system) still works after the + // restore. That is the default when the target has none of these documents yet; into a + // populated instance it would overwrite whatever already has the same id, so there it + // takes --preserve-ids. + // Singletons are left out: they are always written under their own id, and the Studio + // auto-creates them on first visit, so one existing says nothing about the rest. + if !preserveIDsSet { + var listCollections []string + for _, name := range restoreOrder { + backupSchema, inBackup := schemaMap[name] + targetSchema, inTarget := targetSchemaMap[name] + if (inTarget && targetSchema.Singleton) || (inBackup && backupSchema.Singleton) { + continue + } + listCollections = append(listCollections, name) + } + preserveIDs = clean || targetIsEmpty(c, listCollections) + } + switch { + case preserveIDs && preserveIDsSet: + fmt.Fprintln(out, "Document ids: kept from the backup (--preserve-ids); a document with the same id is overwritten") + case preserveIDs && clean: + fmt.Fprintln(out, "Document ids: kept from the backup (--clean empties the target first)") + case preserveIDs: + fmt.Fprintln(out, "Document ids: kept from the backup (the target has none of these documents)") + case preserveIDsSet: + fmt.Fprintln(out, "Document ids: new ids (--preserve-ids=false)") + default: + fmt.Fprintln(out, "Document ids: new ids, because the target already has documents. Pass --preserve-ids to keep the backup's ids (overwrites documents with the same id)") + } + // Document ID mappings (old ID -> new ID), used to rewrite document-to-document // references as later collections are restored. Media mappings are deliberately // kept separate: they only exist after phase 2. @@ -339,8 +381,9 @@ Example: // Rewrite document-to-document references using the mappings built // so far. Media references are left exactly as the backup has them; // phase 3 fixes those once the new media IDs exist. + // With kept ids every mapping is to itself: there is nothing to rewrite docRefs := 0 - if len(idMappings) > 0 { + if len(idMappings) > 0 && !preserveIDs { if hasSchema && len(schema.Fields) > 0 { var refCount int cleanDoc, refCount = backup.UpdateReferences(cleanDoc, schema, idMappings) @@ -353,8 +396,13 @@ Example: // Sanitize cleanDoc = backup.SanitizeDocument(cleanDoc) - // Create/update document - docJSON, err := json.Marshal(map[string]interface{}{"data": cleanDoc}) + // Create/update document. With preserved ids a create names its id, which the + // server keeps (and which overwrites a document that already has it). + payload := map[string]interface{}{"data": cleanDoc} + if preserveIDs && !isSingleton && originalID != "" { + payload["id"] = originalID + } + docJSON, err := json.Marshal(payload) if err != nil { fmt.Fprintf(errOut, "\n Warning: failed to marshal doc: %v\n", err) failed++ @@ -394,7 +442,9 @@ Example: cause = writeErr } fmt.Fprintln(out) - return fmt.Errorf("token lacks content:write (required to restore documents): %w", cause) + // The server's message names what is missing: content:write, or the publish + // permission when a write changes what is published + return fmt.Errorf("the token is not allowed to restore these documents: %w", cause) } fmt.Fprintf(errOut, "\n Warning: failed to create/update doc %s: %v\n", originalID, writeErr) failed++ @@ -744,9 +794,36 @@ func init() { restoreCmd.Flags().Bool("clean", false, "delete existing content before restore") restoreCmd.Flags().Bool("overwrite", false, "overwrite existing documents on conflict") restoreCmd.Flags().Bool("dry-run", false, "preview changes without applying") + restoreCmd.Flags().Bool("preserve-ids", false, "keep the backup's document ids (default: on when the target has none of the documents, or with --clean)") rootCmd.AddCommand(restoreCmd) } +// targetIsEmpty reports whether none of the collections hold a document yet. A collection that +// cannot be listed, or answers in a shape this cannot read, counts as not empty: keeping ids +// overwrites documents with the same id, so it is only the default when known to be safe. +func targetIsEmpty(c *client.Client, collections []string) bool { + for _, name := range collections { + data, err := c.Get("/collections/" + name + "?limit=1") + if err != nil { + return false + } + var list []json.RawMessage + if json.Unmarshal(data, &list) == nil { + if len(list) > 0 { + return false + } + continue + } + var page struct { + Documents *[]json.RawMessage `json:"documents"` + } + if json.Unmarshal(data, &page) != nil || page.Documents == nil || len(*page.Documents) > 0 { + return false + } + } + return true +} + // findMediaEntry resolves the zip entry holding a media item's bytes. It prefers // the path the manifest recorded, then the id-scoped layout, and finally the flat // legacy layout. The second return value reports whether the legacy layout was diff --git a/cmd/restore_test.go b/cmd/restore_test.go index 870dbc2..a191ed8 100644 --- a/cmd/restore_test.go +++ b/cmd/restore_test.go @@ -74,6 +74,13 @@ const ( // one document that references one media file, and that media file. func writeBackupZip(t *testing.T) string { t.Helper() + return writeBackupZipWith(t, false) +} + +// writeBackupZipWith builds the single-post backup, optionally with a "settings" singleton +// collection holding one document, restored before posts. +func writeBackupZipWith(t *testing.T, withSettings bool) string { + t.Helper() manifest := map[string]interface{}{ "version": "2.0", @@ -97,6 +104,15 @@ func writeBackupZip(t *testing.T) string { }, "statistics": map[string]interface{}{"totalDocuments": 1, "totalMedia": 1}, } + if withSettings { + manifest["schemas"] = append(manifest["schemas"].([]map[string]interface{}), map[string]interface{}{ + "name": "settings", + "title": "Settings", + "singleton": true, + "fields": []map[string]interface{}{{"name": "title", "type": "string"}}, + }) + manifest["restoreOrder"] = []string{"settings", "posts"} + } manifestJSON, err := json.Marshal(manifest) if err != nil { t.Fatal(err) @@ -125,6 +141,12 @@ func writeBackupZip(t *testing.T) string { {"collections/posts/doc-old-1.json", docJSON}, {"media/photo.png", []byte("png")}, } + if withSettings { + entries = append(entries, struct { + name string + data []byte + }{"collections/settings/settings.json", []byte(`{"id":"settings","title":"Site"}`)}) + } for _, e := range entries { w, err := zw.Create(e.name) if err != nil { @@ -168,6 +190,13 @@ func runRestore(t *testing.T, args ...string) (string, string, error) { } } + // --preserve-ids defaults by whether it was given at all, so it must also look unset + preserve := restoreCmd.Flags().Lookup("preserve-ids") + if err := preserve.Value.Set("false"); err != nil { + t.Fatal(err) + } + preserve.Changed = false + var stdout, stderr bytes.Buffer rootCmd.SetOut(&stdout) rootCmd.SetErr(&stderr) @@ -242,11 +271,11 @@ func TestRestoreAbortsBeforeUploadingMediaWhenDocumentWritesAreForbidden(t *test stdout, _, err := runRestore(t, "--input", writeBackupZip(t), "--url", server.URL+"/api", "--token", "read-only", "-q") - want := "token lacks content:write (required to restore documents): " + want := "the token is not allowed to restore these documents: " if err == nil { t.Error("expected an error for a token that cannot write documents") - } else if !strings.HasPrefix(err.Error(), want) { - t.Errorf("error = %q, want it to start with %q", err.Error(), want) + } else if !strings.HasPrefix(err.Error(), want) || !strings.Contains(err.Error(), "content:write scope required") { + t.Errorf("error = %q, want it to start with %q and carry the server's reason", err.Error(), want) } if n := rec.countPath("/media/upload"); n != 0 { t.Errorf("%d request(s) hit /media/upload; the restore must abort before uploading anything", n) @@ -650,3 +679,143 @@ func TestRestoreCleanStopsWhenMediaCannotBeDeleted(t *testing.T) { t.Errorf("listed media %d times, want 1", n) } } + +// newIDServer answers a restore of the single-post backup, listing posts as `posts` +// (a raw response body for GET /collections/posts, or "" for a 404). +func newIDServer(rec *recorder, posts string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.record(r) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && apiPath(r) == "/collections": + io.WriteString(w, testCollectionsResponse) + case r.Method == http.MethodGet && apiPath(r) == "/collections/posts": + if posts == "" { + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"success":false,"error":{"message":"not found"}}`) + return + } + io.WriteString(w, posts) + case r.Method == http.MethodPost && apiPath(r) == "/collections/posts": + // The server keeps an id the create names, and mints one otherwise + var body struct { + ID string `json:"id"` + } + json.Unmarshal([]byte(rec.all()[len(rec.all())-1].Body), &body) + id := body.ID + if id == "" { + id = testNewDocID + } + io.WriteString(w, `{"success":true,"data":{"document":{"id":"`+id+`"}}}`) + case r.Method == http.MethodPost && apiPath(r) == "/media/upload": + io.WriteString(w, `{"success":true,"data":{"files":[{"id":"`+testNewMediaID+`"}]}}`) + default: + io.WriteString(w, `{"success":true,"data":{}}`) + } + })) +} + +const ( + emptyPostsPage = `{"success":true,"data":{"documents":[],"pagination":{"total":0}}}` + populatedPostsPage = `{"success":true,"data":{"documents":[{"id":"someone-else"}],"pagination":{"total":1}}}` +) + +func createdPostBody(t *testing.T, rec *recorder) map[string]interface{} { + t.Helper() + for _, req := range rec.all() { + if req.Method == http.MethodPost && req.Path == "/collections/posts" { + var body map[string]interface{} + if err := json.Unmarshal([]byte(req.Body), &body); err != nil { + t.Fatalf("create body is not JSON: %s", req.Body) + } + return body + } + } + t.Fatal("no document was created") + return nil +} + +func TestRestorePreservesDocumentIDs(t *testing.T) { + cases := []struct { + name string + posts string + args []string + wantID bool + wantNotice string + }{ + {"empty target keeps ids by default", emptyPostsPage, nil, true, "kept from the backup (the target has none"}, + {"populated target mints new ids by default", populatedPostsPage, nil, false, "Pass --preserve-ids"}, + {"--preserve-ids keeps ids in a populated target", populatedPostsPage, []string{"--preserve-ids"}, true, "a document with the same id is overwritten"}, + {"--preserve-ids=false mints new ids in an empty target", emptyPostsPage, []string{"--preserve-ids=false"}, false, "(--preserve-ids=false)"}, + {"an unlistable target is not assumed empty", "", nil, false, "Pass --preserve-ids"}, + {"an unreadable listing is not assumed empty", `{"success":true,"data":{"items":[]}}`, nil, false, "Pass --preserve-ids"}, + {"a bare empty array is empty", `{"success":true,"data":[]}`, nil, true, "the target has none"}, + {"--clean keeps ids", populatedPostsPage, []string{"--clean"}, true, "--clean empties the target first"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := &recorder{} + server := newIDServer(rec, tc.posts) + defer server.Close() + + args := append([]string{"--input", writeBackupZip(t), "--url", server.URL + "/api", "--token", "t", "-q"}, tc.args...) + stdout, stderr, err := runRestore(t, args...) + if err != nil { + t.Fatalf("restore failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, tc.wantNotice) { + t.Errorf("output is missing %q:\n%s", tc.wantNotice, stdout) + } + id, named := createdPostBody(t, rec)["id"] + if tc.wantID && id != "doc-old-1" { + t.Errorf("create did not name the backup's id: %v", id) + } + if !tc.wantID && named { + t.Errorf("create named an id (%v), want the server to mint one", id) + } + }) + } +} + +func TestRestoreIgnoresSingletonsWhenDecidingWhetherTheTargetIsEmpty(t *testing.T) { + rec := &recorder{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.record(r) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && apiPath(r) == "/collections": + io.WriteString(w, `{"success":true,"data":{"collections":[{"name":"posts","title":"Posts","fields":[{"name":"title","type":"string"}]},{"name":"settings","title":"Settings","singleton":true,"fields":[{"name":"title","type":"string"}]}]}}`) + case r.Method == http.MethodGet && apiPath(r) == "/collections/settings": + // Auto-created by a first Studio visit + io.WriteString(w, `{"success":true,"data":{"documents":[{"id":"settings"}]}}`) + case r.Method == http.MethodGet && apiPath(r) == "/collections/posts": + io.WriteString(w, emptyPostsPage) + case r.Method == http.MethodPost && apiPath(r) == "/collections/posts": + io.WriteString(w, `{"success":true,"data":{"document":{"id":"doc-old-1"}}}`) + case r.Method == http.MethodPost && apiPath(r) == "/media/upload": + io.WriteString(w, `{"success":true,"data":{"files":[{"id":"`+testNewMediaID+`"}]}}`) + default: + io.WriteString(w, `{"success":true,"data":{}}`) + } + })) + defer server.Close() + + stdout, stderr, err := runRestore(t, "--input", writeBackupZipWith(t, true), "--url", server.URL+"/api", "--token", "t", "-q") + if err != nil { + t.Fatalf("restore failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Restoring settings") { + t.Fatalf("the backup's singleton was not part of the restore:\n%s", stdout) + } + if !strings.Contains(stdout, "kept from the backup (the target has none") { + t.Errorf("an auto-created singleton made the target look populated:\n%s", stdout) + } + for _, req := range rec.all() { + if req.Method == http.MethodGet && req.Path == "/collections/settings" { + t.Errorf("the singleton collection was listed for the emptiness check") + } + } + if id := createdPostBody(t, rec)["id"]; id != "doc-old-1" { + t.Errorf("create did not name the backup's id: %v", id) + } +}