Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@ trokky restore --input backup.zip --dry-run
# Restore with options
trokky restore --input backup.zip --clean --overwrite
trokky restore --input backup.zip --collections posts --with-dependencies

# Keep the backup's document ids in an instance that already has content
# (the default when the target is empty or with --clean; overwrites same-id documents)
trokky restore --input backup.zip --preserve-ids
```

## Project Scaffolding
Expand Down
85 changes: 81 additions & 4 deletions cmd/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ Documents are restored first, media is uploaded second, and references are
rewritten last. If the token cannot write documents the restore aborts before
anything is uploaded, so a half-restored instance is never left behind.

Document ids are kept from the backup when the target has none of the restored
documents (or with --clean), so links and other systems that name a document by
id keep working. Into an instance that already has documents, new ids are minted
and references between documents are rewritten; --preserve-ids keeps the backup's
ids there too, overwriting any document with the same id (on a server older than
Trokky 3.6, an overwritten document's slug may come back with a numeric suffix).
Media files always get new ids, and references to them are rewritten.

Example:
trokky restore --input backup.zip
trokky restore --input backup.zip --collections posts,pages
Expand All @@ -54,6 +62,8 @@ Example:
dryRun, _ := cmd.Flags().GetBool("dry-run")
clean, _ := cmd.Flags().GetBool("clean")
overwrite, _ := cmd.Flags().GetBool("overwrite")
preserveIDs, _ := cmd.Flags().GetBool("preserve-ids")
preserveIDsSet := cmd.Flags().Changed("preserve-ids")

// Open zip
zr, err := zip.OpenReader(input)
Expand Down Expand Up @@ -259,6 +269,38 @@ Example:
fmt.Fprintf(out, " [DRY RUN] Would restore %d media file(s)\n", len(manifest.MediaIndex))
}

// Document ids. Kept as they are in the backup, anything that refers to a document by id
// from outside the CMS (a link, a redirect map, another system) still works after the
// restore. That is the default when the target has none of these documents yet; into a
// populated instance it would overwrite whatever already has the same id, so there it
// takes --preserve-ids.
// Singletons are left out: they are always written under their own id, and the Studio
// auto-creates them on first visit, so one existing says nothing about the rest.
if !preserveIDsSet {
var listCollections []string
for _, name := range restoreOrder {
backupSchema, inBackup := schemaMap[name]
targetSchema, inTarget := targetSchemaMap[name]
if (inTarget && targetSchema.Singleton) || (inBackup && backupSchema.Singleton) {
continue
}
listCollections = append(listCollections, name)
}
preserveIDs = clean || targetIsEmpty(c, listCollections)
}
switch {
case preserveIDs && preserveIDsSet:
fmt.Fprintln(out, "Document ids: kept from the backup (--preserve-ids); a document with the same id is overwritten")
case preserveIDs && clean:
fmt.Fprintln(out, "Document ids: kept from the backup (--clean empties the target first)")
case preserveIDs:
fmt.Fprintln(out, "Document ids: kept from the backup (the target has none of these documents)")
case preserveIDsSet:
fmt.Fprintln(out, "Document ids: new ids (--preserve-ids=false)")
default:
fmt.Fprintln(out, "Document ids: new ids, because the target already has documents. Pass --preserve-ids to keep the backup's ids (overwrites documents with the same id)")
}

// Document ID mappings (old ID -> new ID), used to rewrite document-to-document
// references as later collections are restored. Media mappings are deliberately
// kept separate: they only exist after phase 2.
Expand Down Expand Up @@ -339,8 +381,9 @@ Example:
// Rewrite document-to-document references using the mappings built
// so far. Media references are left exactly as the backup has them;
// phase 3 fixes those once the new media IDs exist.
// With kept ids every mapping is to itself: there is nothing to rewrite
docRefs := 0
if len(idMappings) > 0 {
if len(idMappings) > 0 && !preserveIDs {
if hasSchema && len(schema.Fields) > 0 {
var refCount int
cleanDoc, refCount = backup.UpdateReferences(cleanDoc, schema, idMappings)
Expand All @@ -353,8 +396,13 @@ Example:
// Sanitize
cleanDoc = backup.SanitizeDocument(cleanDoc)

// Create/update document
docJSON, err := json.Marshal(map[string]interface{}{"data": cleanDoc})
// Create/update document. With preserved ids a create names its id, which the
// server keeps (and which overwrites a document that already has it).
payload := map[string]interface{}{"data": cleanDoc}
if preserveIDs && !isSingleton && originalID != "" {
payload["id"] = originalID
}
docJSON, err := json.Marshal(payload)
if err != nil {
fmt.Fprintf(errOut, "\n Warning: failed to marshal doc: %v\n", err)
failed++
Expand Down Expand Up @@ -394,7 +442,9 @@ Example:
cause = writeErr
}
fmt.Fprintln(out)
return fmt.Errorf("token lacks content:write (required to restore documents): %w", cause)
// The server's message names what is missing: content:write, or the publish
// permission when a write changes what is published
return fmt.Errorf("the token is not allowed to restore these documents: %w", cause)
}
fmt.Fprintf(errOut, "\n Warning: failed to create/update doc %s: %v\n", originalID, writeErr)
failed++
Expand Down Expand Up @@ -744,9 +794,36 @@ func init() {
restoreCmd.Flags().Bool("clean", false, "delete existing content before restore")
restoreCmd.Flags().Bool("overwrite", false, "overwrite existing documents on conflict")
restoreCmd.Flags().Bool("dry-run", false, "preview changes without applying")
restoreCmd.Flags().Bool("preserve-ids", false, "keep the backup's document ids (default: on when the target has none of the documents, or with --clean)")
rootCmd.AddCommand(restoreCmd)
}

// targetIsEmpty reports whether none of the collections hold a document yet. A collection that
// cannot be listed, or answers in a shape this cannot read, counts as not empty: keeping ids
// overwrites documents with the same id, so it is only the default when known to be safe.
func targetIsEmpty(c *client.Client, collections []string) bool {
for _, name := range collections {
data, err := c.Get("/collections/" + name + "?limit=1")
if err != nil {
return false
}
var list []json.RawMessage
if json.Unmarshal(data, &list) == nil {
if len(list) > 0 {
return false
}
continue
}
var page struct {
Documents *[]json.RawMessage `json:"documents"`
}
if json.Unmarshal(data, &page) != nil || page.Documents == nil || len(*page.Documents) > 0 {
return false
}
}
return true
}

// findMediaEntry resolves the zip entry holding a media item's bytes. It prefers
// the path the manifest recorded, then the id-scoped layout, and finally the flat
// legacy layout. The second return value reports whether the legacy layout was
Expand Down
175 changes: 172 additions & 3 deletions cmd/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ const (
// one document that references one media file, and that media file.
func writeBackupZip(t *testing.T) string {
t.Helper()
return writeBackupZipWith(t, false)
}

// writeBackupZipWith builds the single-post backup, optionally with a "settings" singleton
// collection holding one document, restored before posts.
func writeBackupZipWith(t *testing.T, withSettings bool) string {
t.Helper()

manifest := map[string]interface{}{
"version": "2.0",
Expand All @@ -97,6 +104,15 @@ func writeBackupZip(t *testing.T) string {
},
"statistics": map[string]interface{}{"totalDocuments": 1, "totalMedia": 1},
}
if withSettings {
manifest["schemas"] = append(manifest["schemas"].([]map[string]interface{}), map[string]interface{}{
"name": "settings",
"title": "Settings",
"singleton": true,
"fields": []map[string]interface{}{{"name": "title", "type": "string"}},
})
manifest["restoreOrder"] = []string{"settings", "posts"}
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -125,6 +141,12 @@ func writeBackupZip(t *testing.T) string {
{"collections/posts/doc-old-1.json", docJSON},
{"media/photo.png", []byte("png")},
}
if withSettings {
entries = append(entries, struct {
name string
data []byte
}{"collections/settings/settings.json", []byte(`{"id":"settings","title":"Site"}`)})
}
for _, e := range entries {
w, err := zw.Create(e.name)
if err != nil {
Expand Down Expand Up @@ -168,6 +190,13 @@ func runRestore(t *testing.T, args ...string) (string, string, error) {
}
}

// --preserve-ids defaults by whether it was given at all, so it must also look unset
preserve := restoreCmd.Flags().Lookup("preserve-ids")
if err := preserve.Value.Set("false"); err != nil {
t.Fatal(err)
}
preserve.Changed = false

var stdout, stderr bytes.Buffer
rootCmd.SetOut(&stdout)
rootCmd.SetErr(&stderr)
Expand Down Expand Up @@ -242,11 +271,11 @@ func TestRestoreAbortsBeforeUploadingMediaWhenDocumentWritesAreForbidden(t *test

stdout, _, err := runRestore(t, "--input", writeBackupZip(t), "--url", server.URL+"/api", "--token", "read-only", "-q")

want := "token lacks content:write (required to restore documents): "
want := "the token is not allowed to restore these documents: "
if err == nil {
t.Error("expected an error for a token that cannot write documents")
} else if !strings.HasPrefix(err.Error(), want) {
t.Errorf("error = %q, want it to start with %q", err.Error(), want)
} else if !strings.HasPrefix(err.Error(), want) || !strings.Contains(err.Error(), "content:write scope required") {
t.Errorf("error = %q, want it to start with %q and carry the server's reason", err.Error(), want)
}
if n := rec.countPath("/media/upload"); n != 0 {
t.Errorf("%d request(s) hit /media/upload; the restore must abort before uploading anything", n)
Expand Down Expand Up @@ -650,3 +679,143 @@ func TestRestoreCleanStopsWhenMediaCannotBeDeleted(t *testing.T) {
t.Errorf("listed media %d times, want 1", n)
}
}

// newIDServer answers a restore of the single-post backup, listing posts as `posts`
// (a raw response body for GET /collections/posts, or "" for a 404).
func newIDServer(rec *recorder, posts string) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rec.record(r)
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodGet && apiPath(r) == "/collections":
io.WriteString(w, testCollectionsResponse)
case r.Method == http.MethodGet && apiPath(r) == "/collections/posts":
if posts == "" {
w.WriteHeader(http.StatusNotFound)
io.WriteString(w, `{"success":false,"error":{"message":"not found"}}`)
return
}
io.WriteString(w, posts)
case r.Method == http.MethodPost && apiPath(r) == "/collections/posts":
// The server keeps an id the create names, and mints one otherwise
var body struct {
ID string `json:"id"`
}
json.Unmarshal([]byte(rec.all()[len(rec.all())-1].Body), &body)
id := body.ID
if id == "" {
id = testNewDocID
}
io.WriteString(w, `{"success":true,"data":{"document":{"id":"`+id+`"}}}`)
case r.Method == http.MethodPost && apiPath(r) == "/media/upload":
io.WriteString(w, `{"success":true,"data":{"files":[{"id":"`+testNewMediaID+`"}]}}`)
default:
io.WriteString(w, `{"success":true,"data":{}}`)
}
}))
}

const (
emptyPostsPage = `{"success":true,"data":{"documents":[],"pagination":{"total":0}}}`
populatedPostsPage = `{"success":true,"data":{"documents":[{"id":"someone-else"}],"pagination":{"total":1}}}`
)

func createdPostBody(t *testing.T, rec *recorder) map[string]interface{} {
t.Helper()
for _, req := range rec.all() {
if req.Method == http.MethodPost && req.Path == "/collections/posts" {
var body map[string]interface{}
if err := json.Unmarshal([]byte(req.Body), &body); err != nil {
t.Fatalf("create body is not JSON: %s", req.Body)
}
return body
}
}
t.Fatal("no document was created")
return nil
}

func TestRestorePreservesDocumentIDs(t *testing.T) {
cases := []struct {
name string
posts string
args []string
wantID bool
wantNotice string
}{
{"empty target keeps ids by default", emptyPostsPage, nil, true, "kept from the backup (the target has none"},
{"populated target mints new ids by default", populatedPostsPage, nil, false, "Pass --preserve-ids"},
{"--preserve-ids keeps ids in a populated target", populatedPostsPage, []string{"--preserve-ids"}, true, "a document with the same id is overwritten"},
{"--preserve-ids=false mints new ids in an empty target", emptyPostsPage, []string{"--preserve-ids=false"}, false, "(--preserve-ids=false)"},
{"an unlistable target is not assumed empty", "", nil, false, "Pass --preserve-ids"},
{"an unreadable listing is not assumed empty", `{"success":true,"data":{"items":[]}}`, nil, false, "Pass --preserve-ids"},
{"a bare empty array is empty", `{"success":true,"data":[]}`, nil, true, "the target has none"},
{"--clean keeps ids", populatedPostsPage, []string{"--clean"}, true, "--clean empties the target first"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rec := &recorder{}
server := newIDServer(rec, tc.posts)
defer server.Close()

args := append([]string{"--input", writeBackupZip(t), "--url", server.URL + "/api", "--token", "t", "-q"}, tc.args...)
stdout, stderr, err := runRestore(t, args...)
if err != nil {
t.Fatalf("restore failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)
}
if !strings.Contains(stdout, tc.wantNotice) {
t.Errorf("output is missing %q:\n%s", tc.wantNotice, stdout)
}
id, named := createdPostBody(t, rec)["id"]
if tc.wantID && id != "doc-old-1" {
t.Errorf("create did not name the backup's id: %v", id)
}
if !tc.wantID && named {
t.Errorf("create named an id (%v), want the server to mint one", id)
}
})
}
}

func TestRestoreIgnoresSingletonsWhenDecidingWhetherTheTargetIsEmpty(t *testing.T) {
rec := &recorder{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rec.record(r)
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodGet && apiPath(r) == "/collections":
io.WriteString(w, `{"success":true,"data":{"collections":[{"name":"posts","title":"Posts","fields":[{"name":"title","type":"string"}]},{"name":"settings","title":"Settings","singleton":true,"fields":[{"name":"title","type":"string"}]}]}}`)
case r.Method == http.MethodGet && apiPath(r) == "/collections/settings":
// Auto-created by a first Studio visit
io.WriteString(w, `{"success":true,"data":{"documents":[{"id":"settings"}]}}`)
case r.Method == http.MethodGet && apiPath(r) == "/collections/posts":
io.WriteString(w, emptyPostsPage)
case r.Method == http.MethodPost && apiPath(r) == "/collections/posts":
io.WriteString(w, `{"success":true,"data":{"document":{"id":"doc-old-1"}}}`)
case r.Method == http.MethodPost && apiPath(r) == "/media/upload":
io.WriteString(w, `{"success":true,"data":{"files":[{"id":"`+testNewMediaID+`"}]}}`)
default:
io.WriteString(w, `{"success":true,"data":{}}`)
}
}))
defer server.Close()

stdout, stderr, err := runRestore(t, "--input", writeBackupZipWith(t, true), "--url", server.URL+"/api", "--token", "t", "-q")
if err != nil {
t.Fatalf("restore failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)
}
if !strings.Contains(stdout, "Restoring settings") {
t.Fatalf("the backup's singleton was not part of the restore:\n%s", stdout)
}
if !strings.Contains(stdout, "kept from the backup (the target has none") {
t.Errorf("an auto-created singleton made the target look populated:\n%s", stdout)
}
for _, req := range rec.all() {
if req.Method == http.MethodGet && req.Path == "/collections/settings" {
t.Errorf("the singleton collection was listed for the emptiness check")
}
}
if id := createdPostBody(t, rec)["id"]; id != "doc-old-1" {
t.Errorf("create did not name the backup's id: %v", id)
}
}
Loading