diff --git a/docs/architecture/backups.md b/docs/architecture/backups.md index 924e561639..65274503f9 100644 --- a/docs/architecture/backups.md +++ b/docs/architecture/backups.md @@ -1879,9 +1879,17 @@ from it whole, so an SMTP host and password changed since the backup stay changed. A unit the live server lacks, and every unit of another deployment's backup or a separate restore, keeps the backup's other values with its secrets cleared and its switch off, and the command lists it for -re-entry. `TestEveryCredentialSettingIsDecided` fails on an encrypted -setting that is neither in a unit nor this server's own (signing keys, -storage, Redis, the web push key pair). +re-entry. When the AI providers' unit is cleared, the job bindings +(`config.AIProviderBindingKeys`), which name entries of the provider list, +go with it, and the list is left empty at the current conversion version +(encrypted when the restore has this server's key), so the next start +builds no provider from the backup's legacy endpoint settings or the +Ollama default (`config.PlanAIProviderConversion`). A binding left without +its list, as a restore with another key leaves it, clears the unit too. A +full restore then takes the list and bindings from the live server with its +server-specific settings (below). `TestEveryCredentialSettingIsDecided` +fails on an encrypted setting that is neither in a unit nor this server's +own (signing keys, storage, Redis, the web push key pair). A full restore also keeps every server-specific setting (`SettingKindServerSpecific`: the server's identity and public URL, storage, Redis, search, AI diff --git a/internal/backup/backup_postgres_test.go b/internal/backup/backup_postgres_test.go index 7e30559575..2ed38a6076 100644 --- a/internal/backup/backup_postgres_test.go +++ b/internal/backup/backup_postgres_test.go @@ -27,7 +27,6 @@ import ( "github.com/Silo-Server/silo-server/internal/backup/backuptest" "github.com/Silo-Server/silo-server/internal/blobstore" "github.com/Silo-Server/silo-server/internal/config" - "github.com/Silo-Server/silo-server/internal/mail" "github.com/Silo-Server/silo-server/internal/plugins" "github.com/Silo-Server/silo-server/internal/s3client/s3test" "github.com/Silo-Server/silo-server/internal/secret" @@ -784,66 +783,6 @@ func checkLiveReleasesAndNodes(t *testing.T, loader PGLoader, pool *pgxpool.Pool } } -// checkOutboundSettings: a full restore keeps the live mail route as a -// unit, an SMTP host and password changed since the backup included; a -// route the live server does not have comes back with its secret cleared -// and turned off, and so does every route of a restore that carries none. -func checkOutboundSettings(t *testing.T, loader PGLoader, pool *pgxpool.Pool, dsn, into string, restored *pgxpool.Pool) { - t.Helper() - ctx := t.Context() - set := func(db *pgxpool.Pool, values map[string]string) { - t.Helper() - for key, value := range values { - if _, err := db.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2) - ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil { - t.Fatal(err) - } - } - } - read := func(key string) string { - t.Helper() - var v string - if err := restored.QueryRow(ctx, `SELECT coalesce((SELECT value FROM server_settings WHERE key = $1), '')`, key).Scan(&v); err != nil { - t.Fatal(err) - } - return v - } - keys := []string{settingSMTPHost, settingSMTPPassword, mail.SettingSMTPUsername, settingEmailEnabled, settingAIBaseURL, settingAIKey} - var saved []string - for _, key := range keys { - var v *string - _ = pool.QueryRow(ctx, `SELECT value FROM server_settings WHERE key = $1`, key).Scan(&v) - if v != nil { - saved = append(saved, key, *v) - } - } - t.Cleanup(func() { - _, _ = pool.Exec(context.Background(), `DELETE FROM server_settings WHERE key = ANY($1)`, keys) - for i := 0; i+1 < len(saved); i += 2 { - _, _ = pool.Exec(context.Background(), `INSERT INTO server_settings (key, value) VALUES ($1, $2)`, saved[i], saved[i+1]) - } - }) - set(restored, map[string]string{settingSMTPHost: "smtp.then.invalid", settingSMTPPassword: "then", mail.SettingSMTPUsername: "then-user", settingEmailEnabled: "true", - settingAIBaseURL: "https://ai.then.invalid", settingAIKey: "then-key"}) - set(pool, map[string]string{settingSMTPHost: "smtp.now.invalid", settingSMTPPassword: "now", mail.SettingSMTPUsername: "now-user", settingEmailEnabled: "true"}) - if _, err := pool.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, []string{settingAIBaseURL, settingAIKey}); err != nil { - t.Fatal(err) - } - if _, _, err := loader.CarryOutboundSettings(ctx, into, dsn); err != nil { - t.Fatal(err) - } - if read(settingSMTPHost) != "smtp.now.invalid" || read(settingSMTPPassword) != "now" || read(mail.SettingSMTPUsername) != "now-user" { - t.Fatalf("mail route %s / %s, want the live one", read(settingSMTPHost), read(settingSMTPPassword)) - } - if read(settingAIBaseURL) != "https://ai.then.invalid" || read(settingAIKey) != "" { - t.Fatalf("a route the live server lacks: %s / %q, want the address kept and the key cleared", read(settingAIBaseURL), read(settingAIKey)) - } - if _, cleared, err := loader.CarryOutboundSettings(ctx, into, ""); err != nil || read(settingSMTPPassword) != "" || read(settingEmailEnabled) != settingOff || - !slices.Contains(cleared, "mail (SMTP)") { - t.Fatalf("carrying none (%v): password %q, enabled %q, cleared %v; want it cleared and off", err, read(settingSMTPPassword), read(settingEmailEnabled), cleared) - } -} - // checkInterruptedWork: after a full restore nothing the backup had queued // or running runs by itself. A storage transition queued when the backup // was taken, and canceled on the live server since, ends canceled, its diff --git a/internal/backup/restore.go b/internal/backup/restore.go index ec19562e5a..9b5932545a 100644 --- a/internal/backup/restore.go +++ b/internal/backup/restore.go @@ -360,8 +360,9 @@ const ( // carry replaces the restored database's credentials, outbound settings, // plugin releases, and policies with those of carryFrom, or with none when // carryFrom is empty. destination is the live database that will be replaced, -// even if the backup came from another deployment. -func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carryFrom, destination string) error { +// even if the backup came from another deployment. cipher is this server's +// SECRET_KEY, or nil. +func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carryFrom, destination string, cipher *secret.Cipher) error { authorityRestricted, err := carryCredentials(ctx, loader, dbURL, carryFrom, p.out) if err := p.strictStep("replacing the backup's credentials", err, "leaving out all of the backup's "+carriedKinds, func() error { @@ -371,7 +372,7 @@ func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carry }); err != nil { return err } - if err := p.step("carrying the outbound settings", carryOutboundSettings(ctx, loader, dbURL, carryFrom, p.out)); err != nil { + if err := p.step("carrying the outbound settings", carryOutboundSettings(ctx, loader, dbURL, carryFrom, cipher, p.out)); err != nil { return err } releasesRestricted, err := carryPluginReleases(ctx, loader, dbURL, carryFrom, p.out) @@ -615,8 +616,9 @@ type DatabaseLoader interface { CarryServerSpecificSettings(ctx context.Context, dbURL, liveURL string) (int, error) // CarryOutboundSettings gives dbURL liveURL's outbound settings, each // bundle as a unit, clearing the secrets of one liveURL lacks ("" for - // none), and returns the bundles carried and cleared. - CarryOutboundSettings(ctx context.Context, dbURL, liveURL string) (carried, cleared []string, err error) + // none), and returns the bundles carried and cleared. cipher (nil for + // none) encrypts what a cleared bundle is left with. + CarryOutboundSettings(ctx context.Context, dbURL, liveURL string, cipher *secret.Cipher) (carried, cleared []string, err error) // InterruptWork ends the work queued or running in the restored // database dbURL and gives it liveURL's storage transition target // (none for ""), returning what it changed. @@ -797,7 +799,7 @@ func RestoreSeparate(ctx context.Context, src *Source, loader DatabaseLoader, op } // A copy is not the live server: it gets none of the backup's // credentials and none of the live ones. - if err := post.carry(ctx, loader, opts.Into, "", ""); err != nil { + if err := post.carry(ctx, loader, opts.Into, "", "", opts.Cipher); err != nil { return RestoreReport{}, err } if err := post.step(stepInterrupt, interruptWork(ctx, loader, opts.Into, "", out)); err != nil { diff --git a/internal/backup/settings_outbound.go b/internal/backup/settings_outbound.go index 65e33e8c3d..6407016e35 100644 --- a/internal/backup/settings_outbound.go +++ b/internal/backup/settings_outbound.go @@ -14,6 +14,8 @@ import ( "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/mail" "github.com/Silo-Server/silo-server/internal/plugins" + "github.com/Silo-Server/silo-server/internal/secret" + "github.com/Silo-Server/silo-server/internal/serversettings" ) // settingOff is a switch setting turned off. @@ -34,10 +36,12 @@ type outboundBundle struct { name string // keys are every setting of the bundle. keys []string - // secrets are the ones cleared where the live server has no bundle. + // secrets are the ones cleared where the live server has no bundle, + // with the settings that only name an entry of one. secrets []string - // enabled is the switch turned off then, or "". - enabled string + // off are the values a cleared bundle is left with: its switch turned + // off, or an AI provider registry with no providers. + off map[string]string } // outboundBundles are the settings a full restore never takes from the @@ -48,10 +52,17 @@ var outboundBundles = func() []outboundBundle { // (mail.SMTPSettingKeys, config.AIConnectionSettingKeys), so a restore // never mixes the live server's values with the backup's. ai := append(config.AIConnectionSettingKeys(), chatgpt.CredentialsKey) + // The job bindings name providers in the registry, so they go with it. + // The registry is left empty but converted, so the next start builds no + // provider from the endpoint settings the backup keeps or the Ollama + // default (config.PlanAIProviderConversion). + aiSecrets := append([]string{config.AIProvidersSettingKey, chatgpt.CredentialsKey, settingAIKey, settingASRKey, + settingSubtitleAIKey, settingEmbeddingToken, settingOpenAIEmbeddingKey}, config.AIProviderBindingKeys()...) + noProviders, _, _ := config.PlanAIProviderConversion(nil) out := []outboundBundle{ - {name: "mail (SMTP)", keys: mail.SMTPSettingKeys(), secrets: []string{settingSMTPPassword}, enabled: settingEmailEnabled}, - {name: "AI providers", keys: ai, secrets: []string{config.AIProvidersSettingKey, chatgpt.CredentialsKey, settingAIKey, settingASRKey, - settingSubtitleAIKey, settingEmbeddingToken, settingOpenAIEmbeddingKey}}, + {name: "mail (SMTP)", keys: mail.SMTPSettingKeys(), secrets: []string{settingSMTPPassword}, off: map[string]string{settingEmailEnabled: settingOff}}, + {name: "AI providers", keys: ai, secrets: aiSecrets, + off: map[string]string{config.AIProvidersSettingKey: noProviders[config.AIProvidersSettingKey]}}, } whole := func(keys ...string) outboundBundle { return outboundBundle{name: keys[0], keys: keys, secrets: keys} } out = append(out, @@ -85,9 +96,10 @@ func outboundKeys() []string { // secrets a service again. A bundle the live server does not have, or // every bundle when liveURL is "" (another deployment, or a separate // restore), keeps the backup's other settings with its secrets cleared and -// its switch off. It returns the bundles it took from the live server and -// those it cleared, by name. -func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL string) (carried, cleared []string, err error) { +// its off values written, a sensitive one encrypted with cipher (nil for +// none). It returns the bundles it took from the live server and those it +// cleared, by name. +func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL string, cipher *secret.Cipher) (carried, cleared []string, err error) { live := map[string]string{} if liveURL != "" { if live, err = readSettingsAt(ctx, liveURL, outboundKeys()); err != nil { @@ -115,7 +127,7 @@ func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL stri if !slices.ContainsFunc(b.secrets, func(k string) bool { return restored[k] != "" }) { continue } - if err := b.clear(ctx, tx); err != nil { + if err := b.clear(ctx, tx, cipher); err != nil { return err } cleared = append(cleared, b.name) @@ -140,23 +152,33 @@ func (b outboundBundle) takeLive(ctx context.Context, tx pgx.Tx, live map[string return nil } -// clear deletes the bundle's secrets in tx and turns its switch off. -func (b outboundBundle) clear(ctx context.Context, tx pgx.Tx) error { +// clear deletes the bundle's secrets in tx and writes its off values, +// encrypting a sensitive one with cipher. Without one it stays plaintext, +// which Silo reads as it is and encrypts when it starts. +func (b outboundBundle) clear(ctx context.Context, tx pgx.Tx, cipher *secret.Cipher) error { if _, err := tx.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, b.secrets); err != nil { return err } - if b.enabled == "" { - return nil + for key, value := range b.off { + if cipher != nil && serversettings.SensitiveSettingKeys[key] { + sealed, err := cipher.Encrypt(value, secret.SettingsAAD(key)) + if err != nil { + return err + } + value = sealed + } + if _, err := tx.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil { + return err + } } - _, err := tx.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2) - ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, b.enabled, settingOff) - return err + return nil } // carryOutboundSettings takes the outbound settings from the live server // (liveURL, or none for "") and says what it cleared. -func carryOutboundSettings(ctx context.Context, loader DatabaseLoader, dbURL, liveURL string, out io.Writer) error { - carried, cleared, err := loader.CarryOutboundSettings(ctx, dbURL, liveURL) +func carryOutboundSettings(ctx context.Context, loader DatabaseLoader, dbURL, liveURL string, cipher *secret.Cipher, out io.Writer) error { + carried, cleared, err := loader.CarryOutboundSettings(ctx, dbURL, liveURL, cipher) if err != nil { return fmt.Errorf("keeping the live server's connections to outside services: %w", err) } diff --git a/internal/backup/settings_outbound_postgres_test.go b/internal/backup/settings_outbound_postgres_test.go new file mode 100644 index 0000000000..e8e5d4d1f3 --- /dev/null +++ b/internal/backup/settings_outbound_postgres_test.go @@ -0,0 +1,110 @@ +package backup + +import ( + "context" + "slices" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/mail" + "github.com/Silo-Server/silo-server/internal/secret" + "github.com/Silo-Server/silo-server/internal/serversettings" +) + +// checkOutboundSettings: a full restore keeps the live mail route as a +// unit, an SMTP host and password changed since the backup included; a +// route the live server does not have comes back with its secret cleared +// and turned off, and so does every route of a restore that carries none. +func checkOutboundSettings(t *testing.T, loader PGLoader, pool *pgxpool.Pool, dsn, into string, restored *pgxpool.Pool) { + t.Helper() + ctx := t.Context() + set := func(db *pgxpool.Pool, values map[string]string) { + t.Helper() + for key, value := range values { + if _, err := db.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil { + t.Fatal(err) + } + } + } + read := func(key string) string { + t.Helper() + var v string + if err := restored.QueryRow(ctx, `SELECT coalesce((SELECT value FROM server_settings WHERE key = $1), '')`, key).Scan(&v); err != nil { + t.Fatal(err) + } + return v + } + keys := []string{settingSMTPHost, settingSMTPPassword, mail.SettingSMTPUsername, settingEmailEnabled, settingAIBaseURL, settingAIKey} + var saved []string + for _, key := range keys { + var v *string + _ = pool.QueryRow(ctx, `SELECT value FROM server_settings WHERE key = $1`, key).Scan(&v) + if v != nil { + saved = append(saved, key, *v) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM server_settings WHERE key = ANY($1)`, keys) + for i := 0; i+1 < len(saved); i += 2 { + _, _ = pool.Exec(context.Background(), `INSERT INTO server_settings (key, value) VALUES ($1, $2)`, saved[i], saved[i+1]) + } + }) + set(restored, map[string]string{settingSMTPHost: "smtp.then.invalid", settingSMTPPassword: "then", mail.SettingSMTPUsername: "then-user", settingEmailEnabled: "true", + settingAIBaseURL: "https://ai.then.invalid", settingAIKey: "then-key", + config.AIProvidersSettingKey: `{"version":2,"providers":[{"id":"custom-then","type":"custom","name":"Then","base_url":"http://ai.then.invalid/v1","api_key":"then-key","capabilities":["chat","embeddings"]}]}`, + config.AITextProviderSettingKey: "custom-then", config.EmbeddingProviderSettingKey: "custom-then", "recommendations.enabled": "true"}) + set(pool, map[string]string{settingSMTPHost: "smtp.now.invalid", settingSMTPPassword: "now", mail.SettingSMTPUsername: "now-user", settingEmailEnabled: "true"}) + if _, err := pool.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, []string{settingAIBaseURL, settingAIKey}); err != nil { + t.Fatal(err) + } + cipher, err := secret.New([]byte("outbound-settings-test-key-0123456789")) + if err != nil { + t.Fatal(err) + } + // The AI providers go as a unit: no job stays bound to a cleared + // provider, and the next start converts no provider back from what is left. + checkAIReset := func(when string) { + t.Helper() + ai, err := serversettings.NewEncryptedSettingsRepo(serversettings.NewServerSettingsRepo(restored), cipher). + GetMany(ctx, append(config.AIConnectionSettingKeys(), "recommendations.enabled")...) + if err != nil { + t.Fatal(err) + } + if err := config.ValidateAIProviderBindings(ai); err != nil { + t.Fatalf("%s: %v; want no job bound to a cleared AI provider", when, err) + } + if writes, _, err := config.PlanAIProviderConversion(ai); err != nil || writes != nil { + t.Fatalf("%s: the next start converts %v (%v), want nothing", when, writes, err) + } + if registry, err := config.ParseAIProviderRegistry(ai[config.AIProvidersSettingKey]); err != nil || len(registry.Providers) != 0 { + t.Fatalf("%s: AI providers %v (%v), want none", when, registry.Providers, err) + } + } + if _, _, err := loader.CarryOutboundSettings(ctx, into, dsn, cipher); err != nil { + t.Fatal(err) + } + if read(settingSMTPHost) != "smtp.now.invalid" || read(settingSMTPPassword) != "now" || read(mail.SettingSMTPUsername) != "now-user" { + t.Fatalf("mail route %s / %s, want the live one", read(settingSMTPHost), read(settingSMTPPassword)) + } + if read(settingAIBaseURL) != "https://ai.then.invalid" || read(settingAIKey) != "" { + t.Fatalf("a route the live server lacks: %s / %q, want the address kept and the key cleared", read(settingAIBaseURL), read(settingAIKey)) + } + checkAIReset("a registry the live server lacks") + if !secret.IsEncrypted(read(config.AIProvidersSettingKey)) { + t.Fatalf("the emptied AI provider registry is stored as %q, want it encrypted", read(config.AIProvidersSettingKey)) + } + // A restore with another key has already dropped the providers it cannot + // read; the bindings left behind still reset them. + if _, err := restored.Exec(ctx, `DELETE FROM server_settings WHERE key = $1`, config.AIProvidersSettingKey); err != nil { + t.Fatal(err) + } + set(restored, map[string]string{config.AISpeechProviderSettingKey: "custom-then"}) + if _, cleared, err := loader.CarryOutboundSettings(ctx, into, "", nil); err != nil || read(settingSMTPPassword) != "" || read(settingEmailEnabled) != settingOff || + !slices.Contains(cleared, "mail (SMTP)") || !slices.Contains(cleared, "AI providers") { + t.Fatalf("carrying none (%v): password %q, enabled %q, cleared %v; want it cleared and off", err, read(settingSMTPPassword), read(settingEmailEnabled), cleared) + } + checkAIReset("bindings without their providers") +} diff --git a/internal/backup/swap.go b/internal/backup/swap.go index 11cde42519..5e47cdd471 100644 --- a/internal/backup/swap.go +++ b/internal/backup/swap.go @@ -287,7 +287,7 @@ func carryLiveState(ctx context.Context, ctl ControlDB, live string, loader Data if !sameDeployment { carryFrom = "" } - if err := post.carry(ctx, loader, restoredURL, carryFrom, opts.LiveURL); err != nil { + if err := post.carry(ctx, loader, restoredURL, carryFrom, opts.LiveURL, opts.Cipher); err != nil { return err } // This server's own settings describe it whichever deployment the diff --git a/internal/backup/swap_test.go b/internal/backup/swap_test.go index 98f8fd483b..12de0fa695 100644 --- a/internal/backup/swap_test.go +++ b/internal/backup/swap_test.go @@ -276,7 +276,7 @@ func (l fakeLoader) FinishPluginProvisionRestore(context.Context, string, string func (l fakeLoader) CarryServerSpecificSettings(context.Context, string, string) (int, error) { return 0, nil } -func (l fakeLoader) CarryOutboundSettings(context.Context, string, string) ([]string, []string, error) { +func (l fakeLoader) CarryOutboundSettings(context.Context, string, string, *secret.Cipher) ([]string, []string, error) { return nil, nil, nil } func (l fakeLoader) InterruptWork(context.Context, string, string) ([]string, error) { return nil, nil }