Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions docs/architecture/backups.md
Original file line number Diff line number Diff line change
Expand Up @@ -1879,9 +1879,17 @@ from it whole, so an SMTP host and password changed since the backup stay
changed. A unit the live server lacks, and every unit of another
deployment's backup or a separate restore, keeps the backup's other values
with its secrets cleared and its switch off, and the command lists it for
re-entry. `TestEveryCredentialSettingIsDecided` fails on an encrypted
setting that is neither in a unit nor this server's own (signing keys,
storage, Redis, the web push key pair).
re-entry. When the AI providers' unit is cleared, the job bindings
(`config.AIProviderBindingKeys`), which name entries of the provider list,
go with it, and the list is left empty at the current conversion version
(encrypted when the restore has this server's key), so the next start
builds no provider from the backup's legacy endpoint settings or the
Ollama default (`config.PlanAIProviderConversion`). A binding left without
its list, as a restore with another key leaves it, clears the unit too. A
full restore then takes the list and bindings from the live server with its
server-specific settings (below). `TestEveryCredentialSettingIsDecided`
fails on an encrypted setting that is neither in a unit nor this server's
own (signing keys, storage, Redis, the web push key pair).

A full restore also keeps every server-specific setting (`SettingKindServerSpecific`:
the server's identity and public URL, storage, Redis, search, AI
Expand Down
61 changes: 0 additions & 61 deletions internal/backup/backup_postgres_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/Silo-Server/silo-server/internal/backup/backuptest"
"github.com/Silo-Server/silo-server/internal/blobstore"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/mail"
"github.com/Silo-Server/silo-server/internal/plugins"
"github.com/Silo-Server/silo-server/internal/s3client/s3test"
"github.com/Silo-Server/silo-server/internal/secret"
Expand Down Expand Up @@ -784,66 +783,6 @@ func checkLiveReleasesAndNodes(t *testing.T, loader PGLoader, pool *pgxpool.Pool
}
}

// checkOutboundSettings: a full restore keeps the live mail route as a
// unit, an SMTP host and password changed since the backup included; a
// route the live server does not have comes back with its secret cleared
// and turned off, and so does every route of a restore that carries none.
func checkOutboundSettings(t *testing.T, loader PGLoader, pool *pgxpool.Pool, dsn, into string, restored *pgxpool.Pool) {
t.Helper()
ctx := t.Context()
set := func(db *pgxpool.Pool, values map[string]string) {
t.Helper()
for key, value := range values {
if _, err := db.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil {
t.Fatal(err)
}
}
}
read := func(key string) string {
t.Helper()
var v string
if err := restored.QueryRow(ctx, `SELECT coalesce((SELECT value FROM server_settings WHERE key = $1), '')`, key).Scan(&v); err != nil {
t.Fatal(err)
}
return v
}
keys := []string{settingSMTPHost, settingSMTPPassword, mail.SettingSMTPUsername, settingEmailEnabled, settingAIBaseURL, settingAIKey}
var saved []string
for _, key := range keys {
var v *string
_ = pool.QueryRow(ctx, `SELECT value FROM server_settings WHERE key = $1`, key).Scan(&v)
if v != nil {
saved = append(saved, key, *v)
}
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM server_settings WHERE key = ANY($1)`, keys)
for i := 0; i+1 < len(saved); i += 2 {
_, _ = pool.Exec(context.Background(), `INSERT INTO server_settings (key, value) VALUES ($1, $2)`, saved[i], saved[i+1])
}
})
set(restored, map[string]string{settingSMTPHost: "smtp.then.invalid", settingSMTPPassword: "then", mail.SettingSMTPUsername: "then-user", settingEmailEnabled: "true",
settingAIBaseURL: "https://ai.then.invalid", settingAIKey: "then-key"})
set(pool, map[string]string{settingSMTPHost: "smtp.now.invalid", settingSMTPPassword: "now", mail.SettingSMTPUsername: "now-user", settingEmailEnabled: "true"})
if _, err := pool.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, []string{settingAIBaseURL, settingAIKey}); err != nil {
t.Fatal(err)
}
if _, _, err := loader.CarryOutboundSettings(ctx, into, dsn); err != nil {
t.Fatal(err)
}
if read(settingSMTPHost) != "smtp.now.invalid" || read(settingSMTPPassword) != "now" || read(mail.SettingSMTPUsername) != "now-user" {
t.Fatalf("mail route %s / %s, want the live one", read(settingSMTPHost), read(settingSMTPPassword))
}
if read(settingAIBaseURL) != "https://ai.then.invalid" || read(settingAIKey) != "" {
t.Fatalf("a route the live server lacks: %s / %q, want the address kept and the key cleared", read(settingAIBaseURL), read(settingAIKey))
}
if _, cleared, err := loader.CarryOutboundSettings(ctx, into, ""); err != nil || read(settingSMTPPassword) != "" || read(settingEmailEnabled) != settingOff ||
!slices.Contains(cleared, "mail (SMTP)") {
t.Fatalf("carrying none (%v): password %q, enabled %q, cleared %v; want it cleared and off", err, read(settingSMTPPassword), read(settingEmailEnabled), cleared)
}
}

// checkInterruptedWork: after a full restore nothing the backup had queued
// or running runs by itself. A storage transition queued when the backup
// was taken, and canceled on the live server since, ends canceled, its
Expand Down
14 changes: 8 additions & 6 deletions internal/backup/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,9 @@ const (
// carry replaces the restored database's credentials, outbound settings,
// plugin releases, and policies with those of carryFrom, or with none when
// carryFrom is empty. destination is the live database that will be replaced,
// even if the backup came from another deployment.
func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carryFrom, destination string) error {
// even if the backup came from another deployment. cipher is this server's
// SECRET_KEY, or nil.
func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carryFrom, destination string, cipher *secret.Cipher) error {
authorityRestricted, err := carryCredentials(ctx, loader, dbURL, carryFrom, p.out)
if err := p.strictStep("replacing the backup's credentials", err,
"leaving out all of the backup's "+carriedKinds, func() error {
Expand All @@ -371,7 +372,7 @@ func (p postLoad) carry(ctx context.Context, loader DatabaseLoader, dbURL, carry
}); err != nil {
return err
}
if err := p.step("carrying the outbound settings", carryOutboundSettings(ctx, loader, dbURL, carryFrom, p.out)); err != nil {
if err := p.step("carrying the outbound settings", carryOutboundSettings(ctx, loader, dbURL, carryFrom, cipher, p.out)); err != nil {
return err
}
releasesRestricted, err := carryPluginReleases(ctx, loader, dbURL, carryFrom, p.out)
Expand Down Expand Up @@ -615,8 +616,9 @@ type DatabaseLoader interface {
CarryServerSpecificSettings(ctx context.Context, dbURL, liveURL string) (int, error)
// CarryOutboundSettings gives dbURL liveURL's outbound settings, each
// bundle as a unit, clearing the secrets of one liveURL lacks ("" for
// none), and returns the bundles carried and cleared.
CarryOutboundSettings(ctx context.Context, dbURL, liveURL string) (carried, cleared []string, err error)
// none), and returns the bundles carried and cleared. cipher (nil for
// none) encrypts what a cleared bundle is left with.
CarryOutboundSettings(ctx context.Context, dbURL, liveURL string, cipher *secret.Cipher) (carried, cleared []string, err error)
// InterruptWork ends the work queued or running in the restored
// database dbURL and gives it liveURL's storage transition target
// (none for ""), returning what it changed.
Expand Down Expand Up @@ -797,7 +799,7 @@ func RestoreSeparate(ctx context.Context, src *Source, loader DatabaseLoader, op
}
// A copy is not the live server: it gets none of the backup's
// credentials and none of the live ones.
if err := post.carry(ctx, loader, opts.Into, "", ""); err != nil {
if err := post.carry(ctx, loader, opts.Into, "", "", opts.Cipher); err != nil {
return RestoreReport{}, err
}
if err := post.step(stepInterrupt, interruptWork(ctx, loader, opts.Into, "", out)); err != nil {
Expand Down
60 changes: 41 additions & 19 deletions internal/backup/settings_outbound.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/mail"
"github.com/Silo-Server/silo-server/internal/plugins"
"github.com/Silo-Server/silo-server/internal/secret"
"github.com/Silo-Server/silo-server/internal/serversettings"
)

// settingOff is a switch setting turned off.
Expand All @@ -34,10 +36,12 @@ type outboundBundle struct {
name string
// keys are every setting of the bundle.
keys []string
// secrets are the ones cleared where the live server has no bundle.
// secrets are the ones cleared where the live server has no bundle,
// with the settings that only name an entry of one.
secrets []string
// enabled is the switch turned off then, or "".
enabled string
// off are the values a cleared bundle is left with: its switch turned
// off, or an AI provider registry with no providers.
off map[string]string
}

// outboundBundles are the settings a full restore never takes from the
Expand All @@ -48,10 +52,17 @@ var outboundBundles = func() []outboundBundle {
// (mail.SMTPSettingKeys, config.AIConnectionSettingKeys), so a restore
// never mixes the live server's values with the backup's.
ai := append(config.AIConnectionSettingKeys(), chatgpt.CredentialsKey)
// The job bindings name providers in the registry, so they go with it.
// The registry is left empty but converted, so the next start builds no
// provider from the endpoint settings the backup keeps or the Ollama
// default (config.PlanAIProviderConversion).
aiSecrets := append([]string{config.AIProvidersSettingKey, chatgpt.CredentialsKey, settingAIKey, settingASRKey,
settingSubtitleAIKey, settingEmbeddingToken, settingOpenAIEmbeddingKey}, config.AIProviderBindingKeys()...)
noProviders, _, _ := config.PlanAIProviderConversion(nil)
out := []outboundBundle{
{name: "mail (SMTP)", keys: mail.SMTPSettingKeys(), secrets: []string{settingSMTPPassword}, enabled: settingEmailEnabled},
{name: "AI providers", keys: ai, secrets: []string{config.AIProvidersSettingKey, chatgpt.CredentialsKey, settingAIKey, settingASRKey,
settingSubtitleAIKey, settingEmbeddingToken, settingOpenAIEmbeddingKey}},
{name: "mail (SMTP)", keys: mail.SMTPSettingKeys(), secrets: []string{settingSMTPPassword}, off: map[string]string{settingEmailEnabled: settingOff}},
{name: "AI providers", keys: ai, secrets: aiSecrets,
off: map[string]string{config.AIProvidersSettingKey: noProviders[config.AIProvidersSettingKey]}},
}
whole := func(keys ...string) outboundBundle { return outboundBundle{name: keys[0], keys: keys, secrets: keys} }
out = append(out,
Expand Down Expand Up @@ -85,9 +96,10 @@ func outboundKeys() []string {
// secrets a service again. A bundle the live server does not have, or
// every bundle when liveURL is "" (another deployment, or a separate
// restore), keeps the backup's other settings with its secrets cleared and
// its switch off. It returns the bundles it took from the live server and
// those it cleared, by name.
func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL string) (carried, cleared []string, err error) {
// its off values written, a sensitive one encrypted with cipher (nil for
// none). It returns the bundles it took from the live server and those it
// cleared, by name.
func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL string, cipher *secret.Cipher) (carried, cleared []string, err error) {
live := map[string]string{}
if liveURL != "" {
if live, err = readSettingsAt(ctx, liveURL, outboundKeys()); err != nil {
Expand Down Expand Up @@ -115,7 +127,7 @@ func (l PGLoader) CarryOutboundSettings(ctx context.Context, dbURL, liveURL stri
if !slices.ContainsFunc(b.secrets, func(k string) bool { return restored[k] != "" }) {
continue
}
if err := b.clear(ctx, tx); err != nil {
if err := b.clear(ctx, tx, cipher); err != nil {
return err
}
cleared = append(cleared, b.name)
Expand All @@ -140,23 +152,33 @@ func (b outboundBundle) takeLive(ctx context.Context, tx pgx.Tx, live map[string
return nil
}

// clear deletes the bundle's secrets in tx and turns its switch off.
func (b outboundBundle) clear(ctx context.Context, tx pgx.Tx) error {
// clear deletes the bundle's secrets in tx and writes its off values,
// encrypting a sensitive one with cipher. Without one it stays plaintext,
// which Silo reads as it is and encrypts when it starts.
func (b outboundBundle) clear(ctx context.Context, tx pgx.Tx, cipher *secret.Cipher) error {
if _, err := tx.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, b.secrets); err != nil {
return err
}
if b.enabled == "" {
return nil
for key, value := range b.off {
if cipher != nil && serversettings.SensitiveSettingKeys[key] {
sealed, err := cipher.Encrypt(value, secret.SettingsAAD(key))
if err != nil {
return err
}
value = sealed
}
if _, err := tx.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil {
return err
}
}
_, err := tx.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, b.enabled, settingOff)
return err
return nil
}

// carryOutboundSettings takes the outbound settings from the live server
// (liveURL, or none for "") and says what it cleared.
func carryOutboundSettings(ctx context.Context, loader DatabaseLoader, dbURL, liveURL string, out io.Writer) error {
carried, cleared, err := loader.CarryOutboundSettings(ctx, dbURL, liveURL)
func carryOutboundSettings(ctx context.Context, loader DatabaseLoader, dbURL, liveURL string, cipher *secret.Cipher, out io.Writer) error {
carried, cleared, err := loader.CarryOutboundSettings(ctx, dbURL, liveURL, cipher)
if err != nil {
return fmt.Errorf("keeping the live server's connections to outside services: %w", err)
}
Expand Down
110 changes: 110 additions & 0 deletions internal/backup/settings_outbound_postgres_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package backup

import (
"context"
"slices"
"testing"

"github.com/jackc/pgx/v5/pgxpool"

"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/mail"
"github.com/Silo-Server/silo-server/internal/secret"
"github.com/Silo-Server/silo-server/internal/serversettings"
)

// checkOutboundSettings: a full restore keeps the live mail route as a
// unit, an SMTP host and password changed since the backup included; a
// route the live server does not have comes back with its secret cleared
// and turned off, and so does every route of a restore that carries none.
func checkOutboundSettings(t *testing.T, loader PGLoader, pool *pgxpool.Pool, dsn, into string, restored *pgxpool.Pool) {
t.Helper()
ctx := t.Context()
set := func(db *pgxpool.Pool, values map[string]string) {
t.Helper()
for key, value := range values {
if _, err := db.Exec(ctx, `INSERT INTO server_settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, key, value); err != nil {
t.Fatal(err)
}
}
}
read := func(key string) string {
t.Helper()
var v string
if err := restored.QueryRow(ctx, `SELECT coalesce((SELECT value FROM server_settings WHERE key = $1), '')`, key).Scan(&v); err != nil {
t.Fatal(err)
}
return v
}
keys := []string{settingSMTPHost, settingSMTPPassword, mail.SettingSMTPUsername, settingEmailEnabled, settingAIBaseURL, settingAIKey}
var saved []string
for _, key := range keys {
var v *string
_ = pool.QueryRow(ctx, `SELECT value FROM server_settings WHERE key = $1`, key).Scan(&v)
if v != nil {
saved = append(saved, key, *v)
}
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM server_settings WHERE key = ANY($1)`, keys)
for i := 0; i+1 < len(saved); i += 2 {
_, _ = pool.Exec(context.Background(), `INSERT INTO server_settings (key, value) VALUES ($1, $2)`, saved[i], saved[i+1])
}
})
set(restored, map[string]string{settingSMTPHost: "smtp.then.invalid", settingSMTPPassword: "then", mail.SettingSMTPUsername: "then-user", settingEmailEnabled: "true",
settingAIBaseURL: "https://ai.then.invalid", settingAIKey: "then-key",
config.AIProvidersSettingKey: `{"version":2,"providers":[{"id":"custom-then","type":"custom","name":"Then","base_url":"http://ai.then.invalid/v1","api_key":"then-key","capabilities":["chat","embeddings"]}]}`,
config.AITextProviderSettingKey: "custom-then", config.EmbeddingProviderSettingKey: "custom-then", "recommendations.enabled": "true"})
set(pool, map[string]string{settingSMTPHost: "smtp.now.invalid", settingSMTPPassword: "now", mail.SettingSMTPUsername: "now-user", settingEmailEnabled: "true"})
if _, err := pool.Exec(ctx, `DELETE FROM server_settings WHERE key = ANY($1)`, []string{settingAIBaseURL, settingAIKey}); err != nil {
t.Fatal(err)
}
cipher, err := secret.New([]byte("outbound-settings-test-key-0123456789"))
if err != nil {
t.Fatal(err)
}
// The AI providers go as a unit: no job stays bound to a cleared
// provider, and the next start converts no provider back from what is left.
checkAIReset := func(when string) {
t.Helper()
ai, err := serversettings.NewEncryptedSettingsRepo(serversettings.NewServerSettingsRepo(restored), cipher).
GetMany(ctx, append(config.AIConnectionSettingKeys(), "recommendations.enabled")...)
if err != nil {
t.Fatal(err)
}
if err := config.ValidateAIProviderBindings(ai); err != nil {
t.Fatalf("%s: %v; want no job bound to a cleared AI provider", when, err)
}
if writes, _, err := config.PlanAIProviderConversion(ai); err != nil || writes != nil {
t.Fatalf("%s: the next start converts %v (%v), want nothing", when, writes, err)
}
if registry, err := config.ParseAIProviderRegistry(ai[config.AIProvidersSettingKey]); err != nil || len(registry.Providers) != 0 {
t.Fatalf("%s: AI providers %v (%v), want none", when, registry.Providers, err)
}
}
if _, _, err := loader.CarryOutboundSettings(ctx, into, dsn, cipher); err != nil {
t.Fatal(err)
}
if read(settingSMTPHost) != "smtp.now.invalid" || read(settingSMTPPassword) != "now" || read(mail.SettingSMTPUsername) != "now-user" {
t.Fatalf("mail route %s / %s, want the live one", read(settingSMTPHost), read(settingSMTPPassword))
}
if read(settingAIBaseURL) != "https://ai.then.invalid" || read(settingAIKey) != "" {
t.Fatalf("a route the live server lacks: %s / %q, want the address kept and the key cleared", read(settingAIBaseURL), read(settingAIKey))
}
checkAIReset("a registry the live server lacks")
if !secret.IsEncrypted(read(config.AIProvidersSettingKey)) {
t.Fatalf("the emptied AI provider registry is stored as %q, want it encrypted", read(config.AIProvidersSettingKey))
}
// A restore with another key has already dropped the providers it cannot
// read; the bindings left behind still reset them.
if _, err := restored.Exec(ctx, `DELETE FROM server_settings WHERE key = $1`, config.AIProvidersSettingKey); err != nil {
t.Fatal(err)
}
set(restored, map[string]string{config.AISpeechProviderSettingKey: "custom-then"})
if _, cleared, err := loader.CarryOutboundSettings(ctx, into, "", nil); err != nil || read(settingSMTPPassword) != "" || read(settingEmailEnabled) != settingOff ||
!slices.Contains(cleared, "mail (SMTP)") || !slices.Contains(cleared, "AI providers") {
t.Fatalf("carrying none (%v): password %q, enabled %q, cleared %v; want it cleared and off", err, read(settingSMTPPassword), read(settingEmailEnabled), cleared)
}
checkAIReset("bindings without their providers")
}
2 changes: 1 addition & 1 deletion internal/backup/swap.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ func carryLiveState(ctx context.Context, ctl ControlDB, live string, loader Data
if !sameDeployment {
carryFrom = ""
}
if err := post.carry(ctx, loader, restoredURL, carryFrom, opts.LiveURL); err != nil {
if err := post.carry(ctx, loader, restoredURL, carryFrom, opts.LiveURL, opts.Cipher); err != nil {
return err
}
// This server's own settings describe it whichever deployment the
Expand Down
Loading