diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go index 6c667ed74..e84956bfc 100644 --- a/archivestore/credentials_test.go +++ b/archivestore/credentials_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" ) @@ -130,3 +131,119 @@ func TestArchiveIncompleteCredentialOverride(t *testing.T) { } } } + +// The server removes the AWS credential variables from its environment once the +// store is built, so that child processes do not inherit them. The SDK must have +// captured by then everything it needs to sign: the static keys, and for the +// container and web-identity providers the endpoint, token and file path they +// fetch credentials from later (their first fetch happens after the scrub here). +func TestReview318CredentialsSurviveEnvScrub(t *testing.T) { + scrubbed := []string{ + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", + } + for _, mode := range []string{"environment", "environment-alternate-names", "container", "web-identity"} { + t.Run(mode, func(t *testing.T) { + isolateAWSCredentials(t) + for _, key := range append([]string{ + "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN", + "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", "ARTEX_ARCHIVE_S3_PREFIX", "AWS_ENDPOINT_URL_STS", + }, scrubbed...) { + t.Setenv(key, "") + } + var envScrubbed atomic.Bool + var fetchedAfterScrub atomic.Int32 + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if envScrubbed.Load() { + fetchedAfterScrub.Add(1) + } + expiration := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + if mode == "container" { + if r.Header.Get("Authorization") != "fixture-container-token" { + t.Error("missing container authorization token") + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret", + "Token": "fixture-session", "Expiration": expiration, + }) + return + } + if err := r.ParseForm(); err != nil || r.PostForm.Get("Action") != "AssumeRoleWithWebIdentity" || + r.PostForm.Get("WebIdentityToken") != "fixture-web-identity-token" || + r.PostForm.Get("RoleArn") != "arn:aws:iam::123456789012:role/archive" { + t.Errorf("unexpected STS request: %v %v", err, r.PostForm) + } + w.Header().Set("Content-Type", "text/xml") + fmt.Fprintf(w, `fixture-keyfixture-secretfixture-session%s`, expiration) + })) + defer provider.Close() + switch mode { + case "environment": + t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "environment-alternate-names": + t.Setenv("AWS_ACCESS_KEY", "fixture-key") + t.Setenv("AWS_SECRET_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "container": + t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", provider.URL+"/credentials") + t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token") + case "web-identity": + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("fixture-web-identity-token"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenFile) + t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/archive") + t.Setenv("AWS_ENDPOINT_URL_STS", provider.URL) + } + var authorization, sessionToken atomic.Value + objects := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + authorization.Store(r.Header.Get("Authorization")) + sessionToken.Store(r.Header.Get("X-Amz-Security-Token")) + w.WriteHeader(http.StatusNotFound) + })) + defer objects.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", objects.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + store, err := FromEnv() + if err != nil || store == nil { + t.Fatalf("store from environment: %v", err) + } + for _, key := range scrubbed { + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + envScrubbed.Store(true) + + // HEAD is the first request of a download. The fixture answers 404, so + // the call fails; only the signature of the request matters here. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + checksum := strings.Repeat("a", 64) + err = store.Download(ctx, store.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1) + if err == nil { + t.Fatal("the fixture has no object: the download must fail") + } + got, _ := authorization.Load().(string) + if !strings.Contains(got, "Credential=fixture-key/") || !strings.Contains(got, "Signature=") { + t.Fatalf("request not signed with the captured key after the scrub: Authorization=%q (%v)", got, err) + } + if token, _ := sessionToken.Load().(string); token != "fixture-session" { + t.Fatalf("session token lost after the scrub: %q", token) + } + if lazy := mode == "container" || mode == "web-identity"; lazy && fetchedAfterScrub.Load() == 0 { + t.Fatal("the provider was expected to fetch its credentials after the scrub") + } + }) + } +} diff --git a/db/task_archives.go b/db/task_archives.go index bb9aae241..bf4fa3668 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,6 +60,7 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` + Storage string `json:"storage,omitempty"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -161,6 +162,12 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } + if item.ArchivePath != "" { + item.Storage = "local" + if strings.HasPrefix(item.ArchivePath, "s3://") { + item.Storage = "s3" + } + } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } @@ -449,7 +456,10 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error if cause != nil { message = cause.Error() } - _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1`, id, failed, message) + // Only a job still in its active state fails. The worker also reports a panic + // it recovered after the job committed: a ready archive marked archive_failed + // could be neither restored nor deleted. + _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1 AND state=$4`, id, failed, message, activeState) return err } diff --git a/db/task_archives_test.go b/db/task_archives_test.go index 727ffc7a3..c6cd934c2 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -204,6 +204,13 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if err != nil { t.Fatal(err) } + if archive.Storage != "" { + t.Fatalf("queued archive without a location has storage %q", archive.Storage) + } + encoded, err := json.Marshal(archive) + if err != nil || bytes.Contains(encoded, []byte(`"storage"`)) { + t.Fatalf("unknown storage must be omitted from JSON: %s, %v", encoded, err) + } claimed, err := d.ClaimTaskArchiveJob(t.Context()) if err != nil || claimed == nil || claimed.ID != archive.ID || claimed.State != Archiving { t.Fatalf("claim = %+v, %v", claimed, err) @@ -242,7 +249,7 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro t.Fatalf("hot compaction left nodes=%d assets=%d usage=%d", nodes, assets, usage) } ready, err := d.GetTaskArchive(archive.ID) - if err != nil || ready == nil || ready.State != ArchiveReady { + if err != nil || ready == nil || ready.State != ArchiveReady || ready.Storage != "local" { t.Fatalf("ready archive = %+v, %v", ready, err) } var stats map[string]any @@ -517,3 +524,45 @@ func TestFillArchiveDefaults(t *testing.T) { t.Fatalf("activity changed: %s err=%v", out, err) } } + +// FailTaskArchiveJob fails a job that is still running. A failure reported after +// the archive committed (the worker recovers a panic in the cleanup that follows +// the commit) must not turn a ready archive into archive_failed: it could then +// be neither restored nor deleted. +func TestFailTaskArchiveJobKeepsCommittedState(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + SkipPGUnavailable(t, err) + } + defer d.Close() + task, err := d.CreateTaskWithOptions("late archive failure", "must stay ready", TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + defer func() { + _, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID) + _ = d.DeleteTask(task.ID) + }() + if err := d.SetPaused(task.ID, true); err != nil { + t.Fatal(err) + } + queued, err := d.QueueTaskArchive(task.ID) + if err != nil { + t.Fatal(err) + } + for _, c := range []struct{ state, want string }{{Archiving, ArchiveFailed}, {ArchiveReady, ArchiveReady}} { + if _, err := d.Exec(`UPDATE task_archives SET state=$2,error='' WHERE id=$1`, queued.ID, c.state); err != nil { + t.Fatal(err) + } + if err := d.FailTaskArchiveJob(queued.ID, Archiving, errors.New("late failure")); err != nil { + t.Fatal(err) + } + item, err := d.GetTaskArchive(queued.ID) + if err != nil || item == nil { + t.Fatalf("archive %d: %v", queued.ID, err) + } + if item.State != c.want || (c.want == ArchiveFailed) != (item.Error == "late failure") { + t.Fatalf("failing a job in state %s: state=%s error=%q, want state %s", c.state, item.State, item.Error, c.want) + } + } +} diff --git a/server/env_scrub.go b/server/env_scrub.go index 308278514..63a548e71 100644 --- a/server/env_scrub.go +++ b/server/env_scrub.go @@ -5,13 +5,23 @@ import "os" // secretEnv lists the credentials read from the environment only at startup: // ARTEX_PG_DSN and ARTEX_SECRET_KEY by NewManager, the API keys by agent.FromEnv inside New. // agent.FromEnv also reads ARTEX_LLM_PROXY and ARTEX_LLM_BASE_URL: both are URLs that -// can carry credentials in their userinfo (http://user:pass@proxy:3128). The ARTEX_ARCHIVE_S3_* -// keys are not read by the server, but the container may carry them and the agent's -// shells would inherit them. +// can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive +// store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub, +// so the client can use them without passing them to agent child processes. +// +// Without that override the AWS SDK's default chain reads the AWS_* names below +// while the store is built (LoadDefaultConfig) and keeps what it needs: the static +// keys, or the container endpoint and token and the web-identity role and token +// file path it fetches credentials from later. Credentials that come from a shared +// credentials or config file, or from the instance metadata service, are not in +// the environment: the scrub does not hide them from child processes. var secretEnv = []string{ "ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN", + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", } // ScrubSecretEnv removes secretEnv from the process environment once startup has diff --git a/server/envseal_linux_test.go b/server/envseal_linux_test.go index deaafa7fb..3a50e0900 100644 --- a/server/envseal_linux_test.go +++ b/server/envseal_linux_test.go @@ -60,6 +60,18 @@ func TestSealedEnvironmentHandoff(t *testing.T) { "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID": "AKIATESTONLY", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY": "s3-secret-test", "ARTEX_ARCHIVE_S3_SESSION_TOKEN": "", + // Ambient AWS credentials the archive store reads when it has no override. + "AWS_ACCESS_KEY_ID": "AKIAAMBIENTTEST", + "AWS_ACCESS_KEY": "AKIAALTERNATETEST", + "AWS_SECRET_ACCESS_KEY": "ambient-secret-test", + "AWS_SECRET_KEY": "alternate-secret-test", + "AWS_SESSION_TOKEN": "", + "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:8080/credentials?id=1", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/v2/credentials/test", + "AWS_CONTAINER_AUTHORIZATION_TOKEN": "container-token-test", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": "/var/run/secrets/token-file", + "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token", + "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test", } env := os.Environ() for k, v := range want { diff --git a/server/server.go b/server/server.go index 09efe3aba..92eb284f5 100644 --- a/server/server.go +++ b/server/server.go @@ -24,6 +24,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -158,9 +159,11 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + archiveStore *archivestore.Store + archiveStoreErr error + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. @@ -302,8 +305,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi } else { log.Printf("[engine] no LLM provider configured — engine idle until set via /api/llm or env") } - // Both env readers have now run (NewManager for the DSN, FromEnv above for the - // keys). Drop the secrets before anything below can spawn a child process: + s.initTaskArchiveStore() + // Startup env readers have now run (NewManager for the DSN, the LLM provider + // and archive store for their credentials). Drop the secrets before anything + // below can spawn a child process: // restoring runtimes resumes workers that run shells, and MCP discovery starts // stdio servers. if s.scrubSecrets { diff --git a/server/sync_dir_other.go b/server/sync_dir_other.go new file mode 100644 index 000000000..54245c7a2 --- /dev/null +++ b/server/sync_dir_other.go @@ -0,0 +1,8 @@ +//go:build !unix + +package server + +// syncDir does nothing here: Windows does not support fsync on a directory +// handle (FlushFileBuffers fails on it), and the contents of the file written in +// the directory are already synced by the caller. +func syncDir(path string) error { return nil } diff --git a/server/sync_dir_unix.go b/server/sync_dir_unix.go new file mode 100644 index 000000000..8518f98bf --- /dev/null +++ b/server/sync_dir_unix.go @@ -0,0 +1,18 @@ +//go:build unix + +package server + +import ( + "errors" + "os" +) + +// syncDir flushes a directory, so that an entry just created or renamed in it +// survives a crash. +func syncDir(path string) error { + d, err := os.Open(path) + if err != nil { + return err + } + return errors.Join(d.Sync(), d.Close()) +} diff --git a/server/sync_dir_unix_test.go b/server/sync_dir_unix_test.go new file mode 100644 index 000000000..3d22034ae --- /dev/null +++ b/server/sync_dir_unix_test.go @@ -0,0 +1,18 @@ +//go:build unix + +package server + +import ( + "path/filepath" + "testing" +) + +func TestSyncDir(t *testing.T) { + dir := t.TempDir() + if err := syncDir(dir); err != nil { + t.Fatalf("sync of an existing directory: %v", err) + } + if err := syncDir(filepath.Join(dir, "missing")); err == nil { + t.Fatal("sync of a missing directory must fail") + } +} diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go new file mode 100644 index 000000000..6c4291550 --- /dev/null +++ b/server/task_archive_interrupt_test.go @@ -0,0 +1,403 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// Recreate persisted states at operation boundaries, then run the same recovery +// functions used at startup. PostgreSQL, files, and HTTP object storage are real +// isolated fixtures; no production failpoint hooks are needed. +func TestRemoteArchiveInterruptionRecovery(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + phases := []string{ + "staged_before_compression", "compression_partial", "compressed_before_journal", "journal_before_upload", + "uploaded_before_verification", "verified_before_database", + "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", + "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", + "delete_journal_before_database", "delete_database_before_object", + } + for _, phase := range phases { + t.Run(phase, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + store, objects := newRecoveryObjectStore(t) + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if phase == "compression_partial" { + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath+".partial", []byte("incomplete compressed package"), 0600); err != nil { + t.Fatal(err) + } + } + var original, compressed int64 + var checksum, uri, journal string + if phase != "staged_before_compression" && phase != "compression_partial" { + original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) + if err != nil { + t.Fatal(err) + } + uri = store.URI(id, job.ID, checksum) + } + if phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" { + journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + } + committed := phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" && + phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" + if phase == "uploaded_before_verification" { + data, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + objects.put(uri, data) + } else if phase == "verified_before_database" || committed { + if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { + t.Fatal(err) + } + } + if committed { + if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { + t.Fatal(err) + } + } + afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || + phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if phase == "package_cleaned_before_stage_cleanup" || afterArchive { + if err := os.Remove(packagePath); err != nil { + t.Fatal(err) + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + } + if afterArchive { + if err := stage.commit(); err != nil { + t.Fatal(err) + } + } + restored := false + remoteDeleted := false + if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { + t.Fatal(err) + } + parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") + if err := os.MkdirAll(parent, 0700); err != nil { + t.Fatal(err) + } + extracted, err := os.MkdirTemp(parent, "interrupted-") + if err != nil { + t.Fatal(err) + } + downloaded := filepath.Join(extracted, "download.tar.zst") + if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { + t.Fatal(err) + } + if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { + t.Fatal(err) + } + if phase != "restore_download_before_install" { + if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID, s.execPriv); err != nil { + t.Fatal(err) + } + } + if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { + t.Fatal(err) + } + restored = true + } + if phase == "restore_object_deleted_before_stub" { + if err := store.Delete(s.ctx, uri); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if deleting { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { + t.Fatal(err) + } + if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { + t.Fatal(err) + } + if phase == "delete_database_before_object" { + if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + if err := m.pg.RecoverTaskArchiveJobs(); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + wantSource := !committed || restored + data, fileErr := os.ReadFile(source) + if wantSource && (fileErr != nil || string(data) != "original task data") { + t.Fatalf("source data lost: %v", fileErr) + } + if !wantSource && !os.IsNotExist(fileErr) { + t.Fatalf("archived source resurrected: %v", fileErr) + } + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatalf("temporary package remains: %v", err) + } + if _, err := os.Stat(packagePath + ".partial"); !os.IsNotExist(err) { + t.Fatalf("partial package remains: %v", err) + } + if got, want := objects.has(uri), committed && !remoteDeleted; got != want { + t.Fatalf("remote object exists=%v, want=%v", got, want) + } + if committed && !deleting { + actual, err := m.pg.IsTaskArchiveRestored(job.ID) + if err != nil || actual != restored { + t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) + } + } + // A second startup must be harmless at every boundary. + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + }) + } +} + +type recoveryObjects struct { + mu sync.Mutex + data map[string][]byte +} + +func (o *recoveryObjects) put(uri string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // Test URIs always use the fixture bucket with path-style addressing. + o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) +} +func (o *recoveryObjects) has(uri string) bool { + o.mu.Lock() + defer o.mu.Unlock() + if uri == "" { + return false + } + _, ok := o.data["/test/"+uri[len("s3://test/"):]] + return ok +} +func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { + t.Helper() + objects := &recoveryObjects{data: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + objects.mu.Lock() + defer objects.mu.Unlock() + data, exists := objects.data[r.URL.Path] + w.Header().Set("ETag", `"recovery-etag"`) + switch r.Method { + case "PUT": + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("missing SSE") + } + objects.data[r.URL.Path], _ = io.ReadAll(r.Body) + case "HEAD": + if !exists { + w.WriteHeader(404) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + case "GET": + if !exists { + w.WriteHeader(404) + return + } + var start, end int + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { + w.WriteHeader(416) + return + } + w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(206) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + delete(objects.data, r.URL.Path) + w.WriteHeader(204) + default: + t.Errorf("unexpected object request %s", r.Method) + w.WriteHeader(400) + } + })) + t.Cleanup(srv.Close) + store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return store, objects +} + +// Neither a rollback conflict nor a temporary-file cleanup failure may discard +// the staging journal: recovery must be safe to retry after the obstruction ends. +func TestArchivePartialRecoveryRetainsJournalOnFailure(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + for _, failure := range []string{"rollback-conflict", "cleanup-failure"} { + t.Run(failure, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("partial recovery "+failure, "retain source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + partial := taskArchivePath(m.dir, job.ID, task.ID) + ".partial" + if err := os.MkdirAll(filepath.Dir(partial), 0700); err != nil { + t.Fatal(err) + } + var obstruction string + if failure == "rollback-conflict" { + if len(stage.journal.Moves) == 0 { + t.Fatal("no staged source") + } + obstruction = stage.journal.Moves[0].Source + if err := os.MkdirAll(obstruction, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(partial, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + } else { + if err := os.Mkdir(partial, 0700); err != nil { + t.Fatal(err) + } + obstruction = filepath.Join(partial, "block-remove") + if err := os.WriteFile(obstruction, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err == nil { + t.Fatal("expected recovery obstruction") + } + if _, err := os.Stat(filepath.Join(stage.root, "journal.json")); err != nil { + t.Fatal("lost retry journal", err) + } + if _, err := os.Stat(partial); err != nil { + t.Fatal("removed partial before successful recovery", err) + } + if failure == "cleanup-failure" { + if got, err := os.ReadFile(source); err != nil || string(got) != "original" { + t.Fatal("source was not rolled back", err) + } + } + if err := os.RemoveAll(obstruction); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(source); err != nil || string(got) != "original" { + t.Fatal("source lost on retry", err) + } + for _, path := range []string{stage.root, partial} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("recovery left %s: %v", path, err) + } + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/server/task_archive_journal_test.go b/server/task_archive_journal_test.go new file mode 100644 index 000000000..486bdf422 --- /dev/null +++ b/server/task_archive_journal_test.go @@ -0,0 +1,27 @@ +package server + +import ( + "encoding/json" + "os" + "testing" +) + +// Writing the journal must succeed on every platform the server builds for. It +// needs no database: on Windows the directory sync it ended with failed, so no +// remote archive or remote deletion could start. +func TestRemoteArchiveJournalWrite(t *testing.T) { + s := &Server{m: &Manager{dir: t.TempDir()}} + want := remoteArchiveJournal{ArchiveID: 7, URI: "s3://bucket/tasks/package.tar.zst", LocalPath: "package.tar.zst", Operation: "upload"} + name, err := s.writeRemoteArchiveJournal(want) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + var got remoteArchiveJournal + if err := json.Unmarshal(raw, &got); err != nil || got != want { + t.Fatalf("journal content %q: %v", raw, err) + } +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index 6efef3fda..496a740af 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -193,6 +193,16 @@ func (s *taskArchiveFileStage) rollback() error { if s == nil || s.done { return nil } + err := s.rollbackFiles() + if err == nil { + err = os.RemoveAll(s.root) + } + s.done = true + return err +} + +// Keep the journal until the caller has also cleaned temporary packages. +func (s *taskArchiveFileStage) rollbackFiles() error { var errs []error for i := len(s.journal.Moves) - 1; i >= 0; i-- { move := s.journal.Moves[i] @@ -218,12 +228,6 @@ func (s *taskArchiveFileStage) rollback() error { errs = append(errs, err) } } - if len(errs) == 0 { - if err := os.RemoveAll(s.root); err != nil { - errs = append(errs, err) - } - } - s.done = true return errors.Join(errs...) } @@ -464,18 +468,33 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.State == pgdb.ArchiveReady { + if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } continue } stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} - if err := stage.rollback(); err != nil { + if err := stage.rollbackFiles(); err != nil { errs = append(errs, err) + continue } - if archive != nil && archive.ArchivePath != "" { - _ = os.Remove(archive.ArchivePath) + // The process can exit after compression but before an upload journal or + // database location exists. Remove that temporary package only after the + // source-file rollback succeeds, for both local and remote backends. + packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) + if err := validateArchivePath(dataDir, packagePath); err != nil { + errs = append(errs, err) + continue + } + // A crash inside compression leaves .partial, before any upload journal. + // Retain this staging journal on cleanup failure so startup can retry. + if err := errors.Join(removeLocalArchivePackage(packagePath), removeLocalArchivePackage(packagePath+".partial")); err != nil { + errs = append(errs, err) + continue + } + if err := os.RemoveAll(root); err != nil { + errs = append(errs, err) } } return errors.Join(errs...) diff --git a/server/task_archive_panic_test.go b/server/task_archive_panic_test.go new file mode 100644 index 000000000..1f3c7cae2 --- /dev/null +++ b/server/task_archive_panic_test.go @@ -0,0 +1,131 @@ +package server + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// An S3-compatible endpoint that answers CreateMultipartUpload with 200 and no +// UploadId makes the SDK's upload manager dereference a nil pointer. The panic +// happens in the archive worker goroutine: it must fail that one job like any +// other upload error, keep the source data, and leave the process running. +func TestReview318ArchiveWorkerSurvivesUploadPanic(t *testing.T) { + skipWithoutDSN(t) + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + + var creates, unexpected atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + switch { + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + creates.Add(1) + w.Header().Set("Content-Type", "application/xml") + _, _ = io.WriteString(w, `test`) + case r.Method == http.MethodDelete: + // The worker's deferred cleanup of the upload that never committed. + w.WriteHeader(http.StatusNoContent) + default: + unexpected.Add(1) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + + task, err := m.CreateTask("multipart panic", "keep the source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + // The package must exceed the 8 MiB part size of archivestore.Upload for the + // upload manager to choose multipart: 9 MiB of random bytes do not compress. + source := make([]byte, 9<<20) + if _, err := rand.Read(source); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "large.bin") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, source, 0600); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + + // Run the server's own worker goroutine, as New starts it. + workerCtx, stopWorker := context.WithCancel(context.Background()) + s.ctx = workerCtx + s.startTaskArchiveWorker() + defer func() { stopWorker(); s.archiveWG.Wait() }() + + deadline := time.Now().Add(2 * time.Minute) + var item *pgdb.TaskArchive + for { + item, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if item == nil { + t.Fatal("archive job disappeared") + } + if item.State != pgdb.ArchiveQueued && item.State != pgdb.Archiving { + break + } + if time.Now().After(deadline) { + t.Fatalf("archive job still %s (%s)", item.State, item.Phase) + } + time.Sleep(50 * time.Millisecond) + } + if creates.Load() != 1 { + t.Fatalf("expected one CreateMultipartUpload, got %d: the package did not take the multipart path", creates.Load()) + } + if item.State != pgdb.ArchiveFailed || !strings.Contains(item.Error, "panic") { + t.Fatalf("job must be archive_failed with the panic as its error: state=%s error=%q", item.State, item.Error) + } + if item.ArchivePath != "" || item.SHA256 != "" || item.ArchivedAt != nil { + t.Fatalf("a location was committed for a failed archive: path=%q sha256=%q archived_at=%v", item.ArchivePath, item.SHA256, item.ArchivedAt) + } + got, err := os.ReadFile(file) + if err != nil || sha256.Sum256(got) != sha256.Sum256(source) { + t.Fatalf("source file lost or changed after the panic: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatalf("task lost after the panic: %v", err) + } + if unexpected.Load() != 0 { + t.Fatalf("%d unexpected S3 requests", unexpected.Load()) + } +} diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go new file mode 100644 index 000000000..ee6b0ea12 --- /dev/null +++ b/server/task_archive_remote.go @@ -0,0 +1,163 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + pgdb "github.com/Autumn-27/artex/db" +) + +// An outbox bridges the local PostgreSQL transaction and object storage. Never +// delete the only package before metadata permits it. Unfinished cleanup is +// retried after a restart (and before subsequent jobs). +type remoteArchiveJournal struct { + ArchiveID int64 `json:"archive_id"` + URI string `json:"uri"` + LocalPath string `json:"local_path,omitempty"` + Operation string `json:"operation"` +} + +func (s *Server) validateStoredArchive(uri string) error { + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Validate(uri) + } + return validateArchivePath(s.m.dir, uri) +} + +func removeLocalArchivePackage(filename string) error { + if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (s *Server) removeStoredArchive(uri string) error { + if err := s.validateStoredArchive(uri); err != nil { + return err + } + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Delete(s.ctx, uri) + } + return removeLocalArchivePackage(uri) +} + +func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + if err := os.MkdirAll(dir, archiveDirMode); err != nil { + return "", err + } + f, err := os.CreateTemp(dir, ".journal-") + if err != nil { + return "", err + } + tmp := f.Name() + defer os.Remove(tmp) + encodeErr := json.NewEncoder(f).Encode(j) + syncErr := f.Sync() + if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { + return "", err + } + name := tmp + ".json" + if err := os.Rename(tmp, name); err != nil { + return "", err + } + // Persist the directory entry before an upload or database deletion starts. + if err := syncDir(dir); err != nil { + return "", err + } + // The .remote directory itself may have just been created. + return name, syncDir(filepath.Dir(dir)) +} + +func (s *Server) recoverRemoteArchiveJournals() error { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + name := filepath.Join(dir, entry.Name()) + if err := s.recoverRemoteArchiveJournal(name); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *Server) recoverRemoteArchiveJournal(name string) error { + raw, err := os.ReadFile(name) + if err != nil { + return err + } + var j remoteArchiveJournal + if err := json.Unmarshal(raw, &j); err != nil { + return err + } + if j.ArchiveID <= 0 { + return errors.New("invalid remote archive journal ID") + } + if err := s.archiveStore.Validate(j.URI); err != nil { + return err + } + item, err := s.m.pg.GetTaskArchive(j.ArchiveID) + if err != nil { + return err + } + switch j.Operation { + case "upload": + if item == nil || item.ArchivePath != j.URI { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { + return err + } + // An old failed upload may outlive a retry after switching back to + // local storage. The retry reuses this pathname: keep its committed + // package instead of treating it as the old upload's temporary copy. + if item == nil || item.ArchivePath != j.LocalPath { + if err := removeLocalArchivePackage(j.LocalPath); err != nil { + return err + } + } + case "delete": + if item == nil { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + default: + return fmt.Errorf("unknown remote archive operation %q", j.Operation) + } + return os.Remove(name) +} + +func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) + if err != nil { + return err + } + if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + // A connection failure can leave the transaction's commit ambiguous. + // Recovery consults PostgreSQL before deciding whether to delete. + return err + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + // The task is gone; the durable outbox retries the remaining object cleanup. + log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) + } + return nil +} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go new file mode 100644 index 000000000..ac2a44d43 --- /dev/null +++ b/server/task_archive_remote_test.go @@ -0,0 +1,419 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +func TestRemoteArchiveLifecycle(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + s.ctx = context.Background() + var mu sync.Mutex + objects := map[string][]byte{} + corrupt, failDelete := false, false + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "PUT": + objects[r.URL.Path], _ = io.ReadAll(r.Body) + w.Header().Set("ETag", `"test"`) + case "HEAD": + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) + case "GET": + data := objects[r.URL.Path] + if corrupt && len(data) > 0 { + data = bytes.Clone(data) + data[0] ^= 0xff + } + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + if failDelete { + w.WriteHeader(403) + return + } + delete(objects, r.URL.Path) + w.WriteHeader(204) + default: + w.WriteHeader(400) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + // A configured-but-invalid backend must fail before moving any source data. + s.archiveStoreErr = errors.New("invalid fixture configuration") + if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { + t.Fatalf("configuration error was not returned: %v", err) + } + if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { + t.Fatalf("bad configuration lost source: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatal("bad configuration lost task", err) + } + s.archiveStoreErr = nil + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + // A corrupt readback must roll back filesystem staging and leave PG intact. + mu.Lock() + corrupt = true + mu.Unlock() + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected integrity failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("upload failure lost source") + } + if task, err := m.pg.GetTask(id); err != nil || task == nil { + t.Fatal("upload failure lost task") + } + mu.Lock() + corrupt = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + job, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { + t.Fatalf("unexpected archive %+v", job) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("workspace not released") + } + if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { + t.Fatal("local package retained") + } + // Simulate the crash window after commit: cleanup keeps the remote object. + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + // A corrupted download must leave the archive and its remote package intact. + mu.Lock() + corrupt = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected corrupted restore to fail") + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("corrupted archive installed workspace files") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { + t.Fatalf("corrupted archive restored database: %v %v", restored, err) + } + mu.Lock() + if len(objects) == 0 { + t.Error("corrupted restore deleted the only remote package") + } + corrupt = false + mu.Unlock() + // Restoration commits files and PG first. Failed remote deletion is retryable. + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected remote delete failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("restored content differs") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { + t.Fatalf("PG restore %v %v", restored, err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { + t.Fatal("restore did not consume stub") + } + var count int + if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { + t.Fatalf("activity restore count=%d err=%v", count, err) + } + // Re-archive, then crash/retry the deletion outbox after PG removed its stub. + job, err = m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + mu.Lock() + remaining := len(objects) + mu.Unlock() + if remaining != 0 { + t.Fatalf("left %d objects", remaining) + } +} + +// Once PostgreSQL has committed an archive location and checksum, a subsequent +// queued operation must not make startup recovery resurrect the old workspace. +func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(taskID) + if err != nil { + t.Fatal(err) + } + for _, backend := range []string{"local", "s3"} { + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(backend+"/"+state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { + t.Fatal(err) + } + location := packagePath + if backend == "s3" { + location = "s3://test/tasks/package.tar.zst" + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if backend == "local" && committed { + if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { + t.Fatal("committed local package lost", err) + } + } else if !committed { + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatal("uncommitted temporary package retained", err) + } + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } + } +} + +// Failed remote cleanup can survive switching to local storage and retrying the +// same archive ID. Its journal must never remove the newly committed local copy. +func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("local retry", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + filename := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + var deletes atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer remote.Close() + store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), + LocalPath: filename, Operation: "upload", + }) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { + t.Fatalf("old upload cleanup removed local retry: %v", err) + } + if deletes.Load() != 1 { + t.Fatal("uncommitted remote object was not removed") + } + if _, err := os.Stat(journal); !os.IsNotExist(err) { + t.Fatalf("finished journal remains: %v", err) + } +} diff --git a/server/task_archive_startup_test.go b/server/task_archive_startup_test.go new file mode 100644 index 000000000..def27d071 --- /dev/null +++ b/server/task_archive_startup_test.go @@ -0,0 +1,175 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestArchiveStoreLoadsCredentialsBeforeSecretScrub(t *testing.T) { + skipWithoutDSN(t) + // New scrubs all startup secrets; restore the test database/key environment + // afterward so the remaining package tests keep their isolated configuration. + for _, key := range secretEnv { + t.Setenv(key, os.Getenv(key)) + } + var signed atomic.Bool + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + signed.Store(strings.Contains(r.Header.Get("Authorization"), "Credential=archive-fixture-key/") && + r.Header.Get("X-Amz-Security-Token") == "archive-fixture-session") + w.WriteHeader(http.StatusNoContent) + })) + defer endpoint.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "archive-fixture-key") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "archive-fixture-secret") + t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "archive-fixture-session") + t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256") + t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "") + // If initialization happens after the scrub, the SDK would silently select + // this different ambient identity instead of the dedicated override. + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CA_BUNDLE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Exercise real startup without leaving archive/scheduler workers alive. + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub()) + s.archiveWG.Wait() + if s.archiveStoreErr != nil || s.archiveStore == nil { + t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr) + } + for _, key := range []string{"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN"} { + if _, present := os.LookupEnv(key); present { + t.Fatalf("%s remains in the child-process environment", key) + } + } + requestCtx, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + if err := s.archiveStore.Delete(requestCtx, s.archiveStore.URI(1, 2, "startup-probe")); err != nil { + t.Fatal(err) + } + if !signed.Load() { + t.Fatal("archive client lost its dedicated credentials during startup scrub") + } +} + +// The archive store may take its identity from the ambient AWS credential +// variables instead of the ARTEX_ARCHIVE_S3_* override. Those variables are as +// sensitive as the override: after startup no child process may inherit them, +// and the S3 client built before the scrub must keep signing with them. +func TestReview318AmbientAWSCredentialsScrubbedAfterArchiveInit(t *testing.T) { + skipWithoutDSN(t) + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh in PATH") + } + awsNames := []string{ + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", + } + // New scrubs all startup secrets; t.Setenv restores them for the tests that follow. + for _, key := range secretEnv { + t.Setenv(key, os.Getenv(key)) + } + for _, key := range awsNames { + t.Setenv(key, "probe-"+key) + } + var authorization, sessionToken atomic.Value + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + authorization.Store(r.Header.Get("Authorization")) + sessionToken.Store(r.Header.Get("X-Amz-Security-Token")) + w.WriteHeader(http.StatusNotFound) + })) + defer endpoint.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + // No dedicated override: the SDK's default chain reads the AWS_* variables. + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "") + t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "") + t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256") + t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "") + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "ambient-fixture-session") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CA_BUNDLE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Exercise real startup without leaving archive/scheduler workers alive. + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub()) + s.archiveWG.Wait() + if s.archiveStoreErr != nil || s.archiveStore == nil { + t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr) + } + + out, err := exec.Command(sh, "-c", "env").Output() + if err != nil { + t.Fatal(err) + } + env := "\n" + string(out) + for _, key := range awsNames { + if strings.Contains(env, "\n"+key+"=") { + t.Errorf("%s still reaches child processes after startup", key) + } + } + + // HEAD is the first request of a download. The fixture answers 404, so the + // call fails; only the signature of the request matters here. + requestCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + checksum := strings.Repeat("a", 64) + err = s.archiveStore.Download(requestCtx, s.archiveStore.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1) + if err == nil { + t.Fatal("the fixture has no object: the download must fail") + } + got, _ := authorization.Load().(string) + if !strings.Contains(got, "Credential=ambient-fixture-key/") || !strings.Contains(got, "Signature=") { + t.Fatalf("archive client did not sign with the ambient key after the scrub: Authorization=%q (%v)", got, err) + } + if token, _ := sessionToken.Load().(string); token != "ambient-fixture-session" { + t.Fatalf("archive client lost the ambient session token after the scrub: %q", token) + } +} diff --git a/server/task_archives.go b/server/task_archives.go index 8a6163247..daca52f9a 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -9,11 +9,13 @@ import ( "net/http" "os" "path/filepath" + "runtime/debug" "strconv" "strings" "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -33,6 +35,19 @@ type archiveBatchItem struct { Archive int64 `json:"archive_id,omitempty"` } +// initTaskArchiveStore captures credentials before New scrubs the process environment. +func (s *Server) initTaskArchiveStore() { + if s.m == nil || s.m.pg == nil { + return + } + s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() + if s.archiveStoreErr != nil { + log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) + } else if s.archiveStore != nil { + log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") + } +} + func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return @@ -88,22 +103,19 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { + if s.ctx.Err() != nil { + return s.ctx.Err() + } + // Retry deferred remote cleanup before claiming more work. Failed cleanup + // keeps its durable journal and does not block unrelated local archives. + if err := s.recoverRemoteArchiveJournals(); err != nil { + log.Printf("[task-archive] deferred object cleanup: %v", err) + } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err } - var runErr error - switch job.State { - case pgdb.Archiving: - runErr = s.archiveTask(job) - case pgdb.Restoring: - runErr = s.restoreTaskArchive(job) - case pgdb.Deleting: - runErr = s.deleteTaskArchive(job) - default: - runErr = fmt.Errorf("unknown claimed archive state %q", job.State) - } - if runErr != nil { + if runErr := s.runClaimedTaskArchiveJob(job); runErr != nil { if failErr := s.m.pg.FailTaskArchiveJob(job.ID, job.State, runErr); failErr != nil { return errors.Join(runErr, failErr) } @@ -114,7 +126,35 @@ func (s *Server) runOneTaskArchiveJob() error { return nil } +// runClaimedTaskArchiveJob runs one claimed job and turns a panic below it into +// an ordinary job failure. The worker goroutine has no other recover: a panic +// there (the S3 upload manager dereferences a nil UploadId when an endpoint +// answers CreateMultipartUpload without one) would end the whole server with the +// job half done. The job's deferred rollbacks have already run when the panic is +// recovered here, so the caller fails the job with its source data in place. +func (s *Server) runClaimedTaskArchiveJob(job *pgdb.TaskArchive) (runErr error) { + defer func() { + if r := recover(); r != nil { + log.Printf("[task-archive] archive job %d (%s) panicked: %v\n%s", job.ID, job.State, r, debug.Stack()) + runErr = fmt.Errorf("archive worker panic: %v", r) + } + }() + switch job.State { + case pgdb.Archiving: + return s.archiveTask(job) + case pgdb.Restoring: + return s.restoreTaskArchive(job) + case pgdb.Deleting: + return s.deleteTaskArchive(job) + default: + return fmt.Errorf("unknown claimed archive state %q", job.State) + } +} + func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { + if s.archiveStoreErr != nil { + return s.archiveStoreErr + } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -199,6 +239,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() + storedPath := archivePath + var remoteJournal string + if s.archiveStore != nil { + storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) + remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", + }) + if err != nil { + return err + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) + if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { + return err + } + } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -213,11 +268,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = false + removePackage = s.archiveStore != nil + if remoteJournal != "" { + // The journal also covers a crash after database commit but before local + // package removal. Keep it if cleanup fails, so the worker can retry. + if err := removeLocalArchivePackage(archivePath); err == nil { + _ = os.Remove(remoteJournal) + } + } if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -239,7 +301,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -247,7 +309,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -266,7 +328,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { + packagePath := job.ArchivePath + if strings.HasPrefix(packagePath, "s3://") { + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) + packagePath = filepath.Join(extracted, "download.tar.zst") + if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { + return err + } + } + if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -356,7 +426,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -393,9 +463,12 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } + if strings.HasPrefix(job.ArchivePath, "s3://") { + return s.deleteRemoteTaskArchive(job) + } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil {