From bdd14ff7f02d544e691593c77a0829b67b2f8cc4 Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Wed, 18 Nov 2026 02:05:18 +0000 Subject: [PATCH 1/7] feat: wire remote task archive lifecycle and recovery --- db/task_archives.go | 5 + server/server.go | 9 +- server/task_archive_interrupt_test.go | 291 ++++++++++++++++++ server/task_archive_package.go | 13 +- server/task_archive_remote.go | 171 +++++++++++ server/task_archive_remote_test.go | 419 ++++++++++++++++++++++++++ server/task_archives.go | 65 +++- 7 files changed, 960 insertions(+), 13 deletions(-) create mode 100644 server/task_archive_interrupt_test.go create mode 100644 server/task_archive_remote.go create mode 100644 server/task_archive_remote_test.go diff --git a/db/task_archives.go b/db/task_archives.go index bb9aae241..9f07341ce 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,6 +60,7 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` + Storage string `json:"storage"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } + item.Storage = "local" + if strings.HasPrefix(item.ArchivePath, "s3://") { + item.Storage = "s3" + } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } diff --git a/server/server.go b/server/server.go index baf4423b1..7d750d325 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -144,9 +145,11 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + archiveStore *archivestore.Store + archiveStoreErr error + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go new file mode 100644 index 000000000..2f4a44e5b --- /dev/null +++ b/server/task_archive_interrupt_test.go @@ -0,0 +1,291 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// Recreate persisted states at operation boundaries, then run the same recovery +// functions used at startup. PostgreSQL, files, and HTTP object storage are real +// isolated fixtures; no production failpoint hooks are needed. +func TestRemoteArchiveInterruptionRecovery(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + phases := []string{ + "staged_before_compression", "compressed_before_journal", "journal_before_upload", + "uploaded_before_verification", "verified_before_database", + "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", + "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", + "delete_journal_before_database", "delete_database_before_object", + } + for _, phase := range phases { + t.Run(phase, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + store, objects := newRecoveryObjectStore(t) + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + var original, compressed int64 + var checksum, uri, journal string + if phase != "staged_before_compression" { + original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) + if err != nil { + t.Fatal(err) + } + uri = store.URI(id, job.ID, checksum) + } + if phase != "staged_before_compression" && phase != "compressed_before_journal" { + journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + } + committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && + phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" + if phase == "uploaded_before_verification" { + data, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + objects.put(uri, data) + } else if phase == "verified_before_database" || committed { + if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { + t.Fatal(err) + } + } + if committed { + if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { + t.Fatal(err) + } + } + afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || + phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if phase == "package_cleaned_before_stage_cleanup" || afterArchive { + if err := os.Remove(packagePath); err != nil { + t.Fatal(err) + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + } + if afterArchive { + if err := stage.commit(); err != nil { + t.Fatal(err) + } + } + restored := false + remoteDeleted := false + if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { + t.Fatal(err) + } + parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") + if err := os.MkdirAll(parent, 0700); err != nil { + t.Fatal(err) + } + extracted, err := os.MkdirTemp(parent, "interrupted-") + if err != nil { + t.Fatal(err) + } + downloaded := filepath.Join(extracted, "download.tar.zst") + if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { + t.Fatal(err) + } + if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { + t.Fatal(err) + } + if phase != "restore_download_before_install" { + if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { + t.Fatal(err) + } + } + if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { + t.Fatal(err) + } + restored = true + } + if phase == "restore_object_deleted_before_stub" { + if err := store.Delete(s.ctx, uri); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if deleting { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { + t.Fatal(err) + } + if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { + t.Fatal(err) + } + if phase == "delete_database_before_object" { + if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + if err := m.pg.RecoverTaskArchiveJobs(); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + wantSource := !committed || restored + data, fileErr := os.ReadFile(source) + if wantSource && (fileErr != nil || string(data) != "original task data") { + t.Fatalf("source data lost: %v", fileErr) + } + if !wantSource && !os.IsNotExist(fileErr) { + t.Fatalf("archived source resurrected: %v", fileErr) + } + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatalf("temporary package remains: %v", err) + } + if got, want := objects.has(uri), committed && !remoteDeleted; got != want { + t.Fatalf("remote object exists=%v, want=%v", got, want) + } + if committed && !deleting { + actual, err := m.pg.IsTaskArchiveRestored(job.ID) + if err != nil || actual != restored { + t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) + } + } + // A second startup must be harmless at every boundary. + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + }) + } +} + +type recoveryObjects struct { + mu sync.Mutex + data map[string][]byte +} + +func (o *recoveryObjects) put(uri string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // Test URIs always use the fixture bucket with path-style addressing. + o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) +} +func (o *recoveryObjects) has(uri string) bool { + o.mu.Lock() + defer o.mu.Unlock() + if uri == "" { + return false + } + _, ok := o.data["/test/"+uri[len("s3://test/"):]] + return ok +} +func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { + t.Helper() + objects := &recoveryObjects{data: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + objects.mu.Lock() + defer objects.mu.Unlock() + data, exists := objects.data[r.URL.Path] + w.Header().Set("ETag", `"recovery-etag"`) + switch r.Method { + case "PUT": + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("missing SSE") + } + objects.data[r.URL.Path], _ = io.ReadAll(r.Body) + case "HEAD": + if !exists { + w.WriteHeader(404) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + case "GET": + if !exists { + w.WriteHeader(404) + return + } + var start, end int + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { + w.WriteHeader(416) + return + } + w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(206) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + delete(objects.data, r.URL.Path) + w.WriteHeader(204) + default: + t.Errorf("unexpected object request %s", r.Method) + w.WriteHeader(400) + } + })) + t.Cleanup(srv.Close) + store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return store, objects +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index f23c4c8bc..e365ea3f5 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.State == pgdb.ArchiveReady { + if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } @@ -437,9 +437,16 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} if err := stage.rollback(); err != nil { errs = append(errs, err) + continue } - if archive != nil && archive.ArchivePath != "" { - _ = os.Remove(archive.ArchivePath) + // The process can exit after compression but before an upload journal or + // database location exists. Remove that temporary package only after the + // source-file rollback succeeds, for both local and remote backends. + packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) + if err := validateArchivePath(dataDir, packagePath); err != nil { + errs = append(errs, err) + } else if err := removeLocalArchivePackage(packagePath); err != nil { + errs = append(errs, err) } } return errors.Join(errs...) diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go new file mode 100644 index 000000000..87e230f23 --- /dev/null +++ b/server/task_archive_remote.go @@ -0,0 +1,171 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + pgdb "github.com/Autumn-27/artex/db" +) + +// An outbox bridges the local PostgreSQL transaction and object storage. Never +// delete the only package before metadata permits it. Unfinished cleanup is +// retried after a restart (and before subsequent jobs). +type remoteArchiveJournal struct { + ArchiveID int64 `json:"archive_id"` + URI string `json:"uri"` + LocalPath string `json:"local_path,omitempty"` + Operation string `json:"operation"` +} + +func (s *Server) validateStoredArchive(uri string) error { + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Validate(uri) + } + return validateArchivePath(s.m.dir, uri) +} + +func removeLocalArchivePackage(filename string) error { + if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (s *Server) removeStoredArchive(uri string) error { + if err := s.validateStoredArchive(uri); err != nil { + return err + } + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Delete(s.ctx, uri) + } + return removeLocalArchivePackage(uri) +} + +func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + if err := os.MkdirAll(dir, archiveDirMode); err != nil { + return "", err + } + f, err := os.CreateTemp(dir, ".journal-") + if err != nil { + return "", err + } + tmp := f.Name() + defer os.Remove(tmp) + encodeErr := json.NewEncoder(f).Encode(j) + syncErr := f.Sync() + if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { + return "", err + } + name := tmp + ".json" + if err := os.Rename(tmp, name); err != nil { + return "", err + } + // Persist the directory entry before an upload or database deletion starts. + d, err := os.Open(dir) + if err != nil { + return "", err + } + if err := errors.Join(d.Sync(), d.Close()); err != nil { + return "", err + } + // The .remote directory itself may have just been created. + parent, err := os.Open(filepath.Dir(dir)) + if err != nil { + return "", err + } + return name, errors.Join(parent.Sync(), parent.Close()) +} + +func (s *Server) recoverRemoteArchiveJournals() error { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + name := filepath.Join(dir, entry.Name()) + if err := s.recoverRemoteArchiveJournal(name); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *Server) recoverRemoteArchiveJournal(name string) error { + raw, err := os.ReadFile(name) + if err != nil { + return err + } + var j remoteArchiveJournal + if err := json.Unmarshal(raw, &j); err != nil { + return err + } + if j.ArchiveID <= 0 { + return errors.New("invalid remote archive journal ID") + } + if err := s.archiveStore.Validate(j.URI); err != nil { + return err + } + item, err := s.m.pg.GetTaskArchive(j.ArchiveID) + if err != nil { + return err + } + switch j.Operation { + case "upload": + if item == nil || item.ArchivePath != j.URI { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { + return err + } + // An old failed upload may outlive a retry after switching back to + // local storage. The retry reuses this pathname: keep its committed + // package instead of treating it as the old upload's temporary copy. + if item == nil || item.ArchivePath != j.LocalPath { + if err := removeLocalArchivePackage(j.LocalPath); err != nil { + return err + } + } + case "delete": + if item == nil { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + default: + return fmt.Errorf("unknown remote archive operation %q", j.Operation) + } + return os.Remove(name) +} + +func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) + if err != nil { + return err + } + if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + // A connection failure can leave the transaction's commit ambiguous. + // Recovery consults PostgreSQL before deciding whether to delete. + return err + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + // The task is gone; the durable outbox retries the remaining object cleanup. + log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) + } + return nil +} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go new file mode 100644 index 000000000..ac2a44d43 --- /dev/null +++ b/server/task_archive_remote_test.go @@ -0,0 +1,419 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +func TestRemoteArchiveLifecycle(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + s.ctx = context.Background() + var mu sync.Mutex + objects := map[string][]byte{} + corrupt, failDelete := false, false + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "PUT": + objects[r.URL.Path], _ = io.ReadAll(r.Body) + w.Header().Set("ETag", `"test"`) + case "HEAD": + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) + case "GET": + data := objects[r.URL.Path] + if corrupt && len(data) > 0 { + data = bytes.Clone(data) + data[0] ^= 0xff + } + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + if failDelete { + w.WriteHeader(403) + return + } + delete(objects, r.URL.Path) + w.WriteHeader(204) + default: + w.WriteHeader(400) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + // A configured-but-invalid backend must fail before moving any source data. + s.archiveStoreErr = errors.New("invalid fixture configuration") + if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { + t.Fatalf("configuration error was not returned: %v", err) + } + if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { + t.Fatalf("bad configuration lost source: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatal("bad configuration lost task", err) + } + s.archiveStoreErr = nil + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + // A corrupt readback must roll back filesystem staging and leave PG intact. + mu.Lock() + corrupt = true + mu.Unlock() + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected integrity failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("upload failure lost source") + } + if task, err := m.pg.GetTask(id); err != nil || task == nil { + t.Fatal("upload failure lost task") + } + mu.Lock() + corrupt = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + job, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { + t.Fatalf("unexpected archive %+v", job) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("workspace not released") + } + if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { + t.Fatal("local package retained") + } + // Simulate the crash window after commit: cleanup keeps the remote object. + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + // A corrupted download must leave the archive and its remote package intact. + mu.Lock() + corrupt = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected corrupted restore to fail") + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("corrupted archive installed workspace files") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { + t.Fatalf("corrupted archive restored database: %v %v", restored, err) + } + mu.Lock() + if len(objects) == 0 { + t.Error("corrupted restore deleted the only remote package") + } + corrupt = false + mu.Unlock() + // Restoration commits files and PG first. Failed remote deletion is retryable. + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected remote delete failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("restored content differs") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { + t.Fatalf("PG restore %v %v", restored, err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { + t.Fatal("restore did not consume stub") + } + var count int + if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { + t.Fatalf("activity restore count=%d err=%v", count, err) + } + // Re-archive, then crash/retry the deletion outbox after PG removed its stub. + job, err = m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + mu.Lock() + remaining := len(objects) + mu.Unlock() + if remaining != 0 { + t.Fatalf("left %d objects", remaining) + } +} + +// Once PostgreSQL has committed an archive location and checksum, a subsequent +// queued operation must not make startup recovery resurrect the old workspace. +func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(taskID) + if err != nil { + t.Fatal(err) + } + for _, backend := range []string{"local", "s3"} { + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(backend+"/"+state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { + t.Fatal(err) + } + location := packagePath + if backend == "s3" { + location = "s3://test/tasks/package.tar.zst" + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if backend == "local" && committed { + if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { + t.Fatal("committed local package lost", err) + } + } else if !committed { + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatal("uncommitted temporary package retained", err) + } + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } + } +} + +// Failed remote cleanup can survive switching to local storage and retrying the +// same archive ID. Its journal must never remove the newly committed local copy. +func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("local retry", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + filename := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + var deletes atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer remote.Close() + store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), + LocalPath: filename, Operation: "upload", + }) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { + t.Fatalf("old upload cleanup removed local retry: %v", err) + } + if deletes.Load() != 1 { + t.Fatal("uncommitted remote object was not removed") + } + if _, err := os.Stat(journal); !os.IsNotExist(err) { + t.Fatalf("finished journal remains: %v", err) + } +} diff --git a/server/task_archives.go b/server/task_archives.go index 836c2d8bb..9a7af523a 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return } + s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() + if s.archiveStoreErr != nil { + log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) + } else if s.archiveStore != nil { + log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") + } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } @@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { + if s.ctx.Err() != nil { + return s.ctx.Err() + } + // Retry deferred remote cleanup before claiming more work. Failed cleanup + // keeps its durable journal and does not block unrelated local archives. + if err := s.recoverRemoteArchiveJournals(); err != nil { + log.Printf("[task-archive] deferred object cleanup: %v", err) + } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err @@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error { } func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { + if s.archiveStoreErr != nil { + return s.archiveStoreErr + } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() + storedPath := archivePath + var remoteJournal string + if s.archiveStore != nil { + storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) + remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", + }) + if err != nil { + return err + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) + if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { + return err + } + } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = false + removePackage = s.archiveStore != nil + if remoteJournal != "" { + // The journal also covers a crash after database commit but before local + // package removal. Keep it if cleanup fails, so the worker can retry. + if err := removeLocalArchivePackage(archivePath); err == nil { + _ = os.Remove(remoteJournal) + } + } if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { + packagePath := job.ArchivePath + if strings.HasPrefix(packagePath, "s3://") { + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) + packagePath = filepath.Join(extracted, "download.tar.zst") + if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { + return err + } + } + if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } + if strings.HasPrefix(job.ArchivePath, "s3://") { + return s.deleteRemoteTaskArchive(job) + } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil { From 672a38e2be61f2f5e46b7dcd55f1e65ce94967bc Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Thu, 19 Nov 2026 03:33:51 +0000 Subject: [PATCH 2/7] fix: recover partial archives and defer unknown storage labels --- db/task_archives.go | 10 ++- db/task_archives_test.go | 9 +- server/task_archive_interrupt_test.go | 120 +++++++++++++++++++++++++- server/task_archive_package.go | 28 ++++-- 4 files changed, 150 insertions(+), 17 deletions(-) diff --git a/db/task_archives.go b/db/task_archives.go index 9f07341ce..a8c810132 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,7 +60,7 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` - Storage string `json:"storage"` + Storage string `json:"storage,omitempty"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -162,9 +162,11 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } - item.Storage = "local" - if strings.HasPrefix(item.ArchivePath, "s3://") { - item.Storage = "s3" + if item.ArchivePath != "" { + item.Storage = "local" + if strings.HasPrefix(item.ArchivePath, "s3://") { + item.Storage = "s3" + } } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err diff --git a/db/task_archives_test.go b/db/task_archives_test.go index 62f51672d..285225aa2 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -204,6 +204,13 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if err != nil { t.Fatal(err) } + if archive.Storage != "" { + t.Fatalf("queued archive without a location has storage %q", archive.Storage) + } + encoded, err := json.Marshal(archive) + if err != nil || bytes.Contains(encoded, []byte(`"storage"`)) { + t.Fatalf("unknown storage must be omitted from JSON: %s, %v", encoded, err) + } claimed, err := d.ClaimTaskArchiveJob(t.Context()) if err != nil || claimed == nil || claimed.ID != archive.ID || claimed.State != Archiving { t.Fatalf("claim = %+v, %v", claimed, err) @@ -242,7 +249,7 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro t.Fatalf("hot compaction left nodes=%d assets=%d usage=%d", nodes, assets, usage) } ready, err := d.GetTaskArchive(archive.ID) - if err != nil || ready == nil || ready.State != ArchiveReady { + if err != nil || ready == nil || ready.State != ArchiveReady || ready.Storage != "local" { t.Fatalf("ready archive = %+v, %v", ready, err) } var stats map[string]any diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go index 2f4a44e5b..09b3e038c 100644 --- a/server/task_archive_interrupt_test.go +++ b/server/task_archive_interrupt_test.go @@ -24,7 +24,7 @@ func TestRemoteArchiveInterruptionRecovery(t *testing.T) { t.Skip("requires isolated PostgreSQL") } phases := []string{ - "staged_before_compression", "compressed_before_journal", "journal_before_upload", + "staged_before_compression", "compression_partial", "compressed_before_journal", "journal_before_upload", "uploaded_before_verification", "verified_before_database", "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", @@ -71,22 +71,30 @@ func TestRemoteArchiveInterruptionRecovery(t *testing.T) { t.Fatal(err) } packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if phase == "compression_partial" { + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath+".partial", []byte("incomplete compressed package"), 0600); err != nil { + t.Fatal(err) + } + } var original, compressed int64 var checksum, uri, journal string - if phase != "staged_before_compression" { + if phase != "staged_before_compression" && phase != "compression_partial" { original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) if err != nil { t.Fatal(err) } uri = store.URI(id, job.ID, checksum) } - if phase != "staged_before_compression" && phase != "compressed_before_journal" { + if phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" { journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) if err != nil { t.Fatal(err) } } - committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && + committed := phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" && phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" if phase == "uploaded_before_verification" { data, err := os.ReadFile(packagePath) @@ -196,6 +204,9 @@ func TestRemoteArchiveInterruptionRecovery(t *testing.T) { if _, err := os.Stat(packagePath); !os.IsNotExist(err) { t.Fatalf("temporary package remains: %v", err) } + if _, err := os.Stat(packagePath + ".partial"); !os.IsNotExist(err) { + t.Fatalf("partial package remains: %v", err) + } if got, want := objects.has(uri), committed && !remoteDeleted; got != want { t.Fatalf("remote object exists=%v, want=%v", got, want) } @@ -289,3 +300,104 @@ func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects } return store, objects } + +// Neither a rollback conflict nor a temporary-file cleanup failure may discard +// the staging journal: recovery must be safe to retry after the obstruction ends. +func TestArchivePartialRecoveryRetainsJournalOnFailure(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + for _, failure := range []string{"rollback-conflict", "cleanup-failure"} { + t.Run(failure, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("partial recovery "+failure, "retain source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + partial := taskArchivePath(m.dir, job.ID, task.ID) + ".partial" + if err := os.MkdirAll(filepath.Dir(partial), 0700); err != nil { + t.Fatal(err) + } + var obstruction string + if failure == "rollback-conflict" { + if len(stage.journal.Moves) == 0 { + t.Fatal("no staged source") + } + obstruction = stage.journal.Moves[0].Source + if err := os.MkdirAll(obstruction, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(partial, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + } else { + if err := os.Mkdir(partial, 0700); err != nil { + t.Fatal(err) + } + obstruction = filepath.Join(partial, "block-remove") + if err := os.WriteFile(obstruction, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err == nil { + t.Fatal("expected recovery obstruction") + } + if _, err := os.Stat(filepath.Join(stage.root, "journal.json")); err != nil { + t.Fatal("lost retry journal", err) + } + if _, err := os.Stat(partial); err != nil { + t.Fatal("removed partial before successful recovery", err) + } + if failure == "cleanup-failure" { + if got, err := os.ReadFile(source); err != nil || string(got) != "original" { + t.Fatal("source was not rolled back", err) + } + } + if err := os.RemoveAll(obstruction); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(source); err != nil || string(got) != "original" { + t.Fatal("source lost on retry", err) + } + for _, path := range []string{stage.root, partial} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("recovery left %s: %v", path, err) + } + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index e365ea3f5..4bce256ec 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -185,6 +185,16 @@ func (s *taskArchiveFileStage) rollback() error { if s == nil || s.done { return nil } + err := s.rollbackFiles() + if err == nil { + err = os.RemoveAll(s.root) + } + s.done = true + return err +} + +// Keep the journal until the caller has also cleaned temporary packages. +func (s *taskArchiveFileStage) rollbackFiles() error { var errs []error for i := len(s.journal.Moves) - 1; i >= 0; i-- { move := s.journal.Moves[i] @@ -210,12 +220,6 @@ func (s *taskArchiveFileStage) rollback() error { errs = append(errs, err) } } - if len(errs) == 0 { - if err := os.RemoveAll(s.root); err != nil { - errs = append(errs, err) - } - } - s.done = true return errors.Join(errs...) } @@ -435,7 +439,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { continue } stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} - if err := stage.rollback(); err != nil { + if err := stage.rollbackFiles(); err != nil { errs = append(errs, err) continue } @@ -445,7 +449,15 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) if err := validateArchivePath(dataDir, packagePath); err != nil { errs = append(errs, err) - } else if err := removeLocalArchivePackage(packagePath); err != nil { + continue + } + // A crash inside compression leaves .partial, before any upload journal. + // Retain this staging journal on cleanup failure so startup can retry. + if err := errors.Join(removeLocalArchivePackage(packagePath), removeLocalArchivePackage(packagePath+".partial")); err != nil { + errs = append(errs, err) + continue + } + if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } } From 64a53ea9c137cc19446944f5eb353ada60b11019 Mon Sep 17 00:00:00 2001 From: renaesop Date: Fri, 20 Nov 2026 14:26:07 +0000 Subject: [PATCH 3/7] fix: initialize archive credentials before startup environment scrub --- server/env_scrub.go | 6 +- server/server.go | 6 +- server/task_archive_interrupt_test.go | 2 +- server/task_archive_startup_test.go | 79 +++++++++++++++++++++++++++ server/task_archives.go | 9 ++- 5 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 server/task_archive_startup_test.go diff --git a/server/env_scrub.go b/server/env_scrub.go index 308278514..2e4d4f7c6 100644 --- a/server/env_scrub.go +++ b/server/env_scrub.go @@ -5,9 +5,9 @@ import "os" // secretEnv lists the credentials read from the environment only at startup: // ARTEX_PG_DSN and ARTEX_SECRET_KEY by NewManager, the API keys by agent.FromEnv inside New. // agent.FromEnv also reads ARTEX_LLM_PROXY and ARTEX_LLM_BASE_URL: both are URLs that -// can carry credentials in their userinfo (http://user:pass@proxy:3128). The ARTEX_ARCHIVE_S3_* -// keys are not read by the server, but the container may carry them and the agent's -// shells would inherit them. +// can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive +// store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub, +// so the client can use them without passing them to agent child processes. var secretEnv = []string{ "ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL", diff --git a/server/server.go b/server/server.go index 20e4dbc68..92eb284f5 100644 --- a/server/server.go +++ b/server/server.go @@ -305,8 +305,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi } else { log.Printf("[engine] no LLM provider configured — engine idle until set via /api/llm or env") } - // Both env readers have now run (NewManager for the DSN, FromEnv above for the - // keys). Drop the secrets before anything below can spawn a child process: + s.initTaskArchiveStore() + // Startup env readers have now run (NewManager for the DSN, the LLM provider + // and archive store for their credentials). Drop the secrets before anything + // below can spawn a child process: // restoring runtimes resumes workers that run shells, and MCP discovery starts // stdio servers. if s.scrubSecrets { diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go index 09b3e038c..6c4291550 100644 --- a/server/task_archive_interrupt_test.go +++ b/server/task_archive_interrupt_test.go @@ -149,7 +149,7 @@ func TestRemoteArchiveInterruptionRecovery(t *testing.T) { t.Fatal(err) } if phase != "restore_download_before_install" { - if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { + if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID, s.execPriv); err != nil { t.Fatal(err) } } diff --git a/server/task_archive_startup_test.go b/server/task_archive_startup_test.go new file mode 100644 index 000000000..96f6c947f --- /dev/null +++ b/server/task_archive_startup_test.go @@ -0,0 +1,79 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestArchiveStoreLoadsCredentialsBeforeSecretScrub(t *testing.T) { + skipWithoutDSN(t) + // New scrubs all startup secrets; restore the test database/key environment + // afterward so the remaining package tests keep their isolated configuration. + for _, key := range secretEnv { + t.Setenv(key, os.Getenv(key)) + } + var signed atomic.Bool + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + signed.Store(strings.Contains(r.Header.Get("Authorization"), "Credential=archive-fixture-key/") && + r.Header.Get("X-Amz-Security-Token") == "archive-fixture-session") + w.WriteHeader(http.StatusNoContent) + })) + defer endpoint.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "archive-fixture-key") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "archive-fixture-secret") + t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "archive-fixture-session") + t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256") + t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "") + // If initialization happens after the scrub, the SDK would silently select + // this different ambient identity instead of the dedicated override. + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CA_BUNDLE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Exercise real startup without leaving archive/scheduler workers alive. + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub()) + s.archiveWG.Wait() + if s.archiveStoreErr != nil || s.archiveStore == nil { + t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr) + } + for _, key := range []string{"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN"} { + if _, present := os.LookupEnv(key); present { + t.Fatalf("%s remains in the child-process environment", key) + } + } + requestCtx, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + if err := s.archiveStore.Delete(requestCtx, s.archiveStore.URI(1, 2, "startup-probe")); err != nil { + t.Fatal(err) + } + if !signed.Load() { + t.Fatal("archive client lost its dedicated credentials during startup scrub") + } +} diff --git a/server/task_archives.go b/server/task_archives.go index baefed92c..68e0d8a88 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -34,7 +34,8 @@ type archiveBatchItem struct { Archive int64 `json:"archive_id,omitempty"` } -func (s *Server) startTaskArchiveWorker() { +// initTaskArchiveStore captures credentials before New scrubs the process environment. +func (s *Server) initTaskArchiveStore() { if s.m == nil || s.m.pg == nil { return } @@ -44,6 +45,12 @@ func (s *Server) startTaskArchiveWorker() { } else if s.archiveStore != nil { log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") } +} + +func (s *Server) startTaskArchiveWorker() { + if s.m == nil || s.m.pg == nil { + return + } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } From 745204423c03f57e787fb11fc32bbcabc37c7ac7 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 21:05:22 +0100 Subject: [PATCH 4/7] fix(server): scrub ambient AWS credential variables from child processes The archive store falls back to the AWS SDK default chain when no ARTEX_ARCHIVE_S3_* override is set. Those AWS_* variables stayed in the environment after startup, so agent shells, custom tools and MCP stdio servers inherited them. Add them to secretEnv: they are scrubbed after the store is built and kept out of /proc//environ by the seal. The SDK captures what it needs at config load. Tests cover static keys (both spellings), container credentials and web identity fetched after the scrub, and a real New with WithSecretScrub. --- archivestore/credentials_test.go | 117 ++++++++++++++++++++++++++++ server/env_scrub.go | 10 +++ server/envseal_linux_test.go | 11 +++ server/task_archive_startup_test.go | 96 +++++++++++++++++++++++ 4 files changed, 234 insertions(+) diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go index 6c667ed74..c549f9d7f 100644 --- a/archivestore/credentials_test.go +++ b/archivestore/credentials_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" ) @@ -130,3 +131,119 @@ func TestArchiveIncompleteCredentialOverride(t *testing.T) { } } } + +// The server removes the AWS credential variables from its environment once the +// store is built, so that child processes do not inherit them. The SDK must have +// captured by then everything it needs to sign: the static keys, and for the +// container and web-identity providers the endpoint, token and file path they +// fetch credentials from later (their first fetch happens after the scrub here). +func TestReview318CredentialsSurviveEnvScrub(t *testing.T) { + scrubbed := []string{ + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", + } + for _, mode := range []string{"environment", "environment-alternate-names", "container", "web-identity"} { + t.Run(mode, func(t *testing.T) { + isolateAWSCredentials(t) + for _, key := range append([]string{ + "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN", + "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", "ARTEX_ARCHIVE_S3_PREFIX", "AWS_ENDPOINT_URL_STS", + }, scrubbed...) { + t.Setenv(key, "") + } + var envScrubbed atomic.Bool + var fetchedAfterScrub atomic.Int32 + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if envScrubbed.Load() { + fetchedAfterScrub.Add(1) + } + expiration := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + if mode == "container" { + if r.Header.Get("Authorization") != "fixture-container-token" { + t.Error("missing container authorization token") + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret", + "Token": "fixture-session", "Expiration": expiration, + }) + return + } + if err := r.ParseForm(); err != nil || r.PostForm.Get("Action") != "AssumeRoleWithWebIdentity" || + r.PostForm.Get("WebIdentityToken") != "fixture-web-identity-token" || + r.PostForm.Get("RoleArn") != "arn:aws:iam::123456789012:role/archive" { + t.Errorf("unexpected STS request: %v %v", err, r.PostForm) + } + w.Header().Set("Content-Type", "text/xml") + fmt.Fprintf(w, `fixture-keyfixture-secretfixture-session%s`, expiration) + })) + defer provider.Close() + switch mode { + case "environment": + t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "environment-alternate-names": + t.Setenv("AWS_ACCESS_KEY", "fixture-key") + t.Setenv("AWS_SECRET_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "container": + t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", provider.URL+"/credentials") + t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token") + case "web-identity": + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("fixture-web-identity-token"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenFile) + t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/archive") + t.Setenv("AWS_ENDPOINT_URL_STS", provider.URL) + } + var authorization, sessionToken atomic.Value + objects := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + authorization.Store(r.Header.Get("Authorization")) + sessionToken.Store(r.Header.Get("X-Amz-Security-Token")) + w.WriteHeader(http.StatusNotFound) + })) + defer objects.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", objects.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + store, err := FromEnv() + if err != nil || store == nil { + t.Fatalf("store from environment: %v", err) + } + for _, key := range scrubbed { + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + envScrubbed.Store(true) + + // HEAD is the first request of a download. The fixture answers 404, so + // the call fails; only the signature of the request matters here. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + checksum := strings.Repeat("a", 64) + err = store.Download(ctx, store.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1) + if err == nil { + t.Fatal("the fixture has no object: the download must fail") + } + got, _ := authorization.Load().(string) + if !strings.Contains(got, "Credential=fixture-key/") || !strings.Contains(got, "Signature=") { + t.Fatalf("request not signed with the captured key after the scrub: Authorization=%q (%v)", got, err) + } + if token, _ := sessionToken.Load().(string); token != "fixture-session" { + t.Fatalf("session token lost after the scrub: %q", token) + } + if lazy := mode == "container" || mode == "web-identity"; lazy && fetchedAfterScrub.Load() == 0 { + t.Fatal("the provider was expected to fetch its credentials after the scrub") + } + }) + } +} diff --git a/server/env_scrub.go b/server/env_scrub.go index 2e4d4f7c6..188bd2a1a 100644 --- a/server/env_scrub.go +++ b/server/env_scrub.go @@ -8,10 +8,20 @@ import "os" // can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive // store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub, // so the client can use them without passing them to agent child processes. +// +// Without that override the AWS SDK's default chain reads the AWS_* names below +// while the store is built (LoadDefaultConfig) and keeps what it needs: the static +// keys, or the container endpoint and token and the web-identity role and token +// file path it fetches credentials from later. Credentials that come from a shared +// credentials or config file, or from the instance metadata service, are not in +// the environment: the scrub does not hide them from child processes. var secretEnv = []string{ "ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN", + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", } // ScrubSecretEnv removes secretEnv from the process environment once startup has diff --git a/server/envseal_linux_test.go b/server/envseal_linux_test.go index deaafa7fb..65f9faf1c 100644 --- a/server/envseal_linux_test.go +++ b/server/envseal_linux_test.go @@ -60,6 +60,17 @@ func TestSealedEnvironmentHandoff(t *testing.T) { "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID": "AKIATESTONLY", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY": "s3-secret-test", "ARTEX_ARCHIVE_S3_SESSION_TOKEN": "", + // Ambient AWS credentials the archive store reads when it has no override. + "AWS_ACCESS_KEY_ID": "AKIAAMBIENTTEST", + "AWS_ACCESS_KEY": "AKIAALTERNATETEST", + "AWS_SECRET_ACCESS_KEY": "ambient-secret-test", + "AWS_SECRET_KEY": "alternate-secret-test", + "AWS_SESSION_TOKEN": "", + "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:8080/credentials?id=1", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/v2/credentials/test", + "AWS_CONTAINER_AUTHORIZATION_TOKEN": "container-token-test", + "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token", + "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test", } env := os.Environ() for k, v := range want { diff --git a/server/task_archive_startup_test.go b/server/task_archive_startup_test.go index 96f6c947f..faccb2b8f 100644 --- a/server/task_archive_startup_test.go +++ b/server/task_archive_startup_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "strings" "sync/atomic" @@ -77,3 +78,98 @@ func TestArchiveStoreLoadsCredentialsBeforeSecretScrub(t *testing.T) { t.Fatal("archive client lost its dedicated credentials during startup scrub") } } + +// The archive store may take its identity from the ambient AWS credential +// variables instead of the ARTEX_ARCHIVE_S3_* override. Those variables are as +// sensitive as the override: after startup no child process may inherit them, +// and the S3 client built before the scrub must keep signing with them. +func TestReview318AmbientAWSCredentialsScrubbedAfterArchiveInit(t *testing.T) { + skipWithoutDSN(t) + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh in PATH") + } + awsNames := []string{ + "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", + } + // New scrubs all startup secrets; t.Setenv restores them for the tests that follow. + for _, key := range secretEnv { + t.Setenv(key, os.Getenv(key)) + } + for _, key := range awsNames { + t.Setenv(key, "probe-"+key) + } + var authorization, sessionToken atomic.Value + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("unexpected S3 operation: %s", r.Method) + } + authorization.Store(r.Header.Get("Authorization")) + sessionToken.Store(r.Header.Get("X-Amz-Security-Token")) + w.WriteHeader(http.StatusNotFound) + })) + defer endpoint.Close() + t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3") + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL) + t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture") + t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + // No dedicated override: the SDK's default chain reads the AWS_* variables. + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "") + t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "") + t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256") + t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "") + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "ambient-fixture-session") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CA_BUNDLE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Exercise real startup without leaving archive/scheduler workers alive. + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub()) + s.archiveWG.Wait() + if s.archiveStoreErr != nil || s.archiveStore == nil { + t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr) + } + + out, err := exec.Command(sh, "-c", "env").Output() + if err != nil { + t.Fatal(err) + } + env := "\n" + string(out) + for _, key := range awsNames { + if strings.Contains(env, "\n"+key+"=") { + t.Errorf("%s still reaches child processes after startup", key) + } + } + + // HEAD is the first request of a download. The fixture answers 404, so the + // call fails; only the signature of the request matters here. + requestCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + checksum := strings.Repeat("a", 64) + err = s.archiveStore.Download(requestCtx, s.archiveStore.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1) + if err == nil { + t.Fatal("the fixture has no object: the download must fail") + } + got, _ := authorization.Load().(string) + if !strings.Contains(got, "Credential=ambient-fixture-key/") || !strings.Contains(got, "Signature=") { + t.Fatalf("archive client did not sign with the ambient key after the scrub: Authorization=%q (%v)", got, err) + } + if token, _ := sessionToken.Load().(string); token != "ambient-fixture-session" { + t.Fatalf("archive client lost the ambient session token after the scrub: %q", token) + } +} From 74116993cc0e51b1d4ec51e0c52d47d4aeb5f2c4 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 21:05:22 +0100 Subject: [PATCH 5/7] fix(server): fail the archive job instead of crashing on a worker panic The archive worker goroutine had no recover. An S3-compatible endpoint that answers CreateMultipartUpload with 200 and no UploadId makes the SDK upload manager dereference a nil pointer, which ended the server. Run each claimed job under a recover: log the panic value and stack and return it as the job error, so the job is failed through the same FailTaskArchiveJob path as any other error. The job's deferred rollbacks have run by then, so the source data stays in place. FailTaskArchiveJob now only fails a job that is still in its active state. A panic recovered after the archive committed would otherwise turn a ready archive into archive_failed, which can be neither restored nor deleted. --- db/task_archives.go | 5 +- db/task_archives_test.go | 42 ++++++++++ server/task_archive_panic_test.go | 131 ++++++++++++++++++++++++++++++ server/task_archives.go | 39 ++++++--- 4 files changed, 204 insertions(+), 13 deletions(-) create mode 100644 server/task_archive_panic_test.go diff --git a/db/task_archives.go b/db/task_archives.go index a8c810132..bf4fa3668 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -456,7 +456,10 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error if cause != nil { message = cause.Error() } - _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1`, id, failed, message) + // Only a job still in its active state fails. The worker also reports a panic + // it recovered after the job committed: a ready archive marked archive_failed + // could be neither restored nor deleted. + _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1 AND state=$4`, id, failed, message, activeState) return err } diff --git a/db/task_archives_test.go b/db/task_archives_test.go index 15198a9c5..c6cd934c2 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -524,3 +524,45 @@ func TestFillArchiveDefaults(t *testing.T) { t.Fatalf("activity changed: %s err=%v", out, err) } } + +// FailTaskArchiveJob fails a job that is still running. A failure reported after +// the archive committed (the worker recovers a panic in the cleanup that follows +// the commit) must not turn a ready archive into archive_failed: it could then +// be neither restored nor deleted. +func TestFailTaskArchiveJobKeepsCommittedState(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + SkipPGUnavailable(t, err) + } + defer d.Close() + task, err := d.CreateTaskWithOptions("late archive failure", "must stay ready", TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + defer func() { + _, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID) + _ = d.DeleteTask(task.ID) + }() + if err := d.SetPaused(task.ID, true); err != nil { + t.Fatal(err) + } + queued, err := d.QueueTaskArchive(task.ID) + if err != nil { + t.Fatal(err) + } + for _, c := range []struct{ state, want string }{{Archiving, ArchiveFailed}, {ArchiveReady, ArchiveReady}} { + if _, err := d.Exec(`UPDATE task_archives SET state=$2,error='' WHERE id=$1`, queued.ID, c.state); err != nil { + t.Fatal(err) + } + if err := d.FailTaskArchiveJob(queued.ID, Archiving, errors.New("late failure")); err != nil { + t.Fatal(err) + } + item, err := d.GetTaskArchive(queued.ID) + if err != nil || item == nil { + t.Fatalf("archive %d: %v", queued.ID, err) + } + if item.State != c.want || (c.want == ArchiveFailed) != (item.Error == "late failure") { + t.Fatalf("failing a job in state %s: state=%s error=%q, want state %s", c.state, item.State, item.Error, c.want) + } + } +} diff --git a/server/task_archive_panic_test.go b/server/task_archive_panic_test.go new file mode 100644 index 000000000..1f3c7cae2 --- /dev/null +++ b/server/task_archive_panic_test.go @@ -0,0 +1,131 @@ +package server + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// An S3-compatible endpoint that answers CreateMultipartUpload with 200 and no +// UploadId makes the SDK's upload manager dereference a nil pointer. The panic +// happens in the archive worker goroutine: it must fail that one job like any +// other upload error, keep the source data, and leave the process running. +func TestReview318ArchiveWorkerSurvivesUploadPanic(t *testing.T) { + skipWithoutDSN(t) + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + + var creates, unexpected atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + switch { + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + creates.Add(1) + w.Header().Set("Content-Type", "application/xml") + _, _ = io.WriteString(w, `test`) + case r.Method == http.MethodDelete: + // The worker's deferred cleanup of the upload that never committed. + w.WriteHeader(http.StatusNoContent) + default: + unexpected.Add(1) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + + task, err := m.CreateTask("multipart panic", "keep the source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + // The package must exceed the 8 MiB part size of archivestore.Upload for the + // upload manager to choose multipart: 9 MiB of random bytes do not compress. + source := make([]byte, 9<<20) + if _, err := rand.Read(source); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "large.bin") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, source, 0600); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + + // Run the server's own worker goroutine, as New starts it. + workerCtx, stopWorker := context.WithCancel(context.Background()) + s.ctx = workerCtx + s.startTaskArchiveWorker() + defer func() { stopWorker(); s.archiveWG.Wait() }() + + deadline := time.Now().Add(2 * time.Minute) + var item *pgdb.TaskArchive + for { + item, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if item == nil { + t.Fatal("archive job disappeared") + } + if item.State != pgdb.ArchiveQueued && item.State != pgdb.Archiving { + break + } + if time.Now().After(deadline) { + t.Fatalf("archive job still %s (%s)", item.State, item.Phase) + } + time.Sleep(50 * time.Millisecond) + } + if creates.Load() != 1 { + t.Fatalf("expected one CreateMultipartUpload, got %d: the package did not take the multipart path", creates.Load()) + } + if item.State != pgdb.ArchiveFailed || !strings.Contains(item.Error, "panic") { + t.Fatalf("job must be archive_failed with the panic as its error: state=%s error=%q", item.State, item.Error) + } + if item.ArchivePath != "" || item.SHA256 != "" || item.ArchivedAt != nil { + t.Fatalf("a location was committed for a failed archive: path=%q sha256=%q archived_at=%v", item.ArchivePath, item.SHA256, item.ArchivedAt) + } + got, err := os.ReadFile(file) + if err != nil || sha256.Sum256(got) != sha256.Sum256(source) { + t.Fatalf("source file lost or changed after the panic: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatalf("task lost after the panic: %v", err) + } + if unexpected.Load() != 0 { + t.Fatalf("%d unexpected S3 requests", unexpected.Load()) + } +} diff --git a/server/task_archives.go b/server/task_archives.go index 68e0d8a88..daca52f9a 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "runtime/debug" "strconv" "strings" "time" @@ -114,18 +115,7 @@ func (s *Server) runOneTaskArchiveJob() error { if err != nil || job == nil { return err } - var runErr error - switch job.State { - case pgdb.Archiving: - runErr = s.archiveTask(job) - case pgdb.Restoring: - runErr = s.restoreTaskArchive(job) - case pgdb.Deleting: - runErr = s.deleteTaskArchive(job) - default: - runErr = fmt.Errorf("unknown claimed archive state %q", job.State) - } - if runErr != nil { + if runErr := s.runClaimedTaskArchiveJob(job); runErr != nil { if failErr := s.m.pg.FailTaskArchiveJob(job.ID, job.State, runErr); failErr != nil { return errors.Join(runErr, failErr) } @@ -136,6 +126,31 @@ func (s *Server) runOneTaskArchiveJob() error { return nil } +// runClaimedTaskArchiveJob runs one claimed job and turns a panic below it into +// an ordinary job failure. The worker goroutine has no other recover: a panic +// there (the S3 upload manager dereferences a nil UploadId when an endpoint +// answers CreateMultipartUpload without one) would end the whole server with the +// job half done. The job's deferred rollbacks have already run when the panic is +// recovered here, so the caller fails the job with its source data in place. +func (s *Server) runClaimedTaskArchiveJob(job *pgdb.TaskArchive) (runErr error) { + defer func() { + if r := recover(); r != nil { + log.Printf("[task-archive] archive job %d (%s) panicked: %v\n%s", job.ID, job.State, r, debug.Stack()) + runErr = fmt.Errorf("archive worker panic: %v", r) + } + }() + switch job.State { + case pgdb.Archiving: + return s.archiveTask(job) + case pgdb.Restoring: + return s.restoreTaskArchive(job) + case pgdb.Deleting: + return s.deleteTaskArchive(job) + default: + return fmt.Errorf("unknown claimed archive state %q", job.State) + } +} + func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { if s.archiveStoreErr != nil { return s.archiveStoreErr From 35932976647098a3dc4e1d3b634f246217353cab Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 21:05:23 +0100 Subject: [PATCH 6/7] fix(server): do not fsync the journal directory on Windows writeRemoteArchiveJournal ended with Sync on two directory handles. Windows cannot flush a directory handle, so the journal write failed there and no remote archive or remote deletion could start. Move the directory sync to syncDir: the unix build keeps the fsync, the other builds return nil (the journal file itself is already synced). --- server/sync_dir_other.go | 8 ++++++++ server/sync_dir_unix.go | 18 ++++++++++++++++++ server/sync_dir_unix_test.go | 18 ++++++++++++++++++ server/task_archive_journal_test.go | 27 +++++++++++++++++++++++++++ server/task_archive_remote.go | 12 ++---------- 5 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 server/sync_dir_other.go create mode 100644 server/sync_dir_unix.go create mode 100644 server/sync_dir_unix_test.go create mode 100644 server/task_archive_journal_test.go diff --git a/server/sync_dir_other.go b/server/sync_dir_other.go new file mode 100644 index 000000000..54245c7a2 --- /dev/null +++ b/server/sync_dir_other.go @@ -0,0 +1,8 @@ +//go:build !unix + +package server + +// syncDir does nothing here: Windows does not support fsync on a directory +// handle (FlushFileBuffers fails on it), and the contents of the file written in +// the directory are already synced by the caller. +func syncDir(path string) error { return nil } diff --git a/server/sync_dir_unix.go b/server/sync_dir_unix.go new file mode 100644 index 000000000..8518f98bf --- /dev/null +++ b/server/sync_dir_unix.go @@ -0,0 +1,18 @@ +//go:build unix + +package server + +import ( + "errors" + "os" +) + +// syncDir flushes a directory, so that an entry just created or renamed in it +// survives a crash. +func syncDir(path string) error { + d, err := os.Open(path) + if err != nil { + return err + } + return errors.Join(d.Sync(), d.Close()) +} diff --git a/server/sync_dir_unix_test.go b/server/sync_dir_unix_test.go new file mode 100644 index 000000000..3d22034ae --- /dev/null +++ b/server/sync_dir_unix_test.go @@ -0,0 +1,18 @@ +//go:build unix + +package server + +import ( + "path/filepath" + "testing" +) + +func TestSyncDir(t *testing.T) { + dir := t.TempDir() + if err := syncDir(dir); err != nil { + t.Fatalf("sync of an existing directory: %v", err) + } + if err := syncDir(filepath.Join(dir, "missing")); err == nil { + t.Fatal("sync of a missing directory must fail") + } +} diff --git a/server/task_archive_journal_test.go b/server/task_archive_journal_test.go new file mode 100644 index 000000000..486bdf422 --- /dev/null +++ b/server/task_archive_journal_test.go @@ -0,0 +1,27 @@ +package server + +import ( + "encoding/json" + "os" + "testing" +) + +// Writing the journal must succeed on every platform the server builds for. It +// needs no database: on Windows the directory sync it ended with failed, so no +// remote archive or remote deletion could start. +func TestRemoteArchiveJournalWrite(t *testing.T) { + s := &Server{m: &Manager{dir: t.TempDir()}} + want := remoteArchiveJournal{ArchiveID: 7, URI: "s3://bucket/tasks/package.tar.zst", LocalPath: "package.tar.zst", Operation: "upload"} + name, err := s.writeRemoteArchiveJournal(want) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + var got remoteArchiveJournal + if err := json.Unmarshal(raw, &got); err != nil || got != want { + t.Fatalf("journal content %q: %v", raw, err) + } +} diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go index 87e230f23..ee6b0ea12 100644 --- a/server/task_archive_remote.go +++ b/server/task_archive_remote.go @@ -67,19 +67,11 @@ func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, erro return "", err } // Persist the directory entry before an upload or database deletion starts. - d, err := os.Open(dir) - if err != nil { - return "", err - } - if err := errors.Join(d.Sync(), d.Close()); err != nil { + if err := syncDir(dir); err != nil { return "", err } // The .remote directory itself may have just been created. - parent, err := os.Open(filepath.Dir(dir)) - if err != nil { - return "", err - } - return name, errors.Join(parent.Sync(), parent.Close()) + return name, syncDir(filepath.Dir(dir)) } func (s *Server) recoverRemoteArchiveJournals() error { From 65876b7abbda1fedbcc4274cb9cd89ac91c25fda Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 21:45:15 +0100 Subject: [PATCH 7/7] fix(server): scrub the container authorization token file path too The SDK reads AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE while LoadDefaultConfig builds the archive store and captures the path; the file itself is read lazily. Adding the name to secretEnv keeps that path out of agent child processes, the same way AWS_WEB_IDENTITY_TOKEN_FILE is already scrubbed, and does not affect the client. --- archivestore/credentials_test.go | 2 +- server/env_scrub.go | 2 +- server/envseal_linux_test.go | 1 + server/task_archive_startup_test.go | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go index c549f9d7f..e84956bfc 100644 --- a/archivestore/credentials_test.go +++ b/archivestore/credentials_test.go @@ -140,7 +140,7 @@ func TestArchiveIncompleteCredentialOverride(t *testing.T) { func TestReview318CredentialsSurviveEnvScrub(t *testing.T) { scrubbed := []string{ "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", - "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", } for _, mode := range []string{"environment", "environment-alternate-names", "container", "web-identity"} { diff --git a/server/env_scrub.go b/server/env_scrub.go index 188bd2a1a..63a548e71 100644 --- a/server/env_scrub.go +++ b/server/env_scrub.go @@ -20,7 +20,7 @@ var secretEnv = []string{ "ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", - "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", } diff --git a/server/envseal_linux_test.go b/server/envseal_linux_test.go index 65f9faf1c..3a50e0900 100644 --- a/server/envseal_linux_test.go +++ b/server/envseal_linux_test.go @@ -69,6 +69,7 @@ func TestSealedEnvironmentHandoff(t *testing.T) { "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:8080/credentials?id=1", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/v2/credentials/test", "AWS_CONTAINER_AUTHORIZATION_TOKEN": "container-token-test", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": "/var/run/secrets/token-file", "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token", "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test", } diff --git a/server/task_archive_startup_test.go b/server/task_archive_startup_test.go index faccb2b8f..def27d071 100644 --- a/server/task_archive_startup_test.go +++ b/server/task_archive_startup_test.go @@ -91,7 +91,7 @@ func TestReview318AmbientAWSCredentialsScrubbedAfterArchiveInit(t *testing.T) { } awsNames := []string{ "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", - "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", } // New scrubs all startup secrets; t.Setenv restores them for the tests that follow.