diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go
index 6c667ed74..e84956bfc 100644
--- a/archivestore/credentials_test.go
+++ b/archivestore/credentials_test.go
@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strings"
+ "sync/atomic"
"testing"
"time"
)
@@ -130,3 +131,119 @@ func TestArchiveIncompleteCredentialOverride(t *testing.T) {
}
}
}
+
+// The server removes the AWS credential variables from its environment once the
+// store is built, so that child processes do not inherit them. The SDK must have
+// captured by then everything it needs to sign: the static keys, and for the
+// container and web-identity providers the endpoint, token and file path they
+// fetch credentials from later (their first fetch happens after the scrub here).
+func TestReview318CredentialsSurviveEnvScrub(t *testing.T) {
+ scrubbed := []string{
+ "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
+ }
+ for _, mode := range []string{"environment", "environment-alternate-names", "container", "web-identity"} {
+ t.Run(mode, func(t *testing.T) {
+ isolateAWSCredentials(t)
+ for _, key := range append([]string{
+ "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN",
+ "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", "ARTEX_ARCHIVE_S3_PREFIX", "AWS_ENDPOINT_URL_STS",
+ }, scrubbed...) {
+ t.Setenv(key, "")
+ }
+ var envScrubbed atomic.Bool
+ var fetchedAfterScrub atomic.Int32
+ provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if envScrubbed.Load() {
+ fetchedAfterScrub.Add(1)
+ }
+ expiration := time.Now().Add(time.Hour).UTC().Format(time.RFC3339)
+ if mode == "container" {
+ if r.Header.Get("Authorization") != "fixture-container-token" {
+ t.Error("missing container authorization token")
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret",
+ "Token": "fixture-session", "Expiration": expiration,
+ })
+ return
+ }
+ if err := r.ParseForm(); err != nil || r.PostForm.Get("Action") != "AssumeRoleWithWebIdentity" ||
+ r.PostForm.Get("WebIdentityToken") != "fixture-web-identity-token" ||
+ r.PostForm.Get("RoleArn") != "arn:aws:iam::123456789012:role/archive" {
+ t.Errorf("unexpected STS request: %v %v", err, r.PostForm)
+ }
+ w.Header().Set("Content-Type", "text/xml")
+ fmt.Fprintf(w, `fixture-keyfixture-secretfixture-session%s`, expiration)
+ }))
+ defer provider.Close()
+ switch mode {
+ case "environment":
+ t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
+ case "environment-alternate-names":
+ t.Setenv("AWS_ACCESS_KEY", "fixture-key")
+ t.Setenv("AWS_SECRET_KEY", "fixture-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
+ case "container":
+ t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", provider.URL+"/credentials")
+ t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token")
+ case "web-identity":
+ tokenFile := filepath.Join(t.TempDir(), "token")
+ if err := os.WriteFile(tokenFile, []byte("fixture-web-identity-token"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenFile)
+ t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/archive")
+ t.Setenv("AWS_ENDPOINT_URL_STS", provider.URL)
+ }
+ var authorization, sessionToken atomic.Value
+ objects := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodHead {
+ t.Errorf("unexpected S3 operation: %s", r.Method)
+ }
+ authorization.Store(r.Header.Get("Authorization"))
+ sessionToken.Store(r.Header.Get("X-Amz-Security-Token"))
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer objects.Close()
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
+ t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", objects.URL)
+ t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
+ t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
+ t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
+ store, err := FromEnv()
+ if err != nil || store == nil {
+ t.Fatalf("store from environment: %v", err)
+ }
+ for _, key := range scrubbed {
+ if err := os.Unsetenv(key); err != nil {
+ t.Fatal(err)
+ }
+ }
+ envScrubbed.Store(true)
+
+ // HEAD is the first request of a download. The fixture answers 404, so
+ // the call fails; only the signature of the request matters here.
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ checksum := strings.Repeat("a", 64)
+ err = store.Download(ctx, store.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1)
+ if err == nil {
+ t.Fatal("the fixture has no object: the download must fail")
+ }
+ got, _ := authorization.Load().(string)
+ if !strings.Contains(got, "Credential=fixture-key/") || !strings.Contains(got, "Signature=") {
+ t.Fatalf("request not signed with the captured key after the scrub: Authorization=%q (%v)", got, err)
+ }
+ if token, _ := sessionToken.Load().(string); token != "fixture-session" {
+ t.Fatalf("session token lost after the scrub: %q", token)
+ }
+ if lazy := mode == "container" || mode == "web-identity"; lazy && fetchedAfterScrub.Load() == 0 {
+ t.Fatal("the provider was expected to fetch its credentials after the scrub")
+ }
+ })
+ }
+}
diff --git a/db/task_archives.go b/db/task_archives.go
index bb9aae241..bf4fa3668 100644
--- a/db/task_archives.go
+++ b/db/task_archives.go
@@ -60,6 +60,7 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
+ Storage string `json:"storage,omitempty"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -161,6 +162,12 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
+ if item.ArchivePath != "" {
+ item.Storage = "local"
+ if strings.HasPrefix(item.ArchivePath, "s3://") {
+ item.Storage = "s3"
+ }
+ }
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
@@ -449,7 +456,10 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error
if cause != nil {
message = cause.Error()
}
- _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1`, id, failed, message)
+ // Only a job still in its active state fails. The worker also reports a panic
+ // it recovered after the job committed: a ready archive marked archive_failed
+ // could be neither restored nor deleted.
+ _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1 AND state=$4`, id, failed, message, activeState)
return err
}
diff --git a/db/task_archives_test.go b/db/task_archives_test.go
index 727ffc7a3..c6cd934c2 100644
--- a/db/task_archives_test.go
+++ b/db/task_archives_test.go
@@ -204,6 +204,13 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro
if err != nil {
t.Fatal(err)
}
+ if archive.Storage != "" {
+ t.Fatalf("queued archive without a location has storage %q", archive.Storage)
+ }
+ encoded, err := json.Marshal(archive)
+ if err != nil || bytes.Contains(encoded, []byte(`"storage"`)) {
+ t.Fatalf("unknown storage must be omitted from JSON: %s, %v", encoded, err)
+ }
claimed, err := d.ClaimTaskArchiveJob(t.Context())
if err != nil || claimed == nil || claimed.ID != archive.ID || claimed.State != Archiving {
t.Fatalf("claim = %+v, %v", claimed, err)
@@ -242,7 +249,7 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro
t.Fatalf("hot compaction left nodes=%d assets=%d usage=%d", nodes, assets, usage)
}
ready, err := d.GetTaskArchive(archive.ID)
- if err != nil || ready == nil || ready.State != ArchiveReady {
+ if err != nil || ready == nil || ready.State != ArchiveReady || ready.Storage != "local" {
t.Fatalf("ready archive = %+v, %v", ready, err)
}
var stats map[string]any
@@ -517,3 +524,45 @@ func TestFillArchiveDefaults(t *testing.T) {
t.Fatalf("activity changed: %s err=%v", out, err)
}
}
+
+// FailTaskArchiveJob fails a job that is still running. A failure reported after
+// the archive committed (the worker recovers a panic in the cleanup that follows
+// the commit) must not turn a ready archive into archive_failed: it could then
+// be neither restored nor deleted.
+func TestFailTaskArchiveJobKeepsCommittedState(t *testing.T) {
+ d, err := Open(testDSN(t))
+ if err != nil {
+ SkipPGUnavailable(t, err)
+ }
+ defer d.Close()
+ task, err := d.CreateTaskWithOptions("late archive failure", "must stay ready", TaskCreateOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ _, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
+ _ = d.DeleteTask(task.ID)
+ }()
+ if err := d.SetPaused(task.ID, true); err != nil {
+ t.Fatal(err)
+ }
+ queued, err := d.QueueTaskArchive(task.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range []struct{ state, want string }{{Archiving, ArchiveFailed}, {ArchiveReady, ArchiveReady}} {
+ if _, err := d.Exec(`UPDATE task_archives SET state=$2,error='' WHERE id=$1`, queued.ID, c.state); err != nil {
+ t.Fatal(err)
+ }
+ if err := d.FailTaskArchiveJob(queued.ID, Archiving, errors.New("late failure")); err != nil {
+ t.Fatal(err)
+ }
+ item, err := d.GetTaskArchive(queued.ID)
+ if err != nil || item == nil {
+ t.Fatalf("archive %d: %v", queued.ID, err)
+ }
+ if item.State != c.want || (c.want == ArchiveFailed) != (item.Error == "late failure") {
+ t.Fatalf("failing a job in state %s: state=%s error=%q, want state %s", c.state, item.State, item.Error, c.want)
+ }
+ }
+}
diff --git a/server/env_scrub.go b/server/env_scrub.go
index 308278514..63a548e71 100644
--- a/server/env_scrub.go
+++ b/server/env_scrub.go
@@ -5,13 +5,23 @@ import "os"
// secretEnv lists the credentials read from the environment only at startup:
// ARTEX_PG_DSN and ARTEX_SECRET_KEY by NewManager, the API keys by agent.FromEnv inside New.
// agent.FromEnv also reads ARTEX_LLM_PROXY and ARTEX_LLM_BASE_URL: both are URLs that
-// can carry credentials in their userinfo (http://user:pass@proxy:3128). The ARTEX_ARCHIVE_S3_*
-// keys are not read by the server, but the container may carry them and the agent's
-// shells would inherit them.
+// can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive
+// store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub,
+// so the client can use them without passing them to agent child processes.
+//
+// Without that override the AWS SDK's default chain reads the AWS_* names below
+// while the store is built (LoadDefaultConfig) and keeps what it needs: the static
+// keys, or the container endpoint and token and the web-identity role and token
+// file path it fetches credentials from later. Credentials that come from a shared
+// credentials or config file, or from the instance metadata service, are not in
+// the environment: the scrub does not hide them from child processes.
var secretEnv = []string{
"ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
"ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL",
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN",
+ "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
}
// ScrubSecretEnv removes secretEnv from the process environment once startup has
diff --git a/server/envseal_linux_test.go b/server/envseal_linux_test.go
index deaafa7fb..3a50e0900 100644
--- a/server/envseal_linux_test.go
+++ b/server/envseal_linux_test.go
@@ -60,6 +60,18 @@ func TestSealedEnvironmentHandoff(t *testing.T) {
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID": "AKIATESTONLY",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY": "s3-secret-test",
"ARTEX_ARCHIVE_S3_SESSION_TOKEN": "",
+ // Ambient AWS credentials the archive store reads when it has no override.
+ "AWS_ACCESS_KEY_ID": "AKIAAMBIENTTEST",
+ "AWS_ACCESS_KEY": "AKIAALTERNATETEST",
+ "AWS_SECRET_ACCESS_KEY": "ambient-secret-test",
+ "AWS_SECRET_KEY": "alternate-secret-test",
+ "AWS_SESSION_TOKEN": "",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:8080/credentials?id=1",
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/v2/credentials/test",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN": "container-token-test",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": "/var/run/secrets/token-file",
+ "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token",
+ "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test",
}
env := os.Environ()
for k, v := range want {
diff --git a/server/server.go b/server/server.go
index 09efe3aba..92eb284f5 100644
--- a/server/server.go
+++ b/server/server.go
@@ -24,6 +24,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
+ "github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -158,9 +159,11 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
- archiveWake chan struct{}
- archiveWG sync.WaitGroup
- side *sideQuestionState
+ archiveWake chan struct{}
+ archiveWG sync.WaitGroup
+ archiveStore *archivestore.Store
+ archiveStoreErr error
+ side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
@@ -302,8 +305,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi
} else {
log.Printf("[engine] no LLM provider configured — engine idle until set via /api/llm or env")
}
- // Both env readers have now run (NewManager for the DSN, FromEnv above for the
- // keys). Drop the secrets before anything below can spawn a child process:
+ s.initTaskArchiveStore()
+ // Startup env readers have now run (NewManager for the DSN, the LLM provider
+ // and archive store for their credentials). Drop the secrets before anything
+ // below can spawn a child process:
// restoring runtimes resumes workers that run shells, and MCP discovery starts
// stdio servers.
if s.scrubSecrets {
diff --git a/server/sync_dir_other.go b/server/sync_dir_other.go
new file mode 100644
index 000000000..54245c7a2
--- /dev/null
+++ b/server/sync_dir_other.go
@@ -0,0 +1,8 @@
+//go:build !unix
+
+package server
+
+// syncDir does nothing here: Windows does not support fsync on a directory
+// handle (FlushFileBuffers fails on it), and the contents of the file written in
+// the directory are already synced by the caller.
+func syncDir(path string) error { return nil }
diff --git a/server/sync_dir_unix.go b/server/sync_dir_unix.go
new file mode 100644
index 000000000..8518f98bf
--- /dev/null
+++ b/server/sync_dir_unix.go
@@ -0,0 +1,18 @@
+//go:build unix
+
+package server
+
+import (
+ "errors"
+ "os"
+)
+
+// syncDir flushes a directory, so that an entry just created or renamed in it
+// survives a crash.
+func syncDir(path string) error {
+ d, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ return errors.Join(d.Sync(), d.Close())
+}
diff --git a/server/sync_dir_unix_test.go b/server/sync_dir_unix_test.go
new file mode 100644
index 000000000..3d22034ae
--- /dev/null
+++ b/server/sync_dir_unix_test.go
@@ -0,0 +1,18 @@
+//go:build unix
+
+package server
+
+import (
+ "path/filepath"
+ "testing"
+)
+
+func TestSyncDir(t *testing.T) {
+ dir := t.TempDir()
+ if err := syncDir(dir); err != nil {
+ t.Fatalf("sync of an existing directory: %v", err)
+ }
+ if err := syncDir(filepath.Join(dir, "missing")); err == nil {
+ t.Fatal("sync of a missing directory must fail")
+ }
+}
diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go
new file mode 100644
index 000000000..6c4291550
--- /dev/null
+++ b/server/task_archive_interrupt_test.go
@@ -0,0 +1,403 @@
+package server
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "testing"
+
+ "github.com/Autumn-27/artex/archivestore"
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+// Recreate persisted states at operation boundaries, then run the same recovery
+// functions used at startup. PostgreSQL, files, and HTTP object storage are real
+// isolated fixtures; no production failpoint hooks are needed.
+func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ phases := []string{
+ "staged_before_compression", "compression_partial", "compressed_before_journal", "journal_before_upload",
+ "uploaded_before_verification", "verified_before_database",
+ "database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
+ "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
+ "delete_journal_before_database", "delete_database_before_object",
+ }
+ for _, phase := range phases {
+ t.Run(phase, func(t *testing.T) {
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ store, objects := newRecoveryObjectStore(t)
+ s := &Server{ctx: context.Background(), m: m, archiveStore: store}
+ task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
+ t.Fatal(err)
+ }
+ source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
+ if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ snapshot, err := m.pg.SnapshotTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ packagePath := taskArchivePath(m.dir, job.ID, task.ID)
+ if phase == "compression_partial" {
+ if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(packagePath+".partial", []byte("incomplete compressed package"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var original, compressed int64
+ var checksum, uri, journal string
+ if phase != "staged_before_compression" && phase != "compression_partial" {
+ original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
+ if err != nil {
+ t.Fatal(err)
+ }
+ uri = store.URI(id, job.ID, checksum)
+ }
+ if phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" {
+ journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ committed := phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" &&
+ phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
+ if phase == "uploaded_before_verification" {
+ data, err := os.ReadFile(packagePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ objects.put(uri, data)
+ } else if phase == "verified_before_database" || committed {
+ if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if committed {
+ if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
+ t.Fatal(err)
+ }
+ }
+ afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
+ phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
+ if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
+ if err := os.Remove(packagePath); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(journal); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if afterArchive {
+ if err := stage.commit(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ restored := false
+ remoteDeleted := false
+ if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
+ t.Fatal(err)
+ }
+ parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
+ if err := os.MkdirAll(parent, 0700); err != nil {
+ t.Fatal(err)
+ }
+ extracted, err := os.MkdirTemp(parent, "interrupted-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ downloaded := filepath.Join(extracted, "download.tar.zst")
+ if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
+ t.Fatal(err)
+ }
+ if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
+ t.Fatal(err)
+ }
+ if phase != "restore_download_before_install" {
+ if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID, s.execPriv); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
+ if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
+ t.Fatal(err)
+ }
+ restored = true
+ }
+ if phase == "restore_object_deleted_before_stub" {
+ if err := store.Delete(s.ctx, uri); err != nil {
+ t.Fatal(err)
+ }
+ remoteDeleted = true
+ }
+ }
+ deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
+ if deleting {
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
+ t.Fatal(err)
+ }
+ if phase == "delete_database_before_object" {
+ if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ remoteDeleted = true
+ }
+ }
+ if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ wantSource := !committed || restored
+ data, fileErr := os.ReadFile(source)
+ if wantSource && (fileErr != nil || string(data) != "original task data") {
+ t.Fatalf("source data lost: %v", fileErr)
+ }
+ if !wantSource && !os.IsNotExist(fileErr) {
+ t.Fatalf("archived source resurrected: %v", fileErr)
+ }
+ if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
+ t.Fatalf("temporary package remains: %v", err)
+ }
+ if _, err := os.Stat(packagePath + ".partial"); !os.IsNotExist(err) {
+ t.Fatalf("partial package remains: %v", err)
+ }
+ if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
+ t.Fatalf("remote object exists=%v, want=%v", got, want)
+ }
+ if committed && !deleting {
+ actual, err := m.pg.IsTaskArchiveRestored(job.ID)
+ if err != nil || actual != restored {
+ t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
+ }
+ }
+ // A second startup must be harmless at every boundary.
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+type recoveryObjects struct {
+ mu sync.Mutex
+ data map[string][]byte
+}
+
+func (o *recoveryObjects) put(uri string, data []byte) {
+ o.mu.Lock()
+ defer o.mu.Unlock()
+ // Test URIs always use the fixture bucket with path-style addressing.
+ o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
+}
+func (o *recoveryObjects) has(uri string) bool {
+ o.mu.Lock()
+ defer o.mu.Unlock()
+ if uri == "" {
+ return false
+ }
+ _, ok := o.data["/test/"+uri[len("s3://test/"):]]
+ return ok
+}
+func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
+ t.Helper()
+ objects := &recoveryObjects{data: map[string][]byte{}}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ objects.mu.Lock()
+ defer objects.mu.Unlock()
+ data, exists := objects.data[r.URL.Path]
+ w.Header().Set("ETag", `"recovery-etag"`)
+ switch r.Method {
+ case "PUT":
+ if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
+ t.Error("missing SSE")
+ }
+ objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
+ case "HEAD":
+ if !exists {
+ w.WriteHeader(404)
+ return
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
+ case "GET":
+ if !exists {
+ w.WriteHeader(404)
+ return
+ }
+ var start, end int
+ if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
+ w.WriteHeader(416)
+ return
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
+ w.WriteHeader(206)
+ _, _ = w.Write(data[start : end+1])
+ case "DELETE":
+ delete(objects.data, r.URL.Path)
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected object request %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ t.Cleanup(srv.Close)
+ store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return store, objects
+}
+
+// Neither a rollback conflict nor a temporary-file cleanup failure may discard
+// the staging journal: recovery must be safe to retry after the obstruction ends.
+func TestArchivePartialRecoveryRetainsJournalOnFailure(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ for _, failure := range []string{"rollback-conflict", "cleanup-failure"} {
+ t.Run(failure, func(t *testing.T) {
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ task, err := m.CreateTask("partial recovery "+failure, "retain source", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
+ if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(source, []byte("original"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ snapshot, err := m.pg.SnapshotTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ partial := taskArchivePath(m.dir, job.ID, task.ID) + ".partial"
+ if err := os.MkdirAll(filepath.Dir(partial), 0700); err != nil {
+ t.Fatal(err)
+ }
+ var obstruction string
+ if failure == "rollback-conflict" {
+ if len(stage.journal.Moves) == 0 {
+ t.Fatal("no staged source")
+ }
+ obstruction = stage.journal.Moves[0].Source
+ if err := os.MkdirAll(obstruction, 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(partial, []byte("partial"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ if err := os.Mkdir(partial, 0700); err != nil {
+ t.Fatal(err)
+ }
+ obstruction = filepath.Join(partial, "block-remove")
+ if err := os.WriteFile(obstruction, []byte("fixture"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err == nil {
+ t.Fatal("expected recovery obstruction")
+ }
+ if _, err := os.Stat(filepath.Join(stage.root, "journal.json")); err != nil {
+ t.Fatal("lost retry journal", err)
+ }
+ if _, err := os.Stat(partial); err != nil {
+ t.Fatal("removed partial before successful recovery", err)
+ }
+ if failure == "cleanup-failure" {
+ if got, err := os.ReadFile(source); err != nil || string(got) != "original" {
+ t.Fatal("source was not rolled back", err)
+ }
+ }
+ if err := os.RemoveAll(obstruction); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if got, err := os.ReadFile(source); err != nil || string(got) != "original" {
+ t.Fatal("source lost on retry", err)
+ }
+ for _, path := range []string{stage.root, partial} {
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ t.Fatalf("recovery left %s: %v", path, err)
+ }
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
diff --git a/server/task_archive_journal_test.go b/server/task_archive_journal_test.go
new file mode 100644
index 000000000..486bdf422
--- /dev/null
+++ b/server/task_archive_journal_test.go
@@ -0,0 +1,27 @@
+package server
+
+import (
+ "encoding/json"
+ "os"
+ "testing"
+)
+
+// Writing the journal must succeed on every platform the server builds for. It
+// needs no database: on Windows the directory sync it ended with failed, so no
+// remote archive or remote deletion could start.
+func TestRemoteArchiveJournalWrite(t *testing.T) {
+ s := &Server{m: &Manager{dir: t.TempDir()}}
+ want := remoteArchiveJournal{ArchiveID: 7, URI: "s3://bucket/tasks/package.tar.zst", LocalPath: "package.tar.zst", Operation: "upload"}
+ name, err := s.writeRemoteArchiveJournal(want)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, err := os.ReadFile(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got remoteArchiveJournal
+ if err := json.Unmarshal(raw, &got); err != nil || got != want {
+ t.Fatalf("journal content %q: %v", raw, err)
+ }
+}
diff --git a/server/task_archive_package.go b/server/task_archive_package.go
index 6efef3fda..496a740af 100644
--- a/server/task_archive_package.go
+++ b/server/task_archive_package.go
@@ -193,6 +193,16 @@ func (s *taskArchiveFileStage) rollback() error {
if s == nil || s.done {
return nil
}
+ err := s.rollbackFiles()
+ if err == nil {
+ err = os.RemoveAll(s.root)
+ }
+ s.done = true
+ return err
+}
+
+// Keep the journal until the caller has also cleaned temporary packages.
+func (s *taskArchiveFileStage) rollbackFiles() error {
var errs []error
for i := len(s.journal.Moves) - 1; i >= 0; i-- {
move := s.journal.Moves[i]
@@ -218,12 +228,6 @@ func (s *taskArchiveFileStage) rollback() error {
errs = append(errs, err)
}
}
- if len(errs) == 0 {
- if err := os.RemoveAll(s.root); err != nil {
- errs = append(errs, err)
- }
- }
- s.done = true
return errors.Join(errs...)
}
@@ -464,18 +468,33 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
- if archive != nil && archive.State == pgdb.ArchiveReady {
+ if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
continue
}
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
- if err := stage.rollback(); err != nil {
+ if err := stage.rollbackFiles(); err != nil {
errs = append(errs, err)
+ continue
}
- if archive != nil && archive.ArchivePath != "" {
- _ = os.Remove(archive.ArchivePath)
+ // The process can exit after compression but before an upload journal or
+ // database location exists. Remove that temporary package only after the
+ // source-file rollback succeeds, for both local and remote backends.
+ packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
+ if err := validateArchivePath(dataDir, packagePath); err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ // A crash inside compression leaves .partial, before any upload journal.
+ // Retain this staging journal on cleanup failure so startup can retry.
+ if err := errors.Join(removeLocalArchivePackage(packagePath), removeLocalArchivePackage(packagePath+".partial")); err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ if err := os.RemoveAll(root); err != nil {
+ errs = append(errs, err)
}
}
return errors.Join(errs...)
diff --git a/server/task_archive_panic_test.go b/server/task_archive_panic_test.go
new file mode 100644
index 000000000..1f3c7cae2
--- /dev/null
+++ b/server/task_archive_panic_test.go
@@ -0,0 +1,131 @@
+package server
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/sha256"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Autumn-27/artex/archivestore"
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+// An S3-compatible endpoint that answers CreateMultipartUpload with 200 and no
+// UploadId makes the SDK's upload manager dereference a nil pointer. The panic
+// happens in the archive worker goroutine: it must fail that one job like any
+// other upload error, keep the source data, and leave the process running.
+func TestReview318ArchiveWorkerSurvivesUploadPanic(t *testing.T) {
+ skipWithoutDSN(t)
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
+ s.archiveWG.Wait()
+
+ var creates, unexpected atomic.Int32
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ switch {
+ case r.Method == http.MethodPost && r.URL.Query().Has("uploads"):
+ creates.Add(1)
+ w.Header().Set("Content-Type", "application/xml")
+ _, _ = io.WriteString(w, `test`)
+ case r.Method == http.MethodDelete:
+ // The worker's deferred cleanup of the upload that never committed.
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ unexpected.Add(1)
+ w.WriteHeader(http.StatusBadRequest)
+ }
+ }))
+ defer remote.Close()
+ s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ task, err := m.CreateTask("multipart panic", "keep the source", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ // The package must exceed the 8 MiB part size of archivestore.Upload for the
+ // upload manager to choose multipart: 9 MiB of random bytes do not compress.
+ source := make([]byte, 9<<20)
+ if _, err := rand.Read(source); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "large.bin")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, source, 0600); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Run the server's own worker goroutine, as New starts it.
+ workerCtx, stopWorker := context.WithCancel(context.Background())
+ s.ctx = workerCtx
+ s.startTaskArchiveWorker()
+ defer func() { stopWorker(); s.archiveWG.Wait() }()
+
+ deadline := time.Now().Add(2 * time.Minute)
+ var item *pgdb.TaskArchive
+ for {
+ item, err = m.pg.GetTaskArchive(job.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if item == nil {
+ t.Fatal("archive job disappeared")
+ }
+ if item.State != pgdb.ArchiveQueued && item.State != pgdb.Archiving {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("archive job still %s (%s)", item.State, item.Phase)
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ if creates.Load() != 1 {
+ t.Fatalf("expected one CreateMultipartUpload, got %d: the package did not take the multipart path", creates.Load())
+ }
+ if item.State != pgdb.ArchiveFailed || !strings.Contains(item.Error, "panic") {
+ t.Fatalf("job must be archive_failed with the panic as its error: state=%s error=%q", item.State, item.Error)
+ }
+ if item.ArchivePath != "" || item.SHA256 != "" || item.ArchivedAt != nil {
+ t.Fatalf("a location was committed for a failed archive: path=%q sha256=%q archived_at=%v", item.ArchivePath, item.SHA256, item.ArchivedAt)
+ }
+ got, err := os.ReadFile(file)
+ if err != nil || sha256.Sum256(got) != sha256.Sum256(source) {
+ t.Fatalf("source file lost or changed after the panic: %v", err)
+ }
+ if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
+ t.Fatalf("task lost after the panic: %v", err)
+ }
+ if unexpected.Load() != 0 {
+ t.Fatalf("%d unexpected S3 requests", unexpected.Load())
+ }
+}
diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go
new file mode 100644
index 000000000..ee6b0ea12
--- /dev/null
+++ b/server/task_archive_remote.go
@@ -0,0 +1,163 @@
+package server
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+// An outbox bridges the local PostgreSQL transaction and object storage. Never
+// delete the only package before metadata permits it. Unfinished cleanup is
+// retried after a restart (and before subsequent jobs).
+type remoteArchiveJournal struct {
+ ArchiveID int64 `json:"archive_id"`
+ URI string `json:"uri"`
+ LocalPath string `json:"local_path,omitempty"`
+ Operation string `json:"operation"`
+}
+
+func (s *Server) validateStoredArchive(uri string) error {
+ if strings.HasPrefix(uri, "s3://") {
+ return s.archiveStore.Validate(uri)
+ }
+ return validateArchivePath(s.m.dir, uri)
+}
+
+func removeLocalArchivePackage(filename string) error {
+ if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ return nil
+}
+
+func (s *Server) removeStoredArchive(uri string) error {
+ if err := s.validateStoredArchive(uri); err != nil {
+ return err
+ }
+ if strings.HasPrefix(uri, "s3://") {
+ return s.archiveStore.Delete(s.ctx, uri)
+ }
+ return removeLocalArchivePackage(uri)
+}
+
+func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
+ dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
+ if err := os.MkdirAll(dir, archiveDirMode); err != nil {
+ return "", err
+ }
+ f, err := os.CreateTemp(dir, ".journal-")
+ if err != nil {
+ return "", err
+ }
+ tmp := f.Name()
+ defer os.Remove(tmp)
+ encodeErr := json.NewEncoder(f).Encode(j)
+ syncErr := f.Sync()
+ if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
+ return "", err
+ }
+ name := tmp + ".json"
+ if err := os.Rename(tmp, name); err != nil {
+ return "", err
+ }
+ // Persist the directory entry before an upload or database deletion starts.
+ if err := syncDir(dir); err != nil {
+ return "", err
+ }
+ // The .remote directory itself may have just been created.
+ return name, syncDir(filepath.Dir(dir))
+}
+
+func (s *Server) recoverRemoteArchiveJournals() error {
+ dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
+ entries, err := os.ReadDir(dir)
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ var errs []error
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ name := filepath.Join(dir, entry.Name())
+ if err := s.recoverRemoteArchiveJournal(name); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *Server) recoverRemoteArchiveJournal(name string) error {
+ raw, err := os.ReadFile(name)
+ if err != nil {
+ return err
+ }
+ var j remoteArchiveJournal
+ if err := json.Unmarshal(raw, &j); err != nil {
+ return err
+ }
+ if j.ArchiveID <= 0 {
+ return errors.New("invalid remote archive journal ID")
+ }
+ if err := s.archiveStore.Validate(j.URI); err != nil {
+ return err
+ }
+ item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
+ if err != nil {
+ return err
+ }
+ switch j.Operation {
+ case "upload":
+ if item == nil || item.ArchivePath != j.URI {
+ if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
+ return err
+ }
+ }
+ if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
+ return err
+ }
+ // An old failed upload may outlive a retry after switching back to
+ // local storage. The retry reuses this pathname: keep its committed
+ // package instead of treating it as the old upload's temporary copy.
+ if item == nil || item.ArchivePath != j.LocalPath {
+ if err := removeLocalArchivePackage(j.LocalPath); err != nil {
+ return err
+ }
+ }
+ case "delete":
+ if item == nil {
+ if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
+ return err
+ }
+ }
+ default:
+ return fmt.Errorf("unknown remote archive operation %q", j.Operation)
+ }
+ return os.Remove(name)
+}
+
+func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
+ if err != nil {
+ return err
+ }
+ if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
+ // A connection failure can leave the transaction's commit ambiguous.
+ // Recovery consults PostgreSQL before deciding whether to delete.
+ return err
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ // The task is gone; the durable outbox retries the remaining object cleanup.
+ log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
+ }
+ return nil
+}
diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go
new file mode 100644
index 000000000..ac2a44d43
--- /dev/null
+++ b/server/task_archive_remote_test.go
@@ -0,0 +1,419 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Autumn-27/artex/archivestore"
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+func TestRemoteArchiveLifecycle(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
+ s.archiveWG.Wait()
+ s.ctx = context.Background()
+ var mu sync.Mutex
+ objects := map[string][]byte{}
+ corrupt, failDelete := false, false
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ switch r.Method {
+ case "PUT":
+ objects[r.URL.Path], _ = io.ReadAll(r.Body)
+ w.Header().Set("ETag", `"test"`)
+ case "HEAD":
+ w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
+ w.Header().Set("ETag", `"test"`)
+ w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
+ case "GET":
+ data := objects[r.URL.Path]
+ if corrupt && len(data) > 0 {
+ data = bytes.Clone(data)
+ data[0] ^= 0xff
+ }
+ var start, end int64
+ if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
+ w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ w.Header().Set("ETag", `"test"`)
+ w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(data[start : end+1])
+ case "DELETE":
+ if failDelete {
+ w.WriteHeader(403)
+ return
+ }
+ delete(objects, r.URL.Path)
+ w.WriteHeader(204)
+ default:
+ w.WriteHeader(400)
+ }
+ }))
+ defer remote.Close()
+ s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ pgTask, err := m.pg.GetTask(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A configured-but-invalid backend must fail before moving any source data.
+ s.archiveStoreErr = errors.New("invalid fixture configuration")
+ if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
+ t.Fatalf("configuration error was not returned: %v", err)
+ }
+ if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
+ t.Fatalf("bad configuration lost source: %v", err)
+ }
+ if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
+ t.Fatal("bad configuration lost task", err)
+ }
+ s.archiveStoreErr = nil
+ if _, err := m.pg.QueueTaskArchive(id); err != nil {
+ t.Fatal(err)
+ }
+ // A corrupt readback must roll back filesystem staging and leave PG intact.
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected integrity failure")
+ }
+ if got, _ := os.ReadFile(file); string(got) != "keep this content" {
+ t.Fatal("upload failure lost source")
+ }
+ if task, err := m.pg.GetTask(id); err != nil || task == nil {
+ t.Fatal("upload failure lost task")
+ }
+ mu.Lock()
+ corrupt = false
+ mu.Unlock()
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.QueueTaskArchive(id); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ job, err = m.pg.GetTaskArchive(job.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
+ t.Fatalf("unexpected archive %+v", job)
+ }
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatal("workspace not released")
+ }
+ if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
+ t.Fatal("local package retained")
+ }
+ // Simulate the crash window after commit: cleanup keeps the remote object.
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ t.Fatal(err)
+ }
+ // A corrupted download must leave the archive and its remote package intact.
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected corrupted restore to fail")
+ }
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatal("corrupted archive installed workspace files")
+ }
+ if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
+ t.Fatalf("corrupted archive restored database: %v %v", restored, err)
+ }
+ mu.Lock()
+ if len(objects) == 0 {
+ t.Error("corrupted restore deleted the only remote package")
+ }
+ corrupt = false
+ mu.Unlock()
+ // Restoration commits files and PG first. Failed remote deletion is retryable.
+ mu.Lock()
+ failDelete = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected remote delete failure")
+ }
+ if got, _ := os.ReadFile(file); string(got) != "keep this content" {
+ t.Fatal("restored content differs")
+ }
+ if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
+ t.Fatalf("PG restore %v %v", restored, err)
+ }
+ mu.Lock()
+ failDelete = false
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
+ t.Fatal("restore did not consume stub")
+ }
+ var count int
+ if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
+ t.Fatalf("activity restore count=%d err=%v", count, err)
+ }
+ // Re-archive, then crash/retry the deletion outbox after PG removed its stub.
+ job, err = m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ failDelete = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ failDelete = false
+ mu.Unlock()
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ remaining := len(objects)
+ mu.Unlock()
+ if remaining != 0 {
+ t.Fatalf("left %d objects", remaining)
+ }
+}
+
+// Once PostgreSQL has committed an archive location and checksum, a subsequent
+// queued operation must not make startup recovery resurrect the old workspace.
+func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() {
+ _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
+ _ = m.pg.DeleteTask(taskID)
+ }()
+ if err := m.pg.SetPaused(taskID, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pgTask, err := m.pg.GetTask(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, backend := range []string{"local", "s3"} {
+ for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
+ t.Run(backend+"/"+state, func(t *testing.T) {
+ committed := state != pgdb.ArchiveQueued
+ checksum := ""
+ if committed {
+ checksum = strings.Repeat("a", 64)
+ }
+ packagePath := taskArchivePath(m.dir, job.ID, task.ID)
+ if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ location := packagePath
+ if backend == "s3" {
+ location = "s3://test/tasks/package.tar.zst"
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
+ t.Fatalf("staging cleanup incomplete: %v", err)
+ }
+ if backend == "local" && committed {
+ if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
+ t.Fatal("committed local package lost", err)
+ }
+ } else if !committed {
+ if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
+ t.Fatal("uncommitted temporary package retained", err)
+ }
+ }
+ if committed {
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatalf("resurrected committed workspace: %v", err)
+ }
+ } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
+ t.Fatalf("uncommitted archive lost source: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// Failed remote cleanup can survive switching to local storage and retrying the
+// same archive ID. Its journal must never remove the newly committed local copy.
+func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ task, err := m.CreateTask("local retry", "test", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() {
+ _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
+ _ = m.pg.DeleteTask(taskID)
+ }()
+ if err := m.pg.SetPaused(taskID, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := taskArchivePath(m.dir, job.ID, task.ID)
+ if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
+ t.Fatal(err)
+ }
+ var deletes atomic.Int32
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete {
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ return
+ }
+ deletes.Add(1)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer remote.Close()
+ store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := &Server{ctx: context.Background(), m: m, archiveStore: store}
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
+ ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
+ LocalPath: filename, Operation: "upload",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ t.Fatal(err)
+ }
+ if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
+ t.Fatalf("old upload cleanup removed local retry: %v", err)
+ }
+ if deletes.Load() != 1 {
+ t.Fatal("uncommitted remote object was not removed")
+ }
+ if _, err := os.Stat(journal); !os.IsNotExist(err) {
+ t.Fatalf("finished journal remains: %v", err)
+ }
+}
diff --git a/server/task_archive_startup_test.go b/server/task_archive_startup_test.go
new file mode 100644
index 000000000..def27d071
--- /dev/null
+++ b/server/task_archive_startup_test.go
@@ -0,0 +1,175 @@
+package server
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestArchiveStoreLoadsCredentialsBeforeSecretScrub(t *testing.T) {
+ skipWithoutDSN(t)
+ // New scrubs all startup secrets; restore the test database/key environment
+ // afterward so the remaining package tests keep their isolated configuration.
+ for _, key := range secretEnv {
+ t.Setenv(key, os.Getenv(key))
+ }
+ var signed atomic.Bool
+ endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete {
+ t.Errorf("unexpected S3 operation: %s", r.Method)
+ }
+ signed.Store(strings.Contains(r.Header.Get("Authorization"), "Credential=archive-fixture-key/") &&
+ r.Header.Get("X-Amz-Security-Token") == "archive-fixture-session")
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer endpoint.Close()
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
+ t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL)
+ t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
+ t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
+ t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives")
+ t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
+ t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "archive-fixture-key")
+ t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "archive-fixture-secret")
+ t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "archive-fixture-session")
+ t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256")
+ t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "")
+ // If initialization happens after the scrub, the SDK would silently select
+ // this different ambient identity instead of the dedicated override.
+ t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "")
+ t.Setenv("AWS_PROFILE", "")
+ t.Setenv("AWS_DEFAULT_PROFILE", "")
+ t.Setenv("AWS_CA_BUNDLE", "")
+ t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config"))
+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials"))
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // Exercise real startup without leaving archive/scheduler workers alive.
+ s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub())
+ s.archiveWG.Wait()
+ if s.archiveStoreErr != nil || s.archiveStore == nil {
+ t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr)
+ }
+ for _, key := range []string{"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN"} {
+ if _, present := os.LookupEnv(key); present {
+ t.Fatalf("%s remains in the child-process environment", key)
+ }
+ }
+ requestCtx, stop := context.WithTimeout(context.Background(), 3*time.Second)
+ defer stop()
+ if err := s.archiveStore.Delete(requestCtx, s.archiveStore.URI(1, 2, "startup-probe")); err != nil {
+ t.Fatal(err)
+ }
+ if !signed.Load() {
+ t.Fatal("archive client lost its dedicated credentials during startup scrub")
+ }
+}
+
+// The archive store may take its identity from the ambient AWS credential
+// variables instead of the ARTEX_ARCHIVE_S3_* override. Those variables are as
+// sensitive as the override: after startup no child process may inherit them,
+// and the S3 client built before the scrub must keep signing with them.
+func TestReview318AmbientAWSCredentialsScrubbedAfterArchiveInit(t *testing.T) {
+ skipWithoutDSN(t)
+ sh, err := exec.LookPath("sh")
+ if err != nil {
+ t.Skip("no sh in PATH")
+ }
+ awsNames := []string{
+ "AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
+ }
+ // New scrubs all startup secrets; t.Setenv restores them for the tests that follow.
+ for _, key := range secretEnv {
+ t.Setenv(key, os.Getenv(key))
+ }
+ for _, key := range awsNames {
+ t.Setenv(key, "probe-"+key)
+ }
+ var authorization, sessionToken atomic.Value
+ endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodHead {
+ t.Errorf("unexpected S3 operation: %s", r.Method)
+ }
+ authorization.Store(r.Header.Get("Authorization"))
+ sessionToken.Store(r.Header.Get("X-Amz-Security-Token"))
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer endpoint.Close()
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
+ t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL)
+ t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
+ t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
+ t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives")
+ t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
+ // No dedicated override: the SDK's default chain reads the AWS_* variables.
+ t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "")
+ t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
+ t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "")
+ t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256")
+ t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "")
+ t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "ambient-fixture-session")
+ t.Setenv("AWS_PROFILE", "")
+ t.Setenv("AWS_DEFAULT_PROFILE", "")
+ t.Setenv("AWS_CA_BUNDLE", "")
+ t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config"))
+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials"))
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // Exercise real startup without leaving archive/scheduler workers alive.
+ s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub())
+ s.archiveWG.Wait()
+ if s.archiveStoreErr != nil || s.archiveStore == nil {
+ t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr)
+ }
+
+ out, err := exec.Command(sh, "-c", "env").Output()
+ if err != nil {
+ t.Fatal(err)
+ }
+ env := "\n" + string(out)
+ for _, key := range awsNames {
+ if strings.Contains(env, "\n"+key+"=") {
+ t.Errorf("%s still reaches child processes after startup", key)
+ }
+ }
+
+ // HEAD is the first request of a download. The fixture answers 404, so the
+ // call fails; only the signature of the request matters here.
+ requestCtx, stop := context.WithTimeout(context.Background(), 5*time.Second)
+ defer stop()
+ checksum := strings.Repeat("a", 64)
+ err = s.archiveStore.Download(requestCtx, s.archiveStore.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1)
+ if err == nil {
+ t.Fatal("the fixture has no object: the download must fail")
+ }
+ got, _ := authorization.Load().(string)
+ if !strings.Contains(got, "Credential=ambient-fixture-key/") || !strings.Contains(got, "Signature=") {
+ t.Fatalf("archive client did not sign with the ambient key after the scrub: Authorization=%q (%v)", got, err)
+ }
+ if token, _ := sessionToken.Load().(string); token != "ambient-fixture-session" {
+ t.Fatalf("archive client lost the ambient session token after the scrub: %q", token)
+ }
+}
diff --git a/server/task_archives.go b/server/task_archives.go
index 8a6163247..daca52f9a 100644
--- a/server/task_archives.go
+++ b/server/task_archives.go
@@ -9,11 +9,13 @@ import (
"net/http"
"os"
"path/filepath"
+ "runtime/debug"
"strconv"
"strings"
"time"
"github.com/Autumn-27/artex/agent"
+ "github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -33,6 +35,19 @@ type archiveBatchItem struct {
Archive int64 `json:"archive_id,omitempty"`
}
+// initTaskArchiveStore captures credentials before New scrubs the process environment.
+func (s *Server) initTaskArchiveStore() {
+ if s.m == nil || s.m.pg == nil {
+ return
+ }
+ s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
+ if s.archiveStoreErr != nil {
+ log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
+ } else if s.archiveStore != nil {
+ log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
+ }
+}
+
func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
@@ -88,22 +103,19 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
+ if s.ctx.Err() != nil {
+ return s.ctx.Err()
+ }
+ // Retry deferred remote cleanup before claiming more work. Failed cleanup
+ // keeps its durable journal and does not block unrelated local archives.
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ log.Printf("[task-archive] deferred object cleanup: %v", err)
+ }
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
}
- var runErr error
- switch job.State {
- case pgdb.Archiving:
- runErr = s.archiveTask(job)
- case pgdb.Restoring:
- runErr = s.restoreTaskArchive(job)
- case pgdb.Deleting:
- runErr = s.deleteTaskArchive(job)
- default:
- runErr = fmt.Errorf("unknown claimed archive state %q", job.State)
- }
- if runErr != nil {
+ if runErr := s.runClaimedTaskArchiveJob(job); runErr != nil {
if failErr := s.m.pg.FailTaskArchiveJob(job.ID, job.State, runErr); failErr != nil {
return errors.Join(runErr, failErr)
}
@@ -114,7 +126,35 @@ func (s *Server) runOneTaskArchiveJob() error {
return nil
}
+// runClaimedTaskArchiveJob runs one claimed job and turns a panic below it into
+// an ordinary job failure. The worker goroutine has no other recover: a panic
+// there (the S3 upload manager dereferences a nil UploadId when an endpoint
+// answers CreateMultipartUpload without one) would end the whole server with the
+// job half done. The job's deferred rollbacks have already run when the panic is
+// recovered here, so the caller fails the job with its source data in place.
+func (s *Server) runClaimedTaskArchiveJob(job *pgdb.TaskArchive) (runErr error) {
+ defer func() {
+ if r := recover(); r != nil {
+ log.Printf("[task-archive] archive job %d (%s) panicked: %v\n%s", job.ID, job.State, r, debug.Stack())
+ runErr = fmt.Errorf("archive worker panic: %v", r)
+ }
+ }()
+ switch job.State {
+ case pgdb.Archiving:
+ return s.archiveTask(job)
+ case pgdb.Restoring:
+ return s.restoreTaskArchive(job)
+ case pgdb.Deleting:
+ return s.deleteTaskArchive(job)
+ default:
+ return fmt.Errorf("unknown claimed archive state %q", job.State)
+ }
+}
+
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
+ if s.archiveStoreErr != nil {
+ return s.archiveStoreErr
+ }
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -199,6 +239,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
+ storedPath := archivePath
+ var remoteJournal string
+ if s.archiveStore != nil {
+ storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
+ remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
+ ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
+ })
+ if err != nil {
+ return err
+ }
+ _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
+ if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
+ return err
+ }
+ }
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -213,11 +268,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
- if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
+ if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
- removePackage = false
+ removePackage = s.archiveStore != nil
+ if remoteJournal != "" {
+ // The journal also covers a crash after database commit but before local
+ // package removal. Keep it if cleanup fails, so the worker can retry.
+ if err := removeLocalArchivePackage(archivePath); err == nil {
+ _ = os.Remove(remoteJournal)
+ }
+ }
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -239,7 +301,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
- if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
+ if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -247,7 +309,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
- if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
+ if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -266,7 +328,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
- if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
+ packagePath := job.ArchivePath
+ if strings.HasPrefix(packagePath, "s3://") {
+ _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
+ packagePath = filepath.Join(extracted, "download.tar.zst")
+ if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
+ return err
+ }
+ }
+ if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -356,7 +426,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
- if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
+ if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -393,9 +463,12 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
- if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
+ if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
+ if strings.HasPrefix(job.ArchivePath, "s3://") {
+ return s.deleteRemoteTaskArchive(job)
+ }
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {