Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18767-session-delete-background.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Deleting a session now runs in the background, shows its progress, and can no longer leave a session listed whose files are gone** ([#18767](https://github.com/hivecommons/hive/issues/18767)). Deleting an ended session looked like it did nothing: the request stayed open for about a minute with no indicator, then the dashboard said "could not remove the session's files" although the files had been removed, and the session stayed in the list. Three things caused it. The sandbox was deleted under the 10-second timeout meant for tmux calls, which a HOME with a CLI's caches outlasts. The cancel that the timeout triggered ran `su-exec root kill`, and the image has no `kill` binary, so the cancel itself failed. The hive then reported that cancel error over a removal that had finished, and kept the record. Now `DELETE /api/sessions/{id}` on an ended or failed session answers 202 at once and the session is `deleting`: the tab shows "Deleting…" with a spinner and no controls, and disappears by itself when the hive is done. One job per session removes the files under its own 15-minute ceiling and removes the record last; what is on disk decides the outcome, so an error from a command that left nothing behind is a success. A real failure puts the session back as it was with the reason on the tab and a **Retry delete** button. The login safety from #18728 is unchanged in what it protects: a session holding a strictly newer, usable Claude login that cannot be adopted is not deleted, and now says so ("this session holds the hive's current sign-in; adopting it failed: ..."); having nothing to adopt never blocks or slows a delete. The cancel of a privileged child now uses the shell's builtin `kill`, so a real timeout ends the child instead of waiting for it. Sessions already left half-deleted are repaired without a manual step: at the next sweep (boot, and every list) an `ended` record whose sandbox and worktree are both gone is dropped, and a delete click on one completes at once. Every delete outcome writes one log line with its duration and a `session_closed` audit entry, failures included. `DELETE ...?remove=true` on a running session closes and deletes it in one call.
21 changes: 18 additions & 3 deletions dashboard/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -45108,17 +45108,23 @@
"tags": [
"Sessions"
],
"summary": "Close a session (creator or owner)",
"description": "Kills tmux and keeps the metadata marked ended; on an ended session removes its metadata.",
"summary": "Close or delete a session (creator or owner)",
"description": "On a running session: kills tmux and keeps the metadata marked ended (with remove=true it is then deleted). On an ended or failed session: starts the delete and answers 202 at once with the session as deleting; the files are removed by a background job, the record goes when they are gone, and a failure puts the previous status back with delete_error.",
"responses": {
"200": {
"description": "OK"
"description": "Closed (running session), or removed=true: nothing was left on disk and the record is gone"
},
"202": {
"description": "The delete is running in the background; body carries the session with status deleting"
},
"403": {
"description": "Not the creator or an owner"
},
"404": {
"description": "Unknown session"
},
"409": {
"description": "The session is still being prepared, or changed state meanwhile"
}
},
"parameters": [
Expand All @@ -45129,6 +45135,15 @@
"schema": {
"type": "string"
}
},
{
"name": "remove",
"in": "query",
"required": false,
"description": "true: delete a running session after closing it",
"schema": {
"type": "string"
}
}
]
}
Expand Down
2 changes: 1 addition & 1 deletion src/crux/agent/usersession_login.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ func (o sessionLoginOps) reconcileClaudeLogin(id string, mode sessionLoginMode)
if sessionClaudeLoginNewer(sess, info.ModTime(), shared, fileModTime(target), now) {
staleShared, perr := promoteClaudeCredential(data, target, now)
if perr != nil {
return none, fmt.Errorf("adopting the session's newer login into the shared login: %w", perr)
return none, fmt.Errorf("%w; adopting it failed: %w", crux.ErrUserSessionHoldsLogin, perr)
}
res := sessionLoginResult{action: sessionLoginAdopted, staleShared: staleShared}
if mode == sessionLoginModeRemove {
Expand Down
70 changes: 15 additions & 55 deletions src/crux/agent/usersession_priv.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,18 +245,32 @@ func newUserSessionPrivCmd(ctx context.Context, user string, stdin io.Reader, ar
return cmd, nil
}

// userSessionRootKillScript signals the process group $1 with KILL. `kill` is
// the shell's builtin: the image ships no kill binary, so `su-exec root kill`
// could never run ("su-exec: kill: No such file or directory"), the cancel
// failed, and Wait kept blocking until the child finished by itself and then
// reported the cancel's error over a command that had succeeded. The -s form
// is the one dash and bash both take with a negative pid; the second form is
// for a shell that only knows the signal-name flag.
const userSessionRootKillScript = `kill -s KILL -- "-$1" 2>/dev/null || kill -KILL "-$1"`

// killUserSessionRootGroup ends a root child (and everything it spawned, via
// its process group) through the setuid helper, the only way the hive's uid
// can signal a root process.
func killUserSessionRootGroup(pid int, env []string) error {
kill := exec.Command(userSuExecBinary, userSessionRootSpec, "kill", "-KILL", "--", "-"+strconv.Itoa(pid))
kill := exec.Command(userSuExecBinary, userSessionRootKillArgv(pid)...)
kill.Env = env
if out, err := kill.CombinedOutput(); err != nil {
return outputErr("killing privileged child", err, out)
}
return nil
}

// userSessionRootKillArgv is the su-exec argv killUserSessionRootGroup runs.
func userSessionRootKillArgv(pid int) []string {
return []string{userSessionRootSpec, "sh", "-c", userSessionRootKillScript, "sh", strconv.Itoa(pid)}
}

// privRun is r.priv bound to the standard command timeout.
func (r *UserSessionRuntime) privRun(user string, stdin io.Reader, argv ...string) ([]byte, error) {
return r.privRunFor(userSessionCmdTimeout, user, stdin, argv...)
Expand Down Expand Up @@ -353,57 +367,3 @@ func (r *UserSessionRuntime) openSocketDirToAttacher(uid int) error {
}
return nil
}

// Remove ends the session and deletes everything it owned: its worktree of the
// shared workspace clone (uncommitted work is auto-saved onto its branch first,
// see userSessionWorktreeRemoveScript), the sandbox (home, work dir, uploads,
// tmp) and the uid's tmux socket dir, so a later session that is allocated the
// same uid starts clean. Directories earlier releases created for the id are
// removed too. A failure is returned and the caller must keep the session
// record (and its uid slot) so the removal is retried.
func (r *UserSessionRuntime) Remove(id string) error {
if err := checkUserSessionID(id); err != nil {
return err
}
uid, uidErr := r.sessionUID(id)
if uidErr == nil {
if err := r.killAs(uid, id); err != nil {
r.logger.Warn("user session kill before removal failed", "id", id, "error", err)
}
}
// The session's CLI is gone (killed above), so its login can no longer
// change. If that login is the newest one on this hive, it goes into the
// shared file BEFORE the sandbox is deleted: removing a session must never
// remove the only valid login. A login that cannot be judged or adopted
// keeps the sandbox, and the caller retries.
// (The CLI creates files and directories with private modes; the hive
// must be able to look at the login path to judge it.)
r.makeHomeReadable(id)
if err := r.syncClaudeLogin(id, sessionLoginTriggerRemove, sessionLoginModeRemove); err != nil {
return fmt.Errorf("keeping the session's login before removal: %w", err)
}
r.workspaceMu.Lock()
err := r.removeWorktrees(id)
r.workspaceMu.Unlock()
if err != nil {
return fmt.Errorf("removing session worktree: %w", err)
}
dir := userSessionSandboxDir(id)
if _, err := os.Lstat(dir); err == nil {
if out, err := r.privRun(userSessionRootSpec, nil, "rm", "-rf", "--one-file-system", "--", dir); err != nil {
return outputErr("removing session sandbox", err, out)
}
}
if uidErr == nil {
sock := userSessionTmuxSocketDir(uid)
if out, err := r.privRun(userSessionRootSpec, nil, "rm", "-rf", "--one-file-system", "--", sock); err != nil {
return outputErr("removing session tmux socket dir", err, out)
}
}
for _, p := range legacyUserSessionPaths(id) {
if err := os.RemoveAll(p); err != nil {
return fmt.Errorf("removing legacy session dir %s: %w", p, err)
}
}
return nil
}
135 changes: 135 additions & 0 deletions src/crux/agent/usersession_remove.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package agent

import (
"errors"
"fmt"
"io/fs"
"os"
"time"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/sharedfile"
)

// Removing a session's files.
//
// The failure this closes (#18767). The sandbox was deleted with the
// 10-second userSessionCmdTimeout that bounds a tmux call. A HOME holding a
// CLI's caches takes longer than that to delete (a minute was seen), so the
// timeout fired, the cancel of the root child could not run (see
// userSessionRootKillScript), `rm` finished by itself, and the caller was
// handed the cancel's error for a removal that had succeeded: the files were
// gone and the session record was kept.
//
// Two rules follow. A removal has its own, generous deadline. And what decides
// whether a tree was removed is the tree: a command that reports an error
// after the directory is gone has removed it.

// userSessionRemoveTimeout bounds deleting one session tree (the sandbox, the
// tmux socket dir). It is a ceiling for a wedged filesystem, not an estimate.
const userSessionRemoveTimeout = 15 * time.Minute

// removeTree deletes dir as root. An error from a command that left nothing
// behind is logged and dropped: the directory is gone, which is what was
// asked for.
func (r *UserSessionRuntime) removeTree(what, dir string) error {
out, err := r.privRunFor(userSessionRemoveTimeout, userSessionRootSpec, nil, "rm", "-rf", "--one-file-system", "--", dir)
if err == nil {
return nil
}
if _, serr := os.Lstat(dir); errors.Is(serr, fs.ErrNotExist) {
r.logger.Info("user session removal: the command reported an error, but the directory is gone; treated as removed", "what", what, "path", dir, "error", err)
return nil
}
return outputErr(what, err, out)
}

// Remove ends the session and deletes everything it owned: its worktree of the
// shared workspace clone (uncommitted work is auto-saved onto its branch first,
// see userSessionWorktreeRemoveScript), the sandbox (home, work dir, uploads,
// tmp) and the uid's tmux socket dir, so a later session that is allocated the
// same uid starts clean. Directories earlier releases created for the id are
// removed too. It is safe to call again after any failure, and on a session
// that is already partly or wholly gone. A failure is returned and the caller
// must keep the session record (and its uid slot) so the removal is retried.
//
// The order puts what can refuse first and what cannot be undone last: the
// login check removes nothing, the worktree step keeps the work on a branch
// (a resume gets the worktree back from it), and only then is the sandbox
// deleted.
func (r *UserSessionRuntime) Remove(id string) error {
if err := checkUserSessionID(id); err != nil {
return err
}
uid, uidErr := r.sessionUID(id)
if uidErr == nil {
if err := r.killAs(uid, id); err != nil {
r.logger.Warn("user session kill before removal failed", "id", id, "error", err)
}
}
// The session's CLI is gone (killed above), so its login can no longer
// change. If that login is the newest one on this hive, it goes into the
// shared file BEFORE the sandbox is deleted: removing a session must never
// remove the only valid login. "Nothing to adopt" (no file, an expired
// copy, a copy without a refresh token, a file that is not a credential)
// is not a failure and costs two stats and one small read.
// (The CLI creates files and directories with private modes; the hive
// must be able to look at the login path to judge it.)
r.makeHomeReadable(id)
if err := r.syncClaudeLogin(id, sessionLoginTriggerRemove, sessionLoginModeRemove); err != nil {
switch {
case errors.Is(err, sharedfile.ErrTooLarge):
// Over the size of any credentials file: not a login, so there
// is nothing to keep. syncClaudeLogin logged it.
case errors.Is(err, crux.ErrUserSessionHoldsLogin):
return err
default:
return fmt.Errorf("this session's sign-in could not be checked, so its files are kept: %w", err)
}
}
r.workspaceMu.Lock()
err := r.removeWorktrees(id)
r.workspaceMu.Unlock()
if err != nil {
return fmt.Errorf("removing session worktree: %w", err)
}
dir := userSessionSandboxDir(id)
if _, err := os.Lstat(dir); err == nil {
if err := r.removeTree("removing session sandbox", dir); err != nil {
return err
}
}
if uidErr == nil {
if err := r.removeTree("removing session tmux socket dir", userSessionTmuxSocketDir(uid)); err != nil {
return err
}
}
for _, p := range legacyUserSessionPaths(id) {
if err := os.RemoveAll(p); err != nil {
return fmt.Errorf("removing legacy session dir %s: %w", p, err)
}
}
return nil
}

// Removed reports whether nothing of session id is left on disk: no sandbox,
// no worktrees and none of the directories earlier releases made. It answers
// false when that cannot be told (the sandbox root itself is missing or
// unreadable, as on a volume that is not mounted), so a caller that drops
// records of removed sessions never drops one whose files it merely cannot
// see.
func (r *UserSessionRuntime) Removed(id string) bool {
if checkUserSessionID(id) != nil {
return false
}
if fi, err := os.Stat(userSessionSandboxRoot()); err != nil || !fi.IsDir() {
return false
}
paths := append([]string{userSessionSandboxDir(id), userSessionWorktreesDir(id)}, legacyUserSessionPaths(id)...)
for _, p := range paths {
if _, err := os.Lstat(p); !errors.Is(err, fs.ErrNotExist) {
return false
}
}
return true
}
Loading
Loading