fix(cli): verify ICMSRequest deletion before reporting kill-all/kill-function success

src/clis/nvcf-cli/README.md

@@ -1742,10 +1742,15 @@ the config change is already persisted and re-running is a no-op.
 ### How kill works
 
 `kill-function` and `kill-all` delete the matching `ICMSRequest` CRs; the NVCA
-reconciler detects the deletion and evicts the workloads. Deletion is
-asynchronous, so the command returns once the delete is accepted. `--force`
-additionally strips finalizers so a request stuck `Terminating` is removed even
-when NVCA is not running to process its finalizer.
+reconciler detects the deletion and evicts the workloads. Deleting a CR only
+accepts the deletion; the object stays `Terminating` behind its finalizer
+until NVCA finishes evicting the workload and removes it. The command polls
+for the CR to actually disappear before reporting success: a request removed
+within `--timeout` (default 60s) is reported `deleted`, and one still present
+when the timeout elapses is reported `terminating` instead, with a non-zero
+exit code. `--force` additionally strips finalizers so a request stuck
+`Terminating` is removed even when NVCA is not running to process its
+finalizer.
 
 ### Confirmation and safety
 
@@ -1760,7 +1765,9 @@ connected cluster. When the cluster has no name, it falls back to the cluster id
 All maintenance commands accept `--dry-run` to preview without mutating, and
 `--expect-cluster-id <id>` to refuse to act unless the connected cluster's id or
 name matches (guards against a wrong `--compute-plane-context`). `kill-function`
-and `kill-all` accept `--reason` for an audit note, and `--json` for automation.
+and `kill-all` accept `--reason` for an audit note, `--timeout` to bound how
+long to wait for NVCA to finish evicting a terminated request (default 60s),
+and `--json` for automation.
 
 These commands need write access to the target cluster: get/update on the
 `agent-config` ConfigMap and the `nvca` Deployment for drain, and list/delete

src/clis/nvcf-cli/cmd/cluster_agent_maintenance.go

@@ -150,6 +150,7 @@ func initClusterAgentMaintenanceCmds() {
 	for _, c := range []*cobra.Command{clusterAgentKillFunctionCmd, clusterAgentKillAllCmd} {
 		c.Flags().String(flagReason, "", "Optional reason recorded in logs for audit")
 		c.Flags().Bool(flagForce, false, "Strip finalizers so requests stuck Terminating are removed")
+		c.Flags().Duration(flagTimeout, clusteragent.DefaultKillTimeout, "How long to wait for NVCA to finish evicting a terminated request before reporting it as still terminating")
 	}
 	clusterAgentKillAllCmd.Flags().String(flagConfirm, "", "Cluster name confirming kill-all (required with --yes)")
 }
@@ -251,6 +252,7 @@ func runClusterAgentKillFunction(cmd *cobra.Command, args []string) error {
 	force, _ := cmd.Flags().GetBool(flagForce)
 	reason, _ := cmd.Flags().GetString(flagReason)
 	expect, _ := cmd.Flags().GetString(flagExpectClusterID)
+	timeout, _ := cmd.Flags().GetDuration(flagTimeout)
 
 	ctx := context.Background()
 
@@ -278,6 +280,7 @@ func runClusterAgentKillFunction(cmd *cobra.Command, args []string) error {
 		Reason:          reason,
 		DryRun:          dryRun,
 		Force:           force,
+		Timeout:         timeout,
 	})
 	return finishKill(cmd, res, err)
 }
@@ -295,6 +298,7 @@ func runClusterAgentKillAll(cmd *cobra.Command, _ []string) error {
 	reason, _ := cmd.Flags().GetString(flagReason)
 	expect, _ := cmd.Flags().GetString(flagExpectClusterID)
 	confirm, _ := cmd.Flags().GetString(flagConfirm)
+	timeout, _ := cmd.Flags().GetDuration(flagTimeout)
 
 	ctx := context.Background()
 
@@ -346,6 +350,7 @@ func runClusterAgentKillAll(cmd *cobra.Command, _ []string) error {
 		Reason:          reason,
 		DryRun:          dryRun,
 		Force:           force,
+		Timeout:         timeout,
 	})
 	return finishKill(cmd, res, err)
 }
@@ -480,21 +485,27 @@ func printKillResult(cmd *cobra.Command, res *clusteragent.KillResult) {
 		prefix = "[dry-run] "
 		verbed = "would terminate"
 	}
-	fmt.Fprintf(w, "%s%s %d request(s) in namespace %s\n", prefix, verbed, len(res.Affected)-res.FailedCount, res.RequestsNamespace)
+	deletedCount := len(res.Affected) - res.FailedCount - res.TerminatingCount
+	fmt.Fprintf(w, "%s%s %d request(s) in namespace %s\n", prefix, verbed, deletedCount, res.RequestsNamespace)
 	if res.Reason != "" {
 		fmt.Fprintf(w, "  reason: %s\n", res.Reason)
 	}
 	for _, r := range res.Affected {
 		status := "deleted"
-		if res.DryRun {
+		switch {
+		case res.DryRun:
 			status = "would delete"
-		}
-		if r.Error != "" {
+		case r.Error != "":
 			status = "FAILED: " + r.Error
+		case r.Terminating:
+			status = "terminating: NVCA has not finished evicting the workload yet"
 		}
 		fmt.Fprintf(w, "  %s/%s  function=%s version=%s  [%s]\n", r.Namespace, r.Name, orDash(r.FunctionID), orDash(r.FunctionVersionID), status)
 	}
 	if res.FailedCount > 0 {
 		fmt.Fprintf(w, "%d of %d request(s) failed\n", res.FailedCount, len(res.Affected))
 	}
+	if res.TerminatingCount > 0 {
+		fmt.Fprintf(w, "%d of %d request(s) still terminating; re-check with cluster agent get-function\n", res.TerminatingCount, len(res.Affected))
+	}
 }

src/clis/nvcf-cli/cmd/cluster_agent_maintenance_test.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"nvcf-cli/internal/clusteragent"
 
@@ -391,6 +392,41 @@ func TestKillFunctionPartialFailureReturnsError(t *testing.T) {
 	}
 }
 
+func TestKillFunctionTimeoutFlagForwarded(t *testing.T) {
+	f := &fakeMaintainer{killResult: &clusteragent.KillResult{RequestsNamespace: "nvcf-backend", Affected: []clusteragent.KilledRequest{{Name: "r1", FunctionID: "fn"}}}}
+	withFakeMaintainer(t, f)
+
+	if _, err := executeMaintenance(t, "", "cluster", "agent", "kill-function", "fn", "--yes", "--timeout", "45s"); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if f.lastKillOpts.Timeout != 45*time.Second {
+		t.Fatalf("Timeout = %s, want 45s", f.lastKillOpts.Timeout)
+	}
+}
+
+func TestKillFunctionTerminatingOutput(t *testing.T) {
+	f := &fakeMaintainer{
+		killResult: &clusteragent.KillResult{
+			RequestsNamespace: "nvcf-backend",
+			Affected:          []clusteragent.KilledRequest{{Name: "r1", FunctionID: "fn", Terminating: true}},
+			TerminatingCount:  1,
+		},
+		killErr: errFakeKill,
+	}
+	withFakeMaintainer(t, f)
+
+	out, err := executeMaintenance(t, "", "cluster", "agent", "kill-function", "fn", "--yes")
+	if err == nil {
+		t.Fatal("expected the aggregate error to propagate")
+	}
+	if !strings.Contains(out, "terminating") {
+		t.Errorf("expected the still-terminating request to be printed, got:\n%s", out)
+	}
+	if strings.Contains(out, "[deleted]") {
+		t.Errorf("a still-terminating request must not be reported as deleted, got:\n%s", out)
+	}
+}
+
 // --- kill-all ---
 
 func TestKillAllTypeInInteractive(t *testing.T) {
@@ -526,6 +562,21 @@ func TestKillAllDryRun(t *testing.T) {
 	}
 }
 
+func TestKillAllTimeoutFlagForwarded(t *testing.T) {
+	f := &fakeMaintainer{
+		target:     &clusteragent.ClusterTarget{ClusterID: "c1", ClusterName: "edge-1", RequestsNamespace: "nvcf-backend"},
+		killResult: &clusteragent.KillResult{RequestsNamespace: "nvcf-backend", Affected: []clusteragent.KilledRequest{{Name: "r1", FunctionID: "fn"}}},
+	}
+	withFakeMaintainer(t, f)
+
+	if _, err := executeMaintenance(t, "", "cluster", "agent", "kill-all", "--yes", "--confirm", "edge-1", "--timeout", "45s"); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if f.lastKillOpts.Timeout != 45*time.Second {
+		t.Fatalf("Timeout = %s, want 45s", f.lastKillOpts.Timeout)
+	}
+}
+
 // --- JSON output ---
 
 func TestDrainJSONOutput(t *testing.T) {
@@ -543,3 +594,26 @@ func TestDrainJSONOutput(t *testing.T) {
 		t.Errorf("unexpected JSON output:\n%s", out)
 	}
 }
+
+func TestKillFunctionJSONOutput(t *testing.T) {
+	f := &fakeMaintainer{
+		killResult: &clusteragent.KillResult{
+			RequestsNamespace: "nvcf-backend",
+			Affected:          []clusteragent.KilledRequest{{Name: "r1", FunctionID: "fn", Terminating: true}},
+			TerminatingCount:  1,
+		},
+		killErr: errFakeKill,
+	}
+	withFakeMaintainer(t, f)
+
+	var err error
+	out := captureMaintStdout(t, func() {
+		_, err = executeMaintenance(t, "", "cluster", "agent", "kill-function", "fn", "--yes", "--json")
+	})
+	if err == nil {
+		t.Fatal("expected the aggregate error to propagate")
+	}
+	if !strings.Contains(out, `"terminatingCount": 1`) || !strings.Contains(out, `"terminating": true`) {
+		t.Errorf("unexpected JSON output:\n%s", out)
+	}
+}