From 42bc19d521ea2153fa6506025d4f9207944341b8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 22 Sep 2026 02:34:14 -0700 Subject: [PATCH] fix(runner): prevent Retry-After duration overflow --- CHANGELOG.md | 1 + docs/benchmarks.md | 6 +++ internal/runner/benchmark.go | 8 ++- internal/runner/benchmark_backoff_test.go | 60 +++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 internal/runner/benchmark_backoff_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd6af4..0e6c178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Prevent large Hugging Face `Retry-After` values from overflowing into immediate retries while preserving server cooldowns and cancellation. - Label worker-owned Docker containers with optional run and command IDs so supervisors can clean up after cancellation. Thanks @jesse-merhi (#55). - Add `--platform` to build smaller npm tarballs for one supported operating system and architecture while retaining the default universal package. - Preserve every scanner report when sanitized target, profile, or custom scanner names collide with each other or generated numeric suffixes. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 3955784..1fdd29e 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -22,6 +22,12 @@ lines must be smaller than the parser's 1 MiB buffer limit. ## Available benchmarks +Hugging Face row requests retry temporary errors up to six attempts and honor +the server's `Retry-After` cooldown, including delays longer than 30 seconds. +Numeric cooldowns beyond Go's duration range saturate at its maximum instead +of wrapping into an immediate retry. Embedded clients can cancel the wait +through `HuggingFaceBenchmarkClient.Context`; CLI users can interrupt the process. + | Benchmark | ID | Source | | --- | --- | --- | | ClawHub Security Signals | `clawhub-security-signals` | [Hugging Face](https://huggingface.co/datasets/OpenClaw/clawhub-security-signals) | diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index fb8d00d..23a169d 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "math" "net/http" "net/url" "os" @@ -954,7 +955,12 @@ func isRetriableHuggingFaceRowsStatus(statusCode int) bool { func huggingFaceRowsBackoff(attempt int, headers http.Header) time.Duration { if retryAfter := headers.Get("Retry-After"); retryAfter != "" { - if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds >= 0 { + seconds, err := strconv.ParseUint(retryAfter, 10, 64) + if err == nil || (errors.Is(err, strconv.ErrRange) && strings.Trim(retryAfter, "0123456789") == "") { + // Saturate before converting seconds to nanoseconds so large cooldowns cannot wrap. + if seconds > uint64(math.MaxInt64/time.Second) { + return time.Duration(math.MaxInt64) + } return time.Duration(seconds) * time.Second } if retryAt, err := http.ParseTime(retryAfter); err == nil { diff --git a/internal/runner/benchmark_backoff_test.go b/internal/runner/benchmark_backoff_test.go new file mode 100644 index 0000000..5ea1fec --- /dev/null +++ b/internal/runner/benchmark_backoff_test.go @@ -0,0 +1,60 @@ +package runner + +import ( + "context" + "errors" + "math" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" +) + +func TestHuggingFaceRowsBackoffAvoidsDurationOverflow(t *testing.T) { + for _, tc := range []struct { + header string + want time.Duration + }{ + {"0", 0}, + {"3600", time.Hour}, + {"9223372036", 9223372036 * time.Second}, + {"9223372037", time.Duration(math.MaxInt64)}, + {strconv.FormatUint(math.MaxUint64, 10), time.Duration(math.MaxInt64)}, + {"18446744073709551616", time.Duration(math.MaxInt64)}, + {"-1", huggingFaceRowsRetryDelay}, + {"invalid", huggingFaceRowsRetryDelay}, + {"18446744073709551616invalid", huggingFaceRowsRetryDelay}, + } { + t.Run(tc.header, func(t *testing.T) { + if got := huggingFaceRowsBackoff(1, http.Header{"Retry-After": {tc.header}}); got != tc.want { + t.Fatalf("backoff = %s, want %s", got, tc.want) + } + }) + } +} + +func TestHuggingFaceRowsLongCooldownWaitsForDeadline(t *testing.T) { + for _, header := range []string{"3600", "9223372037", time.Now().Add(time.Hour).UTC().Format(http.TimeFormat)} { + t.Run(header, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Retry-After", header) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + client := HuggingFaceBenchmarkClient{Endpoint: server.URL, Context: ctx} + _, err := client.FetchOpenClawRows("OpenClaw/clawhub-security-signals", "eval_holdout", 0, 1) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("fetch error = %v, want deadline exceeded", err) + } + if got := requests.Load(); got != 1 { + t.Fatalf("requests = %d, want one request before deadline", got) + } + }) + } +}