Retry PostSyncer file uploads on 429

app/Support/Postsyncer/PostsyncerClient.php

@@ -7,6 +7,7 @@
 use GuzzleHttp\Psr7\UriResolver;
 use Illuminate\Http\Client\Response;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
 use Symfony\Component\Mime\MimeTypes;
 use Throwable;
 
@@ -17,6 +18,13 @@ class PostsyncerClient
 
     private const MAX_DOWNLOAD_REDIRECTS = 5;
 
+    private const UPLOAD_RATE_LIMIT_BACKOFF_SECONDS = [30, 60, 60];
+
+    private const UPLOAD_RATE_LIMIT_MAX_WAIT_SECONDS = 60;
+
+    // Shared by every file in one publish group so retries stay well inside the job timeout.
+    private const UPLOAD_RATE_LIMIT_TOTAL_WAIT_SECONDS = 120;
+
     public function __construct(
         private readonly PostsyncerConfig $config,
         private readonly ?PublicUrlGuard $urlGuard = null,
@@ -196,6 +204,7 @@ public function uploadFromUrls(int|string $workspaceId, array $urls, ?string $id
     public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?string $idempotencyKey = null): array
     {
         $paths = [];
+        $waitBudget = self::UPLOAD_RATE_LIMIT_TOTAL_WAIT_SECONDS;
 
         try {
             foreach ($urls as $url) {
@@ -215,9 +224,7 @@ public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?strin
                     : ['Idempotency-Key' => $idempotencyKey.':'.$index];
 
                 try {
-                    $response = $this->request('post', '/media/upload/file', [
-                        'workspace_id' => $workspaceId,
-                    ], $headers, $path);
+                    $response = $this->uploadFileWithRateLimitRetry($workspaceId, $headers, $path, $waitBudget);
                 } catch (PostsyncerException $exception) {
                     // A definitive rejection before any file was stored left
                     // nothing behind in PostSyncer, so a retry cannot duplicate.
@@ -267,6 +274,46 @@ public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?strin
         }
     }
 
+    /**
+     * PostSyncer's upload throttle answers 429 before storing anything, so the
+     * same file is retried after Retry-After (capped) or a fixed backoff. All
+     * files in one call share a 120 s wait budget, so retries plus re-sent
+     * uploads stay inside the 900-second publish job timeout.
+     * A 429 that outlasts the retries is still a definitive rejection.
+     *
+     * @param  array<string, string>  $headers
+     */
+    private function uploadFileWithRateLimitRetry(int|string $workspaceId, array $headers, string $path, int &$waitBudget): Response
+    {
+        $backoff = self::UPLOAD_RATE_LIMIT_BACKOFF_SECONDS;
+
+        for ($attempt = 0; ; $attempt++) {
+            try {
+                return $this->request('post', '/media/upload/file', [
+                    'workspace_id' => $workspaceId,
+                ], $headers, $path);
+            } catch (PostsyncerException $exception) {
+                if ($exception->getCode() !== 429) {
+                    throw $exception;
+                }
+
+                $wait = min($exception->retryAfter ?? $backoff[$attempt] ?? 0, self::UPLOAD_RATE_LIMIT_MAX_WAIT_SECONDS);
+
+                if ($attempt >= count($backoff) || $wait > $waitBudget) {
+                    throw new PostsyncerException(
+                        $exception->getMessage().' Gave up after '.count($backoff).' retries.',
+                        429,
+                        $exception,
+                        responseReceived: true,
+                    );
+                }
+
+                $waitBudget -= $wait;
+                Sleep::for($wait)->seconds();
+            }
+        }
+    }
+
     private function downloadMedia(string $url, string $path): void
     {
         $guard = $this->urlGuard ?? app(PublicUrlGuard::class);

app/Support/Postsyncer/PostsyncerException.php

@@ -16,6 +16,7 @@ public function __construct(
         public readonly bool $outcomeUnknown = false,
         public readonly bool $responseReceived = false,
         public readonly bool $safeToRetry = false,
+        public readonly ?int $retryAfter = null,
     ) {
         parent::__construct($message, $code, $previous);
     }
@@ -30,6 +31,7 @@ public static function fromResponse(Response $response): self
 
         $status = $response->status();
         $outcomeUnknown = in_array($status, [408, 425, 429], true) || $status >= 500;
+        $retryAfter = $response->header('Retry-After');
 
         return new self(
             "PostSyncer API error {$status}: {$message}",
@@ -38,6 +40,7 @@ public static function fromResponse(Response $response): self
             $outcomeUnknown,
             $outcomeUnknown,
             true,
+            retryAfter: ctype_digit($retryAfter) ? (int) $retryAfter : null,
         );
     }
 }

tests/Unit/Support/Postsyncer/PostsyncerClientTest.php

@@ -14,6 +14,7 @@
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
 use Tests\TestCase;
 
 class PostsyncerClientTest extends TestCase
@@ -194,6 +195,142 @@ public function test_upload_files_from_urls_keeps_a_rejection_after_a_stored_fil
         }
     }
 
+    public function test_upload_files_from_urls_retries_a_rate_limited_file_and_keeps_order(): void
+    {
+        Sleep::fake();
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence()
+                ->push(['media' => ['id' => 915]], 201)
+                ->push(['message' => 'Too Many Attempts.'], 429)
+                ->push(['media' => ['id' => 916]], 201),
+        ]);
+
+        $ids = $this->clientWithKey()->uploadFilesFromUrls(15211, [
+            'https://drive.usercontent.google.com/download?id=video',
+            'https://drive.usercontent.google.com/download?id=cover',
+        ], 'op:0:group:media');
+
+        $this->assertSame([915, 916], $ids);
+        $uploads = Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file'));
+        $this->assertCount(3, $uploads);
+        $this->assertSame(['op:0:group:media:1'], $uploads->last()[0]->header('Idempotency-Key'));
+        Sleep::assertSequence([Sleep::for(30)->seconds()]);
+    }
+
+    public function test_upload_files_from_urls_honors_a_capped_retry_after(): void
+    {
+        Sleep::fake();
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence()
+                ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '7'])
+                ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '3600'])
+                ->push(['media' => ['id' => 915]], 201),
+        ]);
+
+        $ids = $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']);
+
+        $this->assertSame([915], $ids);
+        Sleep::assertSequence([Sleep::for(7)->seconds(), Sleep::for(60)->seconds()]);
+    }
+
+    public function test_upload_files_from_urls_gives_up_on_rate_limit_and_names_stored_ids(): void
+    {
+        Sleep::fake();
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence()
+                ->push(['media' => ['id' => 915]], 201)
+                ->whenEmpty(Http::response(['message' => 'Too Many Attempts.'], 429)),
+        ]);
+
+        try {
+            $this->clientWithKey()->uploadFilesFromUrls(15211, [
+                'https://drive.usercontent.google.com/download?id=video',
+                'https://drive.usercontent.google.com/download?id=cover',
+            ]);
+            $this->fail('Expected the rate limit to throw.');
+        } catch (PostsyncerException $exception) {
+            $this->assertSame(429, $exception->getCode());
+            $this->assertFalse($exception->outcomeUnknown);
+            $this->assertFalse($exception->safeToRetry);
+            $this->assertStringContainsString('PostSyncer API error 429: Too Many Attempts.', $exception->getMessage());
+            $this->assertStringContainsString('Already stored PostSyncer media ids: 915.', $exception->getMessage());
+        }
+
+        // 30 + 60 s spends the shared budget; the third 60 s wait would exceed it.
+        Sleep::assertSequence([Sleep::for(30)->seconds(), Sleep::for(60)->seconds()]);
+        $this->assertCount(4, Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file')));
+    }
+
+    public function test_upload_files_from_urls_shares_one_rate_limit_wait_budget_across_files(): void
+    {
+        Sleep::fake();
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence()
+                ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '60'])
+                ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '60'])
+                ->push(['media' => ['id' => 915]], 201)
+                ->whenEmpty(Http::response(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '1'])),
+        ]);
+
+        try {
+            $this->clientWithKey()->uploadFilesFromUrls(15211, [
+                'https://drive.usercontent.google.com/download?id=video',
+                'https://drive.usercontent.google.com/download?id=cover',
+            ]);
+            $this->fail('Expected the exhausted budget to throw.');
+        } catch (PostsyncerException $exception) {
+            $this->assertSame(429, $exception->getCode());
+            $this->assertStringContainsString('Already stored PostSyncer media ids: 915.', $exception->getMessage());
+        }
+
+        Sleep::assertSequence([Sleep::for(60)->seconds(), Sleep::for(60)->seconds()]);
+        $this->assertCount(4, Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file')));
+    }
+
+    public function test_upload_files_from_urls_gives_up_on_a_first_file_rate_limit_safely(): void
+    {
+        Sleep::fake();
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => Http::response(['message' => 'Too Many Attempts.'], 429),
+        ]);
+
+        try {
+            $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']);
+            $this->fail('Expected the rate limit to throw.');
+        } catch (PostsyncerException $exception) {
+            $this->assertSame(429, $exception->getCode());
+            $this->assertTrue($exception->safeToRetry);
+        }
+    }
+
+    public function test_upload_files_from_urls_does_not_retry_a_connection_failure(): void
+    {
+        Sleep::fake();
+        $attempts = 0;
+        Http::fake([
+            'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200),
+            'upload.postsyncer.com/api/v1/media/upload/file' => function () use (&$attempts) {
+                $attempts++;
+                throw new ConnectionException('timed out');
+            },
+        ]);
+
+        try {
+            $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']);
+            $this->fail('Expected the lost upload response to throw.');
+        } catch (PostsyncerException $exception) {
+            $this->assertTrue($exception->outcomeUnknown);
+        }
+
+        Sleep::assertNeverSlept();
+        $this->assertSame(1, $attempts);
+    }
+
     public function test_upload_files_from_urls_download_failure_sends_nothing_to_postsyncer(): void
     {
         $before = $this->mediaTempFiles();