diff --git a/app/Support/Postsyncer/PostsyncerClient.php b/app/Support/Postsyncer/PostsyncerClient.php index e83f288..b735ed8 100644 --- a/app/Support/Postsyncer/PostsyncerClient.php +++ b/app/Support/Postsyncer/PostsyncerClient.php @@ -7,6 +7,7 @@ use GuzzleHttp\Psr7\UriResolver; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; use Symfony\Component\Mime\MimeTypes; use Throwable; @@ -17,6 +18,13 @@ class PostsyncerClient private const MAX_DOWNLOAD_REDIRECTS = 5; + private const UPLOAD_RATE_LIMIT_BACKOFF_SECONDS = [30, 60, 60]; + + private const UPLOAD_RATE_LIMIT_MAX_WAIT_SECONDS = 60; + + // Shared by every file in one publish group so retries stay well inside the job timeout. + private const UPLOAD_RATE_LIMIT_TOTAL_WAIT_SECONDS = 120; + public function __construct( private readonly PostsyncerConfig $config, private readonly ?PublicUrlGuard $urlGuard = null, @@ -196,6 +204,7 @@ public function uploadFromUrls(int|string $workspaceId, array $urls, ?string $id public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?string $idempotencyKey = null): array { $paths = []; + $waitBudget = self::UPLOAD_RATE_LIMIT_TOTAL_WAIT_SECONDS; try { foreach ($urls as $url) { @@ -215,9 +224,7 @@ public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?strin : ['Idempotency-Key' => $idempotencyKey.':'.$index]; try { - $response = $this->request('post', '/media/upload/file', [ - 'workspace_id' => $workspaceId, - ], $headers, $path); + $response = $this->uploadFileWithRateLimitRetry($workspaceId, $headers, $path, $waitBudget); } catch (PostsyncerException $exception) { // A definitive rejection before any file was stored left // nothing behind in PostSyncer, so a retry cannot duplicate. @@ -267,6 +274,46 @@ public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?strin } } + /** + * PostSyncer's upload throttle answers 429 before storing anything, so the + * same file is retried after Retry-After (capped) or a fixed backoff. All + * files in one call share a 120 s wait budget, so retries plus re-sent + * uploads stay inside the 900-second publish job timeout. + * A 429 that outlasts the retries is still a definitive rejection. + * + * @param array $headers + */ + private function uploadFileWithRateLimitRetry(int|string $workspaceId, array $headers, string $path, int &$waitBudget): Response + { + $backoff = self::UPLOAD_RATE_LIMIT_BACKOFF_SECONDS; + + for ($attempt = 0; ; $attempt++) { + try { + return $this->request('post', '/media/upload/file', [ + 'workspace_id' => $workspaceId, + ], $headers, $path); + } catch (PostsyncerException $exception) { + if ($exception->getCode() !== 429) { + throw $exception; + } + + $wait = min($exception->retryAfter ?? $backoff[$attempt] ?? 0, self::UPLOAD_RATE_LIMIT_MAX_WAIT_SECONDS); + + if ($attempt >= count($backoff) || $wait > $waitBudget) { + throw new PostsyncerException( + $exception->getMessage().' Gave up after '.count($backoff).' retries.', + 429, + $exception, + responseReceived: true, + ); + } + + $waitBudget -= $wait; + Sleep::for($wait)->seconds(); + } + } + } + private function downloadMedia(string $url, string $path): void { $guard = $this->urlGuard ?? app(PublicUrlGuard::class); diff --git a/app/Support/Postsyncer/PostsyncerException.php b/app/Support/Postsyncer/PostsyncerException.php index 0569ee8..ea02e14 100644 --- a/app/Support/Postsyncer/PostsyncerException.php +++ b/app/Support/Postsyncer/PostsyncerException.php @@ -16,6 +16,7 @@ public function __construct( public readonly bool $outcomeUnknown = false, public readonly bool $responseReceived = false, public readonly bool $safeToRetry = false, + public readonly ?int $retryAfter = null, ) { parent::__construct($message, $code, $previous); } @@ -30,6 +31,7 @@ public static function fromResponse(Response $response): self $status = $response->status(); $outcomeUnknown = in_array($status, [408, 425, 429], true) || $status >= 500; + $retryAfter = $response->header('Retry-After'); return new self( "PostSyncer API error {$status}: {$message}", @@ -38,6 +40,7 @@ public static function fromResponse(Response $response): self $outcomeUnknown, $outcomeUnknown, true, + retryAfter: ctype_digit($retryAfter) ? (int) $retryAfter : null, ); } } diff --git a/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php b/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php index 40eef43..e6103e9 100644 --- a/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php +++ b/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php @@ -14,6 +14,7 @@ use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; use Tests\TestCase; class PostsyncerClientTest extends TestCase @@ -194,6 +195,142 @@ public function test_upload_files_from_urls_keeps_a_rejection_after_a_stored_fil } } + public function test_upload_files_from_urls_retries_a_rate_limited_file_and_keeps_order(): void + { + Sleep::fake(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->push(['message' => 'Too Many Attempts.'], 429) + ->push(['media' => ['id' => 916]], 201), + ]); + + $ids = $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ], 'op:0:group:media'); + + $this->assertSame([915, 916], $ids); + $uploads = Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file')); + $this->assertCount(3, $uploads); + $this->assertSame(['op:0:group:media:1'], $uploads->last()[0]->header('Idempotency-Key')); + Sleep::assertSequence([Sleep::for(30)->seconds()]); + } + + public function test_upload_files_from_urls_honors_a_capped_retry_after(): void + { + Sleep::fake(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '7']) + ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '3600']) + ->push(['media' => ['id' => 915]], 201), + ]); + + $ids = $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + + $this->assertSame([915], $ids); + Sleep::assertSequence([Sleep::for(7)->seconds(), Sleep::for(60)->seconds()]); + } + + public function test_upload_files_from_urls_gives_up_on_rate_limit_and_names_stored_ids(): void + { + Sleep::fake(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->whenEmpty(Http::response(['message' => 'Too Many Attempts.'], 429)), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ]); + $this->fail('Expected the rate limit to throw.'); + } catch (PostsyncerException $exception) { + $this->assertSame(429, $exception->getCode()); + $this->assertFalse($exception->outcomeUnknown); + $this->assertFalse($exception->safeToRetry); + $this->assertStringContainsString('PostSyncer API error 429: Too Many Attempts.', $exception->getMessage()); + $this->assertStringContainsString('Already stored PostSyncer media ids: 915.', $exception->getMessage()); + } + + // 30 + 60 s spends the shared budget; the third 60 s wait would exceed it. + Sleep::assertSequence([Sleep::for(30)->seconds(), Sleep::for(60)->seconds()]); + $this->assertCount(4, Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file'))); + } + + public function test_upload_files_from_urls_shares_one_rate_limit_wait_budget_across_files(): void + { + Sleep::fake(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '60']) + ->push(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '60']) + ->push(['media' => ['id' => 915]], 201) + ->whenEmpty(Http::response(['message' => 'Too Many Attempts.'], 429, ['Retry-After' => '1'])), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ]); + $this->fail('Expected the exhausted budget to throw.'); + } catch (PostsyncerException $exception) { + $this->assertSame(429, $exception->getCode()); + $this->assertStringContainsString('Already stored PostSyncer media ids: 915.', $exception->getMessage()); + } + + Sleep::assertSequence([Sleep::for(60)->seconds(), Sleep::for(60)->seconds()]); + $this->assertCount(4, Http::recorded(fn ($request): bool => str_contains($request->url(), '/media/upload/file'))); + } + + public function test_upload_files_from_urls_gives_up_on_a_first_file_rate_limit_safely(): void + { + Sleep::fake(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response(['message' => 'Too Many Attempts.'], 429), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the rate limit to throw.'); + } catch (PostsyncerException $exception) { + $this->assertSame(429, $exception->getCode()); + $this->assertTrue($exception->safeToRetry); + } + } + + public function test_upload_files_from_urls_does_not_retry_a_connection_failure(): void + { + Sleep::fake(); + $attempts = 0; + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => function () use (&$attempts) { + $attempts++; + throw new ConnectionException('timed out'); + }, + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the lost upload response to throw.'); + } catch (PostsyncerException $exception) { + $this->assertTrue($exception->outcomeUnknown); + } + + Sleep::assertNeverSlept(); + $this->assertSame(1, $attempts); + } + public function test_upload_files_from_urls_download_failure_sends_nothing_to_postsyncer(): void { $before = $this->mediaTempFiles();