diff --git a/app/Actions/Postsyncer/PublishVideoAction.php b/app/Actions/Postsyncer/PublishVideoAction.php index 421c2b0..b02f3a1 100644 --- a/app/Actions/Postsyncer/PublishVideoAction.php +++ b/app/Actions/Postsyncer/PublishVideoAction.php @@ -117,7 +117,7 @@ public function handle(Video $video, array $options, ?string $runToken = null): } if ($group->mediaUrls !== []) { - $mediaIds = $client->uploadFromUrls( + $mediaIds = $client->uploadFilesFromUrls( $group->workspaceId, $group->mediaUrls, $idempotencyKey.':media', diff --git a/app/Jobs/PublishVideoJob.php b/app/Jobs/PublishVideoJob.php index b40d9c8..5fe0037 100644 --- a/app/Jobs/PublishVideoJob.php +++ b/app/Jobs/PublishVideoJob.php @@ -18,11 +18,13 @@ class PublishVideoJob implements ShouldBeUnique, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - public const TIMEOUT_SECONDS = 180; + // Video media is downloaded and re-uploaded as files (up to ~300 MB per + // video), so match the post job and the worker's 900-second timeout. + public const TIMEOUT_SECONDS = 900; public const UNIQUE_FOR_SECONDS = 3600; - public const OVERLAP_EXPIRES_AFTER_SECONDS = 240; + public const OVERLAP_EXPIRES_AFTER_SECONDS = 1020; public int $timeout = self::TIMEOUT_SECONDS; diff --git a/app/Support/Postsyncer/PostsyncerClient.php b/app/Support/Postsyncer/PostsyncerClient.php index bbe03ac..e83f288 100644 --- a/app/Support/Postsyncer/PostsyncerClient.php +++ b/app/Support/Postsyncer/PostsyncerClient.php @@ -2,14 +2,24 @@ namespace App\Support\Postsyncer; +use App\Support\LinkResolution\PublicUrlGuard; +use GuzzleHttp\Psr7\Uri; +use GuzzleHttp\Psr7\UriResolver; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Http; +use Symfony\Component\Mime\MimeTypes; use Throwable; class PostsyncerClient { + /** PostSyncer's documented limit for POST /media/upload/file. */ + public const MAX_FILE_UPLOAD_BYTES = 500 * 1024 * 1024; + + private const MAX_DOWNLOAD_REDIRECTS = 5; + public function __construct( private readonly PostsyncerConfig $config, + private readonly ?PublicUrlGuard $urlGuard = null, ) {} /** @@ -173,6 +183,201 @@ public function uploadFromUrls(int|string $workspaceId, array $urls, ?string $id return $ids; } + /** + * Upload media through PostSyncer's multipart file endpoint, one request + * per file, because its URL import stores no videos (2026-10-26). Every + * URL is downloaded first, so a download failure happens before anything + * reaches PostSyncer. + * + * @param list $urls + * @param string|null $idempotencyKey Stable per-operation/group key; each file gets its own suffix. + * @return list Media ids in the same order as $urls. + */ + public function uploadFilesFromUrls(int|string $workspaceId, array $urls, ?string $idempotencyKey = null): array + { + $paths = []; + + try { + foreach ($urls as $url) { + $path = tempnam(sys_get_temp_dir(), 'cm-media-'); + if ($path === false) { + throw $this->safeUploadFailure('Could not create a temporary file for the media download.'); + } + $paths[] = $path; + $this->downloadMedia($url, $path); + } + + $ids = []; + + foreach ($paths as $index => $path) { + $headers = $idempotencyKey === null || trim($idempotencyKey) === '' + ? [] + : ['Idempotency-Key' => $idempotencyKey.':'.$index]; + + try { + $response = $this->request('post', '/media/upload/file', [ + 'workspace_id' => $workspaceId, + ], $headers, $path); + } catch (PostsyncerException $exception) { + // A definitive rejection before any file was stored left + // nothing behind in PostSyncer, so a retry cannot duplicate. + if ($ids === [] && $exception->responseReceived && ! $exception->outcomeUnknown) { + throw new PostsyncerException( + $exception->getMessage(), + $exception->getCode(), + $exception, + responseReceived: true, + safeToRetry: true, + ); + } + + if ($ids !== []) { + // Name the orphaned media so reconciliation can find it. + throw new PostsyncerException( + $exception->getMessage().' Already stored PostSyncer media ids: '.implode(', ', $ids).'.', + $exception->getCode(), + $exception, + $exception->retryable, + $exception->outcomeUnknown, + $exception->responseReceived, + $exception->safeToRetry, + ); + } + + throw $exception; + } + + $id = $this->decodeResponse($response)['media']['id'] ?? null; + + if (! $this->hasMediaId($id)) { + throw new PostsyncerException( + 'PostSyncer returned a media item without a valid id. ' + .'Refusing to publish with invalid media.' + ); + } + + $ids[] = $id; + } + + return $ids; + } finally { + foreach ($paths as $path) { + @unlink($path); + } + } + } + + private function downloadMedia(string $url, string $path): void + { + $guard = $this->urlGuard ?? app(PublicUrlGuard::class); + $overCap = false; + $progress = $this->downloadCapProgress($overCap); + + for ($hop = 0; ; $hop++) { + $curlOptions = $this->isDriveDownloadUrl($url) && $guard->isSafe($url) + ? $guard->curlResolveOptions($url) + : null; + + if ($curlOptions === null) { + throw $this->safeUploadFailure('Refusing to download media from a URL that is not a public Google Drive download.'); + } + + try { + $response = Http::timeout(300) + ->sink($path) + ->withOptions([ + 'allow_redirects' => false, + 'progress' => $progress, + ...$curlOptions, + ]) + ->get($url); + } catch (Throwable $exception) { + throw $overCap + ? $this->oversizeFailure() + : $this->safeUploadFailure('Could not download media before uploading it to PostSyncer: ' + .$exception->getMessage(), $exception); + } + + if (! $response->redirect()) { + break; + } + + $location = $response->header('Location'); + if ($location === '' || $hop >= self::MAX_DOWNLOAD_REDIRECTS) { + throw $this->safeUploadFailure('Could not download media before uploading it to PostSyncer: too many redirects.'); + } + + $url = (string) UriResolver::resolve(new Uri($url), new Uri($location)); + } + + if (! $response->successful()) { + throw $this->safeUploadFailure('Could not download media before uploading it to PostSyncer: HTTP ' + .$response->status().'.'); + } + + $length = $response->header('Content-Length'); + if ($length !== '' && (int) $length > static::MAX_FILE_UPLOAD_BYTES) { + throw $this->oversizeFailure(); + } + + clearstatcache(true, $path); + $size = (int) filesize($path); + + if ($size === 0) { + throw $this->safeUploadFailure('The downloaded media file is empty. Refusing to upload it to PostSyncer.'); + } + + if ($size > static::MAX_FILE_UPLOAD_BYTES) { + throw $this->oversizeFailure(); + } + + // Drive serves an HTML page instead of the file when sharing is off. + if (str_starts_with((string) MimeTypes::getDefault()->guessMimeType($path), 'text/html')) { + throw $this->safeUploadFailure('The media URL returned an HTML page instead of a file. ' + .'Check that the Drive file is shared publicly.'); + } + } + + /** + * Guzzle's progress option: a truthy return makes its cURL handler abort + * the transfer (Guzzle 8 rejects raw CURLOPT_PROGRESSFUNCTION), as soon + * as Content-Length or the received bytes pass the cap. + */ + private function downloadCapProgress(bool &$overCap): \Closure + { + return function (int $expected, int $downloaded) use (&$overCap): bool { + $overCap = $expected > static::MAX_FILE_UPLOAD_BYTES || $downloaded > static::MAX_FILE_UPLOAD_BYTES; + + return $overCap; + }; + } + + private function isDriveDownloadUrl(string $url): bool + { + $parts = parse_url($url); + $host = strtolower((string) ($parts['host'] ?? '')); + + return strtolower((string) ($parts['scheme'] ?? '')) === 'https' + && (in_array($host, ['drive.usercontent.google.com', 'drive.google.com'], true) + || str_ends_with($host, '.googleusercontent.com')); + } + + private function oversizeFailure(): PostsyncerException + { + return $this->safeUploadFailure(sprintf( + 'The media file is over PostSyncer\'s %d MB upload limit.', + static::MAX_FILE_UPLOAD_BYTES / 1024 / 1024, + )); + } + + /** + * Nothing was sent to PostSyncer, so retrying cannot duplicate media. + */ + private function safeUploadFailure(string $message, ?Throwable $previous = null): PostsyncerException + { + return new PostsyncerException($message, 0, $previous, responseReceived: true, safeToRetry: true); + } + /** * @param array $body * @param string|null $idempotencyKey Stable per-operation/group key for providers that support replay deduplication. @@ -287,15 +492,22 @@ public function getPostAnalytics(int|string $id): array * @param array $body * @param array $headers */ - private function request(string $method, string $path, array $body = [], array $headers = []): Response - { + private function request( + string $method, + string $path, + array $body = [], + array $headers = [], + ?string $filePath = null, + ): Response { $apiKey = $this->config->apiKey(); if (! is_string($apiKey) || $apiKey === '') { throw new PostsyncerException('PostSyncer API key is not configured.'); } - $url = rtrim($this->config->apiBase(), '/').$path; + // File uploads must go to the dedicated upload host, never the API host. + $base = $filePath === null ? $this->config->apiBase() : $this->config->uploadBase(); + $url = rtrim($base, '/').$path; $pending = Http::withToken($apiKey) // Multi-platform PostSyncer creates can take several minutes. @@ -305,6 +517,21 @@ private function request(string $method, string $path, array $body = [], array $ ->withOptions(['allow_redirects' => false]) ->withHeaders($headers); + if ($filePath !== null) { + // Guzzle closes this stream when the request is released. + $stream = fopen($filePath, 'r'); + if ($stream === false) { + throw new PostsyncerException( + 'Could not open the downloaded media file for upload.', + responseReceived: true, + safeToRetry: true, + ); + } + $mime = MimeTypes::getDefault()->guessMimeType($filePath) ?? 'application/octet-stream'; + $extension = MimeTypes::getDefault()->getExtensions($mime)[0] ?? 'bin'; + $pending = $pending->attach('file', $stream, 'media.'.$extension, ['Content-Type' => $mime]); + } + try { $response = match ($method) { 'get' => $pending->get($url), diff --git a/docs/guides/postsyncer.md b/docs/guides/postsyncer.md index c82ec07..e3d339e 100644 --- a/docs/guides/postsyncer.md +++ b/docs/guides/postsyncer.md @@ -28,9 +28,11 @@ queue starts to hurt. | `postsyncer:sync-scheduled` | every five minutes | Pulls PostSyncer status back onto scheduled records. | `PublishPostJob` has a 900-second timeout and the dedicated `postsyncer` database -queue connection has a 960-second visibility window. `PublishVideoJob` has a -180-second timeout and uses the same connection, leaving enough time for its -media upload, create, and canonical verification calls. Keep +queue connection has a 960-second visibility window. `PublishVideoJob` also has a +900-second timeout on the same connection: it downloads each video and cover +from Drive and uploads them one by one to PostSyncer's file endpoint +(`upload.postsyncer.com/api/v1/media/upload/file`, 500 MB cap), because the +URL import stores no videos. Keep `POSTSYNCER_QUEUE_RETRY_AFTER` above the longest worker timeout. Both publish jobs are unique for the record and use a shared overlap lock whose expiry is longer than the video timeout. Each queued job also carries the run token it diff --git a/tests/Feature/Api/VideosApiTest.php b/tests/Feature/Api/VideosApiTest.php index fcd3123..52b0d3f 100644 --- a/tests/Feature/Api/VideosApiTest.php +++ b/tests/Feature/Api/VideosApiTest.php @@ -10,6 +10,7 @@ use App\Models\User; use App\Models\Video; use App\Models\Workspace; +use App\Support\LinkResolution\PublicUrlGuard; use App\Support\Postsyncer\PostsyncerConfig; use App\Support\Postsyncer\PostsyncerException; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -29,6 +30,7 @@ protected function setUp(): void { parent::setUp(); + $this->app->instance(PublicUrlGuard::class, new PublicUrlGuard(fn (): array => [['ip' => '142.250.0.1']])); $this->workspace = Workspace::factory()->create(); $this->token = (new CreateWorkspaceApiTokenAction)->handle( $this->workspace, @@ -562,9 +564,9 @@ public function test_reconcile_create_absent_endpoint_makes_verified_empty_creat ]); Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'drive.usercontent.google.com/*' => fn () => Http::response('fake-media-bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'message' => 'temporary outage', @@ -626,7 +628,8 @@ public function test_reconcile_media_absent_endpoint_resets_verified_empty_uploa ]); Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ + 'drive.usercontent.google.com/*' => fn () => Http::response('fake-media-bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ 'message' => 'temporary outage', ], 503), ]); diff --git a/tests/Feature/Console/ReconcileVideoPublishCommandTest.php b/tests/Feature/Console/ReconcileVideoPublishCommandTest.php index 48d7596..7e4910a 100644 --- a/tests/Feature/Console/ReconcileVideoPublishCommandTest.php +++ b/tests/Feature/Console/ReconcileVideoPublishCommandTest.php @@ -5,6 +5,7 @@ use App\Actions\Postsyncer\PublishVideoAction; use App\Models\Video; use App\Models\Workspace; +use App\Support\LinkResolution\PublicUrlGuard; use App\Support\Postsyncer\PostsyncerConfig; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Http; @@ -12,6 +13,15 @@ class ReconcileVideoPublishCommandTest extends TestCase { + protected function setUp(): void + { + parent::setUp(); + + $this->app->instance(PublicUrlGuard::class, new PublicUrlGuard(fn (): array => [['ip' => '142.250.0.1']])); + // Video media is downloaded from Drive before the file upload. + Http::fake(['drive.usercontent.google.com/*' => fn () => Http::response('fake-media-bytes', 200)]); + } + use RefreshDatabase; public function test_it_verifies_and_checkpoints_an_uncertain_video(): void @@ -50,9 +60,8 @@ public function test_it_verifies_and_checkpoints_an_uncertain_video(): void ]; Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'message' => 'gateway timeout', @@ -165,9 +174,8 @@ public function test_it_recovers_a_drifted_video_from_its_stored_payload(): void ]); Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 99, diff --git a/tests/Unit/Actions/Postsyncer/PublishVideoActionTest.php b/tests/Unit/Actions/Postsyncer/PublishVideoActionTest.php index a292f2b..121426a 100644 --- a/tests/Unit/Actions/Postsyncer/PublishVideoActionTest.php +++ b/tests/Unit/Actions/Postsyncer/PublishVideoActionTest.php @@ -6,6 +6,7 @@ use App\Actions\Postsyncer\PublishVideoAction; use App\Models\Video; use App\Models\Workspace; +use App\Support\LinkResolution\PublicUrlGuard; use App\Support\Postsyncer\MediaUrlResolver; use App\Support\Postsyncer\PostsyncerConfig; use App\Support\Postsyncer\PostsyncerException; @@ -82,16 +83,19 @@ protected function setUp(): void { parent::setUp(); + $this->app->instance(PublicUrlGuard::class, new PublicUrlGuard(fn (): array => [['ip' => '142.250.0.1']])); + // Video media is downloaded from Drive before the file upload. + Http::fake(['drive.usercontent.google.com/*' => fn () => Http::response('fake-media-bytes', 200)]); + $this->action = new PublishVideoAction(new VideoPublishPlanner(new MediaUrlResolver)); } public function test_publish_now_sets_succeeded_status_posted_and_groups(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915], ['id' => 916]], - 'count_stored' => 2, - ], 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->push(['media' => ['id' => 916]], 201), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 42, 'status' => 'published', @@ -159,9 +163,10 @@ public function test_publish_now_sets_succeeded_status_posted_and_groups(): void ]], ], $video->postsyncer); - Http::assertSentCount(4); + // Two Drive downloads, two file uploads, the create, and its verification. + Http::assertSentCount(7); $mediaRequest = Http::recorded( - fn ($request): bool => $request->url() === 'https://postsyncer.com/api/v1/media/upload/url', + fn ($request): bool => $request->url() === 'https://upload.postsyncer.com/api/v1/media/upload/file', )->first(); $postRequest = Http::recorded( fn ($request): bool => $request->url() === 'https://postsyncer.com/api/v1/posts', @@ -170,7 +175,7 @@ public function test_publish_now_sets_succeeded_status_posted_and_groups(): void $postKey = $postRequest[0]->header('Idempotency-Key')[0] ?? null; $this->assertIsString($mediaKey); $this->assertIsString($postKey); - $this->assertStringEndsWith(':media', $mediaKey); + $this->assertStringEndsWith(':media:0', $mediaKey); $this->assertStringEndsWith(':post', $postKey); $this->assertNotSame($mediaKey, $postKey); } @@ -178,9 +183,10 @@ public function test_publish_now_sets_succeeded_status_posted_and_groups(): void public function test_bangla_x_leg_publishes_as_a_second_video_only_group(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::sequence() - ->push(['media' => [['id' => 915], ['id' => 916]], 'count_stored' => 2], 200) - ->push(['media' => [['id' => 917]], 'count_stored' => 1], 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->push(['media' => ['id' => 916]], 201) + ->push(['media' => ['id' => 917]], 201), 'postsyncer.com/api/v1/posts' => Http::sequence() ->push(['id' => 42, 'status' => 'published'], 201) ->push(['id' => 43, 'status' => 'published'], 201), @@ -246,11 +252,10 @@ public function test_bangla_x_leg_publishes_as_a_second_video_only_group(): void $this->assertSame(['twitter'], $video->postsyncer['groups'][1]['platforms']); $uploads = Http::recorded( - fn ($request): bool => $request->url() === 'https://postsyncer.com/api/v1/media/upload/url', + fn ($request): bool => $request->url() === 'https://upload.postsyncer.com/api/v1/media/upload/file', ); - $this->assertCount(2, $uploads); - $this->assertCount(2, $uploads->first()[0]->data()['urls']); - $this->assertCount(1, $uploads->last()[0]->data()['urls']); + // Video and cover for the first group, video only for the X group. + $this->assertCount(3, $uploads); $xPost = Http::recorded( fn ($request): bool => $request->url() === 'https://postsyncer.com/api/v1/posts' @@ -287,10 +292,9 @@ public function test_create_polls_an_async_canonical_response_before_checkpointi { $lookups = 0; Http::fake(function ($request) use (&$lookups) { - if ($request->url() === 'https://postsyncer.com/api/v1/media/upload/url') { + if ($request->url() === 'https://upload.postsyncer.com/api/v1/media/upload/file') { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } @@ -336,17 +340,16 @@ public function test_create_polls_an_async_canonical_response_before_checkpointi $video->refresh(); $this->assertSame('succeeded', $video->publish_state); $this->assertSame(2, $lookups); - Http::assertSentCount(4); + Http::assertSentCount(5); } public function test_canonical_connection_failure_is_retried_without_recreating(): void { $lookups = 0; Http::fake(function ($request) use (&$lookups) { - if ($request->url() === 'https://postsyncer.com/api/v1/media/upload/url') { + if ($request->url() === 'https://upload.postsyncer.com/api/v1/media/upload/file') { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } @@ -403,9 +406,8 @@ public function test_canonical_connection_failure_is_retried_without_recreating( public function test_canonical_payload_mismatch_becomes_uncertain_without_recreating(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response(['id' => 44, 'status' => 'published'], 201), 'postsyncer.com/api/v1/posts/44' => Http::response([ @@ -449,9 +451,8 @@ public function test_canonical_payload_mismatch_becomes_uncertain_without_recrea public function test_schedule_sets_status_scheduled(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 99, @@ -505,9 +506,8 @@ public function test_schedule_sets_status_scheduled(): void public function test_schedule_keeps_audit_of_groups_cleared_after_reconcile(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 99, @@ -566,9 +566,8 @@ public function test_schedule_keeps_audit_of_groups_cleared_after_reconcile(): v public function test_naive_when_sends_workspace_timezone_on_schedule_for(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 99, @@ -648,10 +647,9 @@ public function test_empty_plan_fails_and_leaves_status_unchanged(): void public function test_empty_media_upload_response_does_not_create_a_text_only_video(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ 'media' => [], - 'count_stored' => 0, - ], 200), + ], 201), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 42, 'status' => 'published', @@ -675,12 +673,40 @@ public function test_empty_media_upload_response_does_not_create_a_text_only_vid $video->refresh(); $this->assertSame('failed', $video->publish_state); - $this->assertStringContainsString('incomplete media upload response', (string) $video->publish_error); + $this->assertStringContainsString('without a valid id', (string) $video->publish_error); $this->assertSame('recorded', $video->status); $this->assertNull($video->postsyncer); Http::assertNotSent(fn ($request) => $request->url() === 'https://postsyncer.com/api/v1/posts'); } + public function test_drive_download_failure_fails_safely_without_reaching_postsyncer(): void + { + Http::swap(new Factory); + Http::fake(['drive.usercontent.google.com/*' => Http::response('missing', 404)]); + + $workspace = Workspace::factory()->create(); + $this->configureWorkspace($workspace); + $video = Video::factory()->for($workspace)->create([ + 'status' => 'recorded', + 'language' => 'bn', + 'video_drive_url' => 'https://drive.google.com/file/d/video/view', + 'captions' => ['facebook' => 'Reel caption'], + ]); + + $this->action->handle($video, [ + 'platforms' => ['facebook'], + 'confirm_ask' => false, + ]); + + $video->refresh(); + $this->assertSame('failed', $video->publish_state); + $this->assertSame('failed', $video->publish_progress['state']); + $this->assertNull($video->publish_progress['current']); + $this->assertStringContainsString('Could not download media', (string) $video->publish_error); + $this->assertTrue($video->canRetryPublish()); + Http::assertNotSent(fn ($request): bool => str_contains($request->url(), 'postsyncer.com')); + } + public function test_uncertain_media_upload_can_be_reconciled_without_uploading_again(): void { $workspace = Workspace::factory()->create(); @@ -695,7 +721,7 @@ public function test_uncertain_media_upload_can_be_reconciled_without_uploading_ $uploadCalls = 0; Http::fake(function ($request) use (&$uploadCalls) { - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_ends_with($request->url(), '/media/upload/file')) { $uploadCalls++; return $uploadCalls === 1 @@ -761,10 +787,9 @@ public function test_plan_drift_can_be_recovered_from_the_stored_payload_snapsho $changed = false; Http::fake(function ($request) use ($video, &$changed) { - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_ends_with($request->url(), '/media/upload/file')) { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } @@ -811,9 +836,8 @@ public function test_plan_drift_can_be_recovered_from_the_stored_payload_snapsho public function test_reconciliation_replaces_a_duplicate_completed_checkpoint(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'message' => 'gateway timeout', @@ -890,10 +914,9 @@ public function test_failure_keeps_a_newer_media_checkpoint_over_stale_local_pro ]); Http::fake(function ($request) use ($video) { - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_ends_with($request->url(), '/media/upload/file')) { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } @@ -957,9 +980,8 @@ public function test_second_publish_is_refused_when_groups_have_post_ids(): void public function test_empty_groups_without_post_ids_is_treated_as_first_publish(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 77, @@ -1005,7 +1027,7 @@ public function test_empty_groups_without_post_ids_is_treated_as_first_publish() public function test_failure_leaves_pipeline_status_and_sets_publish_error(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ 'message' => 'Invalid workspace', ], 422), ]); @@ -1046,10 +1068,9 @@ public function test_missing_account_mapping_is_retriable_without_uploading_medi return Http::response(['message' => 'Unexpected request'], 500); } - if ($request->url() === 'https://postsyncer.com/api/v1/media/upload/url') { + if ($request->url() === 'https://upload.postsyncer.com/api/v1/media/upload/file') { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } @@ -1125,9 +1146,8 @@ public function test_missing_account_mapping_is_retriable_without_uploading_medi public function test_legacy_missing_account_checkpoint_reuses_registered_media_after_mapping_fix(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 999]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 999], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'id' => 42, @@ -1199,7 +1219,7 @@ public function test_legacy_missing_account_checkpoint_reuses_registered_media_a $video->refresh(); $this->assertSame('succeeded', $video->publish_state); $this->assertSame('42', $video->postsyncer['groups'][0]['post_id']); - Http::assertNotSent(fn ($request): bool => str_ends_with($request->url(), '/media/upload/url')); + Http::assertNotSent(fn ($request): bool => str_ends_with($request->url(), '/media/upload/file')); Http::assertSent(fn ($request): bool => $request->url() === 'https://postsyncer.com/api/v1/posts' && $request['content'][0]['media'] === [915]); } @@ -1352,9 +1372,8 @@ public function test_retryable_media_checkpoint_without_plan_metadata_is_not_res public function test_lost_create_response_records_an_uncertain_current_group(): void { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'message' => 'gateway timeout', @@ -1392,9 +1411,8 @@ public function test_lost_create_response_records_an_uncertain_current_group(): private function uncertainCreateVideo(Workspace $workspace): Video { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response([ 'message' => 'gateway timeout', @@ -1438,10 +1456,10 @@ public function test_verified_absent_create_can_be_reconciled_without_reuploadin // is expected: the retry reuses the media registered before the 500. Http::swap(new Factory); Http::fake(function ($request) use (&$uploadCalls, &$createCalls) { - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_ends_with($request->url(), '/media/upload/file')) { $uploadCalls++; - return Http::response(['media' => [['id' => 915]], 'count_stored' => 1], 200); + return Http::response(['media' => ['id' => 915]], 200); } if ($request->url() === 'https://postsyncer.com/api/v1/posts') { @@ -1581,7 +1599,7 @@ public function test_create_absent_reconciliation_refuses_empty_media_ids_when_t private function uncertainUploadVideo(Workspace $workspace): Video { Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ 'message' => 'temporary outage', ], 503), ]); @@ -1625,10 +1643,14 @@ public function test_verified_absent_media_upload_is_reset_and_uploads_again_on_ Http::swap(new Factory); Http::fake(function ($request) use (&$uploadCalls) { - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_starts_with($request->url(), 'https://drive.usercontent.google.com/')) { + return Http::response('fake-media-bytes', 200); + } + + if (str_ends_with($request->url(), '/media/upload/file')) { $uploadCalls++; - return Http::response(['media' => [['id' => 915]], 'count_stored' => 1], 200); + return Http::response(['media' => ['id' => 915]], 200); } if ($request->url() === 'https://postsyncer.com/api/v1/posts') { @@ -1746,9 +1768,8 @@ public function test_confirm_failed_reconciliation_requires_the_explicit_flag(): ]; Http::fake([ - 'postsyncer.com/api/v1/media/upload/url' => Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response([ + 'media' => ['id' => 915], ], 200), 'postsyncer.com/api/v1/posts' => Http::response(['message' => 'gateway timeout'], 500), ]); @@ -1842,10 +1863,9 @@ public function test_sets_running_before_api_calls(): void $seenRunning = true; } - if (str_ends_with($request->url(), '/media/upload/url')) { + if (str_ends_with($request->url(), '/media/upload/file')) { return Http::response([ - 'media' => [['id' => 915]], - 'count_stored' => 1, + 'media' => ['id' => 915], ], 200); } diff --git a/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php b/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php index bf2eb01..40eef43 100644 --- a/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php +++ b/tests/Unit/Support/Postsyncer/PostsyncerClientTest.php @@ -3,10 +3,16 @@ namespace Tests\Unit\Support\Postsyncer; use App\Models\Workspace; +use App\Support\LinkResolution\PublicUrlGuard; use App\Support\Postsyncer\PostsyncerClient; use App\Support\Postsyncer\PostsyncerConfig; use App\Support\Postsyncer\PostsyncerException; +use GuzzleHttp\Client; +use GuzzleHttp\Exception\TransferException; +use GuzzleHttp\Handler\CurlHandler; +use GuzzleHttp\HandlerStack; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; use Tests\TestCase; @@ -14,6 +20,13 @@ class PostsyncerClientTest extends TestCase { use RefreshDatabase; + protected function setUp(): void + { + parent::setUp(); + + $this->app->instance(PublicUrlGuard::class, new PublicUrlGuard(fn (): array => [['ip' => '142.250.0.1']])); + } + private function clientWithKey(string $apiKey = 'test-api-key'): PostsyncerClient { $workspace = Workspace::factory()->create(['settings' => []]); @@ -82,6 +95,309 @@ public function test_upload_from_urls_rejects_a_media_item_without_a_valid_id(): $client->uploadFromUrls(15211, ['https://example.com/a.png']); } + /** + * @return list + */ + private function mediaTempFiles(): array + { + return glob(sys_get_temp_dir().'/cm-media-*') ?: []; + } + + public function test_upload_files_from_urls_downloads_then_uploads_each_file_in_order(): void + { + $before = $this->mediaTempFiles(); + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response('video-bytes', 200), + 'drive.usercontent.google.com/download?id=cover' => fn () => Http::response('cover-bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->push(['media' => ['id' => 916]], 201), + ]); + + $ids = $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ], 'op:0:group:media'); + + $this->assertSame([915, 916], $ids); + $uploads = Http::recorded(fn ($request): bool => $request->url() + === 'https://upload.postsyncer.com/api/v1/media/upload/file'); + $this->assertCount(2, $uploads); + $first = $uploads->first()[0]; + $this->assertTrue($first->isMultipart()); + $this->assertTrue($first->hasHeader('Authorization', 'Bearer test-api-key')); + $this->assertSame(['op:0:group:media:0'], $first->header('Idempotency-Key')); + $this->assertSame(['op:0:group:media:1'], $uploads->last()[0]->header('Idempotency-Key')); + $parts = collect($first->data())->keyBy('name'); + $this->assertSame('15211', (string) $parts['workspace_id']['contents']); + $this->assertSame('video-bytes', (string) $parts['file']['contents']); + Http::assertNotSent(fn ($request): bool => str_starts_with($request->url(), 'https://postsyncer.com/')); + $this->assertSame($before, $this->mediaTempFiles()); + } + + public function test_upload_files_from_urls_marks_a_lost_upload_response_uncertain(): void + { + $before = $this->mediaTempFiles(); + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response('video-bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => fn () => throw new ConnectionException('timed out'), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the lost upload response to throw.'); + } catch (PostsyncerException $exception) { + $this->assertTrue($exception->outcomeUnknown); + $this->assertFalse($exception->responseReceived); + $this->assertFalse($exception->safeToRetry); + } + + $this->assertSame($before, $this->mediaTempFiles()); + } + + public function test_upload_files_from_urls_treats_a_first_file_rejection_as_safe_to_retry(): void + { + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response('video-bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::response(['message' => 'Unsupported file'], 422), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the rejection to throw.'); + } catch (PostsyncerException $exception) { + $this->assertSame(422, $exception->getCode()); + $this->assertTrue($exception->responseReceived); + $this->assertFalse($exception->outcomeUnknown); + $this->assertTrue($exception->safeToRetry); + } + } + + public function test_upload_files_from_urls_keeps_a_rejection_after_a_stored_file_unsafe(): void + { + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('bytes', 200), + 'upload.postsyncer.com/api/v1/media/upload/file' => Http::sequence() + ->push(['media' => ['id' => 915]], 201) + ->push(['message' => 'Unsupported file'], 422), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ]); + $this->fail('Expected the rejection to throw.'); + } catch (PostsyncerException $exception) { + $this->assertFalse($exception->safeToRetry); + $this->assertStringContainsString('Already stored PostSyncer media ids: 915.', $exception->getMessage()); + } + } + + public function test_upload_files_from_urls_download_failure_sends_nothing_to_postsyncer(): void + { + $before = $this->mediaTempFiles(); + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response('video-bytes', 200), + 'drive.usercontent.google.com/download?id=cover' => Http::response('missing', 404), + 'upload.postsyncer.com/*' => Http::response(['media' => ['id' => 915]], 201), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, [ + 'https://drive.usercontent.google.com/download?id=video', + 'https://drive.usercontent.google.com/download?id=cover', + ]); + $this->fail('Expected the download failure to throw.'); + } catch (PostsyncerException $exception) { + $this->assertStringContainsString('Could not download media', $exception->getMessage()); + $this->assertTrue($exception->safeToRetry); + $this->assertTrue($exception->responseReceived); + $this->assertFalse($exception->outcomeUnknown); + } + + Http::assertNotSent(fn ($request): bool => str_contains($request->url(), 'postsyncer.com')); + $this->assertSame($before, $this->mediaTempFiles()); + } + + public function test_upload_files_from_urls_rejects_an_html_page_instead_of_media(): void + { + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response('Sign in', 200), + ]); + + $this->expectException(PostsyncerException::class); + $this->expectExceptionMessage('HTML page'); + + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + } + + public function test_upload_files_from_urls_rejects_files_over_the_size_cap_before_uploading(): void + { + $before = $this->mediaTempFiles(); + Http::fake([ + 'drive.usercontent.google.com/download?id=video' => fn () => Http::response(str_repeat('x', 11), 200), + ]); + $workspace = Workspace::factory()->create(['settings' => []]); + PostsyncerConfig::write($workspace, ['api_key' => 'test-api-key']); + $client = new class(PostsyncerConfig::fromWorkspace($workspace->refresh())) extends PostsyncerClient + { + public const MAX_FILE_UPLOAD_BYTES = 10; + }; + + try { + $client->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the size cap to throw.'); + } catch (PostsyncerException $exception) { + $this->assertStringContainsString('upload limit', $exception->getMessage()); + $this->assertTrue($exception->safeToRetry); + } + + Http::assertNotSent(fn ($request): bool => str_contains($request->url(), 'postsyncer.com')); + $this->assertSame($before, $this->mediaTempFiles()); + } + + public function test_upload_files_from_urls_refuses_a_non_drive_host(): void + { + Http::fake(); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['http://169.254.169.254/latest/meta-data']); + $this->fail('Expected the non-Drive URL to be refused.'); + } catch (PostsyncerException $exception) { + $this->assertStringContainsString('not a public Google Drive download', $exception->getMessage()); + $this->assertTrue($exception->safeToRetry); + } + + Http::assertNothingSent(); + } + + public function test_upload_files_from_urls_refuses_a_drive_host_resolving_to_a_private_address(): void + { + $this->app->instance(PublicUrlGuard::class, new PublicUrlGuard(fn (): array => [['ip' => '10.0.0.5']])); + Http::fake(); + + $this->expectException(PostsyncerException::class); + $this->expectExceptionMessage('not a public Google Drive download'); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + } finally { + Http::assertNothingSent(); + } + } + + public function test_upload_files_from_urls_follows_redirects_between_drive_hosts(): void + { + Http::fake([ + 'drive.google.com/*' => Http::response('', 302, [ + 'Location' => 'https://drive.usercontent.google.com/download?id=video', + ]), + 'drive.usercontent.google.com/*' => fn () => Http::response('video-bytes', 200), + 'upload.postsyncer.com/*' => Http::response(['media' => ['id' => 915]], 201), + ]); + + $this->assertSame([915], $this->clientWithKey()->uploadFilesFromUrls( + 15211, + ['https://drive.google.com/uc?id=video'], + )); + } + + public function test_upload_files_from_urls_refuses_a_redirect_off_drive(): void + { + Http::fake([ + 'evil.example.com/*' => Http::response('', 302, ['Location' => 'http://127.0.0.1/admin']), + 'drive.usercontent.google.com/*' => Http::response('', 302, [ + 'Location' => 'https://evil.example.com/x', + ]), + ]); + + try { + $this->clientWithKey()->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=other']); + $this->fail('Expected the off-Drive redirect to be refused.'); + } catch (PostsyncerException $exception) { + $this->assertStringContainsString('not a public Google Drive download', $exception->getMessage()); + } + + Http::assertNotSent(fn ($request): bool => str_contains($request->url(), 'evil.example.com') + || str_contains($request->url(), '127.0.0.1')); + } + + public function test_upload_files_from_urls_rejects_an_oversized_content_length(): void + { + $before = $this->mediaTempFiles(); + Http::fake([ + 'drive.usercontent.google.com/*' => fn () => Http::response('tiny', 200, ['Content-Length' => '11']), + ]); + $workspace = Workspace::factory()->create(['settings' => []]); + PostsyncerConfig::write($workspace, ['api_key' => 'test-api-key']); + $client = new class(PostsyncerConfig::fromWorkspace($workspace->refresh())) extends PostsyncerClient + { + public const MAX_FILE_UPLOAD_BYTES = 10; + }; + + try { + $client->uploadFilesFromUrls(15211, ['https://drive.usercontent.google.com/download?id=video']); + $this->fail('Expected the Content-Length cap to throw.'); + } catch (PostsyncerException $exception) { + $this->assertStringContainsString('upload limit', $exception->getMessage()); + $this->assertTrue($exception->safeToRetry); + } + + Http::assertNotSent(fn ($request): bool => str_contains($request->url(), 'postsyncer.com')); + $this->assertSame($before, $this->mediaTempFiles()); + } + + public function test_download_cap_progress_callback_really_aborts_a_curl_transfer(): void + { + $workspace = Workspace::factory()->create(['settings' => []]); + PostsyncerConfig::write($workspace, ['api_key' => 'test-api-key']); + $client = new class(PostsyncerConfig::fromWorkspace($workspace->refresh())) extends PostsyncerClient + { + public const MAX_FILE_UPLOAD_BYTES = 1024; + }; + $overCap = false; + $progress = (new \ReflectionMethod(PostsyncerClient::class, 'downloadCapProgress')) + ->invokeArgs($client, [&$overCap]); + + // A real Guzzle cURL transfer from a local server proves the callback + // stops the download, which Http::fake cannot show. + $root = sys_get_temp_dir().'/cm-cap-'.bin2hex(random_bytes(4)); + mkdir($root); + $size = 32 * 1024 * 1024; + file_put_contents($root.'/big.bin', str_repeat('x', $size)); + $port = 20000 + random_int(0, 20000); + $server = proc_open([PHP_BINARY, '-S', '127.0.0.1:'.$port, '-t', $root], [ + ['pipe', 'r'], ['file', '/dev/null', 'w'], ['file', '/dev/null', 'w'], + ], $pipes); + $sink = $root.'/sink.bin'; + + try { + for ($i = 0; $i < 50 && @fsockopen('127.0.0.1', $port) === false; $i++) { + usleep(100000); + } + + $guzzle = new Client(['handler' => HandlerStack::create(new CurlHandler)]); + + try { + $guzzle->get('http://127.0.0.1:'.$port.'/big.bin', ['sink' => $sink, 'progress' => $progress]); + $this->fail('Expected the transfer to be aborted.'); + } catch (TransferException) { + // Aborted by the cap callback. + } + + $this->assertTrue($overCap); + clearstatcache(); + $this->assertLessThan($size, file_exists($sink) ? filesize($sink) : 0); + } finally { + proc_terminate($server); + proc_close($server); + @unlink($sink); + @unlink($root.'/big.bin'); + @rmdir($root); + } + } + public function test_create_post_returns_post_payload(): void { Http::fake([