fix: stop treating CheckMedia 429s as permanent upload failures

.github/workflows/run-maintenance-script.yml

@@ -0,0 +1,71 @@
+name: Run maintenance script
+
+# Manually triggered one-off script runner. Runs a script from scripts/ inside
+# the migrator image (full source + Payload CLI + node_modules) against a
+# chosen environment's database — the same mechanism promote.yml uses to run
+# `payload migrate` during a release, reused here for one-off data fixes that
+# don't belong in a migration (see scripts/*.ts for examples).
+#
+# Defaults to a dry run: the script only writes data when "Apply changes" is
+# checked. Targeting "production" is gated by the `production` GitHub
+# environment's required reviewers, same as a real release.
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Target environment (selects which database the script runs against)"
+        type: choice
+        options:
+          - development
+          - staging
+          - production
+        required: true
+      script_path:
+        description: "Script path relative to repo root"
+        type: string
+        required: true
+        default: "scripts/clear-checkmedia-rate-limit-errors.ts"
+      apply:
+        description: "Apply changes (unchecked = dry run)"
+        type: boolean
+        default: false
+
+concurrency:
+  group: "run-maintenance-script @ ${{ inputs.environment }}"
+  cancel-in-progress: false
+
+# Least-privilege default for GITHUB_TOKEN; the run authenticates to
+# DockerHub via dedicated secrets, not the workflow token.
+permissions:
+  contents: read
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      # Runs the migrator image built for the commit this workflow was
+      # dispatched against — trigger this from the same ref (branch or tag)
+      # that was actually deployed, so the script sees the code it was
+      # written for. Staging uses distinctly-named secrets (no GitHub
+      # Environment override configured for it); development and production
+      # share secret names but resolve to different values via their
+      # respective GitHub Environments.
+      - name: Run script against ${{ inputs.environment }}
+        env:
+          DATABASE_URI: ${{ inputs.environment == 'staging' && secrets.STAGING_DATABASE_URI || secrets.DATABASE_URI }}
+          PAYLOAD_SECRET: ${{ inputs.environment == 'staging' && secrets.STAGING_PAYLOAD_SECRET || secrets.PAYLOAD_SECRET }}
+        run: |
+          docker run --rm \
+            -e DATABASE_URI \
+            -e PAYLOAD_SECRET \
+            -e TIKA_ENABLED=0 \
+            "codeforafrica/promisetracker-v2-migrator:${{ github.sha }}" \
+            pnpm payload run "${{ inputs.script_path }}" ${{ inputs.apply && '-- --apply' || '' }}

scripts/clear-checkmedia-rate-limit-errors.ts

@@ -0,0 +1,82 @@
+/**
+ * One-off remediation for the pre-fix uploadToMeedan behavior that treated
+ * CheckMedia HTTP 429 (rate limit) responses as permanent failures. Clears
+ * `uploadError` on extractions whose stored error is a 429 so they become
+ * eligible for upload again on the next uploadToMeedan run.
+ *
+ * Non-429 errors (bad source URLs, GraphQL validation errors, missing
+ * entity/tenant, etc.) are left untouched — those are deterministic
+ * failures that would just fail again on retry.
+ *
+ * Dry run (default):
+ *   pnpm payload run scripts/clear-checkmedia-rate-limit-errors.ts
+ * Apply:
+ *   pnpm payload run scripts/clear-checkmedia-rate-limit-errors.ts -- --apply
+ */
+import { getPayload } from "payload";
+import config from "../src/payload.config";
+
+const apply = process.argv.includes("--apply");
+
+const isRateLimitError = (message: string | null | undefined): boolean =>
+  Boolean(message) && /HTTP 429|TooManyRequestsError/i.test(message as string);
+
+const payload = await getPayload({ config });
+
+const { docs: extractionDocs } = await payload.find({
+  collection: "ai-extractions",
+  depth: 0,
+  limit: 0,
+});
+
+console.log(
+  `Found ${extractionDocs.length} ai-extractions docs (apply=${apply})`,
+);
+
+let clearedExtractions = 0;
+let touchedDocs = 0;
+
+for (const doc of extractionDocs) {
+  const extractions = doc.extractions ?? [];
+  const matches = extractions.filter((extraction) =>
+    isRateLimitError(extraction.uploadError),
+  );
+
+  if (matches.length === 0) {
+    continue;
+  }
+
+  touchedDocs += 1;
+  for (const match of matches) {
+    console.log(
+      JSON.stringify({
+        action: apply ? "clear" : "would-clear",
+        extractionDocId: doc.id,
+        extractionUniqueId: match.uniqueId,
+        uploadError: match.uploadError,
+      }),
+    );
+  }
+
+  if (apply) {
+    const updatedExtractions = extractions.map((extraction) =>
+      isRateLimitError(extraction.uploadError)
+        ? { ...extraction, uploadError: null }
+        : extraction,
+    );
+    await payload.update({
+      collection: "ai-extractions",
+      id: doc.id,
+      data: { extractions: updatedExtractions },
+    });
+  }
+
+  clearedExtractions += matches.length;
+}
+
+console.log(
+  `Done. ${apply ? "Cleared" : "Would clear"} ${clearedExtractions} extraction(s) across ${touchedDocs} document(s).`,
+);
+
+await payload.destroy();
+process.exit(0);

src/app/api/run-jobs/route.ts

@@ -9,6 +9,11 @@
  * - You are debugging a specific queue and want an immediate execution cycle.
  * - The everyMinute cron has not fired yet after a fresh deploy.
  *
+ * Query params (both optional; `all` takes precedence over `queue`):
+ * - `queue`: run jobs from this queue instead of Payload's default `default`
+ *   queue (e.g. `?queue=exportSync`).
+ * - `all=true`: run jobs from every queue, ignoring `queue`.
+ *
  * Security: requires an active Payload CMS session (i.e. the caller must be
  * logged in to the admin panel). Unauthenticated requests are rejected with
  * HTTP 401.
@@ -30,12 +35,23 @@ export const GET = async (request: NextRequest) => {
     return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
   }
 
-  payload.jobs.run().catch((error: unknown) => {
-    payload.logger.error({
-      msg: "run-jobs:: Unhandled error in jobs.run()",
-      error,
+  const { searchParams } = new URL(request.url);
+  const allQueues = searchParams.get("all") === "true";
+  const queue = searchParams.get("queue") || undefined;
+
+  payload.jobs
+    .run(allQueues ? { allQueues: true } : { queue })
+    .catch((error: unknown) => {
+      payload.logger.error({
+        msg: "run-jobs:: Unhandled error in jobs.run()",
+        allQueues,
+        queue,
+        error,
+      });
     });
-  });
 
-  return NextResponse.json({ ok: true }, { status: 202 });
+  return NextResponse.json(
+    { ok: true, allQueues, queue: allQueues ? undefined : (queue ?? "default") },
+    { status: 202 },
+  );
 };

src/lib/meedan.ts

@@ -43,6 +43,45 @@ const PUBLISHED_REPORTS_QUERY = `
 const toErrorMessage = (error: unknown): string =>
   error instanceof Error ? error.message : String(error);
 
+// Thrown for any non-OK HTTP response from CheckMedia. Callers use `status`
+// to distinguish transient failures (429, 5xx) from permanent ones so a
+// rate limit doesn't get treated the same as a validation error.
+export class CheckMediaHttpError extends Error {
+  readonly status: number;
+  readonly retryAfterMs: number | null;
+
+  constructor(message: string, status: number, retryAfterMs: number | null) {
+    super(message);
+    this.name = "CheckMediaHttpError";
+    this.status = status;
+    this.retryAfterMs = retryAfterMs;
+  }
+}
+
+// Parses a `Retry-After` header per RFC 9110: either a number of seconds or
+// an HTTP-date. Returns null when absent or unparseable.
+const parseRetryAfterMs = (response: Response): number | null => {
+  const header = response.headers.get("retry-after");
+  if (!header) {
+    return null;
+  }
+
+  const seconds = Number(header);
+  if (Number.isFinite(seconds)) {
+    return Math.max(0, seconds * 1000);
+  }
+
+  const dateMs = Date.parse(header);
+  if (!Number.isNaN(dateMs)) {
+    return Math.max(0, dateMs - Date.now());
+  }
+
+  return null;
+};
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
 const previewResponseBody = (rawBody: string): string => {
   const normalized = rawBody.replace(/\s+/g, " ").trim();
 
@@ -90,8 +129,10 @@ const throwHttpErrorWithBody = async ({
   }
 
   const traceHeaders = getTraceHeaders(response);
-  throw new Error(
+  throw new CheckMediaHttpError(
     `[CheckMedia:${operation}] HTTP ${response.status} ${response.statusText}; traceHeaders=${traceHeaders}; responseBody=${bodyPreview}`,
+    response.status,
+    parseRetryAfterMs(response),
   );
 };
 
@@ -529,6 +570,14 @@ export const fetchPublishedReports = async ({
   return mapPublishedReports(json);
 };
 
+// Bounded inline retry for HTTP 429 only. CheckMedia's rate-limit window is
+// short (seconds), so it's worth waiting once or twice in-process before
+// giving up; anything longer than MAX_RATE_LIMIT_WAIT_MS is left to the next
+// scheduled task run instead of blocking this one.
+const MAX_RATE_LIMIT_RETRIES = 2;
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 2000;
+const MAX_RATE_LIMIT_WAIT_MS = 10_000;
+
 export const postRequest = async ({
   apiKey,
   teamId,
@@ -548,32 +597,44 @@ export const postRequest = async ({
   };
 
   try {
-    const response = await fetch(BASE_URL, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "X-Check-Token": apiKey,
-        "X-Check-Team": teamId,
-      },
-      body: JSON.stringify(requestBody),
-    });
+    for (let attempt = 0; ; attempt += 1) {
+      const response = await fetch(BASE_URL, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "X-Check-Token": apiKey,
+          "X-Check-Team": teamId,
+        },
+        body: JSON.stringify(requestBody),
+      });
+
+      if (response.ok) {
+        const result: CreateProjectMediaResponse = await response.json();
+
+        if (result.errors && result.errors.length > 0) {
+          throw new Error(
+            `[CheckMedia:postRequest.createProjectMedia] GraphQL errors: ${formatGraphQLErrorDetails(result.errors)}`,
+          );
+        }
+
+        return result;
+      }
+
+      if (response.status === 429 && attempt < MAX_RATE_LIMIT_RETRIES) {
+        const retryAfterMs = parseRetryAfterMs(response);
+        const waitMs = Math.min(
+          retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS * (attempt + 1),
+          MAX_RATE_LIMIT_WAIT_MS,
+        );
+        await sleep(waitMs);
+        continue;
+      }
 
-    if (!response.ok) {
       await throwHttpErrorWithBody({
         response,
         operation: "postRequest.createProjectMedia",
       });
     }
-
-    const result: CreateProjectMediaResponse = await response.json();
-
-    if (result.errors && result.errors.length > 0) {
-      throw new Error(
-        `[CheckMedia:postRequest.createProjectMedia] GraphQL errors: ${formatGraphQLErrorDetails(result.errors)}`,
-      );
-    }
-
-    return result;
   } catch (error) {
     console.error("Error making request to CheckMedia:", error);
     throw error;