316 label review race condition

serverless.yml

@@ -201,6 +201,16 @@ functions:
 
 resources:
   Resources:
+    # API Gateway Log Group
+    # Defined explicitly so CloudFormation manages the lifecycle.
+    # Imported via resource-import since API Gateway auto-creates it.
+    ApiGatewayLogGroup:
+      Type: AWS::Logs::LogGroup
+      DeletionPolicy: Retain
+      Properties:
+        LogGroupName: /aws/api-gateway/animl-api-${self:provider.stage}
+        RetentionInDays: 30
+
     # Secrets Manager secret for API key
     apiKey:
       Type: AWS::SecretsManager::Secret

src/scripts/operations.js

@@ -1,3 +1,6 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import appRoot from 'app-root-path';
 import Mongoose from 'mongoose';
 import { DateTime } from 'luxon';
 import fetch from 'node-fetch';
@@ -6,9 +9,77 @@ import Image from '../../.build/api/db/schemas/Image.js';
 import Project from '../../.build/api/db/schemas/Project.js';
 import { getQueryableLabelIds, isImageReviewed } from '../../.build/api/db/models/utils.js';
 import { buildImgUrl } from '../../.build/api/db/models/utils.js';
+import { backupConfig } from './backupConfig.js';
 
 const ObjectId = Mongoose.Types.ObjectId;
 
+// Pipelines for finding images affected by out-of-order mutation race condition
+// See: https://github.com/tnc-ca-geo/animl-api/issues/316
+
+// Case A: Objects that are unlocked but ALL labels have been invalidated
+const pipelineCaseA = [
+  {
+    $match: {
+      objects: {
+        $elemMatch: {
+          locked: false,
+          labels: {
+            $not: {
+              $elemMatch: {
+                $or: [{ validation: null }, { 'validation.validated': true }],
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+];
+
+// Case B: Objects that are unlocked but have at least one validated label
+const pipelineCaseB = [
+  {
+    $match: {
+      objects: {
+        $elemMatch: {
+          locked: false,
+          labels: {
+            $elemMatch: {
+              'validation.validated': true,
+            },
+          },
+        },
+      },
+    },
+  },
+];
+
+async function getAffectedImageIds() {
+  const [caseAResults, caseBResults] = await Promise.all([
+    Image.aggregate([...pipelineCaseA, { $project: { _id: 1, projectId: 1 } }]),
+    Image.aggregate([...pipelineCaseB, { $project: { _id: 1, projectId: 1 } }]),
+  ]);
+
+  // const caseAIds = new Set(caseAResults.map((img) => img._id));
+  // const caseBIds = new Set(caseBResults.map((img) => img._id));
+
+  // Deduplicate and tag each image with which case(s) it matches
+  const allIds = new Map();
+  for (const img of caseAResults) {
+    allIds.set(img._id, { _id: img._id, projectId: img.projectId, case: 'A' });
+  }
+  for (const img of caseBResults) {
+    const existing = allIds.get(img._id);
+    if (existing) {
+      existing.case = 'A+B';
+    } else {
+      allIds.set(img._id, { _id: img._id, projectId: img.projectId, case: 'B' });
+    }
+  }
+
+  return { entries: [...allIds.values()], ids: [...allIds.keys()] };
+}
+
 const pipeline = [
   // {
   //   $match: {
@@ -531,6 +602,108 @@ const operations = {
       }
     },
   },
+
+  'audit-unlocked-objects-with-reviewed-labels': {
+    // Generate a CSV report and JSON backup of images affected by the
+    // out-of-order mutation race condition (issue #316)
+    getIds: async () => {
+      const { ids } = await getAffectedImageIds();
+      return ids;
+    },
+    update: async () => {
+      console.log('Auditing images with incorrectly unlocked objects...');
+      const { entries, ids } = await getAffectedImageIds();
+
+      if (ids.length === 0) {
+        console.log('No affected images found.');
+        return { nModified: 0 };
+      }
+
+      // Fetch full image records for JSON backup
+      const images = await Image.find({ _id: { $in: ids } });
+
+      // Write outputs
+      const stage = process.env.STAGE || 'dev';
+      const dt = DateTime.now().setZone('utc').toFormat("yyyy-LL-dd'T'HHmm'Z'");
+      const backupsRoot = path.join(appRoot.path, backupConfig.BACKUP_DIR);
+      const outDir = path.join(backupsRoot, 'issue-316-backfill');
+
+      if (!fs.existsSync(outDir)) {
+        fs.mkdirSync(outDir, { recursive: true });
+      }
+
+      // CSV report
+      const csvHeader = '_id,projectId,case';
+      const csvRows = entries.map((e) => `${e._id},${e.projectId},${e.case}`);
+      const csvContent = [csvHeader, ...csvRows].join('\n');
+      const csvPath = path.join(outDir, `${stage}_affected-images_${dt}.csv`);
+      fs.writeFileSync(csvPath, csvContent, 'utf8');
+      console.log(`CSV report written to: ${csvPath}`);
+      console.log(
+        `  Case A (all labels invalidated, unlocked): ${entries.filter((e) => e.case.includes('A')).length}`,
+      );
+      console.log(
+        `  Case B (has validated label, unlocked): ${entries.filter((e) => e.case.includes('B')).length}`,
+      );
+
+      // JSON backup of full records
+      const jsonPath = path.join(outDir, `${stage}_affected-images-backup_${dt}.json`);
+      fs.writeFileSync(jsonPath, JSON.stringify(images, null, 2), 'utf8');
+      console.log(`JSON backup written to: ${jsonPath}`);
+
+      return { nModified: ids.length };
+    },
+  },
+
+  'fix-unlocked-objects-with-reviewed-labels': {
+    // Lock objects that should be locked and update image review status
+    // for images affected by the out-of-order mutation race condition (issue #316)
+    getIds: async () => {
+      const { ids } = await getAffectedImageIds();
+      return ids;
+    },
+    update: async () => {
+      console.log('Fixing images with incorrectly unlocked objects...');
+      const { ids } = await getAffectedImageIds();
+
+      if (ids.length === 0) {
+        console.log('No affected images found.');
+        return { nModified: 0 };
+      }
+
+      try {
+        const res = { nModified: 0 };
+        for (const id of ids) {
+          const image = await Image.findOne({ _id: id });
+          if (!image) continue;
+
+          for (const obj of image.objects) {
+            if (obj.locked) continue;
+
+            const hasValidatedLabel = obj.labels.some(
+              (lbl) => lbl.validation && lbl.validation.validated === true,
+            );
+            const allLabelsInvalidated =
+              obj.labels.length > 0 &&
+              obj.labels.every((lbl) => lbl.validation && lbl.validation.validated === false);
+
+            if (hasValidatedLabel || allLabelsInvalidated) {
+              obj.locked = true;
+            }
+          }
+
+          image.reviewed = isImageReviewed(image);
+          image.queryableLabelIds = getQueryableLabelIds(image);
+          await image.save();
+          res.nModified++;
+        }
+        return res;
+      } catch (err) {
+        console.error(err);
+        throw err;
+      }
+    },
+  },
 };
 
 export { operations };