From 6af69a301a24fd043696d9e1732edbed681aef85 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 3 Sep 2026 10:00:07 -0400 Subject: [PATCH 01/17] KMS-703: Move metadata-correction auditing to DocumentDB Serverless --- README.md | 40 +- bin/documentdb/start.sh | 65 +++ bin/documentdb/stop.sh | 22 + bin/env/local_env.sh | 3 + bin/start-local.sh | 2 +- cdk/app/lib/CmrEventProcessingStack.ts | 14 +- cdk/app/lib/KmsStack.ts | 11 +- cdk/app/lib/MetadataCorrectionAuditStack.ts | 128 ++++++ cdk/app/lib/helper/DocumentDbLambdaConfig.ts | 75 +++ cdk/app/lib/helper/KmsLambdaFunctions.ts | 36 +- .../MetadataCorrectionAuditDatabaseSetup.ts | 122 +++++ cdk/app/lib/helper/MetadataCorrectionSetup.ts | 29 +- cdk/bin/main.ts | 36 +- config/metadataCorrectionAuditIndexes.json | 57 +++ package-lock.json | 426 ++++++++++++------ package.json | 4 + .../initialize_metadata_correction_audit.mjs | 28 ++ scripts/local/mock_cmr_server.mjs | 48 +- ...etadata_correction_applied_audit_smoke.mjs | 82 ++-- ...orrection_consumer_metrics_async_smoke.mjs | 50 +- ...ion_consumer_metrics_manual_sync_smoke.mjs | 52 +-- ...metadata_correction_failed_audit_smoke.mjs | 78 ++-- ...correction_partial_batch_failure_smoke.mjs | 12 +- ...etadata_correction_request_delay_smoke.mjs | 59 +-- .../run_metadata_correction_sync_smoke.mjs | 85 ++-- ...n_platform_keyword_event_mapping_smoke.mjs | 10 +- scripts/local/run_rdf_export_smoke.sh | 8 + .../show_metadata_correction_audit_log.mjs | 150 ++---- serverless/certs/us-east-1-bundle.pem | 76 ++++ .../__tests__/handler.test.js | 3 + .../src/cmrKeywordEventsListener/handler.js | 4 +- serverless/src/getCapabilities/handler.js | 2 +- .../__tests__/handler.test.js | 48 +- .../src/getMetadataCorrectionAudit/handler.js | 45 +- .../__tests__/handler.test.js | 72 +++ .../handler.js | 48 ++ .../__tests__/handler.test.js | 48 +- .../src/metadataCorrectionService/handler.js | 1 + .../src/publisher/__tests__/handler.test.js | 1 + serverless/src/publisher/handler.js | 5 +- .../__tests__/handler.test.js | 8 + .../src/requestMetadataCorrection/handler.js | 3 + .../__tests__/handler.test.js | 8 + .../src/runMetadataCorrection/handler.js | 4 + .../src/shared/__tests__/awsClients.test.js | 38 +- .../shared/__tests__/documentDbClient.test.js | 204 +++++++++ .../getMetadataCorrectionAuditLog.test.js | 362 +++++---------- .../persistMetadataCorrectionAuditLog.test.js | 348 +++++++------- .../runCollectionMetadataCorrection.test.js | 314 +++++++------ serverless/src/shared/awsClients.js | 16 + serverless/src/shared/documentDbClient.js | 180 ++++++++ .../shared/getMetadataCorrectionAuditLog.js | 389 ++++++++-------- .../src/shared/metadataCorrectionAudit.js | 38 -- .../persistMetadataCorrectionAuditLog.js | 364 ++++++++++----- .../shared/runCollectionMetadataCorrection.js | 145 +++++- .../src/shared/writeCorrectedMetadataToCmr.js | 2 +- 56 files changed, 3003 insertions(+), 1505 deletions(-) create mode 100755 bin/documentdb/start.sh create mode 100755 bin/documentdb/stop.sh create mode 100644 cdk/app/lib/MetadataCorrectionAuditStack.ts create mode 100644 cdk/app/lib/helper/DocumentDbLambdaConfig.ts create mode 100644 cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts create mode 100644 config/metadataCorrectionAuditIndexes.json create mode 100644 scripts/local/initialize_metadata_correction_audit.mjs create mode 100644 serverless/certs/us-east-1-bundle.pem create mode 100644 serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js create mode 100644 serverless/src/initializeMetadataCorrectionAudit/handler.js create mode 100644 serverless/src/shared/__tests__/documentDbClient.test.js create mode 100644 serverless/src/shared/documentDbClient.js delete mode 100644 serverless/src/shared/metadataCorrectionAudit.js diff --git a/README.md b/README.md index 7e349899..ec4b7228 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,15 @@ npm run localstack:start By default, `start-local` enables Redis with the local container settings from `bin/env/local_env.sh`, so the normal local startup path is: ```bash npm run redis:start +npm run documentdb:start npm run start-local ``` +The local MongoDB 8 container provides a DocumentDB-compatible metadata-correction audit store. +It listens on `localhost:27018` for host scripts and uses the shared KMS Docker network for SAM. +Starting it also creates or verifies the metadata-correction audit indexes used in AWS. +Stop it with `npm run documentdb:stop`. + If you do not need Redis for your local test, start local with Redis disabled: ```bash REDIS_ENABLED=false npm run start-local @@ -71,7 +77,8 @@ Local development intentionally splits responsibilities between SAM and LocalSta - SAM runs the API Gateway and Lambda side of KMS locally. - LocalStack emulates AWS-managed services that SAM does not model end-to-end for this repo, especially SNS and SQS. -- RDF4J and Redis remain separate local services because they are not AWS services. +- RDF4J, Redis, and the MongoDB-compatible audit database remain separate local services because + they are not modeled by SAM. We do not run the entire application stack inside LocalStack because the existing SAM flow is simpler for day-to-day Lambda/API development, while LocalStack is most useful here for the managed messaging pieces. For keyword event processing, `npm run start-local` also starts `scripts/localstack/run_bridge.sh`, which runs `scripts/localstack/bridge.js`. @@ -297,7 +304,7 @@ Internally, the correction flow is now object-first: key construction - XML and UMM delegates work from `oldKeywordObject` / `newKeywordObject` - joined `oldKeywordPath` / `newKeywordPath` strings are now primarily boundary values for Redis, - logs, and audit records + logs, and audit documents The important distinction is: @@ -440,6 +447,35 @@ Resolved corrections are also object-first now: Audit logging still derives `oldKeywordPath` / `newKeywordPath` strings for readability, but the runtime correction and delegate flow works from normalized keyword objects. +Each collection-correction run is stored as one audit document. Its `statusHistory` records the +`checked`, `pending`, and terminal `applied` or `failed` transitions. The audit document also +links to the current CMR collection record and records the prior and resulting CMR revision IDs. +The audit API is: + +- `GET /metadata_correction_audit` for newest-first, token-paginated audit searches. Supported + filters include collection, keyword UUID, action, scheme, status, native format, KMS version, + source, and date range. + +Publisher events carry the published KMS version through the queue into this document. Manual +correction endpoints look up the current published version before starting the run, so the +metadata-correction consumer and audit API do not query RDF4J. + +Deployed Lambdas use the public AWS `us-east-1` CA bundle to validate DocumentDB TLS connections. +The checked-in `serverless/certs/us-east-1-bundle.pem` was downloaded from the +[AWS certificate trust store](https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem): + +```bash +curl --fail --location \ + https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem \ + --output serverless/certs/us-east-1-bundle.pem +``` + +AWS documents the CA-bundle download requirement in +[Connecting programmatically to Amazon DocumentDB](https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html). +That example uses the global bundle; KMS uses the equivalent regional bundle listed for +US East (N. Virginia) in the +[AWS regional certificate bundle table](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html). + ## Setting up the RDF Database for local development In order to run KMS locally, you first need to setup a RDF database. ### Prerequisites diff --git a/bin/documentdb/start.sh b/bin/documentdb/start.sh new file mode 100755 index 00000000..f7ecf12e --- /dev/null +++ b/bin/documentdb/start.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=bin/env/local_env.sh +source "${SCRIPT_DIR}/../env/local_env.sh" + +IMAGE="${DOCUMENTDB_LOCAL_IMAGE:-mongo:8.0}" +CONTAINER_PORT="27017" +VOLUME_NAME="${DOCUMENTDB_LOCAL_VOLUME:-kms-documentdb-local-data}" + +if ! docker network inspect "${KMS_DOCKER_NETWORK}" >/dev/null 2>&1; then + docker network create "${KMS_DOCKER_NETWORK}" >/dev/null + echo "Created docker network '${KMS_DOCKER_NETWORK}'" +fi + +existing_id="$(docker ps -aq --filter "name=^${DOCUMENTDB_CONTAINER_NAME}$")" +if [[ -n "${existing_id}" ]]; then + running_id="$(docker ps -q --filter "name=^${DOCUMENTDB_CONTAINER_NAME}$")" + if [[ -n "${running_id}" ]]; then + echo "Mongo container '${DOCUMENTDB_CONTAINER_NAME}' is already running" + else + docker start "${DOCUMENTDB_CONTAINER_NAME}" >/dev/null + echo "Started existing Mongo container '${DOCUMENTDB_CONTAINER_NAME}'" + fi +else + docker run -d \ + --name "${DOCUMENTDB_CONTAINER_NAME}" \ + --network "${KMS_DOCKER_NETWORK}" \ + --network-alias "${DOCUMENTDB_CONTAINER_NAME}" \ + -p "${DOCUMENTDB_HOST_PORT}:${CONTAINER_PORT}" \ + -v "${VOLUME_NAME}:/data/db" \ + "${IMAGE}" >/dev/null + + echo "Started Mongo container '${DOCUMENTDB_CONTAINER_NAME}' on ${DOCUMENTDB_HOST_PORT}->${CONTAINER_PORT}" +fi + +mongo_ready=false +for _ in {1..30}; do + if docker exec "${DOCUMENTDB_CONTAINER_NAME}" \ + mongosh --quiet --eval 'db.runCommand({ ping: 1 }).ok' 2>/dev/null | grep -q '1'; then + mongo_ready=true + break + fi + + sleep 1 +done + +if [[ "${mongo_ready}" != "true" ]]; then + echo "Mongo container '${DOCUMENTDB_CONTAINER_NAME}' did not become ready" >&2 + exit 1 +fi + +( + cd "${PROJECT_ROOT}" + DOCUMENTDB_URI="mongodb://127.0.0.1:${DOCUMENTDB_HOST_PORT}/?directConnection=true" \ + ./node_modules/.bin/vite-node \ + --config vite.config.js \ + scripts/local/initialize_metadata_correction_audit.mjs +) + +echo "Connect from host using mongodb://localhost:${DOCUMENTDB_HOST_PORT}" +echo "Connect from SAM using ${DOCUMENTDB_URI}" diff --git a/bin/documentdb/stop.sh b/bin/documentdb/stop.sh new file mode 100755 index 00000000..c489a645 --- /dev/null +++ b/bin/documentdb/stop.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/env/local_env.sh +source "${SCRIPT_DIR}/../env/local_env.sh" + +container_id="$(docker ps -aq --filter "name=^${DOCUMENTDB_CONTAINER_NAME}$")" +if [[ -z "${container_id}" ]]; then + echo "Mongo container '${DOCUMENTDB_CONTAINER_NAME}' does not exist" + exit 0 +fi + +running_id="$(docker ps -q --filter "name=^${DOCUMENTDB_CONTAINER_NAME}$")" +if [[ -z "${running_id}" ]]; then + echo "Mongo container '${DOCUMENTDB_CONTAINER_NAME}' is already stopped" + exit 0 +fi + +docker stop "${DOCUMENTDB_CONTAINER_NAME}" >/dev/null +echo "Stopped Mongo container '${DOCUMENTDB_CONTAINER_NAME}'" diff --git a/bin/env/local_env.sh b/bin/env/local_env.sh index 30f61ba2..f98c80bc 100644 --- a/bin/env/local_env.sh +++ b/bin/env/local_env.sh @@ -19,5 +19,8 @@ export LOCALSTACK_CONTAINER_NAME="${LOCALSTACK_CONTAINER_NAME:-kms-localstack}" export LOCALSTACK_IMAGE="${LOCALSTACK_IMAGE:-localstack/localstack:3.8.1}" export LOCALSTACK_PORT="${LOCALSTACK_PORT:-4566}" export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://localstack:${LOCALSTACK_PORT}}" +export DOCUMENTDB_URI="${DOCUMENTDB_URI:-mongodb://kms-documentdb-local:27017/?directConnection=true}" +export DOCUMENTDB_CONTAINER_NAME="${DOCUMENTDB_CONTAINER_NAME:-kms-documentdb-local}" +export DOCUMENTDB_HOST_PORT="${DOCUMENTDB_HOST_PORT:-27018}" export SAM_WARM_CONTAINERS="${SAM_WARM_CONTAINERS:-LAZY}" export SAM_LOCAL_WATCH="${SAM_LOCAL_WATCH:-false}" diff --git a/bin/start-local.sh b/bin/start-local.sh index 952afd27..cd3b18bd 100755 --- a/bin/start-local.sh +++ b/bin/start-local.sh @@ -25,7 +25,7 @@ clearStaleSAMContainers() { echo "Clearing stale SAM containers..." docker ps --format '{{.ID}} {{.Image}}' \ - | awk '$2 ~ /public\.ecr\.aws\/lambda\/nodejs:22-rapid-/ { print $1 }' \ + | awk '$2 ~ /public\.ecr\.aws\/lambda\/nodejs:[0-9]+-rapid-/ { print $1 }' \ | xargs -r docker rm -f >/dev/null 2>&1 || true } diff --git a/cdk/app/lib/CmrEventProcessingStack.ts b/cdk/app/lib/CmrEventProcessingStack.ts index 7e12a0cf..b830b55b 100644 --- a/cdk/app/lib/CmrEventProcessingStack.ts +++ b/cdk/app/lib/CmrEventProcessingStack.ts @@ -1,5 +1,6 @@ import * as cdk from 'aws-cdk-lib' import * as ec2 from 'aws-cdk-lib/aws-ec2' +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager' import * as sns from 'aws-cdk-lib/aws-sns' import { Construct } from 'constructs' @@ -20,12 +21,12 @@ export interface CmrEventProcessingStackProps extends cdk.StackProps { cmrWritebackProviders?: string cmrWritebackValidateKeywords?: string cmrWritebackValidateUmmC?: string + metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup + metadataCorrectionAuditEnvironment: Record + metadataCorrectionAuditSecret?: secretsmanager.ISecret redisEnabled?: string redisHost?: string redisPort?: string - rdf4jPassword: string - rdf4jServiceUrl: string - rdf4jUserName: string prefix: string stage: string topicArn: string @@ -73,13 +74,14 @@ export class CmrEventProcessingStack extends cdk.Stack { cmrWritebackProviders: props.cmrWritebackProviders, cmrWritebackValidateKeywords: props.cmrWritebackValidateKeywords, cmrWritebackValidateUmmC: props.cmrWritebackValidateUmmC, + metadataCorrectionAuditClientSecurityGroup: + props.metadataCorrectionAuditClientSecurityGroup, + metadataCorrectionAuditEnvironment: props.metadataCorrectionAuditEnvironment, + metadataCorrectionAuditSecret: props.metadataCorrectionAuditSecret, prefix: props.prefix, redisEnabled: props.redisEnabled, redisHost: props.redisHost, redisPort: props.redisPort, - rdf4jPassword: props.rdf4jPassword, - rdf4jServiceUrl: props.rdf4jServiceUrl, - rdf4jUserName: props.rdf4jUserName, stage: props.stage, securityGroup: this.securityGroup, useLocalstack, diff --git a/cdk/app/lib/KmsStack.ts b/cdk/app/lib/KmsStack.ts index 9e3878aa..c2775c9e 100644 --- a/cdk/app/lib/KmsStack.ts +++ b/cdk/app/lib/KmsStack.ts @@ -3,6 +3,7 @@ import * as cdk from 'aws-cdk-lib' import * as apigateway from 'aws-cdk-lib/aws-apigateway' import * as ec2 from 'aws-cdk-lib/aws-ec2' import * as iam from 'aws-cdk-lib/aws-iam' +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager' import * as sns from 'aws-cdk-lib/aws-sns' import { Construct } from 'constructs' @@ -25,6 +26,9 @@ export interface KmsStackProps extends cdk.StackProps { cmrWriterToken?: string existingApiId: string | undefined keywordSyncAlarmEmails?: string[] + metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup + metadataCorrectionAuditEnvironment: Record + metadataCorrectionAuditSecret?: secretsmanager.ISecret prefix: string rootResourceId: string | undefined stage: string @@ -124,6 +128,8 @@ export class KmsStack extends cdk.Stack { ) this.lambdaRole = iamSetup.lambdaRole + props.metadataCorrectionAuditSecret?.grantRead(this.lambdaRole) + this.keywordEventsTopic = new sns.Topic(this, 'KeywordEventsTopic', { topicName: keywordEventsTopicName }) @@ -179,8 +185,11 @@ export class KmsStack extends cdk.Stack { CMR_WRITEBACK_PROVIDERS: props.cmrWritebackProviders || '', CMR_WRITEBACK_VALIDATE_KEYWORDS: props.cmrWritebackValidateKeywords || '', CMR_WRITEBACK_VALIDATE_UMM_C: props.cmrWritebackValidateUmmC || '', - METADATA_CORRECTION_REQUESTS_TOPIC_ARN: metadataCorrectionRequestsTopicArn + METADATA_CORRECTION_REQUESTS_TOPIC_ARN: metadataCorrectionRequestsTopicArn, + ...props.metadataCorrectionAuditEnvironment }, + metadataCorrectionAuditClientSecurityGroup: + props.metadataCorrectionAuditClientSecurityGroup, prefix, securityGroup: this.securityGroup, stage: this.stage, diff --git a/cdk/app/lib/MetadataCorrectionAuditStack.ts b/cdk/app/lib/MetadataCorrectionAuditStack.ts new file mode 100644 index 00000000..55eed05e --- /dev/null +++ b/cdk/app/lib/MetadataCorrectionAuditStack.ts @@ -0,0 +1,128 @@ +import * as fs from 'fs' +import * as path from 'path' + +import * as cdk from 'aws-cdk-lib' +import * as docdb from 'aws-cdk-lib/aws-docdb' +import * as ec2 from 'aws-cdk-lib/aws-ec2' +import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs' +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager' +import * as customResources from 'aws-cdk-lib/custom-resources' +import { Construct } from 'constructs' + +import { getDocumentDbCertificateBundling } from './helper/DocumentDbLambdaConfig' +import { MetadataCorrectionAuditDatabaseSetup } from './helper/MetadataCorrectionAuditDatabaseSetup' +import { NODE_LAMBDA_RUNTIME } from './helper/NodeLambdaRuntime' + +const PROJECT_ROOT = path.join(__dirname, '../../..') +const METADATA_CORRECTION_AUDIT_INDEXES = JSON.parse(fs.readFileSync( + path.join(PROJECT_ROOT, 'config/metadataCorrectionAuditIndexes.json'), + 'utf8' +)) + +export interface MetadataCorrectionAuditStackProps extends cdk.StackProps { + localUri?: string + maxCapacity?: number + minCapacity?: number + prefix: string + stage: string + useLocalstack: boolean + vpcId: string +} + +/** + * Shared DocumentDB infrastructure for metadata-correction audit readers and writers. + */ +export class MetadataCorrectionAuditStack extends cdk.Stack { + public readonly cluster?: docdb.DatabaseCluster + + public readonly clientSecurityGroup?: ec2.SecurityGroup + + public readonly connectionEnvironment: Record + + public readonly secret?: secretsmanager.ISecret + + /** + * Creates the shared audit database resources and exposes their Lambda connection settings. + * + * @param scope Parent CDK construct. + * @param id Stack identifier. + * @param props Environment, VPC, and serverless capacity configuration. + */ + constructor( + scope: Construct, + id: string, + props: MetadataCorrectionAuditStackProps + ) { + super(scope, id, props) + + const vpc = props.useLocalstack + ? undefined + : ec2.Vpc.fromLookup(this, 'Vpc', { vpcId: props.vpcId }) + const databaseSetup = new MetadataCorrectionAuditDatabaseSetup( + this, + 'Database', + { + localUri: props.localUri, + maxCapacity: props.maxCapacity, + minCapacity: props.minCapacity, + prefix: props.prefix, + stage: props.stage, + useLocalstack: props.useLocalstack, + vpc + } + ) + + this.cluster = databaseSetup.cluster + this.clientSecurityGroup = databaseSetup.clientSecurityGroup + this.connectionEnvironment = databaseSetup.environment + this.secret = databaseSetup.secret + + if ( + !props.useLocalstack + && vpc + && this.cluster + && this.clientSecurityGroup + && this.secret + ) { + // Define the deployment Lambda that connects to DocumentDB and creates the indexes. + const indexInitializer = new NodejsFunction(this, 'IndexInitializer', { + functionName: `${props.prefix}-${props.stage}-metadata-correction-audit-indexes`, + entry: path.join( + PROJECT_ROOT, + 'serverless/src/initializeMetadataCorrectionAudit/handler.js' + ), + handler: 'initializeMetadataCorrectionAudit', + runtime: NODE_LAMBDA_RUNTIME, + timeout: cdk.Duration.minutes(5), + memorySize: 512, + environment: this.connectionEnvironment, + ...getDocumentDbCertificateBundling(this.connectionEnvironment), + depsLockFilePath: path.join(PROJECT_ROOT, 'package-lock.json'), + projectRoot: PROJECT_ROOT, + vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS + }, + securityGroups: [this.clientSecurityGroup] + }) + this.secret.grantRead(indexInitializer) + + // Register the initializer as the handler for CloudFormation custom-resource events. + const indexProvider = new customResources.Provider(this, 'IndexProvider', { + onEventHandler: indexInitializer + }) + + // Invoke the provider when this stack creates or updates the audit index resource. + const indexResource = new cdk.CustomResource(this, 'Indexes', { + serviceToken: indexProvider.serviceToken, + properties: { + ClusterEndpoint: this.cluster.clusterEndpoint.hostname, + IndexDefinitions: METADATA_CORRECTION_AUDIT_INDEXES + } + }) + indexResource.node.addDependency(this.cluster) + } + } +} + +export default MetadataCorrectionAuditStack diff --git a/cdk/app/lib/helper/DocumentDbLambdaConfig.ts b/cdk/app/lib/helper/DocumentDbLambdaConfig.ts new file mode 100644 index 00000000..50725cad --- /dev/null +++ b/cdk/app/lib/helper/DocumentDbLambdaConfig.ts @@ -0,0 +1,75 @@ +import * as ec2 from 'aws-cdk-lib/aws-ec2' +import { NodejsFunctionProps } from 'aws-cdk-lib/aws-lambda-nodejs' + +const DOCUMENTDB_TLS_CA_FILE_NAME = 'us-east-1-bundle.pem' + +/** + * Bundles DocumentDB Lambda dependencies and copies the CA certificate when TLS is configured. + * + * @example + * getDocumentDbCertificateBundling({ + * DOCUMENTDB_DATABASE_NAME: 'kms', + * DOCUMENTDB_TLS_CA_FILE: '/var/task/us-east-1-bundle.pem' + * }) + * // { bundling: { externalModules: [], commandHooks: { ...copy the CA... } } } + * + * @param environment Lambda environment containing the database name and optional TLS path. + * @returns Bundling props for DocumentDB-enabled Lambdas, otherwise an empty object. + */ +export const getDocumentDbCertificateBundling = ( + environment: { + DOCUMENTDB_DATABASE_NAME?: string + DOCUMENTDB_TLS_CA_FILE?: string + } +): Pick => ( + environment.DOCUMENTDB_DATABASE_NAME + ? { + bundling: { + // Node.js 24 SAM images do not supply AWS SDK packages to local Lambda containers. + externalModules: [], + ...(environment.DOCUMENTDB_TLS_CA_FILE + ? { + commandHooks: { + beforeBundling: () => [], + beforeInstall: () => [], + afterBundling: (inputDir: string, outputDir: string) => [ + `cp "${inputDir}/serverless/certs/${DOCUMENTDB_TLS_CA_FILE_NAME}" "${outputDir}/${DOCUMENTDB_TLS_CA_FILE_NAME}"` + ] + } + } + : {}) + } + } + : {} +) + +/** + * Adds the DocumentDB client security group to a Lambda when the database is configured. + * + * @example + * getDocumentDbLambdaSecurityGroups({ + * securityGroup: baseGroup, + * clientSecurityGroup: documentDbGroup, + * environment: { DOCUMENTDB_DATABASE_NAME: 'kms' } + * }) + * // [baseGroup, documentDbGroup] + * + * @param props Security groups and DocumentDB environment for the Lambda. + * @returns The base Lambda group plus the DocumentDB client group when configured. + */ +export const getDocumentDbLambdaSecurityGroups = ({ + clientSecurityGroup, + environment, + securityGroup +}: { + clientSecurityGroup?: ec2.ISecurityGroup + environment: { DOCUMENTDB_DATABASE_NAME?: string } + securityGroup: ec2.ISecurityGroup +}): ec2.ISecurityGroup[] => [ + securityGroup, + ...(environment.DOCUMENTDB_DATABASE_NAME && clientSecurityGroup + ? [clientSecurityGroup] + : []) +] + +export default getDocumentDbCertificateBundling diff --git a/cdk/app/lib/helper/KmsLambdaFunctions.ts b/cdk/app/lib/helper/KmsLambdaFunctions.ts index f32e7300..d11682d2 100644 --- a/cdk/app/lib/helper/KmsLambdaFunctions.ts +++ b/cdk/app/lib/helper/KmsLambdaFunctions.ts @@ -11,6 +11,10 @@ import { NodejsFunction, NodejsFunctionProps } from 'aws-cdk-lib/aws-lambda-node import { Construct } from 'constructs' import { ApiResources } from './ApiResources' +import { + getDocumentDbCertificateBundling, + getDocumentDbLambdaSecurityGroups +} from './DocumentDbLambdaConfig' import { NODE_LAMBDA_RUNTIME } from './NodeLambdaRuntime' /** @@ -20,12 +24,21 @@ interface LambdaFunctionsProps { api: apigateway.IRestApi; apiResources: ApiResources; lambdaRole: iam.Role; + metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup; metadataCorrectionEnvironment?: { CMR_SYSTEM_TOKEN_PARAMETER_NAME?: string; CMR_WRITER_TOKEN: string; CMR_WRITEBACK_PROVIDERS: string; CMR_WRITEBACK_VALIDATE_KEYWORDS: string; CMR_WRITEBACK_VALIDATE_UMM_C: string; + DOCUMENTDB_AUDIT_COLLECTION_NAME?: string; + DOCUMENTDB_DATABASE_NAME?: string; + DOCUMENTDB_HOST?: string; + DOCUMENTDB_MAX_POOL_SIZE?: string; + DOCUMENTDB_PORT?: string; + DOCUMENTDB_SECRET_ARN?: string; + DOCUMENTDB_TLS_CA_FILE?: string; + DOCUMENTDB_URI?: string; METADATA_CORRECTION_REQUESTS_TOPIC_ARN?: string; }; prefix: string; @@ -327,12 +340,16 @@ export class LambdaFunctions { ) this.createApiLambda( - scope, - 'getMetadataCorrectionAudit/handler.js', - 'get-metadata-correction-audit', - 'getMetadataCorrectionAudit', - '/metadata_correction_audit', - 'GET' + scope, // CDK construct scope + 'getMetadataCorrectionAudit/handler.js', // Lambda handler path + 'get-metadata-correction-audit', // Lambda function name + 'getMetadataCorrectionAudit', // Exported handler name + '/metadata_correction_audit', // API resource path + 'GET', // HTTP method + false, // Do not use the EDL authorizer + Duration.seconds(30), // Lambda timeout + 1024, // Lambda memory in MB + this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables ) this.createApiLambda( @@ -784,13 +801,18 @@ export class LambdaFunctions { ...this.props.environment, ...additionalEnvironment }, + ...getDocumentDbCertificateBundling(additionalEnvironment), // Conditionally add VPC configuration ...(this.useLocalstack ? {} : { vpc: this.props.vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, - securityGroups: [this.props.securityGroup] + securityGroups: getDocumentDbLambdaSecurityGroups({ + clientSecurityGroup: this.props.metadataCorrectionAuditClientSecurityGroup, + environment: additionalEnvironment, + securityGroup: this.props.securityGroup + }) }) } diff --git a/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts new file mode 100644 index 00000000..3c9dae19 --- /dev/null +++ b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts @@ -0,0 +1,122 @@ +import * as cdk from 'aws-cdk-lib' +import * as docdb from 'aws-cdk-lib/aws-docdb' +import * as ec2 from 'aws-cdk-lib/aws-ec2' +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager' +import { Construct } from 'constructs' + +interface MetadataCorrectionAuditDatabaseSetupProps { + databaseName?: string + localUri?: string + maxCapacity?: number + minCapacity?: number + prefix: string + stage: string + useLocalstack: boolean + vpc?: ec2.IVpc +} + +/** + * Provisions the shared DocumentDB Serverless cluster used by metadata-correction auditing. + */ +export class MetadataCorrectionAuditDatabaseSetup extends Construct { + public readonly cluster?: docdb.DatabaseCluster + + public readonly clientSecurityGroup?: ec2.SecurityGroup + + public readonly environment: Record + + public readonly secret?: secretsmanager.ISecret + + /** + * Configures a local MongoDB URI for LocalStack, or provisions the deployed DocumentDB cluster, + * secret, TLS environment, and paired database/client security groups. + * + * @param scope Parent CDK construct. + * @param id Construct identifier. + * @param props Local or deployed database configuration. + */ + constructor( + scope: Construct, + id: string, + props: MetadataCorrectionAuditDatabaseSetupProps + ) { + super(scope, id) + + const databaseName = props.databaseName || 'kms' + const commonEnvironment = { + DOCUMENTDB_DATABASE_NAME: databaseName, + DOCUMENTDB_AUDIT_COLLECTION_NAME: 'metadataCorrectionAudits', + DOCUMENTDB_MAX_POOL_SIZE: '5' + } + + if (props.useLocalstack) { + this.environment = { + ...commonEnvironment, + DOCUMENTDB_URI: props.localUri || 'mongodb://kms-documentdb-local:27017/?directConnection=true' + } + + return + } + + if (!props.vpc) { + throw new Error('A VPC is required for a deployed DocumentDB cluster') + } + + const databaseSecurityGroup = new ec2.SecurityGroup(this, 'DatabaseSecurityGroup', { + vpc: props.vpc, + allowAllOutbound: true, + description: 'DocumentDB access for KMS metadata-correction audit Lambdas' + }) + this.clientSecurityGroup = new ec2.SecurityGroup(this, 'ClientSecurityGroup', { + vpc: props.vpc, + allowAllOutbound: true, + description: 'Shared client access to the KMS metadata-correction audit database' + }) + + databaseSecurityGroup.addIngressRule( + this.clientSecurityGroup, + ec2.Port.tcp(27017), + 'Allow metadata-correction audit clients' + ) + + this.cluster = new docdb.DatabaseCluster(this, 'Cluster', { + dbClusterName: `${props.prefix}-${props.stage}-metadata-correction-audit`, + engineVersion: '8.0.0', + masterUser: { + username: 'kms_audit', + secretName: `${props.prefix}/${props.stage}/metadata-correction-audit/documentdb` + }, + serverlessV2ScalingConfiguration: { + minCapacity: props.minCapacity || 0.5, + maxCapacity: props.maxCapacity || 4 + }, + backup: { + retention: cdk.Duration.days(7) + }, + deletionProtection: ['ops', 'prod'].includes(props.stage.toLowerCase()), + removalPolicy: cdk.RemovalPolicy.RETAIN, + storageEncrypted: true, + securityGroup: databaseSecurityGroup, + vpc: props.vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS + } + }) + + this.secret = this.cluster.secret + + if (!this.secret) { + throw new Error('DocumentDB did not create a master-user secret') + } + + this.environment = { + ...commonEnvironment, + DOCUMENTDB_HOST: this.cluster.clusterEndpoint.hostname, + DOCUMENTDB_PORT: this.cluster.clusterEndpoint.port.toString(), + DOCUMENTDB_SECRET_ARN: this.secret.secretArn, + DOCUMENTDB_TLS_CA_FILE: '/var/task/us-east-1-bundle.pem' + } + } +} + +export default MetadataCorrectionAuditDatabaseSetup diff --git a/cdk/app/lib/helper/MetadataCorrectionSetup.ts b/cdk/app/lib/helper/MetadataCorrectionSetup.ts index 7e400678..39c56924 100644 --- a/cdk/app/lib/helper/MetadataCorrectionSetup.ts +++ b/cdk/app/lib/helper/MetadataCorrectionSetup.ts @@ -5,11 +5,16 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2' import * as iam from 'aws-cdk-lib/aws-iam' import * as eventsources from 'aws-cdk-lib/aws-lambda-event-sources' import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs' +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager' import * as sns from 'aws-cdk-lib/aws-sns' import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions' import * as sqs from 'aws-cdk-lib/aws-sqs' import { Construct } from 'constructs' +import { + getDocumentDbCertificateBundling, + getDocumentDbLambdaSecurityGroups +} from './DocumentDbLambdaConfig' import { NODE_LAMBDA_RUNTIME } from './NodeLambdaRuntime' /** @@ -24,13 +29,13 @@ interface MetadataCorrectionSetupProps { cmrWritebackProviders?: string cmrWritebackValidateKeywords?: string cmrWritebackValidateUmmC?: string + metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup + metadataCorrectionAuditEnvironment: Record + metadataCorrectionAuditSecret?: secretsmanager.ISecret prefix: string redisEnabled?: string redisHost?: string redisPort?: string - rdf4jPassword: string - rdf4jServiceUrl: string - rdf4jUserName: string securityGroup: ec2.SecurityGroup stage: string useLocalstack: boolean @@ -78,13 +83,13 @@ export class MetadataCorrectionSetup extends Construct { cmrWritebackProviders, cmrWritebackValidateKeywords, cmrWritebackValidateUmmC, + metadataCorrectionAuditClientSecurityGroup, + metadataCorrectionAuditEnvironment, + metadataCorrectionAuditSecret, prefix, redisEnabled, redisHost, redisPort, - rdf4jPassword, - rdf4jServiceUrl, - rdf4jUserName, securityGroup, stage, useLocalstack, @@ -168,10 +173,9 @@ export class MetadataCorrectionSetup extends Construct { ...(metadataCorrectionRequestDelayMs ? { METADATA_CORRECTION_REQUEST_DELAY_MS: metadataCorrectionRequestDelayMs } : {}), - RDF4J_PASSWORD: rdf4jPassword, - RDF4J_SERVICE_URL: rdf4jServiceUrl, - RDF4J_USER_NAME: rdf4jUserName + ...metadataCorrectionAuditEnvironment }, + ...getDocumentDbCertificateBundling(metadataCorrectionAuditEnvironment), depsLockFilePath: path.join(projectRoot, 'package-lock.json'), projectRoot, ...(useLocalstack ? {} : { @@ -179,7 +183,11 @@ export class MetadataCorrectionSetup extends Construct { vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, - securityGroups: [securityGroup] + securityGroups: getDocumentDbLambdaSecurityGroups({ + clientSecurityGroup: metadataCorrectionAuditClientSecurityGroup, + environment: metadataCorrectionAuditEnvironment, + securityGroup + }) }) } ) @@ -193,6 +201,7 @@ export class MetadataCorrectionSetup extends Construct { )) this.metadataCorrectionRequestsQueue.grantConsumeMessages(this.metadataCorrectionServiceLambda) + metadataCorrectionAuditSecret?.grantRead(this.metadataCorrectionServiceLambda) this.metadataCorrectionServiceLambda.addToRolePolicy(new iam.PolicyStatement({ actions: ['cloudwatch:PutMetricData'], resources: ['*'], diff --git a/cdk/bin/main.ts b/cdk/bin/main.ts index a8c0dbbb..b31de3d6 100644 --- a/cdk/bin/main.ts +++ b/cdk/bin/main.ts @@ -3,6 +3,7 @@ import * as cdk from 'aws-cdk-lib' import { CmrEventProcessingStack } from '../app/lib/CmrEventProcessingStack' import { KmsStack, KmsStackProps } from '../app/lib/KmsStack' +import { MetadataCorrectionAuditStack } from '../app/lib/MetadataCorrectionAuditStack' import { RedisStack } from '../app/lib/RedisStack' import { EbsStack } from '../rdfdb/lib/EbsStack' import { EcsStack } from '../rdfdb/lib/EcsStack' @@ -180,6 +181,25 @@ async function main() { ? String(localRedisEnabled) : String(redisConfigured) + const metadataCorrectionAuditStack = new MetadataCorrectionAuditStack( + app, // CDK application scope + 'MetadataCorrectionAuditStack', // Construct ID + { + env, // Target AWS account and region + localUri: useLocalstack + ? process.env.DOCUMENTDB_URI + || 'mongodb://kms-documentdb-local:27017/?directConnection=true' + : undefined, // Local MongoDB URI; unused in AWS + maxCapacity: Number(process.env.DOCUMENTDB_MAX_CAPACITY || 4), // Maximum serverless capacity + minCapacity: Number(process.env.DOCUMENTDB_MIN_CAPACITY || 0.5), // Minimum serverless capacity + prefix, // Resource naming prefix + stage, // Deployment environment + stackName: `${prefix}-MetadataCorrectionAuditStack`, // CloudFormation stack name + useLocalstack, // Whether to use local service substitutes + vpcId // VPC containing the deployed cluster + } + ) + // Create KmsStack const kmsStackProps: KmsStackProps = { cmrSystemTokenParameterName: process.env.CMR_SYSTEM_TOKEN_PARAMETER_NAME || '', @@ -187,6 +207,10 @@ async function main() { cmrWritebackProviders: process.env.CMR_WRITEBACK_PROVIDERS || '', cmrWritebackValidateKeywords: process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS || '', cmrWritebackValidateUmmC: process.env.CMR_WRITEBACK_VALIDATE_UMM_C || '', + metadataCorrectionAuditClientSecurityGroup: + metadataCorrectionAuditStack.clientSecurityGroup, + metadataCorrectionAuditEnvironment: metadataCorrectionAuditStack.connectionEnvironment, + metadataCorrectionAuditSecret: metadataCorrectionAuditStack.secret, prefix, env, vpcId, @@ -229,6 +253,8 @@ async function main() { kmsStack.addDependency(redisStack) } + kmsStack.addDependency(metadataCorrectionAuditStack) + const cmrEventProcessingStack = new CmrEventProcessingStack(app, 'CmrEventProcessingStack', { cmrBaseUrl, cmrSystemTokenParameterName: process.env.CMR_SYSTEM_TOKEN_PARAMETER_NAME || '', @@ -239,16 +265,15 @@ async function main() { cmrWritebackProviders: process.env.CMR_WRITEBACK_PROVIDERS || '', cmrWritebackValidateKeywords: process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS || '', cmrWritebackValidateUmmC: process.env.CMR_WRITEBACK_VALIDATE_UMM_C || '', + metadataCorrectionAuditClientSecurityGroup: + metadataCorrectionAuditStack.clientSecurityGroup, + metadataCorrectionAuditEnvironment: metadataCorrectionAuditStack.connectionEnvironment, + metadataCorrectionAuditSecret: metadataCorrectionAuditStack.secret, env, prefix, redisEnabled: redisEnabledValue, redisHost: useLocalstack ? localRedisHost : redisStack?.endpointAddress, redisPort: useLocalstack ? localRedisPort : redisStack?.endpointPort, - rdf4jPassword: process.env.RDF4J_PASSWORD || 'rdf4j', - rdf4jServiceUrl: useLocalstack - ? 'http://rdf4j-server:8080' - : (lbStack?.rdf4jServiceUrl || process.env.RDF4J_SERVICE_URL || 'http://localhost:8081'), - rdf4jUserName: process.env.RDF4J_USER_NAME || 'rdf4j', stage, stackName: `${prefix}-CmrEventProcessingStack`, topicArn: kmsStack.keywordEventsTopic.topicArn, @@ -257,6 +282,7 @@ async function main() { }) cmrEventProcessingStack.addDependency(kmsStack) + cmrEventProcessingStack.addDependency(metadataCorrectionAuditStack) app.synth() } diff --git a/config/metadataCorrectionAuditIndexes.json b/config/metadataCorrectionAuditIndexes.json new file mode 100644 index 00000000..bc615141 --- /dev/null +++ b/config/metadataCorrectionAuditIndexes.json @@ -0,0 +1,57 @@ +[ + { + "key": { + "createdAt": -1, + "_id": -1 + }, + "name": "createdAt_desc" + }, + { + "key": { + "collectionConceptId": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "collection_createdAt_desc" + }, + { + "key": { + "status": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "status_createdAt_desc" + }, + { + "key": { + "publishedVersionName": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "version_createdAt_desc" + }, + { + "key": { + "trigger.scheme": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "triggerScheme_createdAt_desc" + }, + { + "key": { + "corrections.scheme": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "correctionScheme_createdAt_desc" + }, + { + "key": { + "corrections.keywordConceptUuid": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "keywordUuid_createdAt_desc" + } +] diff --git a/package-lock.json b/package-lock.json index fc1e860f..a27400e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-eventbridge": "^3.997.0", "@aws-sdk/client-lambda": "^3.775.0", "@aws-sdk/client-s3": "^3.540.0", + "@aws-sdk/client-secrets-manager": "^3.1124.0", "@aws-sdk/client-sfn": "^3.775.0", "@aws-sdk/client-sns": "^3.997.0", "@aws-sdk/client-sqs": "^3.997.0", @@ -26,6 +27,7 @@ "html-escaper": "^3.0.3", "install": "^0.13.0", "lodash": "^4.17.21", + "mongodb": "^6.21.0", "node-fetch": "^2.7.0", "redis": "^4.7.1", "remove": "^0.1.5", @@ -644,6 +646,25 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-secrets-manager": { + "version": "3.1124.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1124.0.tgz", + "integrity": "sha512-+RE3Gu0JRcO4mMPSezG9VyQEyORXtFTGGUkVt/12Rcmv0+CzsDYfHbj4BRk8LJQE8yYGZMALR3P6OeCj8VvlgA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-sfn": { "version": "3.981.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sfn/-/client-sfn-3.981.0.tgz", @@ -847,17 +868,17 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.977.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.7.tgz", - "integrity": "sha512-I88Iov89NVmjSmJLKSv7Cn9M2J+a2942OkA8nZCbz+sl4ZeY4zEOcoLOrbt1GRfQ8zEQKnjAJdXixA3J/p1fDQ==", + "version": "3.977.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.9.tgz", + "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.974.3", - "@aws-sdk/xml-builder": "^3.972.38", + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", - "@smithy/core": "^3.31.1", + "@smithy/core": "^3.33.3", "@smithy/signature-v4": "^5.6.12", - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" }, @@ -888,15 +909,15 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.62", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.62.tgz", - "integrity": "sha512-BkDrk2cNjed31IKin/Oksb2ziF+gfuyRskFVuT4EU9Mep7M8Y/d8DJG4+anHme4Vuse7CwaEscwEfGyR6mzBhQ==", + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.70.tgz", + "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -904,17 +925,17 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.64", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.64.tgz", - "integrity": "sha512-Wj1FGK2IxY5EccQCvH+niTYhIvDoDujJf2CpRRgS3NpYNEgiFNVItNbJYQjINRlu7fG7jSsXkKV0UWKriEplrw==", + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.72.tgz", + "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/fetch-http-handler": "^5.6.10", - "@smithy/node-http-handler": "^4.9.10", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -922,23 +943,23 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.973.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.7.tgz", - "integrity": "sha512-2CefB8cCxDu52P24B8Ay93/cTT199bcSvNHQ8e2f4BjSCF83yErBnTIZEBo0VeIgCfmw+PJKFUXnlQWxm2dkug==", + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.15.tgz", + "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/credential-provider-env": "^3.972.62", - "@aws-sdk/credential-provider-http": "^3.972.64", - "@aws-sdk/credential-provider-login": "^3.972.69", - "@aws-sdk/credential-provider-process": "^3.972.62", - "@aws-sdk/credential-provider-sso": "^3.973.6", - "@aws-sdk/credential-provider-web-identity": "^3.972.68", - "@aws-sdk/nested-clients": "^3.997.36", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/credential-provider-imds": "^4.4.13", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-login": "^3.972.77", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -946,16 +967,16 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.69", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.69.tgz", - "integrity": "sha512-gM3j0Ie9+FoLNTYODY+QWbg3vCRBc7mR9cRdntxTMkFYIrwfRmuucfavP6HNBlYSuaYww54TNJGej4GFgoPZAg==", + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.77.tgz", + "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/nested-clients": "^3.997.36", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -963,21 +984,21 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.73", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.73.tgz", - "integrity": "sha512-VTzdbf8Ukjdb9yUubZzRI678CWZvKovhE8Nv3qihwhC187sRMGls+r9N8Wuht5q1xjKx2nmpS48ar8ppupjkCA==", + "version": "3.972.82", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.82.tgz", + "integrity": "sha512-znDkEOGXB8W3kG1LJUKP3foBZY/9qLM0eil/DxWXSp37XsdsRLQHE/d/OaCGGVgKpA6znR38h/+INk8do1FjiA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.62", - "@aws-sdk/credential-provider-http": "^3.972.64", - "@aws-sdk/credential-provider-ini": "^3.973.7", - "@aws-sdk/credential-provider-process": "^3.972.62", - "@aws-sdk/credential-provider-sso": "^3.973.6", - "@aws-sdk/credential-provider-web-identity": "^3.972.68", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/credential-provider-imds": "^4.4.13", - "@smithy/types": "^4.16.1", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-ini": "^3.973.15", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -985,15 +1006,15 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.62", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.62.tgz", - "integrity": "sha512-zXYU9UWNL66gtMgNLhmxlrvEokuI7r6G2q7FRGu41Bya4iS30JLelUipJX9SV4zhyCPWJhI9Li54R1d9H8Tq6A==", + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.70.tgz", + "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1001,17 +1022,17 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.973.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.6.tgz", - "integrity": "sha512-DobZggy3K49xdCpjeyMou0FQhkoYbluVGNydL6D+lcxF8GoAsttFX0xnH5GmiQ89We5dB6TRpW+CD/VowBH6HQ==", + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.14.tgz", + "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/nested-clients": "^3.997.36", - "@aws-sdk/token-providers": "3.1096.0", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/token-providers": "3.1116.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1019,16 +1040,16 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.68", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.68.tgz", - "integrity": "sha512-bq+yTt+uWJx60VVp/OIAX5xqUAu/K2Uc3eknWnWl+KtfcU2CQe0uNw6lySrn2t5GKHq7jsV0Z63HiBGVtzr/lg==", + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.76.tgz", + "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/nested-clients": "^3.997.36", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1278,18 +1299,18 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.36", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.36.tgz", - "integrity": "sha512-b71Suv7L+DnhM0MsQHU4WO42I32kxLZi96PbVhZbxMYIoKnEZz3v+LSrG8fupAoA4cBSshCk1Dl/PeRz49qUSg==", + "version": "3.997.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.44.tgz", + "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/signature-v4-multi-region": "^3.996.42", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/fetch-http-handler": "^5.6.10", - "@smithy/node-http-handler": "^4.9.10", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1297,14 +1318,14 @@ } }, "node_modules/@aws-sdk/nested-clients/node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.42", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.42.tgz", - "integrity": "sha512-DBV4naZP6HYBlAvPpoQzOP12Wvfou/5rN8yJPXjBTBylU5qwCbh/tXr2MddHoIjgoRkEl/eS+IljiUqvmwey1Q==", + "version": "3.996.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.974.2", - "@smithy/signature-v4": "^5.6.9", - "@smithy/types": "^4.16.1", + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1363,16 +1384,16 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1096.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1096.0.tgz", - "integrity": "sha512-hdUS2hDppy3vkWeFl5y86RLNU6OWH2mQB09yOSsRefwhhGTSFPkaZvfLDD/9vFcvMzlr8QFQFw3fw2FtrurVQA==", + "version": "3.1116.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1116.0.tgz", + "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.977.1", - "@aws-sdk/nested-clients": "^3.997.36", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.8", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1380,12 +1401,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.974.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.3.tgz", - "integrity": "sha512-ECAqfpNsef+7MO8qtR0h9KcFIBAygaE7Cm6UOiQl+ft+uVap+1G7bNEjs4mdJE2OnA4m6k7i8peH8uGIAsOMGw==", + "version": "3.974.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -1482,12 +1503,12 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.38", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.38.tgz", - "integrity": "sha512-grf7mzfVxBS5AlsuTvBN7uDpzqohFww9fRPCO+EBSUdvtsYMcPSKdz54h/7XiscqNcUM1Ae1MF7JLHmiYYuzbQ==", + "version": "3.972.40", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -3960,6 +3981,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.5.0.tgz", + "integrity": "sha512-Hk1SKJCMcCos38+vqDnZzlIo4XRj9yCGzYkjB4LcqpeXRIYfia1UWTz+VrueLxoU+uSRJzgkufxoRZg8gi52YA==", + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.1.tgz", @@ -4546,12 +4576,12 @@ } }, "node_modules/@smithy/core": { - "version": "3.32.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.32.0.tgz", - "integrity": "sha512-NAiCSC78fzbNIEWoheoF74Ob5ZorLijCHpMY26Fqvqg/+9LuyIqMfHDg2p8Yk1rqOyowtiL3y7WX0AW+teL6zw==", + "version": "3.33.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.17.0", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -4559,13 +4589,13 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.4.15", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.15.tgz", - "integrity": "sha512-xYVGrisQqTJWhOnScUhbx8s9H63TMtoxzuUoxG6mP8J+B/YbX3vZxVsgV0xDf43abJnJP0fjP7BkQh7OESwuRA==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.31.0", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -4638,13 +4668,13 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.6.12", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.12.tgz", - "integrity": "sha512-OpQgP6IGH4j0NJ2zjfYZLjQL85ai+Wi/q51EmZJovXsEwKSvu89qiXUq77Q6EmwZ/hSl7fKpn2Z9mhiDN6OM+Q==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.2.tgz", + "integrity": "sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.31.0", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -4851,13 +4881,13 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.9.12", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.12.tgz", - "integrity": "sha512-dWW5KRt4mnEvjNzbGqGeCuAvgum85Y9ZoyuMQqcTEfapndyVJ1k9BEHK7kdXJZ32enyRmmwcFjMwlB/KgLKI3Q==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.0.tgz", + "integrity": "sha512-0mq1pHadfyXCYCqm2cNpbjNIT+fbaUpNxewZb/YNr2L0IrEVMOb8gM/Fl4K6XvHCW3uSNDFwPl/+iKm0bx9jYg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.31.0", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -4980,9 +5010,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.17.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.0.tgz", - "integrity": "sha512-Aw4joiM0ZdErpo39lCj8phT2lxoiKZV+KZzBxnnQhWVtU2Is/WffQSL04uUWRcXUse9Ln8vXZK6V/FwqRVnQpg==", + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.2.tgz", + "integrity": "sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -5705,6 +5735,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.58.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.0.tgz", @@ -7206,6 +7251,15 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/bson": { + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "license": "Apache-2.0", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/cac": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/cac/-/cac-7.0.0.tgz", @@ -11315,6 +11369,12 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "license": "MIT" + }, "node_modules/meow": { "version": "10.1.5", "resolved": "https://registry.npmjs.org/meow/-/meow-10.1.5.tgz", @@ -11516,6 +11576,87 @@ "node": ">= 6" } }, + "node_modules/mongodb": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", + "integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.2" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.3.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -12312,7 +12453,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -13121,6 +13261,15 @@ "node": ">=0.10.0" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/spdx-correct": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", @@ -14591,7 +14740,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=12" diff --git a/package.json b/package.json index b20b5c28..c33d7a7c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "redis:stop": "bash bin/redis/stop.sh", "redis:connect": "bash bin/redis/connect.sh", "redis:memory_used": "bash bin/redis/memory_used.sh", + "documentdb:start": "bash bin/documentdb/start.sh", + "documentdb:stop": "bash bin/documentdb/stop.sh", "prime-cache:invoke-local": "bash scripts/local/invoke_prime_concepts_cache.sh", "export-data": "vite-node --config vite.config.js setup/scripts/exportData.js -all", "create-rdf-files": "vite-node --config vite.config.js setup/scripts/createRdfFiles.js", @@ -38,6 +40,7 @@ "@aws-sdk/client-eventbridge": "^3.997.0", "@aws-sdk/client-lambda": "^3.775.0", "@aws-sdk/client-s3": "^3.540.0", + "@aws-sdk/client-secrets-manager": "^3.1124.0", "@aws-sdk/client-sfn": "^3.775.0", "@aws-sdk/client-sns": "^3.997.0", "@aws-sdk/client-sqs": "^3.997.0", @@ -52,6 +55,7 @@ "html-escaper": "^3.0.3", "install": "^0.13.0", "lodash": "^4.17.21", + "mongodb": "^6.21.0", "node-fetch": "^2.7.0", "redis": "^4.7.1", "remove": "^0.1.5", diff --git a/scripts/local/initialize_metadata_correction_audit.mjs b/scripts/local/initialize_metadata_correction_audit.mjs new file mode 100644 index 00000000..616b158f --- /dev/null +++ b/scripts/local/initialize_metadata_correction_audit.mjs @@ -0,0 +1,28 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +import { + initializeMetadataCorrectionAudit +} from '../../serverless/src/initializeMetadataCorrectionAudit/handler' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)) +const projectRoot = path.resolve(scriptDirectory, '../..') +const indexDefinitions = JSON.parse(fs.readFileSync( + path.join(projectRoot, 'config/metadataCorrectionAuditIndexes.json'), + 'utf8' +)) + +try { + const result = await initializeMetadataCorrectionAudit({ + RequestType: 'Create', + ResourceProperties: { IndexDefinitions: indexDefinitions } + }) + + console.log( + `[initialize-metadata-correction-audit] Verified ${result.Data.IndexCount} local audit indexes` + ) +} finally { + await closeDocumentDbClient() +} diff --git a/scripts/local/mock_cmr_server.mjs b/scripts/local/mock_cmr_server.mjs index e8d8ca79..7aa8f14b 100644 --- a/scripts/local/mock_cmr_server.mjs +++ b/scripts/local/mock_cmr_server.mjs @@ -89,6 +89,33 @@ const getNativeMetadataContentType = (collection) => { return format || 'application/octet-stream' } +const nativeRevisionsByConceptId = new Map() + +/** + * Snapshots a collection's current native payload by revision for revision-specific GET requests. + * + * @example + * saveNativeRevision({ conceptId: 'C123-PROV', revisionId: 2, nativeMetadata: '' }) + * // nativeRevisionsByConceptId.get('C123-PROV').get('2').nativeMetadata === '' + * + * @param {Object} collection Mutable fixture collection to snapshot. + * @returns {void} + */ +const saveNativeRevision = (collection) => { + const revisions = nativeRevisionsByConceptId.get(collection.conceptId) || new Map() + const nativeMetadata = getNativeMetadataPayload(collection) + + revisions.set(String(collection.revisionId), { + contentType: getNativeMetadataContentType(collection), + nativeMetadata: typeof nativeMetadata === 'string' + ? nativeMetadata + : structuredClone(nativeMetadata) + }) + nativeRevisionsByConceptId.set(collection.conceptId, revisions) +} + +fixture.cmr?.collections?.forEach(saveNativeRevision) + // Keep the concept-id index in sync after local updates. const updateCollectionIndexes = (collection) => { collectionsByConceptId.set(collection.conceptId, collection) @@ -328,7 +355,14 @@ const handleNativeCollectionLookupRequest = (conceptId, revisionId, response) => return } - if (revisionId !== undefined && String(collection.revisionId) !== String(revisionId)) { + const revision = revisionId === undefined + ? { + contentType: getNativeMetadataContentType(collection), + nativeMetadata: getNativeMetadataPayload(collection) + } + : nativeRevisionsByConceptId.get(conceptId)?.get(String(revisionId)) + + if (!revision) { sendJson(response, 404, { errors: [`Revision ${revisionId} not found for collection concept id: ${conceptId}`] }) @@ -336,16 +370,14 @@ const handleNativeCollectionLookupRequest = (conceptId, revisionId, response) => return } - const nativeMetadata = getNativeMetadataPayload(collection) - response.writeHead(200, { - 'Content-Type': getNativeMetadataContentType(collection) + 'Content-Type': revision.contentType }) response.end( - typeof nativeMetadata === 'string' - ? nativeMetadata - : JSON.stringify(nativeMetadata) + typeof revision.nativeMetadata === 'string' + ? revision.nativeMetadata + : JSON.stringify(revision.nativeMetadata) ) } @@ -380,6 +412,7 @@ const handleLocalCollectionUpdateRequest = async (request, response, conceptId) collection.revisionId = Number(collection.revisionId || 0) + 1 updateCollectionIndexes(collection) + saveNativeRevision(collection) sendJson(response, 200, { updated: true, @@ -450,6 +483,7 @@ const handleIngestCollectionWriteRequest = async (request, response, providerId, collection.revisionId = Number(collection.revisionId || 0) + 1 updateCollectionIndexes(collection) + saveNativeRevision(collection) sendJson(response, 200, { 'concept-id': collection.conceptId, diff --git a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs index 83e07ae8..75195d9b 100644 --- a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs @@ -4,15 +4,18 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + /** * Local end-to-end audit smoke for metadata correction. * * This script exercises the real metadataCorrectionService handler against: * - the local mock CMR server * - local Redis keyword caches - * - local RDF4J audit persistence + * - local MongoDB-compatible DocumentDB audit persistence * - * It verifies that a successful correction run writes both audit lifecycle states: + * It verifies that a successful correction run records all audit lifecycle states: + * - `checked` after validation and resolution * - `pending` before writeback * - `applied` after writeback succeeds * @@ -147,40 +150,20 @@ const seedKeywordCaches = async () => { return redisClient } +/** + * Removes prior audit documents for the smoke collection so assertions start from a clean state. + * + * @returns {Promise} Resolves after matching local audit documents are deleted. + */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` - - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() + + await auditCollection.deleteMany({ collectionConceptId }) } let mockServerProcess @@ -206,7 +189,7 @@ try { process.env.CMR_BASE_URL = baseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() await clearAuditRowsForCollection() @@ -214,7 +197,7 @@ try { const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') - const beforeRows = await getMetadataCorrectionAuditLog({ + const { items: beforeRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) @@ -226,20 +209,23 @@ try { body: JSON.stringify({ source: 'local-smoke', collectionConceptId, + publishedVersionName: 'local-published', keywordEvent: normalizeKeywordEvent(rawKeywordEvent) }) } ] }) - const afterRows = await getMetadataCorrectionAuditLog({ + const { items: afterRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.map((row) => row.status))] + const statuses = [...new Set(afterRows.flatMap((row) => ( + row.statusHistory?.map(({ status }) => status) || [row.status] + )))] if (beforeRows.length !== 0) { - throw new Error(`Expected no starting audit rows for ${collectionConceptId}, found ${beforeRows.length}`) + throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) } if (!statuses.includes('pending')) { @@ -250,6 +236,20 @@ try { throw new Error(`Missing applied audit status for ${collectionConceptId}`) } + const appliedRow = afterRows.find(({ status }) => status === 'applied') + if (appliedRow?.publishedVersionName !== 'local-published') { + throw new Error(`Missing published KMS version for ${collectionConceptId}`) + } + + if (appliedRow.priorRevisionId !== 1 || appliedRow.resultingRevisionId !== 2) { + throw new Error(`Missing the expected CMR revision IDs for ${collectionConceptId}`) + } + + const expectedCollectionUri = `${baseUrl}/search/concepts/${encodeURIComponent(collectionConceptId)}` + if (appliedRow.collectionUri !== expectedCollectionUri) { + throw new Error(`Missing the expected CMR collection URI for ${collectionConceptId}`) + } + await fs.mkdir(outputDir, { recursive: true }) await fs.writeFile(outputPath, JSON.stringify({ collectionConceptId, @@ -272,6 +272,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs b/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs index f96e8448..d61cd7c1 100644 --- a/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs +++ b/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs @@ -4,6 +4,7 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' import { CONSUMER_METRIC_NAMES, CONSUMER_METRIC_NAMESPACE @@ -20,7 +21,7 @@ import { * Prerequisites: * - LocalStack is running on `http://127.0.0.1:4566` * - local Redis is running - * - local RDF4J is running + * - local MongoDB-compatible audit storage is running * * Run with: * npx vite-node --config vite.config.js scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs @@ -200,44 +201,19 @@ const seedKeywordCaches = async () => { } /** - * Removes any existing audit rows for the smoke collection. + * Removes any existing audit documents for the smoke collection. * - * @returns {Promise} Resolves once prior audit rows are deleted. + * @returns {Promise} Resolves once prior audit documents are deleted. */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + await auditCollection.deleteMany({ collectionConceptId }) } /** @@ -551,7 +527,7 @@ try { process.env.CMR_BASE_URL = cmrBaseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' process.env.AWS_ENDPOINT_URL = cloudWatchEndpoint redisClient = await seedKeywordCaches() @@ -667,6 +643,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs b/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs index d1f94487..9efa28b5 100644 --- a/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs +++ b/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs @@ -4,6 +4,8 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + import { CONSUMER_METRIC_NAMES, CONSUMER_METRIC_NAMESPACE @@ -21,7 +23,8 @@ import { * Prerequisites: * - LocalStack is running on `http://127.0.0.1:4566` * - local Redis is running - * - local RDF4J is running + * - local RDF4J is running for the manual request's published-version lookup + * - local MongoDB-compatible audit storage is running * * Run with: * npx vite-node --config vite.config.js scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs @@ -186,44 +189,19 @@ const seedKeywordCaches = async () => { } /** - * Removes any existing audit rows for the smoke collection. + * Removes any existing audit documents for the smoke collection. * - * @returns {Promise} Resolves once prior audit rows are deleted. + * @returns {Promise} Resolves once prior audit documents are deleted. */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + await auditCollection.deleteMany({ collectionConceptId }) } /** @@ -537,7 +515,7 @@ try { process.env.CMR_BASE_URL = cmrBaseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' process.env.AWS_ENDPOINT_URL = cloudWatchEndpoint redisClient = await seedKeywordCaches() @@ -678,6 +656,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs index aaf75bab..84a58f2f 100644 --- a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs @@ -4,18 +4,20 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + /** * Local end-to-end audit smoke for failed metadata-correction writeback. * * This script exercises the real metadataCorrectionService handler against: * - the local mock CMR server, configured to fail ingest/writeback * - local Redis keyword caches - * - local RDF4J audit persistence + * - local MongoDB-compatible DocumentDB audit persistence * * It verifies that a failed correction run writes: * - `pending` before writeback * - `failed` after CMR ingest rejects the corrected metadata - * - `writebackErrorMessage` on the failed audit row + * - structured error details on the failed audit document * * Run with: * npx vite-node --config vite.config.js scripts/local/run_metadata_correction_failed_audit_smoke.mjs @@ -154,40 +156,20 @@ const seedKeywordCaches = async () => { return redisClient } +/** + * Removes prior audit documents for the smoke collection so assertions start from a clean state. + * + * @returns {Promise} Resolves after matching local audit documents are deleted. + */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` - - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() + + await auditCollection.deleteMany({ collectionConceptId }) } let mockServerProcess @@ -215,7 +197,7 @@ try { process.env.CMR_BASE_URL = baseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() await clearAuditRowsForCollection() @@ -223,7 +205,7 @@ try { const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') - const beforeRows = await getMetadataCorrectionAuditLog({ + const { items: beforeRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) @@ -241,15 +223,17 @@ try { ] }) - const afterRows = await getMetadataCorrectionAuditLog({ + const { items: afterRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.map((row) => row.status))] + const statuses = [...new Set(afterRows.flatMap((row) => ( + row.statusHistory?.map(({ status }) => status) || [row.status] + )))] const failedRow = afterRows.find((row) => row.status === 'failed') if (beforeRows.length !== 0) { - throw new Error(`Expected no starting audit rows for ${collectionConceptId}, found ${beforeRows.length}`) + throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) } if (JSON.stringify(response?.batchItemFailures) !== JSON.stringify([ @@ -273,14 +257,14 @@ try { throw new Error(`Did not expect applied audit status for failed writeback on ${collectionConceptId}`) } - if (!failedRow?.writebackErrorMessage) { - throw new Error(`Missing writebackErrorMessage on failed audit row for ${collectionConceptId}`) + if (!failedRow?.error?.message) { + throw new Error(`Missing error details on failed audit document for ${collectionConceptId}`) } - if (failedRow.writebackErrorMessage !== mockIngestErrorBody) { + if (JSON.stringify(failedRow.error.cmrResponseBody) !== mockIngestErrorBody) { throw new Error( - 'Expected failed audit writebackErrorMessage to match the mock ingest response body. ' - + `Received ${failedRow.writebackErrorMessage}` + 'Expected failed audit response details to match the mock ingest response body. ' + + `Received ${JSON.stringify(failedRow.error.cmrResponseBody)}` ) } @@ -304,10 +288,12 @@ try { baseUrl, afterCount: afterRows.length, statuses, - failedWritebackErrorMessage: failedRow.writebackErrorMessage, + failedWritebackErrorMessage: failedRow.error.message, outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_partial_batch_failure_smoke.mjs b/scripts/local/run_metadata_correction_partial_batch_failure_smoke.mjs index f9bfb0db..d5fbc182 100644 --- a/scripts/local/run_metadata_correction_partial_batch_failure_smoke.mjs +++ b/scripts/local/run_metadata_correction_partial_batch_failure_smoke.mjs @@ -4,6 +4,8 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + /** * Local end-to-end smoke for metadata-correction consumer partial batch failure handling. * @@ -19,6 +21,7 @@ import path from 'node:path' * * Prerequisites: * - local Redis is running + * - local MongoDB-compatible audit storage is running * - LocalStack is optional; if present, set AWS_ENDPOINT_URL to avoid metric emission errors * * Run with: @@ -192,11 +195,10 @@ try { process.env.CMR_BASE_URL = cmrBaseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' process.env.AWS_ENDPOINT_URL = process.env.AWS_ENDPOINT_URL || 'http://127.0.0.1:4566' - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` redisClient = await seedKeywordCaches() @@ -264,6 +266,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_request_delay_smoke.mjs b/scripts/local/run_metadata_correction_request_delay_smoke.mjs index e9178449..c5700933 100644 --- a/scripts/local/run_metadata_correction_request_delay_smoke.mjs +++ b/scripts/local/run_metadata_correction_request_delay_smoke.mjs @@ -4,6 +4,8 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + /** * Local end-to-end smoke for the queued manual-request delay path. * @@ -19,7 +21,7 @@ import path from 'node:path' * * Prerequisites: * - local Redis is running - * - local RDF4J is running + * - local MongoDB-compatible audit storage is running * - LocalStack is optional; if present, set AWS_ENDPOINT_URL to avoid metric * emission errors in logs * @@ -182,44 +184,19 @@ const seedKeywordCaches = async () => { } /** - * Removes any existing audit rows for the smoke collection so assertions start clean. + * Removes any existing audit documents for the smoke collection so assertions start clean. * - * @returns {Promise} Resolves once prior audit rows have been deleted. + * @returns {Promise} Resolves once prior audit documents have been deleted. */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` - - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() + + await auditCollection.deleteMany({ collectionConceptId }) } let mockServerProcess @@ -245,7 +222,7 @@ try { process.env.CMR_BASE_URL = cmrBaseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' process.env.METADATA_CORRECTION_REQUEST_DELAY_MS = String(configuredDelayMs) process.env.AWS_ENDPOINT_URL = process.env.AWS_ENDPOINT_URL || 'http://127.0.0.1:4566' @@ -313,11 +290,13 @@ try { ) } - const auditRows = await getMetadataCorrectionAuditLog({ + const { items: auditRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) - const statuses = [...new Set(auditRows.map((row) => row.status))] + const statuses = [...new Set(auditRows.flatMap((row) => ( + row.statusHistory?.map(({ status }) => status) || [row.status] + )))] if (!statuses.includes('pending')) { throw new Error(`Missing pending audit status for ${collectionConceptId}`) @@ -355,6 +334,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_metadata_correction_sync_smoke.mjs b/scripts/local/run_metadata_correction_sync_smoke.mjs index f9a0f946..6d61a5e9 100644 --- a/scripts/local/run_metadata_correction_sync_smoke.mjs +++ b/scripts/local/run_metadata_correction_sync_smoke.mjs @@ -4,17 +4,19 @@ import { spawn } from 'node:child_process' import fs from 'node:fs/promises' import path from 'node:path' +import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' + /** * Local end-to-end smoke for the synchronous metadata-correction endpoint. * * This smoke test drives the new `runMetadataCorrection` API handler directly with an - * API-Gateway-like event. It uses the checked-in mock CMR fixture plus local Redis/RDF4J so - * the full correction path runs end to end: + * API-Gateway-like event. It uses the checked-in mock CMR fixture plus local Redis, RDF4J, and + * MongoDB so the full correction path runs end to end: * - fetch collection UMM/native metadata from the mock CMR server * - validate keyword problems against the seeded Redis caches * - resolve corrections * - apply the UMM delegate - * - persist audit rows + * - persist one audit document with lifecycle history * - write the corrected metadata back to the mock CMR ingest route * * The checked-in fixture currently resolves a `platforms` correction from @@ -178,44 +180,19 @@ const seedKeywordCaches = async () => { } /** - * Removes any existing audit rows for the smoke collection so assertions start clean. + * Removes any existing audit documents for the smoke collection so assertions start clean. * - * @returns {Promise} Resolves once prior audit rows have been deleted. + * @returns {Promise} Resolves once prior audit documents have been deleted. */ const clearAuditRowsForCollection = async () => { - process.env.RDF4J_SERVICE_URL = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' - process.env.RDF4J_USER_NAME = process.env.RDF4J_USER_NAME || 'rdf4j' - process.env.RDF4J_PASSWORD = process.env.RDF4J_PASSWORD || 'rdf4j' - - const { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH - } = await import('../../serverless/src/shared/metadataCorrectionAudit') - const { sparqlRequest } = await import('../../serverless/src/shared/sparqlRequest') - - const query = ` - PREFIX gcmd: - - DELETE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record ?predicate ?object . - } - } - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" ; - ?predicate ?object . - } - } - ` - - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const { getMetadataCorrectionAuditCollection } = await import( + '../../serverless/src/shared/documentDbClient' + ) + const auditCollection = await getMetadataCorrectionAuditCollection() + + await auditCollection.deleteMany({ collectionConceptId }) } let mockServerProcess @@ -241,7 +218,7 @@ try { process.env.CMR_BASE_URL = baseUrl process.env.CMR_WRITEBACK_PROVIDERS = process.env.CMR_WRITEBACK_PROVIDERS || providerId - process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'local-writer-token' + process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() await clearAuditRowsForCollection() @@ -250,7 +227,7 @@ try { const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') const { getCmrCollectionNativeMetadata } = await import('../../serverless/src/shared/getCmrCollectionNativeMetadata') - const beforeRows = await getMetadataCorrectionAuditLog({ + const { items: beforeRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) @@ -295,12 +272,22 @@ try { throw new Error(`Expected successful mock CMR writeback for ${collectionConceptId}`) } - if ((responseBody.auditResults?.pending?.insertedCount || 0) < 1) { - throw new Error(`Expected pending audit rows for ${collectionConceptId}`) + if (responseBody.auditResults?.pending?.status !== 'pending') { + throw new Error(`Expected pending audit status for ${collectionConceptId}`) } - if ((responseBody.auditResults?.applied?.insertedCount || 0) < 1) { - throw new Error(`Expected applied audit rows for ${collectionConceptId}`) + if (responseBody.auditResults?.applied?.status !== 'applied') { + throw new Error(`Expected applied audit status for ${collectionConceptId}`) + } + + const auditRunIds = [ + responseBody.auditResults?.checked?.runId, + responseBody.auditResults?.pending?.runId, + responseBody.auditResults?.applied?.runId + ] + + if (new Set(auditRunIds).size !== 1 || !auditRunIds[0]) { + throw new Error(`Expected one audit run across all lifecycle states for ${collectionConceptId}`) } const resolvedCorrection = responseBody.resolvedCorrections[0] @@ -335,14 +322,16 @@ try { throw new Error('Expected corrected UMM Platforms[0].Instruments[0].ShortName to remain Legacy MODIS') } - const afterRows = await getMetadataCorrectionAuditLog({ + const { items: afterRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.map((row) => row.status))] + const statuses = [...new Set(afterRows.flatMap((row) => ( + row.statusHistory?.map(({ status }) => status) || [row.status] + )))] if (beforeRows.length !== 0) { - throw new Error(`Expected no starting audit rows for ${collectionConceptId}, found ${beforeRows.length}`) + throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) } if (!statuses.includes('pending')) { @@ -381,6 +370,8 @@ try { outputPath }, null, 2)) } finally { + await closeDocumentDbClient() + if (redisClient) { await redisClient.quit() } diff --git a/scripts/local/run_platform_keyword_event_mapping_smoke.mjs b/scripts/local/run_platform_keyword_event_mapping_smoke.mjs index eb864896..a1aa9b84 100644 --- a/scripts/local/run_platform_keyword_event_mapping_smoke.mjs +++ b/scripts/local/run_platform_keyword_event_mapping_smoke.mjs @@ -20,14 +20,14 @@ const defaultLongName = 'Greenhouse Gases Observing Satellite' const createPlatformsCsv = (shortName, longName = defaultLongName) => [ '"Keyword Version: smoke"', - '"Category","Class","Type","Short_Name","Long_Name","UUID"', - `"Platforms","Space-based Platforms","Earth Observation Satellites","${shortName}","${longName}","${keywordUuid}"` + '"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"', + `"Space-based Platforms","Earth Observation Satellites",,"${shortName}","${longName}","${keywordUuid}"` ].join('\n') const buildExpectedKeywordObject = (shortName, longName = defaultLongName) => ({ - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', + SubCategory: '', ShortName: shortName, LongName: longName }) diff --git a/scripts/local/run_rdf_export_smoke.sh b/scripts/local/run_rdf_export_smoke.sh index 10f82d88..2ffc7169 100755 --- a/scripts/local/run_rdf_export_smoke.sh +++ b/scripts/local/run_rdf_export_smoke.sh @@ -70,11 +70,19 @@ download_export() { local download_url download_url="$(jq --exit-status --raw-output '.downloadUrl' "$response_file")" + local download_curl_args=() + if [[ "$BASE_URL" =~ ^https?://(127\.0\.0\.1|localhost)(:|/) ]] \ + && [[ "$download_url" == http://localstack:4566/* ]]; then + # Preserve the signed Host header while resolving Docker's LocalStack hostname from the host. + download_curl_args+=(--resolve 'localstack:4566:127.0.0.1') + fi + curl \ --silent \ --show-error \ --fail \ --location \ + "${download_curl_args[@]}" \ "$download_url" \ --output "$gzip_file" diff --git a/scripts/local/show_metadata_correction_audit_log.mjs b/scripts/local/show_metadata_correction_audit_log.mjs index c93ab08f..958f2561 100644 --- a/scripts/local/show_metadata_correction_audit_log.mjs +++ b/scripts/local/show_metadata_correction_audit_log.mjs @@ -1,124 +1,48 @@ #!/usr/bin/env node -/** - * Local audit-log inspector for the metadata-correction smoke flow. - * - * This script queries the RDF4J audit graph used by the metadata-correction service and prints - * a small summary table for one collection concept id. It is mainly a convenience tool for the - * local smoke test so we can quickly confirm which corrections were written, in what order, and - * with what final status. - */ -const collectionConceptId = process.env.COLLECTION_CONCEPT_ID || 'C1234567890-LOCAL' -const rdf4jUserName = process.env.RDF4J_USER_NAME || 'rdf4j' -const rdf4jPassword = process.env.RDF4J_PASSWORD || 'rdf4j' -const rdf4jServiceUrl = process.env.RDF4J_SERVICE_URL || 'http://localhost:8081' -const rdf4jRepository = process.env.RDF4J_REPOSITORY || 'kms' -const rdf4jRepositoryUrl = `${rdf4jServiceUrl.replace(/\/$/, '')}/rdf4j-server/repositories/${rdf4jRepository}` - -// Build the basic-auth header expected by the local RDF4J container. -const createAuthHeader = () => ( - `Basic ${Buffer.from(`${rdf4jUserName}:${rdf4jPassword}`).toString('base64')}` -) - -// Execute a SPARQL query against the configured RDF4J repository and return JSON bindings. -const executeSparqlQuery = async (query) => { - const response = await fetch(rdf4jRepositoryUrl, { - method: 'POST', - headers: { - Authorization: createAuthHeader(), - 'Content-Type': 'application/sparql-query', - Accept: 'application/sparql-results+json' - }, - body: query - }) - - if (!response.ok) { - const responseText = await response.text() - - throw new Error(`RDF4J query failed: ${response.status} ${responseText}`) - } - - return response.json() -} - -// Flatten SPARQL JSON bindings into plain row objects for easier post-processing. -const parseBindings = (results = []) => results.map((binding) => Object.fromEntries( - Object.entries(binding).map(([key, value]) => [key, value?.value || '']) -)) - -// Count how many audit records exist for the requested collection concept id. -const getAuditRowCount = async () => { - const responseBody = await executeSparqlQuery(` - PREFIX gcmd: - - SELECT (COUNT(?record) AS ?count) - WHERE { - GRAPH { - ?record a gcmd:MetadataCorrectionAuditRecord ; - gcmd:collectionConceptId "${collectionConceptId}" . - } - } - `) - - const parsedRows = parseBindings(responseBody?.results?.bindings || []) - - return Number(parsedRows[0]?.count || 0) -} - -// Fetch the detailed audit rows we want to display in the local smoke summary table. -const getAuditRows = async () => { - const responseBody = await executeSparqlQuery(` - PREFIX gcmd: - PREFIX dcterms: - - SELECT ?timestamp ?publishedVersionName ?collectionConceptId ?scheme ?action ?oldKeywordPath ?newKeywordPath ?status - WHERE { - GRAPH { - ?record a gcmd:MetadataCorrectionAuditRecord ; - dcterms:created ?timestamp ; - gcmd:publishedVersionName ?publishedVersionName ; - gcmd:collectionConceptId ?collectionConceptId ; - gcmd:scheme ?scheme ; - gcmd:action ?action ; - gcmd:status ?status . - OPTIONAL { ?record gcmd:oldKeywordPath ?oldKeywordPath } - OPTIONAL { ?record gcmd:newKeywordPath ?newKeywordPath } - FILTER(?collectionConceptId = "${collectionConceptId}") - } - } - ORDER BY DESC(?timestamp) - `) - - return parseBindings(responseBody?.results?.bindings || []).map((row) => ({ - timestamp: row.timestamp, - version: row.publishedVersionName, - conceptId: row.collectionConceptId, - scheme: row.scheme, - action: row.action, - status: row.status, - oldKeywordPath: row.oldKeywordPath, - newKeywordPath: row.newKeywordPath - })) -} +import { MongoClient } from 'mongodb' /** - * Queries RDF4J for audit rows and prints a simple summary table for local smoke verification. + * Prints the local DocumentDB-compatible audit documents for one collection. + * + * Input comes from `COLLECTION_CONCEPT_ID` and the `DOCUMENTDB_*` environment variables; output is + * a console table with one row per matching audit run. * - * @returns {Promise} + * @returns {Promise} Resolves after the query results have been printed and the client closes. */ const main = async () => { - const count = await getAuditRowCount() - const rows = await getAuditRows() - - console.log(`Metadata correction audit rows for ${collectionConceptId}: ${count}`) - - if (rows.length === 0) { - console.log('No audit rows found.') - - return + const collectionConceptId = process.env.COLLECTION_CONCEPT_ID || 'C1234567890-LOCAL' + const uri = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + const databaseName = process.env.DOCUMENTDB_DATABASE_NAME || 'kms' + const collectionName = process.env.DOCUMENTDB_AUDIT_COLLECTION_NAME + || 'metadataCorrectionAudits' + const client = new MongoClient(uri) + + try { + await client.connect() + const documents = await client.db(databaseName) + .collection(collectionName) + .find({ collectionConceptId }) + .sort({ createdAt: -1, _id: -1 }) + .toArray() + + console.log(`Metadata correction audit runs for ${collectionConceptId}: ${documents.length}`) + console.table(documents.map((document) => ({ + runId: document.runId, + collectionUri: document.collectionUri, + createdAt: document.createdAt, + version: document.publishedVersionName, + status: document.status, + nativeFormat: document.nativeFormat, + corrections: document.corrections?.length || 0, + priorRevisionId: document.priorRevisionId, + resultingRevisionId: document.resultingRevisionId, + error: document.error?.message + }))) + } finally { + await client.close() } - - console.table(rows) } main().catch((error) => { diff --git a/serverless/certs/us-east-1-bundle.pem b/serverless/certs/us-east-1-bundle.pem new file mode 100644 index 00000000..f8fb3755 --- /dev/null +++ b/serverless/certs/us-east-1-bundle.pem @@ -0,0 +1,76 @@ +-----BEGIN CERTIFICATE----- +MIID/zCCAuegAwIBAgIRAPVSMfFitmM5PhmbaOFoGfUwDQYJKoZIhvcNAQELBQAw +gZcxCzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJ +bmMuMRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEwMC4GA1UEAwwn +QW1hem9uIFJEUyB1cy1lYXN0LTEgUm9vdCBDQSBSU0EyMDQ4IEcxMRAwDgYDVQQH +DAdTZWF0dGxlMCAXDTIxMDUyNTIyMzQ1N1oYDzIwNjEwNTI1MjMzNDU3WjCBlzEL +MAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIEluYy4x +EzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdBbWF6 +b24gUkRTIHVzLWVhc3QtMSBSb290IENBIFJTQTIwNDggRzExEDAOBgNVBAcMB1Nl +YXR0bGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDu9H7TBeGoDzMr +dxN6H8COntJX4IR6dbyhnj5qMD4xl/IWvp50lt0VpmMd+z2PNZzx8RazeGC5IniV +5nrLg0AKWRQ2A/lGGXbUrGXCSe09brMQCxWBSIYe1WZZ1iU1IJ/6Bp4D2YEHpXrW +bPkOq5x3YPcsoitgm1Xh8ygz6vb7PsvJvPbvRMnkDg5IqEThapPjmKb8ZJWyEFEE +QRrkCIRueB1EqQtJw0fvP4PKDlCJAKBEs/y049FoOqYpT3pRy0WKqPhWve+hScMd +6obq8kxTFy1IHACjHc51nrGII5Bt76/MpTWhnJIJrCnq1/Uc3Qs8IVeb+sLaFC8K +DI69Sw6bAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFE7PCopt +lyOgtXX0Y1lObBUxuKaCMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOC +AQEAFj+bX8gLmMNefr5jRJfHjrL3iuZCjf7YEZgn89pS4z8408mjj9z6Q5D1H7yS +jNETVV8QaJip1qyhh5gRzRaArgGAYvi2/r0zPsy+Tgf7v1KGL5Lh8NT8iCEGGXwF +g3Ir+Nl3e+9XUp0eyyzBIjHtjLBm6yy8rGk9p6OtFDQnKF5OxwbAgip42CD75r/q +p421maEDDvvRFR4D+99JZxgAYDBGqRRceUoe16qDzbMvlz0A9paCZFclxeftAxv6 +QlR5rItMz/XdzpBJUpYhdzM0gCzAzdQuVO5tjJxmXhkSMcDP+8Q+Uv6FA9k2VpUV +E/O5jgpqUJJ2Hc/5rs9VkAPXeA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF/jCCA+agAwIBAgIQaRHaEqqacXN20e8zZJtmDDANBgkqhkiG9w0BAQwFADCB +lzELMAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIElu +Yy4xEzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdB +bWF6b24gUkRTIHVzLWVhc3QtMSBSb290IENBIFJTQTQwOTYgRzExEDAOBgNVBAcM +B1NlYXR0bGUwIBcNMjEwNTI1MjIzODM1WhgPMjEyMTA1MjUyMzM4MzVaMIGXMQsw +CQYDVQQGEwJVUzEiMCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjET +MBEGA1UECwwKQW1hem9uIFJEUzELMAkGA1UECAwCV0ExMDAuBgNVBAMMJ0FtYXpv +biBSRFMgdXMtZWFzdC0xIFJvb3QgQ0EgUlNBNDA5NiBHMTEQMA4GA1UEBwwHU2Vh +dHRsZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAInfBCaHuvj6Rb5c +L5Wmn1jv2PHtEGMHm+7Z8dYosdwouG8VG2A+BCYCZfij9lIGszrTXkY4O7vnXgru +JUNdxh0Q3M83p4X+bg+gODUs3jf+Z3Oeq7nTOk/2UYvQLcxP4FEXILxDInbQFcIx +yen1ESHggGrjEodgn6nbKQNRfIhjhW+TKYaewfsVWH7EF2pfj+cjbJ6njjgZ0/M9 +VZifJFBgat6XUTOf3jwHwkCBh7T6rDpgy19A61laImJCQhdTnHKvzTpxcxiLRh69 +ZObypR7W04OAUmFS88V7IotlPmCL8xf7kwxG+gQfvx31+A9IDMsiTqJ1Cc4fYEKg +bL+Vo+2Ii4W2esCTGVYmHm73drznfeKwL+kmIC/Bq+DrZ+veTqKFYwSkpHRyJCEe +U4Zym6POqQ/4LBSKwDUhWLJIlq99bjKX+hNTJykB+Lbcx0ScOP4IAZQoxmDxGWxN +S+lQj+Cx2pwU3S/7+OxlRndZAX/FKgk7xSMkg88HykUZaZ/ozIiqJqSnGpgXCtED +oQ4OJw5ozAr+/wudOawaMwUWQl5asD8fuy/hl5S1nv9XxIc842QJOtJFxhyeMIXt +LVECVw/dPekhMjS3Zo3wwRgYbnKG7YXXT5WMxJEnHu8+cYpMiRClzq2BEP6/MtI2 +AZQQUFu2yFjRGL2OZA6IYjxnXYiRAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8w +HQYDVR0OBBYEFADCcQCPX2HmkqQcmuHfiQ2jjqnrMA4GA1UdDwEB/wQEAwIBhjAN +BgkqhkiG9w0BAQwFAAOCAgEASXkGQ2eUmudIKPeOIF7RBryCoPmMOsqP0+1qxF8l +pGkwmrgNDGpmd9s0ArfIVBTc1jmpgB3oiRW9c6n2OmwBKL4UPuQ8O3KwSP0iD2sZ +KMXoMEyphCEzW1I2GRvYDugL3Z9MWrnHkoaoH2l8YyTYvszTvdgxBPpM2x4pSkp+ +76d4/eRpJ5mVuQ93nC+YG0wXCxSq63hX4kyZgPxgCdAA+qgFfKIGyNqUIqWgeyTP +n5OgKaboYk2141Rf2hGMD3/hsGm0rrJh7g3C0ZirPws3eeJfulvAOIy2IZzqHUSY +jkFzraz6LEH3IlArT3jUPvWKqvh2lJWnnp56aqxBR7qHH5voD49UpJWY1K0BjGnS +OHcurpp0Yt/BIs4VZeWdCZwI7JaSeDcPMaMDBvND3Ia5Fga0thgYQTG6dE+N5fgF +z+hRaujXO2nb0LmddVyvE8prYlWRMuYFv+Co8hcMdJ0lEZlfVNu0jbm9/GmwAZ+l +9umeYO9yz/uC7edC8XJBglMAKUmVK9wNtOckUWAcCfnPWYLbYa/PqtXBYcxrso5j +iaS/A7iEW51uteHBGrViCy1afGG+hiUWwFlesli+Rq4dNstX3h6h2baWABaAxEVJ +y1RnTQSz6mROT1VmZSgSVO37rgIyY0Hf0872ogcTS+FfvXgBxCxsNWEbiQ/XXva4 +0Ws= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICrjCCAjSgAwIBAgIRAPAlEk8VJPmEzVRRaWvTh2AwCgYIKoZIzj0EAwMwgZYx +CzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJbmMu +MRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEvMC0GA1UEAwwmQW1h +em9uIFJEUyB1cy1lYXN0LTEgUm9vdCBDQSBFQ0MzODQgRzExEDAOBgNVBAcMB1Nl +YXR0bGUwIBcNMjEwNTI1MjI0MTU1WhgPMjEyMTA1MjUyMzQxNTVaMIGWMQswCQYD +VQQGEwJVUzEiMCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEG +A1UECwwKQW1hem9uIFJEUzELMAkGA1UECAwCV0ExLzAtBgNVBAMMJkFtYXpvbiBS +RFMgdXMtZWFzdC0xIFJvb3QgQ0EgRUNDMzg0IEcxMRAwDgYDVQQHDAdTZWF0dGxl +MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEx5xjrup8II4HOJw15NTnS3H5yMrQGlbj +EDA5MMGnE9DmHp5dACIxmPXPMe/99nO7wNdl7G71OYPCgEvWm0FhdvVUeTb3LVnV +BnaXt32Ek7/oxGk1T+Df03C+W0vmuJ+wo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBTGXmqBWN/1tkSea4pNw0oHrjk2UDAOBgNVHQ8BAf8EBAMCAYYwCgYI +KoZIzj0EAwMDaAAwZQIxAIqqZWCSrIkZ7zsv/FygtAusW6yvlL935YAWYPVXU30m +jkMFLM+/RJ9GMvnO8jHfCgIwB+whlkcItzE9CRQ6CsMo/d5cEHDUu/QW6jSIh9BR +OGh9pTYPVkUbBiKPA7lVVhre +-----END CERTIFICATE----- diff --git a/serverless/src/cmrKeywordEventsListener/__tests__/handler.test.js b/serverless/src/cmrKeywordEventsListener/__tests__/handler.test.js index 79f23bd5..e0b6c981 100644 --- a/serverless/src/cmrKeywordEventsListener/__tests__/handler.test.js +++ b/serverless/src/cmrKeywordEventsListener/__tests__/handler.test.js @@ -86,6 +86,7 @@ describe('when the CMR keyword events processor is invoked', () => { EventType: 'UPDATED', Scheme: 'sciencekeywords', UUID: '1234', + VersionName: '20.1', OldKeywordObject: OLD_SCIENCE_KEYWORD_OBJECT, NewKeywordObject: NEW_SCIENCE_KEYWORD_OBJECT, Timestamp: '2026-04-21T00:00:00.000Z' @@ -138,6 +139,7 @@ describe('when the CMR keyword events processor is invoked', () => { expect(publishMetadataCorrectionRequest).toHaveBeenNthCalledWith(1, { source: 'cmrKeywordEventsListener', collectionConceptId: 'C1000000000-PROV', + publishedVersionName: '20.1', keywordEvent: { eventType: 'UPDATED', scheme: 'sciencekeywords', @@ -151,6 +153,7 @@ describe('when the CMR keyword events processor is invoked', () => { expect(publishMetadataCorrectionRequest).toHaveBeenNthCalledWith(2, { source: 'cmrKeywordEventsListener', collectionConceptId: 'C2000000000-PROV', + publishedVersionName: '20.1', keywordEvent: { eventType: 'UPDATED', scheme: 'sciencekeywords', diff --git a/serverless/src/cmrKeywordEventsListener/handler.js b/serverless/src/cmrKeywordEventsListener/handler.js index 4772479d..1bfdb832 100644 --- a/serverless/src/cmrKeywordEventsListener/handler.js +++ b/serverless/src/cmrKeywordEventsListener/handler.js @@ -36,12 +36,14 @@ const buildMetadataCorrectionRequest = (collectionConceptId, keywordEvent) => { UUID: uuid, OldKeywordObject: oldKeywordObject, NewKeywordObject: newKeywordObject, - Timestamp: timestamp + Timestamp: timestamp, + VersionName: versionName } = keywordEvent return { source: 'cmrKeywordEventsListener', collectionConceptId, + publishedVersionName: versionName, keywordEvent: { eventType, scheme, diff --git a/serverless/src/getCapabilities/handler.js b/serverless/src/getCapabilities/handler.js index 3ca18c0e..329874dd 100644 --- a/serverless/src/getCapabilities/handler.js +++ b/serverless/src/getCapabilities/handler.js @@ -141,7 +141,7 @@ export const getCapabilities = async () => { ':@': { name: 'get_metadata_correction_audit', href: '/metadata_correction_audit', - params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&limit=', + params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&limit=', action: 'GET' } }, diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index 5ee35c2a..38c348aa 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -35,19 +35,23 @@ describe('getMetadataCorrectionAudit', () => { vi.clearAllMocks() }) - test('returns audit rows as json', async () => { - vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue([ - { - recordUri: 'https://example.org/audit/1', + test('returns audit documents as json', async () => { + vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({ + items: [{ + runId: 'run-1', collectionConceptId: 'C1234567890-LOCAL', - action: 'UPDATED' - } - ]) + collectionUri: 'https://cmr.example.com/search/concepts/C1234567890-LOCAL', + status: 'applied', + trigger: { + eventType: 'UPDATED' + } + }], + nextPaginationToken: null + }) const result = await getMetadataCorrectionAudit({ queryStringParameters: { collectionConceptId: 'C1234567890-LOCAL', - latestOnly: 'true', limit: '10' } }) @@ -56,9 +60,14 @@ describe('getMetadataCorrectionAudit', () => { collectionConceptId: 'C1234567890-LOCAL', keywordConceptUuid: undefined, action: undefined, + paginationToken: undefined, + endDate: undefined, scheme: undefined, status: undefined, - latestOnly: 'true', + nativeFormat: undefined, + publishedVersionName: undefined, + source: undefined, + startDate: undefined, limit: '10' }) @@ -68,11 +77,16 @@ describe('getMetadataCorrectionAudit', () => { expect(JSON.parse(result.body)).toEqual({ items: [ { - recordUri: 'https://example.org/audit/1', + runId: 'run-1', collectionConceptId: 'C1234567890-LOCAL', - action: 'UPDATED' + collectionUri: 'https://cmr.example.com/search/concepts/C1234567890-LOCAL', + status: 'applied', + trigger: { + eventType: 'UPDATED' + } } - ] + ], + nextPaginationToken: null }) }) @@ -87,4 +101,14 @@ describe('getMetadataCorrectionAudit', () => { error: 'Error: Audit query failed' }) }) + + test('returns 400 for invalid filters', async () => { + vi.mocked(getMetadataCorrectionAuditLog).mockRejectedValue( + new Error('Invalid metadata correction audit paginationToken') + ) + + const result = await getMetadataCorrectionAudit({}) + + expect(result.statusCode).toBe(400) + }) }) diff --git a/serverless/src/getMetadataCorrectionAudit/handler.js b/serverless/src/getMetadataCorrectionAudit/handler.js index c01a71e6..e2b6e3f9 100644 --- a/serverless/src/getMetadataCorrectionAudit/handler.js +++ b/serverless/src/getMetadataCorrectionAudit/handler.js @@ -6,10 +6,8 @@ import { logger } from '@/shared/logger' /** * Read-side audit endpoint for metadata-correction activity. * - * The metadata-correction service writes one audit record per resolved correction into RDF4J. - * This handler exposes those records through an API so we can inspect what corrections were - * attempted, which collection they applied to, what keyword uuid/path was involved, and whether - * the result is still pending or has been applied. + * The metadata-correction service stores one DocumentDB document per collection-correction run. + * This handler exposes those runs for MMT audit search, reporting, and troubleshooting. * * In practice this is useful for: * - local smoke-test verification @@ -18,7 +16,7 @@ import { logger } from '@/shared/logger' */ /** - * Retrieves metadata-correction audit rows from RDF4J. + * Retrieves metadata-correction audit runs from DocumentDB. * * Supported query parameters: * - collectionConceptId @@ -26,12 +24,22 @@ import { logger } from '@/shared/logger' * - action * - scheme * - status - * - latestOnly + * - nativeFormat + * - publishedVersionName + * - source + * - startDate / endDate + * - paginationToken * - limit * * @param {object} event - API Gateway event. * @param {object} context - Lambda context. * @returns {Promise} API Gateway response object. + * + * @example + * await getMetadataCorrectionAudit({ + * queryStringParameters: { status: 'applied', limit: '25' } + * }, context) + * // { statusCode: 200, body: '{"items":[...],"nextPaginationToken":null}' } */ export const getMetadataCorrectionAudit = async (event, context) => { const { defaultResponseHeaders } = getApplicationConfig() @@ -47,18 +55,28 @@ export const getMetadataCorrectionAudit = async (event, context) => { action, scheme, status, - latestOnly, + nativeFormat, + publishedVersionName, + source, + startDate, + endDate, + paginationToken, limit } = event?.queryStringParameters || {} try { - const items = await getMetadataCorrectionAuditLog({ + const auditPage = await getMetadataCorrectionAuditLog({ collectionConceptId, keywordConceptUuid, action, scheme, status, - latestOnly, + nativeFormat, + publishedVersionName, + source, + startDate, + endDate, + paginationToken, limit }) @@ -68,19 +86,20 @@ export const getMetadataCorrectionAudit = async (event, context) => { ...defaultResponseHeaders, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - items - }, null, 2) + body: JSON.stringify(auditPage, null, 2) } } catch (error) { logger.error(`Error retrieving metadata correction audit log, error=${error.toString()}`) + const isClientError = String(error?.message || '') + .startsWith('Invalid metadata correction audit') + return { headers: { ...defaultResponseHeaders, 'Content-Type': 'application/json' }, - statusCode: 500, + statusCode: isClientError ? 400 : 500, body: JSON.stringify({ error: error.toString() }) diff --git a/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js new file mode 100644 index 00000000..cda81208 --- /dev/null +++ b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js @@ -0,0 +1,72 @@ +import { + beforeEach, + describe, + expect, + test, + vi +} from 'vitest' + +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' + +import { initializeMetadataCorrectionAudit } from '../handler' + +vi.mock('@/shared/documentDbClient', () => ({ + getMetadataCorrectionAuditCollection: vi.fn() +})) + +describe('initializeMetadataCorrectionAudit', () => { + const createIndexes = vi.fn() + const consoleLog = vi.spyOn(console, 'log').mockImplementation(() => {}) + const indexDefinitions = [ + { + key: { + createdAt: -1, + _id: -1 + }, + name: 'createdAt_desc' + } + ] + + beforeEach(() => { + vi.clearAllMocks() + createIndexes.mockResolvedValue(['createdAt_desc']) + vi.mocked(getMetadataCorrectionAuditCollection).mockResolvedValue({ createIndexes }) + }) + + test.each(['Create', 'Update'])('creates indexes for a %s deployment event', async (requestType) => { + const result = await initializeMetadataCorrectionAudit({ + RequestType: requestType, + ResourceProperties: { IndexDefinitions: indexDefinitions } + }) + + expect(createIndexes).toHaveBeenCalledWith(indexDefinitions) + expect(consoleLog).toHaveBeenCalledWith( + 'Metadata correction audit indexes are ready', + { indexNames: ['createdAt_desc'] } + ) + + expect(result).toEqual({ + PhysicalResourceId: 'metadata-correction-audit-indexes', + Data: { IndexCount: 1 } + }) + }) + + test('does not change the retained database during stack deletion', async () => { + const result = await initializeMetadataCorrectionAudit({ + RequestType: 'Delete', + PhysicalResourceId: 'existing-audit-indexes' + }) + + expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() + expect(result).toEqual({ PhysicalResourceId: 'existing-audit-indexes' }) + }) + + test('rejects a deployment without index definitions', async () => { + await expect(initializeMetadataCorrectionAudit({ + RequestType: 'Create', + ResourceProperties: {} + })).rejects.toThrow('Metadata correction audit index definitions are required') + + expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() + }) +}) diff --git a/serverless/src/initializeMetadataCorrectionAudit/handler.js b/serverless/src/initializeMetadataCorrectionAudit/handler.js new file mode 100644 index 00000000..e4f26dd8 --- /dev/null +++ b/serverless/src/initializeMetadataCorrectionAudit/handler.js @@ -0,0 +1,48 @@ +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' + +const PHYSICAL_RESOURCE_ID = 'metadata-correction-audit-indexes' + +/** + * Creates the metadata-correction audit indexes during CloudFormation deployment. + * Delete events intentionally leave indexes in place because the audit database is retained. + * + * @example + * await initializeMetadataCorrectionAudit({ + * RequestType: 'Create', + * ResourceProperties: { + * IndexDefinitions: [{ key: { createdAt: -1 }, name: 'createdAt_desc' }] + * } + * }) + * // { PhysicalResourceId: 'metadata-correction-audit-indexes', Data: { IndexCount: 1 } } + * + * @param {Object} event CloudFormation custom-resource event. + * @returns {Promise} Stable resource identity and deployment details. + */ +export const initializeMetadataCorrectionAudit = async (event) => { + // The custom-resource provider forwards CloudFormation Create, Update, and Delete events here. + const physicalResourceId = event.PhysicalResourceId || PHYSICAL_RESOURCE_ID + + // The DocumentDB cluster is retained, so stack deletion must not remove its data or indexes. + if (event.RequestType === 'Delete') { + return { PhysicalResourceId: physicalResourceId } + } + + const indexDefinitions = event.ResourceProperties?.IndexDefinitions + + if (!Array.isArray(indexDefinitions) || indexDefinitions.length === 0) { + throw new Error('Metadata correction audit index definitions are required') + } + + const collection = await getMetadataCorrectionAuditCollection() + const indexNames = await collection.createIndexes(indexDefinitions) + console.log('Metadata correction audit indexes are ready', { indexNames }) + + return { + PhysicalResourceId: physicalResourceId, + Data: { + IndexCount: indexDefinitions.length + } + } +} + +export default initializeMetadataCorrectionAudit diff --git a/serverless/src/metadataCorrectionService/__tests__/handler.test.js b/serverless/src/metadataCorrectionService/__tests__/handler.test.js index 6b6f359c..3fb31850 100644 --- a/serverless/src/metadataCorrectionService/__tests__/handler.test.js +++ b/serverless/src/metadataCorrectionService/__tests__/handler.test.js @@ -146,11 +146,14 @@ describe('when the metadata correction service is invoked', () => { correctedMetadataBytes: 21 }) - vi.mocked(persistMetadataCorrectionAuditLog).mockResolvedValue({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) + vi.mocked(persistMetadataCorrectionAuditLog).mockImplementation(async ({ + runId = 'audit-run-1', + status + }) => ({ + runId, + status, + created: status === 'checked' + })) }) describe('when the invocation is successful', () => { @@ -240,6 +243,7 @@ describe('when the metadata correction service is invoked', () => { body: JSON.stringify({ source: 'cmrKeywordEventsListener', collectionConceptId: 'C123-PROV', + publishedVersionName: '20.1', keywordEvent: { eventType: 'UPDATED', scheme: 'sciencekeywords', @@ -304,7 +308,7 @@ describe('when the metadata correction service is invoked', () => { ] }) - expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledWith({ + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(2, expect.objectContaining({ collectionConceptId: 'C123-PROV', keywordEvent: { eventType: 'UPDATED', @@ -315,6 +319,7 @@ describe('when the metadata correction service is invoked', () => { }, nativeFormat: 'DIF10', delegateName: 'dif10', + publishedVersionName: '20.1', corrections: [ { scheme: 'sciencekeywords', @@ -325,8 +330,9 @@ describe('when the metadata correction service is invoked', () => { ummPath: ['ScienceKeywords', 0] } ], + runId: 'message-collection-1', status: 'pending' - }) + })) expect(writeCorrectedMetadataToCmr).toHaveBeenCalledWith({ collectionConceptId: 'C123-PROV', @@ -350,7 +356,7 @@ describe('when the metadata correction service is invoked', () => { }) }) - test('should append an applied audit record after a successful writeback update', async () => { + test('should update the audit document to applied after a successful writeback', async () => { vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ collectionConceptId: 'C123-PROV', providerId: 'PROV', @@ -457,14 +463,14 @@ describe('when the metadata correction service is invoked', () => { ] }) - expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(1, expect.objectContaining({ + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(2, expect.objectContaining({ collectionConceptId: 'C123-PROV', nativeFormat: 'DIF10', delegateName: 'dif10', status: 'pending' })) - expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(2, expect.objectContaining({ + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(3, expect.objectContaining({ collectionConceptId: 'C123-PROV', nativeFormat: 'DIF10', delegateName: 'dif10', @@ -472,7 +478,7 @@ describe('when the metadata correction service is invoked', () => { })) }) - test('should fall back to the normalized native format when applied audit delegateName is absent', async () => { + test('should use the normalized native format as delegateName when updating the audit document to applied', async () => { vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ collectionConceptId: 'C123-PROV', providerId: 'PROV', @@ -578,7 +584,7 @@ describe('when the metadata correction service is invoked', () => { ] }) - expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(2, expect.objectContaining({ + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith(3, expect.objectContaining({ collectionConceptId: 'C123-PROV', nativeFormat: 'DIF10', delegateName: 'dif10', @@ -893,7 +899,14 @@ describe('when the metadata correction service is invoked', () => { }) expect(invokeMetadataCorrectionDelegate).not.toHaveBeenCalled() - expect(persistMetadataCorrectionAuditLog).not.toHaveBeenCalled() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledOnce() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledWith( + expect.objectContaining({ + outcome: 'no-resolved-corrections', + status: 'checked' + }) + ) + expect(writeCorrectedMetadataToCmr).not.toHaveBeenCalled() expect(logger.info).toHaveBeenCalledWith( @@ -952,7 +965,14 @@ describe('when the metadata correction service is invoked', () => { expect(resolveOldKeywordConceptUuid).not.toHaveBeenCalled() expect(invokeMetadataCorrectionDelegate).not.toHaveBeenCalled() - expect(persistMetadataCorrectionAuditLog).not.toHaveBeenCalled() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledOnce() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledWith( + expect.objectContaining({ + outcome: 'no-keyword-issues', + status: 'checked' + }) + ) + expect(writeCorrectedMetadataToCmr).not.toHaveBeenCalled() expect(logger.info).toHaveBeenCalledWith( diff --git a/serverless/src/metadataCorrectionService/handler.js b/serverless/src/metadataCorrectionService/handler.js index 81d63439..89198096 100644 --- a/serverless/src/metadataCorrectionService/handler.js +++ b/serverless/src/metadataCorrectionService/handler.js @@ -140,6 +140,7 @@ export const metadataCorrectionService = async (event) => { collectionConceptId: metadataCorrectionRequest.collectionConceptId, keywordEvent: metadataCorrectionRequest.keywordEvent, messageId: record.messageId, + publishedVersionName: metadataCorrectionRequest.publishedVersionName, source: metadataCorrectionRequest.source }) } catch (error) { diff --git a/serverless/src/publisher/__tests__/handler.test.js b/serverless/src/publisher/__tests__/handler.test.js index 6dda78d0..a32b2fe2 100644 --- a/serverless/src/publisher/__tests__/handler.test.js +++ b/serverless/src/publisher/__tests__/handler.test.js @@ -214,6 +214,7 @@ describe('publisher handler', () => { EventType: 'INSERTED', Scheme: 'sciencekeywords', UUID: 'uuid1', + VersionName: 'v1.0.0', NewKeywordObject: SCIENCE_PATH_KEYWORD })) diff --git a/serverless/src/publisher/handler.js b/serverless/src/publisher/handler.js index efc785dc..cec030cf 100644 --- a/serverless/src/publisher/handler.js +++ b/serverless/src/publisher/handler.js @@ -146,7 +146,10 @@ const publishKeywordEvents = async (keywordEvents, versionName) => keywordEvents } // eslint-disable-next-line no-await-in-loop - const publishResult = await publishKeywordEvent(keywordEvent) + const publishResult = await publishKeywordEvent({ + ...keywordEvent, + VersionName: versionName + }) logger.info('[publisher] Published keyword event', { versionName, diff --git a/serverless/src/requestMetadataCorrection/__tests__/handler.test.js b/serverless/src/requestMetadataCorrection/__tests__/handler.test.js index 7bbccf77..1d0b8b5e 100644 --- a/serverless/src/requestMetadataCorrection/__tests__/handler.test.js +++ b/serverless/src/requestMetadataCorrection/__tests__/handler.test.js @@ -6,6 +6,7 @@ import { vi } from 'vitest' +import { getVersionMetadata } from '@/shared/getVersionMetadata' import { logger } from '@/shared/logger' import { publishMetadataCorrectionRequest } from '@/shared/publishMetadataCorrectionRequest' @@ -21,6 +22,10 @@ vi.mock('@/shared/logAnalyticsData', () => ({ logAnalyticsData: vi.fn() })) +vi.mock('@/shared/getVersionMetadata', () => ({ + getVersionMetadata: vi.fn() +})) + vi.mock('@/shared/logger', () => ({ logger: { info: vi.fn(), @@ -35,6 +40,7 @@ vi.mock('@/shared/publishMetadataCorrectionRequest', () => ({ describe('requestMetadataCorrection', () => { beforeEach(() => { vi.clearAllMocks() + vi.mocked(getVersionMetadata).mockResolvedValue({ versionName: '20.1' }) }) test('returns 202 and publishes one deduplicated message per collection concept id', async () => { @@ -62,12 +68,14 @@ describe('requestMetadataCorrection', () => { expect(publishMetadataCorrectionRequest).toHaveBeenNthCalledWith(1, { source: 'metadataCorrectionApi', collectionConceptId: 'C123-PROV', + publishedVersionName: '20.1', requestedAt: expect.any(String) }) expect(publishMetadataCorrectionRequest).toHaveBeenNthCalledWith(2, { source: 'metadataCorrectionApi', collectionConceptId: 'C456-PROV', + publishedVersionName: '20.1', requestedAt: expect.any(String) }) diff --git a/serverless/src/requestMetadataCorrection/handler.js b/serverless/src/requestMetadataCorrection/handler.js index 5fb615ee..cc22fa7a 100644 --- a/serverless/src/requestMetadataCorrection/handler.js +++ b/serverless/src/requestMetadataCorrection/handler.js @@ -1,4 +1,5 @@ import { getApplicationConfig } from '@/shared/getConfig' +import { getVersionMetadata } from '@/shared/getVersionMetadata' import { logAnalyticsData } from '@/shared/logAnalyticsData' import { logger } from '@/shared/logger' import { publishMetadataCorrectionRequest } from '@/shared/publishMetadataCorrectionRequest' @@ -113,6 +114,7 @@ export const requestMetadataCorrection = async (event, context) => { acceptedCollectionConceptIds } = normalizeCollectionConceptIds(requestBody.collectionConceptIds) const requestedAt = new Date().toISOString() + const { versionName: publishedVersionName } = await getVersionMetadata('published') logger.info('[metadata-correction] Received asynchronous metadata correction request', { requestedCount, @@ -125,6 +127,7 @@ export const requestMetadataCorrection = async (event, context) => { const publishResult = await publishMetadataCorrectionRequest({ source: 'metadataCorrectionApi', collectionConceptId, + publishedVersionName, requestedAt }) diff --git a/serverless/src/runMetadataCorrection/__tests__/handler.test.js b/serverless/src/runMetadataCorrection/__tests__/handler.test.js index 4f728403..fff255fe 100644 --- a/serverless/src/runMetadataCorrection/__tests__/handler.test.js +++ b/serverless/src/runMetadataCorrection/__tests__/handler.test.js @@ -6,6 +6,7 @@ import { vi } from 'vitest' +import { getVersionMetadata } from '@/shared/getVersionMetadata' import { logger } from '@/shared/logger' import { runCollectionMetadataCorrection } from '@/shared/runCollectionMetadataCorrection' @@ -21,6 +22,10 @@ vi.mock('@/shared/logAnalyticsData', () => ({ logAnalyticsData: vi.fn() })) +vi.mock('@/shared/getVersionMetadata', () => ({ + getVersionMetadata: vi.fn() +})) + vi.mock('@/shared/logger', () => ({ logger: { info: vi.fn(), @@ -35,6 +40,7 @@ vi.mock('@/shared/runCollectionMetadataCorrection', () => ({ describe('runMetadataCorrection', () => { beforeEach(() => { vi.clearAllMocks() + vi.mocked(getVersionMetadata).mockResolvedValue({ versionName: '20.1' }) }) test('returns the synchronous correction summary as json', async () => { @@ -83,6 +89,7 @@ describe('runMetadataCorrection', () => { expect(runCollectionMetadataCorrection).toHaveBeenCalledWith({ collectionConceptId: 'C1234567890-PROV', + publishedVersionName: '20.1', source: 'metadataCorrectionApi' }) @@ -140,6 +147,7 @@ describe('runMetadataCorrection', () => { expect(runCollectionMetadataCorrection).toHaveBeenCalledWith({ collectionConceptId: 'C1234567890+PROV', + publishedVersionName: '20.1', source: 'metadataCorrectionApi' }) }) diff --git a/serverless/src/runMetadataCorrection/handler.js b/serverless/src/runMetadataCorrection/handler.js index 525b0e48..530db80f 100644 --- a/serverless/src/runMetadataCorrection/handler.js +++ b/serverless/src/runMetadataCorrection/handler.js @@ -1,4 +1,5 @@ import { getApplicationConfig } from '@/shared/getConfig' +import { getVersionMetadata } from '@/shared/getVersionMetadata' import { logAnalyticsData } from '@/shared/logAnalyticsData' import { logger } from '@/shared/logger' import { runCollectionMetadataCorrection } from '@/shared/runCollectionMetadataCorrection' @@ -63,8 +64,11 @@ export const runMetadataCorrection = async (event, context) => { collectionConceptId }) + const { versionName: publishedVersionName } = await getVersionMetadata('published') + const result = await runCollectionMetadataCorrection({ collectionConceptId, + publishedVersionName, source: 'metadataCorrectionApi' }) diff --git a/serverless/src/shared/__tests__/awsClients.test.js b/serverless/src/shared/__tests__/awsClients.test.js index 9766f022..acb3b3c8 100644 --- a/serverless/src/shared/__tests__/awsClients.test.js +++ b/serverless/src/shared/__tests__/awsClients.test.js @@ -8,14 +8,24 @@ import { } from 'vitest' // Mock the AWS SDK clients before any imports -const { s3ClientMock, eventBridgeClientMock, snsClientMock } = vi.hoisted(() => ({ +const { + s3ClientMock, + eventBridgeClientMock, + secretsManagerClientMock, + snsClientMock +} = vi.hoisted(() => ({ s3ClientMock: vi.fn(), eventBridgeClientMock: vi.fn(), + secretsManagerClientMock: vi.fn(), snsClientMock: vi.fn() })) vi.mock('@aws-sdk/client-s3', () => ({ S3Client: s3ClientMock })) vi.mock('@aws-sdk/client-eventbridge', () => ({ EventBridgeClient: eventBridgeClientMock })) +vi.mock('@aws-sdk/client-secrets-manager', () => ({ + SecretsManagerClient: secretsManagerClientMock +})) + vi.mock('@aws-sdk/client-sns', () => ({ SNSClient: snsClientMock })) describe('awsClients', () => { @@ -55,6 +65,14 @@ describe('awsClients', () => { expect(snsClientMock).toHaveBeenCalledWith({}) }) + + test('getSecretsManagerClient should create a client with default config', async () => { + const { getSecretsManagerClient } = await import('../awsClients') + + getSecretsManagerClient() + + expect(secretsManagerClientMock).toHaveBeenCalledWith({}) + }) }) describe('when AWS_ENDPOINT_URL is set (LocalStack)', () => { @@ -96,6 +114,14 @@ describe('awsClients', () => { expect(snsClientMock).toHaveBeenCalledWith(expectedConfig) }) + + test('getSecretsManagerClient should create a client with LocalStack config', async () => { + const { getSecretsManagerClient } = await import('../awsClients') + + getSecretsManagerClient() + + expect(secretsManagerClientMock).toHaveBeenCalledWith(expectedConfig) + }) }) describe('singleton behavior', () => { @@ -128,5 +154,15 @@ describe('awsClients', () => { expect(snsClientMock).toHaveBeenCalledTimes(1) expect(client1).toBe(client2) }) + + test('getSecretsManagerClient should only create one instance', async () => { + const { getSecretsManagerClient } = await import('../awsClients') + + const client1 = getSecretsManagerClient() + const client2 = getSecretsManagerClient() + + expect(secretsManagerClientMock).toHaveBeenCalledTimes(1) + expect(client1).toBe(client2) + }) }) }) diff --git a/serverless/src/shared/__tests__/documentDbClient.test.js b/serverless/src/shared/__tests__/documentDbClient.test.js new file mode 100644 index 00000000..5b5595bd --- /dev/null +++ b/serverless/src/shared/__tests__/documentDbClient.test.js @@ -0,0 +1,204 @@ +import { + afterEach, + beforeEach, + describe, + expect, + test, + vi +} from 'vitest' + +const { + collection, + close, + connect, + db, + mongoClient, + mongoClientConstructor, + secretsManagerSend +} = vi.hoisted(() => { + const collectionMock = { findOne: vi.fn() } + const dbMock = vi.fn(() => ({ + collection: vi.fn(() => collectionMock) + })) + const closeMock = vi.fn() + const client = { + close: closeMock, + db: dbMock + } + + return { + collection: collectionMock, + close: closeMock, + connect: vi.fn().mockResolvedValue(client), + db: dbMock, + mongoClient: client, + mongoClientConstructor: vi.fn(), + secretsManagerSend: vi.fn() + } +}) + +vi.mock('mongodb', () => ({ + MongoClient: mongoClientConstructor.mockImplementation(function mockMongoClient(uri, options) { + this.uri = uri + this.options = options + this.connect = connect + }) +})) + +vi.mock('@aws-sdk/client-secrets-manager', () => ({ + GetSecretValueCommand: vi.fn(function mockGetSecretValueCommand(input) { + this.input = input + }) +})) + +vi.mock('@/shared/awsClients', () => ({ + getSecretsManagerClient: vi.fn(() => ({ send: secretsManagerSend })) +})) + +const DOCUMENTDB_ENVIRONMENT_VARIABLES = [ + 'DOCUMENTDB_AUDIT_COLLECTION_NAME', + 'DOCUMENTDB_DATABASE_NAME', + 'DOCUMENTDB_HOST', + 'DOCUMENTDB_MAX_POOL_SIZE', + 'DOCUMENTDB_PORT', + 'DOCUMENTDB_SECRET_ARN', + 'DOCUMENTDB_TLS_CA_FILE', + 'DOCUMENTDB_URI' +] + +describe('documentDbClient', () => { + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + connect.mockResolvedValue(mongoClient) + DOCUMENTDB_ENVIRONMENT_VARIABLES.forEach((name) => delete process.env[name]) + }) + + afterEach(() => { + DOCUMENTDB_ENVIRONMENT_VARIABLES.forEach((name) => delete process.env[name]) + }) + + test('reuses a local MongoDB connection and returns the configured collection', async () => { + process.env.DOCUMENTDB_URI = 'mongodb://localhost:27018/?directConnection=true' + process.env.DOCUMENTDB_DATABASE_NAME = 'test-kms' + process.env.DOCUMENTDB_AUDIT_COLLECTION_NAME = 'audits' + process.env.DOCUMENTDB_MAX_POOL_SIZE = '9' + const { + getDocumentDbClient, + getMetadataCorrectionAuditCollection + } = await import('../documentDbClient') + + await expect(getDocumentDbClient()).resolves.toBe(mongoClient) + await expect(getDocumentDbClient()).resolves.toBe(mongoClient) + await expect(getMetadataCorrectionAuditCollection()).resolves.toBe(collection) + + expect(mongoClientConstructor).toHaveBeenCalledOnce() + expect(mongoClientConstructor).toHaveBeenCalledWith( + 'mongodb://localhost:27018/?directConnection=true', + expect.objectContaining({ + maxPoolSize: 9, + minPoolSize: 0 + }) + ) + + expect(connect).toHaveBeenCalledOnce() + expect(db).toHaveBeenCalledWith('test-kms') + }) + + test('closes and resets the shared connection', async () => { + process.env.DOCUMENTDB_URI = 'mongodb://localhost:27018' + const { + closeDocumentDbClient, + getDocumentDbClient + } = await import('../documentDbClient') + + await closeDocumentDbClient() + await getDocumentDbClient() + await closeDocumentDbClient() + await getDocumentDbClient() + + expect(close).toHaveBeenCalledOnce() + expect(mongoClientConstructor).toHaveBeenCalledTimes(2) + }) + + test('builds the deployed TLS connection from Secrets Manager credentials', async () => { + process.env.DOCUMENTDB_HOST = 'audit.cluster.docdb.amazonaws.com' + process.env.DOCUMENTDB_PORT = '27017' + process.env.DOCUMENTDB_SECRET_ARN = 'arn:aws:secretsmanager:secret:audit' + process.env.DOCUMENTDB_TLS_CA_FILE = '/var/task/us-east-1-bundle.pem' + secretsManagerSend.mockResolvedValue({ + SecretString: JSON.stringify({ + username: 'user@example.com', + password: 'password/with spaces' + }) + }) + + const { getDocumentDbClient } = await import('../documentDbClient') + + await getDocumentDbClient() + + expect(secretsManagerSend).toHaveBeenCalledWith(expect.objectContaining({ + input: { SecretId: 'arn:aws:secretsmanager:secret:audit' } + })) + + expect(mongoClientConstructor).toHaveBeenCalledWith( + 'mongodb://user%40example.com:password%2Fwith%20spaces@audit.cluster.docdb.amazonaws.com:27017/?tls=true&replicaSet=rs0&readPreference=primary&retryWrites=false&authSource=admin', + expect.objectContaining({ + maxPoolSize: 5, + tlsCAFile: '/var/task/us-east-1-bundle.pem' + }) + ) + }) + + test('validates deployed connection configuration and credentials', async () => { + let documentDbClient = await import('../documentDbClient') + await expect(documentDbClient.getDocumentDbClient()).rejects.toThrow( + 'Missing DOCUMENTDB_HOST' + ) + + vi.resetModules() + process.env.DOCUMENTDB_HOST = 'audit.cluster.docdb.amazonaws.com' + documentDbClient = await import('../documentDbClient') + await expect(documentDbClient.getDocumentDbClient()).rejects.toThrow( + 'Missing DOCUMENTDB_TLS_CA_FILE' + ) + + vi.resetModules() + process.env.DOCUMENTDB_TLS_CA_FILE = '/tmp/ca.pem' + documentDbClient = await import('../documentDbClient') + await expect(documentDbClient.getDocumentDbClient()).rejects.toThrow( + 'Missing DOCUMENTDB_SECRET_ARN' + ) + + vi.resetModules() + process.env.DOCUMENTDB_SECRET_ARN = 'arn:aws:secretsmanager:secret:audit' + secretsManagerSend.mockResolvedValue({}) + documentDbClient = await import('../documentDbClient') + await expect(documentDbClient.getDocumentDbClient()).rejects.toThrow( + 'DocumentDB secret does not contain SecretString credentials' + ) + + vi.resetModules() + secretsManagerSend.mockResolvedValue({ + SecretString: JSON.stringify({ username: 'kms_audit' }) + }) + + documentDbClient = await import('../documentDbClient') + await expect(documentDbClient.getDocumentDbClient()).rejects.toThrow( + 'DocumentDB secret is missing username or password' + ) + }) + + test('retries connection creation after a failed connection', async () => { + process.env.DOCUMENTDB_URI = 'mongodb://localhost:27018' + connect + .mockRejectedValueOnce(new Error('connection failed')) + .mockResolvedValueOnce(mongoClient) + + const { getDocumentDbClient } = await import('../documentDbClient') + + await expect(getDocumentDbClient()).rejects.toThrow('connection failed') + await expect(getDocumentDbClient()).resolves.toBe(mongoClient) + expect(mongoClientConstructor).toHaveBeenCalledTimes(2) + }) +}) diff --git a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js index bdf2a45f..9ed768cf 100644 --- a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js @@ -6,279 +6,163 @@ import { vi } from 'vitest' -import { sparqlRequest } from '@/shared/sparqlRequest' +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' import { getMetadataCorrectionAuditLog } from '../getMetadataCorrectionAuditLog' -vi.mock('@/shared/sparqlRequest', () => ({ - sparqlRequest: vi.fn() +vi.mock('@/shared/documentDbClient', () => ({ + getMetadataCorrectionAuditCollection: vi.fn() })) -describe('getMetadataCorrectionAuditLog', () => { +describe('metadata correction audit queries', () => { + let collection + let mongoCursor + beforeEach(() => { vi.clearAllMocks() + mongoCursor = { + sort: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + toArray: vi.fn().mockResolvedValue([]) + } + + collection = { + find: vi.fn().mockReturnValue(mongoCursor), + findOne: vi.fn().mockResolvedValue(null) + } + + vi.mocked(getMetadataCorrectionAuditCollection).mockResolvedValue(collection) }) - test('queries the audit graph and maps bindings into audit rows', async () => { - vi.mocked(sparqlRequest).mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - results: { - bindings: [ - { - record: { value: 'https://gcmd.earthdata.nasa.gov/kms/metadata-correction-audit/audit-1' }, - timestamp: { value: '2026-05-06T18:00:00.000Z' }, - publishedVersionName: { value: '9.1.5' }, - collectionConceptId: { value: 'C1234567890-LOCAL' }, - keywordConceptUuid: { value: 'uuid-1' }, - scheme: { value: 'sciencekeywords' }, - action: { value: 'UPDATED' }, - oldKeywordPath: { value: 'EARTH SCIENCE > ATMOSPHERE' }, - newKeywordPath: { value: 'EARTH SCIENCE > OCEANS' }, - nativeFormat: { value: 'UMM' }, - delegateName: { value: 'umm' }, - status: { value: 'pending' }, - writebackErrorMessage: { value: 'CMR writeback failed with status 400: {"errors":["boom"]}' }, - triggerScheme: { value: 'sciencekeywords' }, - triggerKeywordUuid: { value: 'uuid-trigger' } - } - ] - } - }) - }) + test('filters and returns newest-first audit documents', async () => { + const createdAt = new Date('2026-09-02T12:00:00.000Z') + mongoCursor.toArray.mockResolvedValue([{ + _id: 'run-1', + runId: 'run-1', + collectionConceptId: 'C123-PROV', + createdAt, + status: 'applied' + }]) const result = await getMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', action: 'UPDATED', - status: 'pending', - limit: '25' + collectionConceptId: 'C123-PROV', + endDate: '2026-09-03', + keywordConceptUuid: 'keyword-1', + limit: '25', + nativeFormat: 'UMM', + publishedVersionName: '20.1', + scheme: 'platforms', + source: 'cmrKeywordEventsListener', + startDate: '2026-09-01', + status: 'applied' }) - expect(result).toEqual([ - { - recordUri: 'https://gcmd.earthdata.nasa.gov/kms/metadata-correction-audit/audit-1', - timestamp: '2026-05-06T18:00:00.000Z', - publishedVersionName: '9.1.5', - collectionConceptId: 'C1234567890-LOCAL', - keywordConceptUuid: 'uuid-1', - scheme: 'sciencekeywords', - action: 'UPDATED', - oldKeywordPath: 'EARTH SCIENCE > ATMOSPHERE', - newKeywordPath: 'EARTH SCIENCE > OCEANS', - nativeFormat: 'UMM', - delegateName: 'umm', - status: 'pending', - writebackErrorMessage: 'CMR writeback failed with status 400: {"errors":["boom"]}', - triggerScheme: 'sciencekeywords', - triggerKeywordUuid: 'uuid-trigger' + expect(collection.find).toHaveBeenCalledWith({ + collectionConceptId: 'C123-PROV', + 'trigger.eventType': 'UPDATED', + 'corrections.keywordConceptUuid': 'keyword-1', + nativeFormat: 'UMM', + publishedVersionName: '20.1', + $or: [ + { 'corrections.scheme': 'platforms' }, + { 'trigger.scheme': 'platforms' } + ], + source: 'cmrKeywordEventsListener', + status: 'applied', + createdAt: { + $gte: new Date('2026-09-01'), + $lte: new Date('2026-09-03') } - ]) - - expect(sparqlRequest).toHaveBeenCalledWith({ - method: 'POST', - body: expect.stringContaining('GRAPH '), - contentType: 'application/sparql-query', - accept: 'application/sparql-results+json' }) - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('FILTER(?collectionConceptId = "C1234567890-LOCAL")') - expect(sparqlCall.body).toContain('FILTER(?action = "UPDATED")') - expect(sparqlCall.body).toContain('FILTER(?status = "pending")') - expect(sparqlCall.body).toContain('LIMIT 25') - }) - - test('normalizes invalid limits, applies keyword and scheme filters, and leaves optional fields undefined when absent', async () => { - vi.mocked(sparqlRequest).mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - results: { - bindings: [ - { - record: { value: 'https://gcmd.earthdata.nasa.gov/kms/metadata-correction-audit/audit-2' }, - timestamp: { value: '2026-05-07T18:00:00.000Z' }, - publishedVersionName: { value: '9.1.6' }, - collectionConceptId: { value: 'C0000000002-LOCAL' }, - keywordConceptUuid: { value: 'uuid-2' }, - scheme: { value: 'platforms' }, - action: { value: 'UPDATED' }, - oldKeywordPath: { value: 'OLD PLATFORM' }, - newKeywordPath: { value: 'NEW PLATFORM' }, - nativeFormat: { value: 'DIF10' }, - delegateName: { value: 'dif10' }, - status: { value: 'applied' } - } - ] - } - }) + expect(mongoCursor.sort).toHaveBeenCalledWith({ + createdAt: -1, + _id: -1 }) - const result = await getMetadataCorrectionAuditLog({ - keywordConceptUuid: 'uuid-2', - scheme: 'platforms', - limit: 'not-a-number' + expect(mongoCursor.limit).toHaveBeenCalledWith(26) + expect(result).toEqual({ + items: [{ + runId: 'run-1', + collectionConceptId: 'C123-PROV', + createdAt, + status: 'applied' + }], + nextPaginationToken: null }) + }) - expect(result).toEqual([ + test('returns a pagination token when another page exists and applies it to the next query', async () => { + const documents = [ { - recordUri: 'https://gcmd.earthdata.nasa.gov/kms/metadata-correction-audit/audit-2', - timestamp: '2026-05-07T18:00:00.000Z', - publishedVersionName: '9.1.6', - collectionConceptId: 'C0000000002-LOCAL', - keywordConceptUuid: 'uuid-2', - scheme: 'platforms', - action: 'UPDATED', - oldKeywordPath: 'OLD PLATFORM', - newKeywordPath: 'NEW PLATFORM', - nativeFormat: 'DIF10', - delegateName: 'dif10', - status: 'applied', - writebackErrorMessage: undefined, - triggerScheme: undefined, - triggerKeywordUuid: undefined + _id: 'run-3', + runId: 'run-3', + createdAt: new Date('2026-09-03') + }, + { + _id: 'run-2', + runId: 'run-2', + createdAt: new Date('2026-09-02') + }, + { + _id: 'run-1', + runId: 'run-1', + createdAt: new Date('2026-09-01') } - ]) - - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('FILTER(?keywordConceptUuid = "uuid-2")') - expect(sparqlCall.body).toContain('FILTER(?scheme = "platforms")') - expect(sparqlCall.body).toContain('LIMIT 100') - expect(sparqlCall.body).not.toContain('FILTER(?collectionConceptId =') - expect(sparqlCall.body).not.toContain('FILTER(?action =') - expect(sparqlCall.body).not.toContain('FILTER(?status =') - }) + ] + mongoCursor.toArray.mockResolvedValue(documents) - test('keeps large explicit limits instead of clamping them', async () => { - vi.mocked(sparqlRequest).mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - results: { - bindings: [] - } - }) - }) - - await expect(getMetadataCorrectionAuditLog({ - limit: '5000' - })).resolves.toEqual([]) + const firstPage = await getMetadataCorrectionAuditLog({ limit: '2' }) - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('LIMIT 5000') - }) + expect(firstPage.items).toHaveLength(2) + expect(firstPage.nextPaginationToken).toEqual(expect.any(String)) - test('uses default filters and returns an empty array when the query result has no bindings', async () => { - vi.mocked(sparqlRequest).mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({}) + mongoCursor.toArray.mockResolvedValue([]) + await getMetadataCorrectionAuditLog({ + paginationToken: firstPage.nextPaginationToken, + limit: '2', + status: 'checked' }) - await expect(getMetadataCorrectionAuditLog()).resolves.toEqual([]) - - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('LIMIT 100') - expect(sparqlCall.body).not.toContain('FILTER(?collectionConceptId =') - expect(sparqlCall.body).not.toContain('FILTER(?keywordConceptUuid =') - expect(sparqlCall.body).not.toContain('FILTER(?scheme =') - }) - - test('collapses duplicate pending and applied lifecycle rows when latestOnly is enabled', async () => { - vi.mocked(sparqlRequest).mockResolvedValue({ - ok: true, - json: vi.fn().mockResolvedValue({ - results: { - bindings: [ - { - record: { value: 'https://example.org/audit/applied-1' }, - timestamp: { value: '2026-06-17T12:00:01.000Z' }, - publishedVersionName: { value: '9.1.6' }, - collectionConceptId: { value: 'C1234567890-LOCAL' }, - keywordConceptUuid: { value: 'uuid-1' }, - scheme: { value: 'sciencekeywords' }, - action: { value: 'UPDATED' }, - oldKeywordPath: { value: 'EARTH SCIENCE > ATMOSPHERE' }, - newKeywordPath: { value: 'EARTH SCIENCE > OCEANS' }, - nativeFormat: { value: 'DIF10' }, - delegateName: { value: 'dif10' }, - status: { value: 'applied' }, - writebackErrorMessage: { value: 'CMR writeback failed with status 400: {"errors":["boom"]}' }, - triggerScheme: { value: 'sciencekeywords' }, - triggerKeywordUuid: { value: 'uuid-trigger' } - }, + expect(collection.find).toHaveBeenLastCalledWith({ + $and: [ + { status: 'checked' }, + { + $or: [ + { createdAt: { $lt: new Date('2026-09-02') } }, { - record: { value: 'https://example.org/audit/pending-1' }, - timestamp: { value: '2026-06-17T12:00:00.000Z' }, - publishedVersionName: { value: '9.1.6' }, - collectionConceptId: { value: 'C1234567890-LOCAL' }, - keywordConceptUuid: { value: 'uuid-1' }, - scheme: { value: 'sciencekeywords' }, - action: { value: 'UPDATED' }, - oldKeywordPath: { value: 'EARTH SCIENCE > ATMOSPHERE' }, - newKeywordPath: { value: 'EARTH SCIENCE > OCEANS' }, - nativeFormat: { value: 'DIF10' }, - delegateName: { value: 'dif10' }, - status: { value: 'pending' }, - writebackErrorMessage: undefined, - triggerScheme: { value: 'sciencekeywords' }, - triggerKeywordUuid: { value: 'uuid-trigger' } - }, - { - record: { value: 'https://example.org/audit/pending-2' }, - timestamp: { value: '2026-06-17T11:59:59.000Z' }, - publishedVersionName: { value: '9.1.6' }, - collectionConceptId: { value: 'C9999999999-LOCAL' }, - keywordConceptUuid: { value: 'uuid-2' }, - scheme: { value: 'platforms' }, - action: { value: 'UPDATED' }, - oldKeywordPath: { value: 'OLD PLATFORM' }, - newKeywordPath: { value: 'NEW PLATFORM' }, - nativeFormat: { value: 'UMM' }, - delegateName: { value: 'umm' }, - status: { value: 'pending' } + createdAt: new Date('2026-09-02'), + _id: { $lt: 'run-2' } } ] } - }) + ] }) + }) - const result = await getMetadataCorrectionAuditLog({ - latestOnly: 'true' - }) + test('uses bounded limits and validates filters', async () => { + await getMetadataCorrectionAuditLog({ limit: '5000' }) + expect(mongoCursor.limit).toHaveBeenCalledWith(251) - expect(result).toEqual([ - { - recordUri: 'https://example.org/audit/applied-1', - timestamp: '2026-06-17T12:00:01.000Z', - publishedVersionName: '9.1.6', - collectionConceptId: 'C1234567890-LOCAL', - keywordConceptUuid: 'uuid-1', - scheme: 'sciencekeywords', - action: 'UPDATED', - oldKeywordPath: 'EARTH SCIENCE > ATMOSPHERE', - newKeywordPath: 'EARTH SCIENCE > OCEANS', - nativeFormat: 'DIF10', - delegateName: 'dif10', - status: 'applied', - writebackErrorMessage: 'CMR writeback failed with status 400: {"errors":["boom"]}', - triggerScheme: 'sciencekeywords', - triggerKeywordUuid: 'uuid-trigger' - }, - { - recordUri: 'https://example.org/audit/pending-2', - timestamp: '2026-06-17T11:59:59.000Z', - publishedVersionName: '9.1.6', - collectionConceptId: 'C9999999999-LOCAL', - keywordConceptUuid: 'uuid-2', - scheme: 'platforms', - action: 'UPDATED', - oldKeywordPath: 'OLD PLATFORM', - newKeywordPath: 'NEW PLATFORM', - nativeFormat: 'UMM', - delegateName: 'umm', - status: 'pending', - writebackErrorMessage: undefined, - triggerScheme: undefined, - triggerKeywordUuid: undefined - } - ]) + await expect(getMetadataCorrectionAuditLog({ + status: 'unknown' + })).rejects.toThrow('Invalid metadata correction audit status: unknown') + + await expect(getMetadataCorrectionAuditLog({ + startDate: 'not-a-date' + })).rejects.toThrow('Invalid metadata correction audit startDate') + + await expect(getMetadataCorrectionAuditLog({ + paginationToken: 'not-a-pagination-token' + })).rejects.toThrow('Invalid metadata correction audit paginationToken') + + const invalidPaginationToken = Buffer.from(JSON.stringify({ + createdAt: '2026-09-02T12:00:00.000Z', + runId: '' + })).toString('base64url') + await expect(getMetadataCorrectionAuditLog({ + paginationToken: invalidPaginationToken + })).rejects.toThrow('Invalid metadata correction audit paginationToken') }) }) diff --git a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js index 530e666a..de42299d 100644 --- a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js @@ -1,4 +1,5 @@ import { + afterEach, beforeEach, describe, expect, @@ -6,242 +7,215 @@ import { vi } from 'vitest' -import { getVersionMetadata } from '@/shared/getVersionMetadata' -import { sparqlRequest } from '@/shared/sparqlRequest' +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' import { persistMetadataCorrectionAuditLog } from '../persistMetadataCorrectionAuditLog' vi.mock('uuid', () => ({ - v4: vi.fn(() => 'audit-record-123') + v4: vi.fn(() => 'generated-run-id') })) -vi.mock('@/shared/getVersionMetadata', () => ({ - getVersionMetadata: vi.fn() +vi.mock('@/shared/documentDbClient', () => ({ + getMetadataCorrectionAuditCollection: vi.fn() })) -vi.mock('@/shared/sparqlRequest', () => ({ - sparqlRequest: vi.fn() -})) +const buildCorrection = () => ({ + scheme: 'platforms', + keywordConceptUuid: 'platform-uuid', + oldKeywordObject: { + Basis: 'Platforms', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', + ShortName: 'GOSAT' + }, + newKeywordObject: { + Basis: 'Platforms', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', + ShortName: 'GOSAT - Test1' + } +}) describe('persistMetadataCorrectionAuditLog', () => { + let collection + beforeEach(() => { vi.clearAllMocks() - vi.mocked(getVersionMetadata).mockResolvedValue({ - version: 'published', - versionName: '9.1.5', - versionType: 'published', - created: '2026-01-01T00:00:00Z', - lastSynced: null - }) + process.env.CMR_BASE_URL = 'https://cmr.example.com/' + collection = { + findOne: vi.fn().mockResolvedValue(null), + updateOne: vi.fn().mockResolvedValue({ acknowledged: true }) + } - vi.mocked(sparqlRequest).mockResolvedValue({ ok: true }) + vi.mocked(getMetadataCorrectionAuditCollection).mockResolvedValue(collection) }) - test('persists one audit row per correction with pending status', async () => { + afterEach(() => { + delete process.env.CMR_BASE_URL + }) + + test('creates one checked audit document with corrections and status history', async () => { const result = await persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', + collectionConceptId: 'C123-PROV', + corrections: [buildCorrection()], keywordEvent: { eventType: 'UPDATED', - scheme: 'sciencekeywords', - uuid: '2e5a401b-1507-4f57-82b8-36557c13b154' + scheme: 'platforms', + uuid: 'platform-uuid' }, nativeFormat: 'UMM', - delegateName: 'umm', - corrections: [ - { - scheme: 'sciencekeywords', - keywordConceptUuid: '2e5a401b-1507-4f57-82b8-36557c13b154', - oldKeywordObject: { - Category: 'EARTH SCIENCE', - Topic: 'ATMOSPHERE', - Term: 'AEROSOLS', - VariableLevel1: 'LEGACY AEROSOLS', - VariableLevel2: '', - VariableLevel3: '', - DetailedVariable: '' - }, - newKeywordObject: { - Category: 'EARTH SCIENCE', - Topic: 'ATMOSPHERE', - Term: 'AEROSOLS', - VariableLevel1: '', - VariableLevel2: '', - VariableLevel3: '', - DetailedVariable: '' - } - } - ], - status: 'pending', - timestamp: '2026-05-06T18:00:00.000Z' + priorRevisionId: 7, + providerId: 'PROV', + publishedVersionName: '20.1', + status: 'checked', + timestamp: '2026-09-02T12:00:00.000Z' }) expect(result).toEqual({ - insertedCount: 1, - publishedVersionName: '9.1.5', - status: 'pending' + runId: 'generated-run-id', + status: 'checked', + created: true }) - expect(getVersionMetadata).toHaveBeenCalledWith('published') - expect(sparqlRequest).toHaveBeenCalledWith(expect.objectContaining({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: expect.stringContaining('GRAPH ') - })) + expect(collection.updateOne).toHaveBeenCalledWith( + { _id: 'generated-run-id' }, + expect.objectContaining({ + $set: expect.objectContaining({ + collectionConceptId: 'C123-PROV', + collectionUri: 'https://cmr.example.com/search/concepts/C123-PROV', + publishedVersionName: '20.1', + priorRevisionId: 7, + status: 'checked', + corrections: [expect.objectContaining({ + keywordConceptUuid: 'platform-uuid', + oldKeywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', + newKeywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT - Test1' + })] + }), + $setOnInsert: { + _id: 'generated-run-id', + runId: 'generated-run-id', + createdAt: new Date('2026-09-02T12:00:00.000Z') + }, + $push: { + statusHistory: { + status: 'checked', + timestamp: new Date('2026-09-02T12:00:00.000Z') + } + }, + $unset: { error: '' } + }), + { upsert: true } + ) + }) + + test('omits the collection URI when the CMR base URL is not configured', async () => { + delete process.env.CMR_BASE_URL - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('gcmd:MetadataCorrectionAuditRecord') - expect(sparqlCall.body).toContain('gcmd:publishedVersionName "9.1.5"') - expect(sparqlCall.body).toContain('gcmd:collectionConceptId "C1234567890-LOCAL"') - expect(sparqlCall.body).toContain('gcmd:action "UPDATED"') - expect(sparqlCall.body).toContain('gcmd:scheme "sciencekeywords"') - expect(sparqlCall.body).toContain('gcmd:status "pending"') - expect(sparqlCall.body).toContain('gcmd:triggerScheme "sciencekeywords"') - expect(sparqlCall.body).toContain('gcmd:triggerKeywordUuid "2e5a401b-1507-4f57-82b8-36557c13b154"') - expect(sparqlCall.body).toContain('gcmd:oldKeywordPath "EARTH SCIENCE > ATMOSPHERE > AEROSOLS > LEGACY AEROSOLS > > > "') - expect(sparqlCall.body).toContain('gcmd:newKeywordPath "EARTH SCIENCE > ATMOSPHERE > AEROSOLS > > > > "') - expect(sparqlCall.body).not.toContain('gcmd:oldKeywordObject') - expect(sparqlCall.body).not.toContain('gcmd:newKeywordObject') - expect(sparqlCall.body).not.toContain('gcmd:writebackErrorMessage') - expect(sparqlCall.body).toContain('metadata-correction-audit/audit-record-123') + await persistMetadataCorrectionAuditLog({ + collectionConceptId: 'C123-PROV' + }) + + expect(collection.updateOne.mock.calls[0][1].$set).not.toHaveProperty('collectionUri') }) - test('persists the writeback error message for failed audit rows', async () => { + test('updates the same run to failed and records structured error details', async () => { + collection.findOne.mockResolvedValue({ status: 'pending' }) + const error = Object.assign(new Error('CMR rejected metadata'), { + status: 400, + cmrResponseBody: { errors: ['invalid'] } + }) + await persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', - keywordEvent: { - eventType: 'UPDATED' - }, - nativeFormat: 'UMM', - delegateName: 'umm', - corrections: [ - { - scheme: 'sciencekeywords', - keywordConceptUuid: 'uuid-failed', - oldKeywordObject: { - Category: 'EARTH SCIENCE', - Topic: 'ATMOSPHERE', - Term: 'AEROSOLS' - }, - newKeywordObject: { - Category: 'EARTH SCIENCE', - Topic: 'ATMOSPHERE', - Term: 'AEROSOLS' - } - } - ], + runId: 'run-1', + collectionConceptId: 'C123-PROV', + error, + outcome: 'writeback-failed', status: 'failed', - writebackErrorMessage: 'CMR writeback failed with status 400: {"errors":["boom"]}' + timestamp: '2026-09-02T12:01:00.000Z' }) - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('gcmd:status "failed"') - expect(sparqlCall.body).toContain( - 'gcmd:writebackErrorMessage "CMR writeback failed with status 400: {\\"errors\\":[\\"boom\\"]}"' - ) - }) + const update = collection.updateOne.mock.calls[0][1] + expect(update.$set).toEqual(expect.objectContaining({ + status: 'failed', + outcome: 'writeback-failed', + error: { + message: 'CMR rejected metadata', + status: 400, + cmrResponseBody: { errors: ['invalid'] } + } + })) - test('returns without writing when there are no corrections', async () => { - const result = await persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', - nativeFormat: 'UMM', - delegateName: 'umm' + expect(update.$push.statusHistory).toEqual({ + status: 'failed', + timestamp: new Date('2026-09-02T12:01:00.000Z'), + outcome: 'writeback-failed', + error: 'CMR rejected metadata' }) - expect(result).toEqual({ - insertedCount: 0, - publishedVersionName: 'published', - status: 'pending' + expect(update.$unset).toBeUndefined() + }) + + test('does not regress pending status when a retried run is checked again', async () => { + collection.findOne.mockResolvedValue({ status: 'pending' }) + + const result = await persistMetadataCorrectionAuditLog({ + runId: 'run-1', + collectionConceptId: 'C123-PROV', + status: 'checked' }) - expect(getVersionMetadata).not.toHaveBeenCalled() - expect(sparqlRequest).not.toHaveBeenCalled() + expect(result.status).toBe('pending') + const update = collection.updateOne.mock.calls[0][1] + expect(update.$set.status).toBe('pending') + expect(update.$set['timestamps.pendingAt']).toBeUndefined() + expect(update.$push).toBeUndefined() }) - test('throws when required audit fields are missing', async () => { - await expect(persistMetadataCorrectionAuditLog({ - nativeFormat: 'UMM', - delegateName: 'umm', - corrections: [{}] - })).rejects.toThrow('Missing collectionConceptId for metadata correction audit persistence') + test('does not update an applied run during a retry', async () => { + collection.findOne.mockResolvedValue({ status: 'applied' }) await expect(persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', - delegateName: 'umm', - corrections: [{}] - })).rejects.toThrow('Missing nativeFormat for metadata correction audit persistence') + runId: 'run-1', + collectionConceptId: 'C123-PROV', + status: 'checked' + })).resolves.toEqual({ + runId: 'run-1', + status: 'applied', + created: false + }) - await expect(persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C1234567890-LOCAL', - nativeFormat: 'UMM', - corrections: [{}] - })).rejects.toThrow('Missing delegateName for metadata correction audit persistence') + expect(collection.updateOne).not.toHaveBeenCalled() }) - test('defaults published version, timestamp, and action while omitting optional trigger triples', async () => { - vi.mocked(getVersionMetadata).mockResolvedValue({ - version: 'published', - versionName: '', - versionType: 'published', - created: '2026-01-01T00:00:00Z', - lastSynced: null - }) + test('allows a failed run to restart and clears its previous error', async () => { + collection.findOne.mockResolvedValue({ status: 'failed' }) - await persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C2222222222-LOCAL', - nativeFormat: 'DIF10', - delegateName: 'dif10', - corrections: [ - { - scheme: 'platforms', - keywordConceptUuid: 'uuid-optional', - oldKeywordObject: { - Basis: 'Platforms', - Category: 'Space-based Platforms', - SubCategory: 'Earth Observation Satellites', - ShortName: 'OLD PLATFORM' - }, - newKeywordObject: { - Basis: 'Platforms', - Category: 'Space-based Platforms', - SubCategory: 'Earth Observation Satellites', - ShortName: 'NEW PLATFORM' - } - } - ] + const result = await persistMetadataCorrectionAuditLog({ + runId: 'run-1', + collectionConceptId: 'C123-PROV', + status: 'checked' }) - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).toContain('gcmd:publishedVersionName "published"') - expect(sparqlCall.body).toContain('gcmd:action "UNKNOWN"') - expect(sparqlCall.body).toContain('gcmd:delegateName "dif10"') - expect(sparqlCall.body).toContain('gcmd:nativeFormat "DIF10"') - expect(sparqlCall.body).toContain('gcmd:oldKeywordPath "Platforms > Space-based Platforms > Earth Observation Satellites > OLD PLATFORM"') - expect(sparqlCall.body).toContain('gcmd:newKeywordPath "Platforms > Space-based Platforms > Earth Observation Satellites > NEW PLATFORM"') - expect(sparqlCall.body).not.toContain('gcmd:oldKeywordObject') - expect(sparqlCall.body).not.toContain('gcmd:newKeywordObject') - expect(sparqlCall.body).toContain('^^xsd:dateTime') - expect(sparqlCall.body).not.toContain('gcmd:triggerScheme') - expect(sparqlCall.body).not.toContain('gcmd:triggerKeywordUuid') + expect(result.status).toBe('checked') + expect(collection.updateOne.mock.calls[0][1]).toEqual(expect.objectContaining({ + $unset: { error: '' }, + $push: expect.objectContaining({ + statusHistory: expect.objectContaining({ status: 'checked' }) + }) + })) }) - test('omits keyword-path triples when the correction objects do not produce meaningful paths', async () => { - await persistMetadataCorrectionAuditLog({ - collectionConceptId: 'C3333333333-LOCAL', - nativeFormat: 'UMM', - delegateName: 'umm', - corrections: [ - { - scheme: 'platforms', - keywordConceptUuid: 'uuid-empty-paths', - oldKeywordObject: {}, - newKeywordObject: {} - } - ] - }) + test('validates required fields and lifecycle status', async () => { + await expect(persistMetadataCorrectionAuditLog({ + status: 'checked' + })).rejects.toThrow('Missing collectionConceptId') + + await expect(persistMetadataCorrectionAuditLog({ + collectionConceptId: 'C123-PROV', + status: 'unknown' + })).rejects.toThrow('Invalid metadata correction audit status: unknown') - const sparqlCall = vi.mocked(sparqlRequest).mock.calls[0][0] - expect(sparqlCall.body).not.toContain('gcmd:oldKeywordPath') - expect(sparqlCall.body).not.toContain('gcmd:newKeywordPath') + expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() }) }) diff --git a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js index 7de4dbdd..6591088f 100644 --- a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js +++ b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js @@ -91,14 +91,114 @@ vi.mock('@/shared/writeCorrectedMetadataToCmr', () => ({ describe('runCollectionMetadataCorrection', () => { beforeEach(() => { vi.clearAllMocks() + vi.mocked(persistMetadataCorrectionAuditLog).mockImplementation(async ({ + runId = 'audit-run-1', + status + }) => ({ + runId, + status, + created: status === 'checked' + })) }) + const arrangeResolvableCorrection = () => { + vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ + collectionConceptId: 'C1234567890-PROV', + providerId: 'PROV', + nativeId: 'native-123', + revisionId: 7, + format: 'application/dif10+xml', + umm: {} + }) + + vi.mocked(validateCmrCollectionUmm).mockResolvedValue({ + status: 200, + errors: ['invalid keyword'], + warnings: [] + }) + + vi.mocked(extractKeywordValidationFailures).mockReturnValue([{ + scheme: 'sciencekeywords', + path: ['ScienceKeywords', 0], + keywordValue: { Category: 'EARTH SCIENCE' } + }]) + + vi.mocked(resolveOldKeywordConceptUuid).mockResolvedValue({ + action: 'replace', + keywordConceptUuid: 'uuid-1', + oldKeywordObject: { Category: 'EARTH SCIENCE' }, + newKeywordObject: { Category: 'EARTH SCIENCE - UPDATED' } + }) + } + test('throws when invoked without arguments', async () => { await expect(runCollectionMetadataCorrection()).rejects.toThrow( 'Incomplete metadata correction request: missing collectionConceptId' ) }) + test('does not repeat CMR writeback when an audit run is already applied', async () => { + arrangeResolvableCorrection() + vi.mocked(persistMetadataCorrectionAuditLog).mockResolvedValueOnce({ + runId: 'message-1', + status: 'applied', + created: false + }) + + await expect(runCollectionMetadataCorrection({ + collectionConceptId: 'C1234567890-PROV', + messageId: 'message-1' + })).resolves.toEqual(expect.objectContaining({ + outcome: 'already-applied', + auditResults: { + checked: { + runId: 'message-1', + status: 'applied', + created: false + }, + pending: null, + applied: { + runId: 'message-1', + status: 'applied', + created: false + } + }, + writeResult: null + })) + + expect(getCmrCollectionNativeMetadata).not.toHaveBeenCalled() + expect(invokeMetadataCorrectionDelegate).not.toHaveBeenCalled() + expect(writeCorrectedMetadataToCmr).not.toHaveBeenCalled() + }) + + test('records a failed run when native metadata cannot be retrieved', async () => { + arrangeResolvableCorrection() + const nativeMetadataError = new Error('CMR native metadata unavailable') + vi.mocked(getCmrCollectionNativeMetadata).mockRejectedValue(nativeMetadataError) + + await expect(runCollectionMetadataCorrection({ + collectionConceptId: 'C1234567890-PROV', + messageId: 'message-1', + publishedVersionName: '20.1' + })).rejects.toBe(nativeMetadataError) + + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + runId: 'message-1', + collectionConceptId: 'C1234567890-PROV', + error: nativeMetadataError, + outcome: 'correction-failed', + priorRevisionId: 7, + publishedVersionName: '20.1', + status: 'failed' + }) + ) + + expect(invokeMetadataCorrectionDelegate).not.toHaveBeenCalled() + expect(writeCorrectedMetadataToCmr).not.toHaveBeenCalled() + }) + test('uses the default source when a collection has no keyword issues', async () => { vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ collectionConceptId: 'C1234567890-PROV', @@ -136,6 +236,11 @@ describe('runCollectionMetadataCorrection', () => { resolvedCorrections: [], correctionResult: null, auditResults: { + checked: { + runId: 'audit-run-1', + status: 'checked', + created: true + }, pending: null, applied: null }, @@ -226,6 +331,11 @@ describe('runCollectionMetadataCorrection', () => { resolvedCorrections: [], correctionResult: null, auditResults: { + checked: { + runId: 'audit-run-1', + status: 'checked', + created: true + }, pending: null, applied: null }, @@ -234,7 +344,7 @@ describe('runCollectionMetadataCorrection', () => { }) }) - test('marks audit actions as MANUAL for the synchronous concept-id correction flow', async () => { + test('records a MANUAL trigger for the synchronous concept-id correction flow', async () => { vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ collectionConceptId: 'C1234567890-PROV', providerId: 'PROV', @@ -323,18 +433,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'applied' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { enabled: true, @@ -379,12 +477,22 @@ describe('runCollectionMetadataCorrection', () => { keywordEvent: { eventType: 'MANUAL' }, - status: 'pending' + status: 'checked' }) ) expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( 2, + expect.objectContaining({ + keywordEvent: { + eventType: 'MANUAL' + }, + status: 'pending' + }) + ) + + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( + 3, expect.objectContaining({ keywordEvent: { eventType: 'MANUAL' @@ -460,15 +568,20 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadataProduced: false }, auditResults: { + checked: { + runId: 'audit-run-1', + status: 'checked', + created: true + }, pending: null, applied: null } })) - expect(persistMetadataCorrectionAuditLog).not.toHaveBeenCalled() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledOnce() }) - test('persists a failed audit row with the writeback error message when CMR writeback fails', async () => { + test('updates the audit document to failed with the writeback error when CMR writeback fails', async () => { const writebackError = new Error('CMR writeback failed with status 400: {"errors":["boom"]}') writebackError.cmrResponseBody = { errors: ['boom'] @@ -562,18 +675,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'failed' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) await expect(runCollectionMetadataCorrection({ @@ -587,18 +688,28 @@ describe('runCollectionMetadataCorrection', () => { keywordEvent: { eventType: 'MANUAL' }, - status: 'pending' + status: 'checked' }) ) expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( 2, + expect.objectContaining({ + keywordEvent: { + eventType: 'MANUAL' + }, + status: 'pending' + }) + ) + + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( + 3, expect.objectContaining({ keywordEvent: { eventType: 'MANUAL' }, status: 'failed', - writebackErrorMessage: '{"errors":["boom"]}' + error: writebackError }) ) @@ -664,18 +775,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'failed' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) await expect(runCollectionMetadataCorrection({ @@ -683,10 +782,10 @@ describe('runCollectionMetadataCorrection', () => { })).rejects.toBe(writebackError) expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( - 2, + 3, expect.objectContaining({ status: 'failed', - writebackErrorMessage: 'raw-body' + error: writebackError }) ) }) @@ -751,18 +850,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'failed' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) await expect(runCollectionMetadataCorrection({ @@ -770,10 +857,10 @@ describe('runCollectionMetadataCorrection', () => { })).rejects.toBe(writebackError) expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( - 2, + 3, expect.objectContaining({ status: 'failed', - writebackErrorMessage: 'plain-object-writeback-failure' + error: writebackError }) ) }) @@ -840,18 +927,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'failed' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) await expect(runCollectionMetadataCorrection({ @@ -859,15 +934,15 @@ describe('runCollectionMetadataCorrection', () => { })).rejects.toBe(writebackError) expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( - 2, + 3, expect.objectContaining({ status: 'failed', - writebackErrorMessage: '[object Object]' + error: writebackError }) ) }) - test('does not persist failed audit rows when writeback fails before any corrections were applied', async () => { + test('does not update the audit document to failed when no corrections were applied', async () => { const writebackError = new Error('CMR writeback failed before corrections were applied') vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ @@ -922,10 +997,13 @@ describe('runCollectionMetadataCorrection', () => { collectionConceptId: 'C1234567890-PROV' })).rejects.toBe(writebackError) - expect(persistMetadataCorrectionAuditLog).not.toHaveBeenCalled() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledOnce() + expect(persistMetadataCorrectionAuditLog).toHaveBeenCalledWith( + expect.objectContaining({ status: 'checked' }) + ) }) - test('logs and rethrows the original writeback error when failed-audit persistence also fails', async () => { + test('rethrows the writeback error when updating the audit document to failed also fails', async () => { const writebackError = new Error('CMR writeback failed with status 400: {"errors":["boom"]}') const auditError = new Error('failed audit persistence') @@ -985,11 +1063,16 @@ describe('runCollectionMetadataCorrection', () => { }) vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) + .mockImplementationOnce(async ({ runId = 'audit-run-1', status }) => ({ + runId, + status, + created: true + })) + .mockImplementationOnce(async ({ runId, status }) => ({ + runId, + status, + created: false + })) .mockRejectedValueOnce(auditError) vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) @@ -1009,7 +1092,7 @@ describe('runCollectionMetadataCorrection', () => { ) }) - test('falls back to String(auditError) when failed-audit persistence rejects without a message property', async () => { + test('stringifies an error without a message when updating the audit document to failed', async () => { const writebackError = new Error('CMR writeback failed with status 400: {"errors":["boom"]}') const auditError = { toString: () => 'failed-audit-persistence-fallback' @@ -1071,11 +1154,16 @@ describe('runCollectionMetadataCorrection', () => { }) vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) + .mockImplementationOnce(async ({ runId = 'audit-run-1', status }) => ({ + runId, + status, + created: true + })) + .mockImplementationOnce(async ({ runId, status }) => ({ + runId, + status, + created: false + })) .mockRejectedValueOnce(auditError) vi.mocked(writeCorrectedMetadataToCmr).mockRejectedValue(writebackError) @@ -1209,18 +1297,6 @@ describe('runCollectionMetadataCorrection', () => { } }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'applied' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { enabled: true, @@ -1335,18 +1411,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'applied' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { updated: true @@ -1419,12 +1483,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog).mockResolvedValue({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { enabled: false, @@ -1517,18 +1575,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'applied' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { enabled: true, @@ -1626,18 +1672,6 @@ describe('runCollectionMetadataCorrection', () => { correctedMetadata: 'corrected' }) - vi.mocked(persistMetadataCorrectionAuditLog) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'pending' - }) - .mockResolvedValueOnce({ - insertedCount: 1, - publishedVersionName: 'published', - status: 'applied' - }) - vi.mocked(writeCorrectedMetadataToCmr).mockResolvedValue({ ingestResult: { enabled: true, diff --git a/serverless/src/shared/awsClients.js b/serverless/src/shared/awsClients.js index adf64083..9fcc14cb 100644 --- a/serverless/src/shared/awsClients.js +++ b/serverless/src/shared/awsClients.js @@ -1,5 +1,6 @@ import { EventBridgeClient } from '@aws-sdk/client-eventbridge' import { S3Client } from '@aws-sdk/client-s3' +import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager' import { SNSClient } from '@aws-sdk/client-sns' /** @@ -75,3 +76,18 @@ export const getSnsClient = () => { return snsClient } + +let secretsManagerClient + +/** + * Returns a shared Secrets Manager client instance for the current process. + * + * @returns {SecretsManagerClient} Lazily created Secrets Manager client. + */ +export const getSecretsManagerClient = () => { + if (!secretsManagerClient) { + secretsManagerClient = new SecretsManagerClient(getClientConfig()) + } + + return secretsManagerClient +} diff --git a/serverless/src/shared/documentDbClient.js b/serverless/src/shared/documentDbClient.js new file mode 100644 index 00000000..15e136b2 --- /dev/null +++ b/serverless/src/shared/documentDbClient.js @@ -0,0 +1,180 @@ +import { GetSecretValueCommand } from '@aws-sdk/client-secrets-manager' +import { MongoClient } from 'mongodb' + +import { getSecretsManagerClient } from '@/shared/awsClients' + +const DEFAULT_DATABASE_NAME = 'kms' +const DEFAULT_AUDIT_COLLECTION_NAME = 'metadataCorrectionAudits' +const DEFAULT_MAX_POOL_SIZE = 5 + +let mongoClientPromise + +/** + * Parses a positive integer setting while retaining a safe default for missing/invalid values. + * + * @example + * parsePositiveInteger('9', 5) // 9 + * parsePositiveInteger('0', 5) // 5 + * + * @param {unknown} value Configured value. + * @param {number} fallback Value used when the input is not a positive integer. + * @returns {number} Parsed value or fallback. + */ +const parsePositiveInteger = (value, fallback) => { + const parsedValue = Number.parseInt(value, 10) + + return Number.isInteger(parsedValue) && parsedValue > 0 + ? parsedValue + : fallback +} + +/** + * Loads and validates the DocumentDB username/password JSON managed by Secrets Manager. + * + * @returns {Promise<{username: string, password: string}>} DocumentDB credentials. + */ +const getDocumentDbSecret = async () => { + const secretArn = process.env.DOCUMENTDB_SECRET_ARN + + if (!secretArn) { + throw new Error('Missing DOCUMENTDB_SECRET_ARN') + } + + const response = await getSecretsManagerClient().send(new GetSecretValueCommand({ + SecretId: secretArn + })) + + if (!response.SecretString) { + throw new Error('DocumentDB secret does not contain SecretString credentials') + } + + const credentials = JSON.parse(response.SecretString) + + if (!credentials.username || !credentials.password) { + throw new Error('DocumentDB secret is missing username or password') + } + + return credentials +} + +/** + * Builds MongoClient connection inputs for local MongoDB or deployed DocumentDB. + * + * @example + * // DOCUMENTDB_URI=mongodb://localhost:27018 + * await buildDocumentDbConnection() + * // { uri: 'mongodb://localhost:27018', options: { maxPoolSize: 5, ... } } + * + * @example + * // With DOCUMENTDB_HOST, DOCUMENTDB_SECRET_ARN, and DOCUMENTDB_TLS_CA_FILE configured: + * await buildDocumentDbConnection() + * // { uri: 'mongodb://@:27017/?tls=true&...', options: { tlsCAFile, ... } } + * + * @returns {Promise<{uri: string, options: import('mongodb').MongoClientOptions}>} + * MongoClient constructor arguments. + */ +const buildDocumentDbConnection = async () => { + const localUri = process.env.DOCUMENTDB_URI + const options = { + maxPoolSize: parsePositiveInteger( + process.env.DOCUMENTDB_MAX_POOL_SIZE, + DEFAULT_MAX_POOL_SIZE + ), + minPoolSize: 0, + serverSelectionTimeoutMS: 5_000, + connectTimeoutMS: 5_000 + } + + if (localUri) { + return { + uri: localUri, + options + } + } + + const host = process.env.DOCUMENTDB_HOST + const port = process.env.DOCUMENTDB_PORT || '27017' + const tlsCAFile = process.env.DOCUMENTDB_TLS_CA_FILE + + if (!host) { + throw new Error('Missing DOCUMENTDB_HOST') + } + + if (!tlsCAFile) { + throw new Error('Missing DOCUMENTDB_TLS_CA_FILE') + } + + const { username, password } = await getDocumentDbSecret() + const encodedUsername = encodeURIComponent(username) + const encodedPassword = encodeURIComponent(password) + + return { + uri: `mongodb://${encodedUsername}:${encodedPassword}@${host}:${port}/?tls=true&replicaSet=rs0&readPreference=primary&retryWrites=false&authSource=admin`, + options: { + ...options, + tlsCAFile + } + } +} + +/** + * Returns the shared MongoDB client used for DocumentDB access. + * + * The connection promise is reused across warm Lambda invocations to keep the number of + * DocumentDB connections bounded. + * + * @returns {Promise} Connected MongoDB client. + * + * @example + * const client = await getDocumentDbClient() + * const database = client.db('kms') + */ +export const getDocumentDbClient = async () => { + if (!mongoClientPromise) { + mongoClientPromise = buildDocumentDbConnection() + .then(({ uri, options }) => new MongoClient(uri, options).connect()) + .catch((error) => { + mongoClientPromise = undefined + throw error + }) + } + + return mongoClientPromise +} + +/** + * Closes the shared client, primarily for local scripts and test teardown. + * + * Lambda handlers intentionally leave the client open so warm invocations can reuse it. + * + * @returns {Promise} Resolves after the client has closed. + */ +export const closeDocumentDbClient = async () => { + const clientPromise = mongoClientPromise + mongoClientPromise = undefined + + if (!clientPromise) return + + const client = await clientPromise + await client.close() +} + +/** + * Returns the metadata-correction audit collection. + * + * @returns {Promise} Audit collection. + * + * @example + * const audits = await getMetadataCorrectionAuditCollection() + * await audits.findOne({ runId: 'run-1' }) + */ +export const getMetadataCorrectionAuditCollection = async () => { + const client = await getDocumentDbClient() + const databaseName = process.env.DOCUMENTDB_DATABASE_NAME || DEFAULT_DATABASE_NAME + const collectionName = process.env.DOCUMENTDB_AUDIT_COLLECTION_NAME + || DEFAULT_AUDIT_COLLECTION_NAME + + return client.db(databaseName).collection(collectionName) +} + +export default getDocumentDbClient diff --git a/serverless/src/shared/getMetadataCorrectionAuditLog.js b/serverless/src/shared/getMetadataCorrectionAuditLog.js index 69812c68..68819a83 100644 --- a/serverless/src/shared/getMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/getMetadataCorrectionAuditLog.js @@ -1,215 +1,248 @@ -import { - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH -} from '@/shared/metadataCorrectionAudit' -import { sparqlRequest } from '@/shared/sparqlRequest' +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' +import { METADATA_CORRECTION_AUDIT_STATUSES } from '@/shared/persistMetadataCorrectionAuditLog' + +const DEFAULT_LIMIT = 100 +const MAX_LIMIT = 250 /** - * Normalizes a query-string boolean flag. + * Converts the API limit to an integer within the supported page-size range. + * + * @example + * normalizeLimit('500') // 250 + * normalizeLimit(undefined) // 100 * - * @param {unknown} value - Raw query-string value. - * @returns {boolean} True when the caller explicitly enabled the flag. + * @param {unknown} limit Requested page size. + * @returns {number} A page size from 1 through 250. */ -const normalizeBoolean = (value) => ['1', 'true', 'yes'].includes( - String(value || '').toLowerCase() -) - const normalizeLimit = (limit) => { - const parsed = Number.parseInt(limit, 10) + const parsedLimit = Number.parseInt(limit, 10) - if (Number.isNaN(parsed)) { - return 100 + if (Number.isNaN(parsedLimit)) { + return DEFAULT_LIMIT } - return Math.max(1, parsed) + return Math.min(MAX_LIMIT, Math.max(1, parsedLimit)) } /** - * Builds a stable collapse key for one logical correction row. + * Parses an optional date filter and reports which request field was invalid. * - * The append-only audit writer persists separate `pending` and `applied` rows for the same - * logical correction. This key intentionally ignores row-specific fields like `recordUri`, - * `timestamp`, and `status` so the read path can collapse those lifecycle rows into the newest - * effective state when requested. + * @example + * normalizeDate('2026-09-02', 'startDate') // Date for 2026-09-02 + * normalizeDate(undefined, 'startDate') // undefined * - * @param {object} item - Normalized audit row. - * @returns {string} Stable key used to identify duplicate lifecycle rows. + * @param {unknown} value Date-compatible filter value. + * @param {string} fieldName Filter name used in validation errors. + * @returns {Date|undefined} Parsed date when supplied. */ -const buildCollapsedAuditKey = (item) => JSON.stringify([ - item.publishedVersionName, - item.collectionConceptId, - item.keywordConceptUuid, - item.scheme, - item.action, - item.oldKeywordPath, - item.newKeywordPath, - item.nativeFormat, - item.delegateName, - item.triggerScheme, - item.triggerKeywordUuid -]) +const normalizeDate = (value, fieldName) => { + if (!value) { + return undefined + } + + const date = new Date(value) + + if (Number.isNaN(date.getTime())) { + throw new Error(`Invalid metadata correction audit ${fieldName}`) + } + + return date +} /** - * Collapses append-only lifecycle rows into a latest-only view. + * Decodes the opaque API pagination token into its keyset cursor values. * - * The SPARQL query already returns rows newest-first, so keeping the first row for each - * correction key preserves the most recent status while hiding older duplicate lifecycle rows. + * @example + * decodePaginationToken(encodePaginationToken({ + * createdAt: new Date('2026-09-02T12:00:00.000Z'), + * runId: 'run-2' + * })) + * // { createdAt: Date('2026-09-02T12:00:00.000Z'), runId: 'run-2' } * - * @param {Array} items - Normalized audit rows ordered newest-first. - * @returns {Array} Collapsed audit rows. + * @param {string|undefined} paginationToken Base64url token returned by an earlier query. + * @returns {{createdAt: Date, runId: string}|undefined} Decoded cursor values. */ -const collapseAuditRows = (items) => { - const seenKeys = new Set() +const decodePaginationToken = (paginationToken) => { + if (!paginationToken) { + return undefined + } - return items.filter((item) => { - const collapseKey = buildCollapsedAuditKey(item) + try { + const parsedToken = JSON.parse(Buffer.from(paginationToken, 'base64url').toString('utf8')) + const createdAt = normalizeDate(parsedToken.createdAt, 'paginationToken') - if (seenKeys.has(collapseKey)) { - return false + if (!createdAt || typeof parsedToken.runId !== 'string' || !parsedToken.runId) { + throw new Error('invalid pagination token payload') } - seenKeys.add(collapseKey) - - return true - }) + return { + createdAt, + runId: parsedToken.runId + } + } catch { + throw new Error('Invalid metadata correction audit paginationToken') + } } /** - * Reads metadata-correction audit rows from the dedicated RDF4J audit graph. - * - * This helper is the read-side pair to `persistMetadataCorrectionAuditLog`. It builds a SPARQL - * query against the dedicated audit graph, applies any optional filters the caller supplied, and - * returns a normalized array of audit records ordered newest-first. - * - * The returned rows describe resolved metadata corrections, not raw keyword events. Each row - * represents one correction the service decided to apply (or mark pending), including the - * collection it targeted, the resolved keyword UUID/path information, the delegate/native format - * used, and the triggering event metadata when present. - * - * These audit rows expose the canonical UUID plus the derived human-readable keyword paths. - * Keyword objects themselves are intentionally not stored in the audit graph. - * - * @param {object} [filters={}] - Optional query filters. - * @param {string} [filters.collectionConceptId] - Filter by collection concept id. - * @param {string} [filters.keywordConceptUuid] - Filter by resolved keyword UUID. - * @param {string} [filters.action] - Filter by triggering event action. - * @param {string} [filters.scheme] - Filter by corrected keyword scheme. - * @param {string} [filters.status] - Filter by audit status. - * @param {string|boolean} [filters.latestOnly=false] - When truthy, collapses duplicate - * append-only lifecycle rows so only the newest row for each logical correction is returned. - * @param {string|number} [filters.limit=100] - Maximum number of rows to return. Values are - * normalized to a minimum of `1`, with invalid values falling back to `100`. - * @returns {Promise>} Audit log rows ordered newest-first. + * Encodes the last audit document on a page as an opaque keyset pagination token. + * + * @example + * encodePaginationToken({ + * createdAt: new Date('2026-09-02T12:00:00.000Z'), + * runId: 'run-2' + * }) + * // Base64url for {"createdAt":"2026-09-02T12:00:00.000Z","runId":"run-2"} + * + * @param {{createdAt: Date, runId: string}} document Last document returned on a page. + * @returns {string} Opaque Base64url pagination token. + */ +const encodePaginationToken = (document) => Buffer.from(JSON.stringify({ + createdAt: document.createdAt.toISOString(), + runId: document.runId +})).toString('base64url') + +/** + * Maps supported API filters to the fixed MongoDB fields used by audit documents. + * + * @example + * buildAuditQuery({ scheme: 'platforms', status: 'applied' }) + * // { + * // $or: [{ 'corrections.scheme': 'platforms' }, { 'trigger.scheme': 'platforms' }], + * // status: 'applied' + * // } + * + * @param {Object} filters Validated request filter values. + * @returns {Object} MongoDB query document. */ -export const getMetadataCorrectionAuditLog = async (filters = {}) => { +const buildAuditQuery = (filters) => { + const query = {} const { + action, collectionConceptId, + endDate, keywordConceptUuid, - action, + nativeFormat, + publishedVersionName, scheme, - status, - latestOnly = false, - limit = 100 + source, + startDate, + status } = filters - const filterClauses = [ - collectionConceptId ? `FILTER(?collectionConceptId = "${escapeSparqlLiteral(collectionConceptId)}")` : '', - keywordConceptUuid ? `FILTER(?keywordConceptUuid = "${escapeSparqlLiteral(keywordConceptUuid)}")` : '', - action ? `FILTER(?action = "${escapeSparqlLiteral(action)}")` : '', - scheme ? `FILTER(?scheme = "${escapeSparqlLiteral(scheme)}")` : '', - status ? `FILTER(?status = "${escapeSparqlLiteral(status)}")` : '' - ].filter(Boolean).join('\n ') - - const query = ` - PREFIX gcmd: - PREFIX dcterms: - - SELECT - ?record - ?timestamp - ?publishedVersionName - ?collectionConceptId - ?keywordConceptUuid - ?scheme - ?action - ?oldKeywordPath - ?newKeywordPath - ?nativeFormat - ?delegateName - ?status - ?writebackErrorMessage - ?triggerScheme - ?triggerKeywordUuid - WHERE { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { - ?record a gcmd:MetadataCorrectionAuditRecord ; - dcterms:created ?timestamp ; - gcmd:publishedVersionName ?publishedVersionName ; - gcmd:collectionConceptId ?collectionConceptId ; - gcmd:keywordConceptUuid ?keywordConceptUuid ; - gcmd:scheme ?scheme ; - gcmd:action ?action ; - gcmd:nativeFormat ?nativeFormat ; - gcmd:delegateName ?delegateName ; - gcmd:status ?status . - OPTIONAL { ?record gcmd:oldKeywordPath ?oldKeywordPath } - OPTIONAL { ?record gcmd:newKeywordPath ?newKeywordPath } - OPTIONAL { ?record gcmd:writebackErrorMessage ?writebackErrorMessage } - OPTIONAL { ?record gcmd:triggerScheme ?triggerScheme } - OPTIONAL { ?record gcmd:triggerKeywordUuid ?triggerKeywordUuid } - } - ${filterClauses} + if (collectionConceptId) query.collectionConceptId = collectionConceptId + if (action) query['trigger.eventType'] = action + if (keywordConceptUuid) query['corrections.keywordConceptUuid'] = keywordConceptUuid + if (nativeFormat) query.nativeFormat = nativeFormat + if (publishedVersionName) query.publishedVersionName = publishedVersionName + if (scheme) { + query.$or = [ + { 'corrections.scheme': scheme }, + { 'trigger.scheme': scheme } + ] + } + + if (source) query.source = source + + if (status) { + if (!METADATA_CORRECTION_AUDIT_STATUSES.includes(status)) { + throw new Error(`Invalid metadata correction audit status: ${status}`) + } + + query.status = status + } + + const normalizedStartDate = normalizeDate(startDate, 'startDate') + const normalizedEndDate = normalizeDate(endDate, 'endDate') + + if (normalizedStartDate || normalizedEndDate) { + query.createdAt = { + ...(normalizedStartDate ? { $gte: normalizedStartDate } : {}), + ...(normalizedEndDate ? { $lte: normalizedEndDate } : {}) } - ORDER BY DESC(?timestamp) - LIMIT ${normalizeLimit(limit)} - ` - - const response = await sparqlRequest({ - method: 'POST', - body: query, - contentType: 'application/sparql-query', - accept: 'application/sparql-results+json' - }) - - const result = await response.json() - const bindings = result?.results?.bindings || [] - const items = bindings.map((binding) => ({ - recordUri: binding.record?.value, - timestamp: binding.timestamp?.value, - publishedVersionName: binding.publishedVersionName?.value, - collectionConceptId: binding.collectionConceptId?.value, - keywordConceptUuid: binding.keywordConceptUuid?.value, - scheme: binding.scheme?.value, - action: binding.action?.value, - oldKeywordPath: binding.oldKeywordPath?.value, - newKeywordPath: binding.newKeywordPath?.value, - nativeFormat: binding.nativeFormat?.value, - delegateName: binding.delegateName?.value, - status: binding.status?.value, - writebackErrorMessage: binding.writebackErrorMessage?.value, - triggerScheme: binding.triggerScheme?.value, - triggerKeywordUuid: binding.triggerKeywordUuid?.value - })) - - return normalizeBoolean(latestOnly) - ? collapseAuditRows(items) - : items + } + + return query +} + +/** + * Adds a newest-first keyset boundary to a MongoDB audit query. + * + * For a token representing `{ createdAt: 2026-09-02, runId: 'run-2' }`, the added condition + * selects documents older than that date, or lower run ids at the exact same date. + * + * @param {Object} query Existing field/date query. + * @param {string|undefined} paginationToken Opaque cursor from the previous page. + * @returns {Object} Original query or a query combined with the keyset boundary. + */ +const addPaginationTokenToQuery = (query, paginationToken) => { + const decodedToken = decodePaginationToken(paginationToken) + + if (!decodedToken) { + return query + } + + return { + $and: [ + query, + { + $or: [ + { createdAt: { $lt: decodedToken.createdAt } }, + { + createdAt: decodedToken.createdAt, + _id: { $lt: decodedToken.runId } + } + ] + } + ] + } +} + +/** + * Removes MongoDB's internal `_id` field from the public API representation. + * + * @example + * normalizeAuditDocument({ _id: 'run-1', runId: 'run-1', status: 'applied' }) + * // { runId: 'run-1', status: 'applied' } + * + * @param {Object} document Stored audit document. + * @returns {Object} Public audit document. + */ +const normalizeAuditDocument = (document) => Object.fromEntries( + Object.entries(document).filter(([key]) => key !== '_id') +) + +/** + * Returns metadata-correction audit runs using newest-first token pagination. + * + * @param {Object} [filters={}] Supported field, date, and pagination filters. + * @returns {Promise<{items: Array, nextPaginationToken: string|null}>} Audit page. + * + * @example + * await getMetadataCorrectionAuditLog({ status: 'applied', limit: 25 }) + * // { items: [{ runId: '...', status: 'applied', ... }], nextPaginationToken: '...' } + */ +export const getMetadataCorrectionAuditLog = async (filters = {}) => { + const collection = await getMetadataCorrectionAuditCollection() + + const limit = normalizeLimit(filters.limit) + const query = addPaginationTokenToQuery(buildAuditQuery(filters), filters.paginationToken) + const documents = await collection.find(query) + .sort({ + createdAt: -1, + _id: -1 + }) + .limit(limit + 1) + .toArray() + const hasNextPage = documents.length > limit + const pageDocuments = hasNextPage ? documents.slice(0, limit) : documents + + return { + items: pageDocuments.map(normalizeAuditDocument), + nextPaginationToken: hasNextPage + ? encodePaginationToken(pageDocuments[pageDocuments.length - 1]) + : null + } } export default getMetadataCorrectionAuditLog diff --git a/serverless/src/shared/metadataCorrectionAudit.js b/serverless/src/shared/metadataCorrectionAudit.js deleted file mode 100644 index 5cbefba7..00000000 --- a/serverless/src/shared/metadataCorrectionAudit.js +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Shared constants and helpers for metadata-correction audit records. - * - * The metadata-correction pipeline stores one RDF audit record per resolved correction in a dedicated - * RDF4J graph. This module centralizes the graph URI, the record URI base, and the tiny helper - * functions used by both the audit-write and audit-read paths so they stay aligned. - */ -export const METADATA_CORRECTION_AUDIT_GRAPH = 'https://gcmd.earthdata.nasa.gov/kms/audit/metadata-corrections' -export const METADATA_CORRECTION_AUDIT_RECORD_BASE_URI = 'https://gcmd.earthdata.nasa.gov/kms/metadata-correction-audit/' - -/** - * Escapes a string so it can be safely embedded as a SPARQL string literal. - * - * @param {unknown} value - Value to serialize into a SPARQL-safe string literal. - * @returns {string} Escaped literal text. - */ -export const escapeSparqlLiteral = (value) => String(value) - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - -/** - * Builds the canonical audit record URI for one metadata-correction audit row. - * - * @param {string} recordId - Unique identifier for the audit record. - * @returns {string} Fully qualified audit record URI. - */ -export const createMetadataCorrectionAuditRecordUri = (recordId) => ( - `${METADATA_CORRECTION_AUDIT_RECORD_BASE_URI}${recordId}` -) - -export default { - METADATA_CORRECTION_AUDIT_GRAPH, - METADATA_CORRECTION_AUDIT_RECORD_BASE_URI, - escapeSparqlLiteral, - createMetadataCorrectionAuditRecordUri -} diff --git a/serverless/src/shared/persistMetadataCorrectionAuditLog.js b/serverless/src/shared/persistMetadataCorrectionAuditLog.js index e0644bf6..b70efe36 100644 --- a/serverless/src/shared/persistMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/persistMetadataCorrectionAuditLog.js @@ -1,43 +1,78 @@ import { v4 as uuidv4 } from 'uuid' -import { getVersionMetadata } from '@/shared/getVersionMetadata' -import { - createMetadataCorrectionAuditRecordUri, - escapeSparqlLiteral, - METADATA_CORRECTION_AUDIT_GRAPH -} from '@/shared/metadataCorrectionAudit' +import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' import { getKeywordPathFromKeywordObject } from '@/shared/redis-path-store/getKeywordPathFromKeywordObject' -import { sparqlRequest } from '@/shared/sparqlRequest' + +export const METADATA_CORRECTION_AUDIT_STATUSES = Object.freeze([ + 'checked', + 'pending', + 'applied', + 'failed' +]) + +const TERMINAL_STATUS = 'applied' +const STATUS_ORDER = Object.freeze({ + checked: 0, + pending: 1, + failed: 2, + applied: 3 +}) /** - * RDF4J audit-log writer for metadata-correction activity. + * Removes only undefined values so meaningful `null`, false, and empty-string values are retained. + * + * @example + * compactObject({ status: 'checked', error: undefined, outcome: null }) + * // { status: 'checked', outcome: null } * - * This module is the write-side counterpart to `getMetadataCorrectionAuditLog`. It takes the - * resolved corrections produced by the metadata-correction service and appends one RDF audit - * record per correction into the dedicated metadata-correction audit graph. + * @param {Object} value Source object. + * @returns {Object} Object without undefined entries. + */ +const compactObject = (value) => Object.fromEntries( + Object.entries(value).filter(([, entryValue]) => entryValue !== undefined) +) + +/** + * Builds a link to the latest CMR record for an audited collection. * - * The audit records are intentionally append-only and correction-centric. That means one keyword - * event affecting one collection can produce multiple audit rows when several resolved corrections - * are applied during the same run. + * @example + * // With CMR_BASE_URL=https://cmr.earthdata.nasa.gov/ + * buildCmrCollectionUri('C123-PROV') + * // 'https://cmr.earthdata.nasa.gov/search/concepts/C123-PROV' * - * The delegate correction contract can now carry optional long-name metadata for some short-name - * schemes. The audit log does not persist those keyword objects directly. Instead, it derives and - * stores only the human-readable keyword paths needed for audit inspection. + * @param {string} collectionConceptId CMR collection concept id. + * @returns {string|undefined} CMR concept URI, or undefined when CMR is not configured. */ +const buildCmrCollectionUri = (collectionConceptId) => { + const cmrBaseUrl = String(process.env.CMR_BASE_URL || '').trim().replace(/\/+$/, '') -// Emits a triple only when the optional value is present so audit rows stay compact. -const optionalLiteralTriple = (subject, predicate, value) => { - if (value === undefined || value === null || value === '') { - return '' - } + if (!cmrBaseUrl) return undefined - return ` <${subject}> ${predicate} "${escapeSparqlLiteral(value)}" .\n` + return `${cmrBaseUrl}/search/concepts/${encodeURIComponent(collectionConceptId)}` } -// Reconstructs a human-readable keyword path from the normalized keyword object when the object -// contains enough non-blank values to form a meaningful path. +/** + * Builds the readable CSV-shaped keyword path stored alongside a correction. + * + * @example + * buildAuditKeywordPath({ + * scheme: 'platforms', + * keywordObject: { + * Basis: 'Platforms', + * Category: 'Space-based Platforms', + * SubCategory: 'Earth Observation Satellites', + * ShortName: 'GOSAT' + * } + * }) + * // 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT' + * + * @param {Object} params Path inputs. + * @param {string} params.scheme Keyword scheme. + * @param {Object} params.keywordObject CSV-shaped keyword object. + * @returns {string} Human-readable keyword path, or an empty string when unavailable. + */ const buildAuditKeywordPath = ({ scheme, keywordObject @@ -47,117 +82,218 @@ const buildAuditKeywordPath = ({ }) || '' /** - * Persists append-only metadata-correction audit rows to a dedicated RDF4J graph. - * - * Current behavior defaults each resolved correction to `pending`, but callers can persist - * `applied` immediately once metadata write-back succeeds. - * - * @param {object} params - Audit persistence parameters. - * @param {string} params.collectionConceptId - CMR collection concept id. - * @param {{ eventType?: string, scheme?: string, uuid?: string }} [params.keywordEvent={}] - Triggering keyword event. - * @param {string} params.nativeFormat - Normalized native format used for delegate selection. - * @param {string} params.delegateName - Delegate name returned by the correction delegate. - * @param {Array<{ - * scheme: string, - * keywordConceptUuid: string, - * oldKeywordObject: Record, - * newKeywordObject?: Record - * }>} params.corrections - Fully resolved corrections to persist. Audit rows store only the - * derived keyword paths, not the original keyword objects. - * @param {string} [params.status='pending'] - Audit lifecycle status. - * @param {string} [params.writebackErrorMessage] - Optional CMR ingest/writeback error message for - * failed writeback attempts. - * @param {string} [params.timestamp] - ISO timestamp override for tests. - * @returns {Promise<{ insertedCount: number, publishedVersionName: string, status: string }>} - * Insert summary for logging/verification. + * Preserves each correction and adds readable old/new paths for filtering and display. + * + * @example + * normalizeCorrections([{ + * scheme: 'dataformat', + * oldKeywordObject: { ShortName: 'NetCDF' }, + * newKeywordObject: { ShortName: 'NetCDF-4' } + * }]) + * // [{ ..., oldKeywordPath: 'NetCDF', newKeywordPath: 'NetCDF-4' }] + * + * @param {Array} corrections Resolved correction objects. + * @returns {Array} Corrections enriched with audit paths. */ -export const persistMetadataCorrectionAuditLog = async ({ +const normalizeCorrections = (corrections) => corrections.map((correction) => compactObject({ + ...correction, + oldKeywordPath: buildAuditKeywordPath({ + scheme: correction.scheme, + keywordObject: correction.oldKeywordObject + }), + newKeywordPath: buildAuditKeywordPath({ + scheme: correction.scheme, + keywordObject: correction.newKeywordObject + }) +})) + +/** + * Maps a lifecycle status to its timestamp field in the audit document. + * + * @example + * buildStatusTimestampField('pending') // 'timestamps.pendingAt' + * + * @param {string} status Audit lifecycle status. + * @returns {string} Dot-notation MongoDB field name. + */ +const buildStatusTimestampField = (status) => `timestamps.${status}At` + +/** + * Converts correction-run inputs into the stable fields stored on one audit document. + * + * Error instances are reduced to serializable diagnostic fields, and keyword events are reduced + * to the trigger fields needed for filtering and traceability. + * + * @example + * buildAuditPatch({ + * collectionConceptId: 'C123-PROV', + * keywordEvent: { eventType: 'UPDATED', scheme: 'platforms', uuid: 'platform-uuid' } + * }) + * // { + * // collectionConceptId: 'C123-PROV', + * // trigger: { eventType: 'UPDATED', scheme: 'platforms', keywordConceptUuid: 'platform-uuid' } + * // } + * + * @param {Object} auditFields Correction-run audit values. + * @returns {Object} Serializable partial audit document for `$set`. + */ +const buildAuditPatch = ({ collectionConceptId, - keywordEvent = {}, + corrections, + delegateName, + error, + keywordEvent, + keywordValidationFailures, + messageId, + nativeFormat, + outcome, + priorRevisionId, + providerId, + publishedVersionName, + resultingRevisionId, + source +}) => compactObject({ + collectionConceptId, + collectionUri: buildCmrCollectionUri(collectionConceptId), + providerId, + publishedVersionName: publishedVersionName || null, nativeFormat, delegateName, - corrections = [], - status = 'pending', - writebackErrorMessage, - timestamp + source, + messageId, + trigger: keywordEvent && Object.keys(keywordEvent).length > 0 + ? compactObject({ + eventType: keywordEvent.eventType, + scheme: keywordEvent.scheme, + keywordConceptUuid: keywordEvent.uuid, + timestamp: keywordEvent.timestamp + }) + : undefined, + corrections: Array.isArray(corrections) + ? normalizeCorrections(corrections) + : undefined, + keywordValidationFailures: Array.isArray(keywordValidationFailures) + ? keywordValidationFailures + : undefined, + keywordValidationFailureCount: Array.isArray(keywordValidationFailures) + ? keywordValidationFailures.length + : undefined, + outcome, + error: error ? compactObject({ + message: error.message || String(error), + status: error.status, + statusText: error.statusText, + url: error.url, + cmrRequest: error.cmrRequest, + cmrResponseBody: error.cmrResponseBody + }) : undefined, + priorRevisionId, + resultingRevisionId +}) + +/** + * Creates or updates the single DocumentDB audit document for a correction run. + * + * Repeated lifecycle calls use the same `runId`, update the current status, and append a status + * history entry only when the status changes. An applied run cannot be regressed by an SQS retry. + * + * @param {Object} params Audit run fields. + * @param {string} [params.runId] Stable run identifier. Generated when omitted. + * @param {'checked'|'pending'|'applied'|'failed'} [params.status='checked'] Lifecycle status. + * @param {string} params.collectionConceptId CMR collection concept id. + * @param {string} [params.timestamp] ISO timestamp override for tests. + * @returns {Promise<{runId: string, status: string, created: boolean}>} Persistence summary. + * + * @example + * await persistMetadataCorrectionAuditLog({ + * runId: 'run-1', + * collectionConceptId: 'C123-PROV', + * priorRevisionId: 7, + * status: 'pending' + * }) + * // { runId: 'run-1', status: 'pending', created: true } + */ +export const persistMetadataCorrectionAuditLog = async ({ + runId = uuidv4(), + status = 'checked', + timestamp, + ...auditFields }) => { - if (!collectionConceptId) { + if (!auditFields.collectionConceptId) { throw new Error('Missing collectionConceptId for metadata correction audit persistence') } - if (!nativeFormat) { - throw new Error('Missing nativeFormat for metadata correction audit persistence') + if (!METADATA_CORRECTION_AUDIT_STATUSES.includes(status)) { + throw new Error(`Invalid metadata correction audit status: ${status}`) } - if (!delegateName) { - throw new Error('Missing delegateName for metadata correction audit persistence') - } + const collection = await getMetadataCorrectionAuditCollection() + const auditTimestamp = new Date(timestamp || Date.now()) + const existingDocument = await collection.findOne( + { _id: runId }, + { projection: { status: 1 } } + ) - if (!Array.isArray(corrections) || corrections.length === 0) { + if (existingDocument?.status === TERMINAL_STATUS && status !== TERMINAL_STATUS) { return { - insertedCount: 0, - publishedVersionName: 'published', - status + runId, + status: existingDocument.status, + created: false } } - const publishedVersionMetadata = await getVersionMetadata('published') - const publishedVersionName = publishedVersionMetadata?.versionName || 'published' - const auditTimestamp = timestamp || new Date().toISOString() + const effectiveStatus = existingDocument?.status !== 'failed' + && STATUS_ORDER[existingDocument?.status] > STATUS_ORDER[status] + ? existingDocument.status + : status + const auditPatch = buildAuditPatch(auditFields) + const statusChanged = existingDocument?.status !== effectiveStatus + const setFields = { + ...auditPatch, + status: effectiveStatus, + updatedAt: auditTimestamp + } - const triples = corrections.map((correction) => { - const recordUri = createMetadataCorrectionAuditRecordUri(uuidv4()) - const oldKeywordPath = buildAuditKeywordPath({ - scheme: correction.scheme, - keywordObject: correction.oldKeywordObject - }) - const newKeywordPath = buildAuditKeywordPath({ - scheme: correction.scheme, - keywordObject: correction.newKeywordObject - }) + if (statusChanged) { + setFields[buildStatusTimestampField(effectiveStatus)] = auditTimestamp + } + + const update = { + $set: setFields, + $setOnInsert: { + _id: runId, + runId, + createdAt: auditTimestamp + } + } - return [ - ` <${recordUri}> a gcmd:MetadataCorrectionAuditRecord .`, - ` <${recordUri}> dcterms:created "${escapeSparqlLiteral(auditTimestamp)}"^^xsd:dateTime .`, - ` <${recordUri}> gcmd:publishedVersionName "${escapeSparqlLiteral(publishedVersionName)}" .`, - ` <${recordUri}> gcmd:collectionConceptId "${escapeSparqlLiteral(collectionConceptId)}" .`, - ` <${recordUri}> gcmd:keywordConceptUuid "${escapeSparqlLiteral(correction.keywordConceptUuid)}" .`, - ` <${recordUri}> gcmd:scheme "${escapeSparqlLiteral(correction.scheme)}" .`, - ` <${recordUri}> gcmd:action "${escapeSparqlLiteral(keywordEvent.eventType || 'UNKNOWN')}" .`, - optionalLiteralTriple(recordUri, 'gcmd:oldKeywordPath', oldKeywordPath), - optionalLiteralTriple(recordUri, 'gcmd:newKeywordPath', newKeywordPath), - ` <${recordUri}> gcmd:nativeFormat "${escapeSparqlLiteral(nativeFormat)}" .`, - ` <${recordUri}> gcmd:delegateName "${escapeSparqlLiteral(delegateName)}" .`, - ` <${recordUri}> gcmd:status "${escapeSparqlLiteral(status)}" .`, - optionalLiteralTriple(recordUri, 'gcmd:writebackErrorMessage', writebackErrorMessage), - optionalLiteralTriple(recordUri, 'gcmd:triggerScheme', keywordEvent.scheme), - optionalLiteralTriple(recordUri, 'gcmd:triggerKeywordUuid', keywordEvent.uuid) - ].join('\n') - }).join('\n') - - const query = ` - PREFIX gcmd: - PREFIX dcterms: - PREFIX xsd: - - INSERT DATA { - GRAPH <${METADATA_CORRECTION_AUDIT_GRAPH}> { -${triples} + if (statusChanged) { + update.$push = { + statusHistory: { + status: effectiveStatus, + timestamp: auditTimestamp, + ...(auditFields.outcome ? { outcome: auditFields.outcome } : {}), + ...(auditFields.error + ? { error: auditFields.error.message || String(auditFields.error) } + : {}) } } - ` + } - await sparqlRequest({ - method: 'POST', - contentType: 'application/sparql-update', - accept: 'application/json', - body: query - }) + if (effectiveStatus !== 'failed' && !auditFields.error) { + update.$unset = { error: '' } + } + + await collection.updateOne( + { _id: runId }, + update, + { upsert: true } + ) return { - insertedCount: corrections.length, - publishedVersionName, - status + runId, + status: effectiveStatus, + created: !existingDocument } } diff --git a/serverless/src/shared/runCollectionMetadataCorrection.js b/serverless/src/shared/runCollectionMetadataCorrection.js index c4687bfd..53c1d2f4 100644 --- a/serverless/src/shared/runCollectionMetadataCorrection.js +++ b/serverless/src/shared/runCollectionMetadataCorrection.js @@ -49,10 +49,10 @@ const normalizeKeywordEvent = (keywordEvent) => ( ) /** - * Normalizes the audit trigger metadata used when persisting correction audit rows. + * Normalizes the trigger metadata stored on a correction-run audit document. * * The synchronous concept-id endpoint does not originate from a keyword event, so we stamp those - * audit rows with a synthetic `MANUAL` action to distinguish them from event-driven runs. + * the audit document with a synthetic `MANUAL` action to distinguish it from event-driven runs. * * @param {Object} params Normalization inputs. * @param {Object} params.keywordEvent Optional triggering keyword event context. @@ -266,6 +266,7 @@ const getWritebackErrorMessage = (error) => { * @param {string} params.collectionConceptId Collection concept id to correct. * @param {Object} [params.keywordEvent] Optional triggering keyword event context. * @param {string} [params.messageId] Optional request/message identifier for logging. + * @param {string} [params.publishedVersionName] Published KMS version associated with the run. * @param {string} [params.source='metadataCorrectionService'] Source label for audit/writeback telemetry. * @returns {Promise} Rich per-collection correction result. */ @@ -273,6 +274,7 @@ export const runCollectionMetadataCorrection = async ({ collectionConceptId, keywordEvent: rawKeywordEvent, messageId, + publishedVersionName, source = 'metadataCorrectionService' } = {}) => { validateMetadataCorrectionRequest({ @@ -302,6 +304,54 @@ export const runCollectionMetadataCorrection = async ({ collectionDetails, keywordEvent }) + const checkedOutcome = resolvedCorrections.length === 0 + ? determineNoOpOutcome(keywordValidationFailures) + : 'corrections-resolved' + const checkedAuditResult = await persistMetadataCorrectionAuditLog({ + runId: messageId, + collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + publishedVersionName, + keywordEvent: auditKeywordEvent, + nativeFormat, + corrections: resolvedCorrections, + keywordValidationFailures, + priorRevisionId: collectionDetails.revisionId, + outcome: checkedOutcome, + source, + messageId, + status: 'checked' + }) + const auditRunId = checkedAuditResult.runId + + if (checkedAuditResult.status === 'applied') { + logger.info('[metadata-correction] Skipping an already-applied correction run', { + collectionConceptId: collectionDetails.collectionConceptId, + messageId, + runId: auditRunId + }) + + return { + outcome: 'already-applied', + collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + nativeId: collectionDetails.nativeId, + revisionId: collectionDetails.revisionId, + nativeFormat, + keywordValidationFailureCount: keywordValidationFailures.length, + keywordValidationFailures, + resolvedCorrectionCount: resolvedCorrections.length, + resolvedCorrections, + correctionResult: null, + auditResults: { + checked: checkedAuditResult, + pending: null, + applied: checkedAuditResult + }, + writeResult: null, + source + } + } logger.info('[metadata-correction] Resolved metadata corrections from collection UMM', { collectionConceptId: collectionDetails.collectionConceptId, @@ -349,6 +399,7 @@ export const runCollectionMetadataCorrection = async ({ resolvedCorrections: [], correctionResult: null, auditResults: { + checked: checkedAuditResult, pending: null, applied: null }, @@ -364,26 +415,52 @@ export const runCollectionMetadataCorrection = async ({ resolvedCorrections }) - const nativeMetadataResponse = await getCmrCollectionNativeMetadata({ - collectionConceptId: collectionDetails.collectionConceptId, - revisionId: collectionDetails.revisionId, - includeResponseMetadata: nativeFormat === 'UMM' - }) - const metadataPayload = nativeFormat === 'UMM' - ? nativeMetadataResponse.metadataPayload - : nativeMetadataResponse + let nativeMetadataResponse + let rawCorrectionResult + + try { + nativeMetadataResponse = await getCmrCollectionNativeMetadata({ + collectionConceptId: collectionDetails.collectionConceptId, + revisionId: collectionDetails.revisionId, + includeResponseMetadata: nativeFormat === 'UMM' + }) + + const metadataPayload = nativeFormat === 'UMM' + ? nativeMetadataResponse.metadataPayload + : nativeMetadataResponse + + rawCorrectionResult = await invokeMetadataCorrectionDelegate({ + collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + nativeId: collectionDetails.nativeId, + nativeFormat, + metadataPayload, + corrections: resolvedCorrections + }) + } catch (error) { + await persistMetadataCorrectionAuditLog({ + runId: auditRunId, + collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + publishedVersionName, + keywordEvent: auditKeywordEvent, + nativeFormat, + corrections: resolvedCorrections, + keywordValidationFailures, + priorRevisionId: collectionDetails.revisionId, + outcome: 'correction-failed', + error, + source, + messageId, + status: 'failed' + }) + + throw error + } + const nativeMetadataContentType = nativeFormat === 'UMM' ? nativeMetadataResponse.contentType : String(collectionDetails.format || '') - - const rawCorrectionResult = await invokeMetadataCorrectionDelegate({ - collectionConceptId: collectionDetails.collectionConceptId, - providerId: collectionDetails.providerId, - nativeId: collectionDetails.nativeId, - nativeFormat, - metadataPayload, - corrections: resolvedCorrections - }) const delegateName = rawCorrectionResult.delegateName || nativeFormat.toLowerCase() const correctionsApplied = Array.isArray(rawCorrectionResult.correctionsApplied) ? rawCorrectionResult.correctionsApplied @@ -395,11 +472,19 @@ export const runCollectionMetadataCorrection = async ({ if (correctionsApplied.length > 0) { pendingAuditResult = await persistMetadataCorrectionAuditLog({ + runId: auditRunId, collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + publishedVersionName, keywordEvent: auditKeywordEvent, nativeFormat, delegateName, corrections: correctionsApplied, + keywordValidationFailures, + priorRevisionId: collectionDetails.revisionId, + outcome: 'writeback-pending', + source, + messageId, status: 'pending' }) @@ -438,13 +523,21 @@ export const runCollectionMetadataCorrection = async ({ if (correctionsApplied.length > 0) { try { await persistMetadataCorrectionAuditLog({ + runId: auditRunId, collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + publishedVersionName, keywordEvent: auditKeywordEvent, nativeFormat, delegateName, corrections: correctionsApplied, - status: 'failed', - writebackErrorMessage: getWritebackErrorMessage(error) + keywordValidationFailures, + priorRevisionId: collectionDetails.revisionId, + outcome: 'writeback-failed', + error, + source, + messageId, + status: 'failed' }) logger.debug('[metadata-correction] Persisted failed metadata correction audit log', { @@ -469,11 +562,20 @@ export const runCollectionMetadataCorrection = async ({ if (correctionsApplied.length > 0 && writeResult?.ingestResult?.updated === true) { appliedAuditResult = await persistMetadataCorrectionAuditLog({ + runId: auditRunId, collectionConceptId: collectionDetails.collectionConceptId, + providerId: collectionDetails.providerId, + publishedVersionName, keywordEvent: auditKeywordEvent, nativeFormat, delegateName, corrections: correctionsApplied, + keywordValidationFailures, + priorRevisionId: collectionDetails.revisionId, + resultingRevisionId: writeResult.ingestResult.revisionId, + outcome: 'writeback-applied', + source, + messageId, status: 'applied' }) @@ -531,6 +633,7 @@ export const runCollectionMetadataCorrection = async ({ && rawCorrectionResult.correctedMetadata !== null }, auditResults: { + checked: checkedAuditResult, pending: pendingAuditResult, applied: appliedAuditResult }, diff --git a/serverless/src/shared/writeCorrectedMetadataToCmr.js b/serverless/src/shared/writeCorrectedMetadataToCmr.js index 8dff6a58..8324ca21 100644 --- a/serverless/src/shared/writeCorrectedMetadataToCmr.js +++ b/serverless/src/shared/writeCorrectedMetadataToCmr.js @@ -3,7 +3,7 @@ import { getCmrWriterToken } from './getCmrWriterToken' import { logger } from './logger' // Keep the writeback timeout comfortably inside the metadataCorrectionService Lambda's -// 30s timeout so a stalled ingest request can still be recorded as a failed audit row. +// 30s timeout so a stalled ingest request can still be recorded as a failed audit status. const CMR_WRITEBACK_TIMEOUT_MS = 10_000 const CMR_WRITEBACK_CLIENT_ID = 'kms-metadata-correction-service' From eed9c38f1658168298a76f9feb6a4fd59262c8a6 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 3 Sep 2026 10:15:19 -0400 Subject: [PATCH 02/17] KMS-704: Validate and expand metadata correction audit filtering --- README.md | 3 +- config/metadataCorrectionAuditIndexes.json | 16 +++ .../getMetadataCorrectionAuditLog.test.js | 98 +++++++++++--- .../shared/getMetadataCorrectionAuditLog.js | 122 +++++++++++++++--- 4 files changed, 202 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index ec4b7228..d8d57a52 100644 --- a/README.md +++ b/README.md @@ -454,7 +454,8 @@ The audit API is: - `GET /metadata_correction_audit` for newest-first, token-paginated audit searches. Supported filters include collection, keyword UUID, action, scheme, status, native format, KMS version, - source, and date range. + source, and date range. Supplied actions and schemes must be recognized KMS values, limits must + be integers from 1 through 250, and `startDate` must not be after `endDate`. Publisher events carry the published KMS version through the queue into this document. Manual correction endpoints look up the current published version before starting the run, so the diff --git a/config/metadataCorrectionAuditIndexes.json b/config/metadataCorrectionAuditIndexes.json index bc615141..13546c74 100644 --- a/config/metadataCorrectionAuditIndexes.json +++ b/config/metadataCorrectionAuditIndexes.json @@ -22,6 +22,14 @@ }, "name": "status_createdAt_desc" }, + { + "key": { + "trigger.eventType": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "triggerAction_createdAt_desc" + }, { "key": { "publishedVersionName": 1, @@ -53,5 +61,13 @@ "_id": -1 }, "name": "keywordUuid_createdAt_desc" + }, + { + "key": { + "trigger.keywordConceptUuid": 1, + "createdAt": -1, + "_id": -1 + }, + "name": "triggerKeywordUuid_createdAt_desc" } ] diff --git a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js index 9ed768cf..94ba3dc9 100644 --- a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js @@ -45,35 +45,46 @@ describe('metadata correction audit queries', () => { }]) const result = await getMetadataCorrectionAuditLog({ - action: 'UPDATED', + action: 'updated', collectionConceptId: 'C123-PROV', endDate: '2026-09-03', keywordConceptUuid: 'keyword-1', limit: '25', nativeFormat: 'UMM', publishedVersionName: '20.1', - scheme: 'platforms', + scheme: 'dataformat', source: 'cmrKeywordEventsListener', startDate: '2026-09-01', status: 'applied' }) expect(collection.find).toHaveBeenCalledWith({ - collectionConceptId: 'C123-PROV', - 'trigger.eventType': 'UPDATED', - 'corrections.keywordConceptUuid': 'keyword-1', - nativeFormat: 'UMM', - publishedVersionName: '20.1', - $or: [ - { 'corrections.scheme': 'platforms' }, - { 'trigger.scheme': 'platforms' } - ], - source: 'cmrKeywordEventsListener', - status: 'applied', - createdAt: { - $gte: new Date('2026-09-01'), - $lte: new Date('2026-09-03') - } + $and: [ + { + collectionConceptId: 'C123-PROV', + 'trigger.eventType': 'UPDATED', + nativeFormat: 'UMM', + publishedVersionName: '20.1', + source: 'cmrKeywordEventsListener', + status: 'applied', + createdAt: { + $gte: new Date('2026-09-01'), + $lte: new Date('2026-09-03') + } + }, + { + $or: [ + { 'corrections.keywordConceptUuid': 'keyword-1' }, + { 'trigger.keywordConceptUuid': 'keyword-1' } + ] + }, + { + $or: [ + { 'corrections.scheme': { $in: ['DataFormat', 'dataformat'] } }, + { 'trigger.scheme': { $in: ['DataFormat', 'dataformat'] } } + ] + } + ] }) expect(mongoCursor.sort).toHaveBeenCalledWith({ @@ -141,10 +152,32 @@ describe('metadata correction audit queries', () => { }) }) - test('uses bounded limits and validates filters', async () => { - await getMetadataCorrectionAuditLog({ limit: '5000' }) - expect(mongoCursor.limit).toHaveBeenCalledWith(251) + test('supports default filters, one-sided date ranges, and lowercase scheme storage', async () => { + await getMetadataCorrectionAuditLog() + + expect(collection.find).toHaveBeenLastCalledWith({}) + expect(mongoCursor.limit).toHaveBeenLastCalledWith(101) + + await getMetadataCorrectionAuditLog({ scheme: 'PLATFORMS' }) + expect(collection.find).toHaveBeenLastCalledWith({ + $or: [ + { 'corrections.scheme': 'platforms' }, + { 'trigger.scheme': 'platforms' } + ] + }) + + await getMetadataCorrectionAuditLog({ startDate: '2026-09-01' }) + expect(collection.find).toHaveBeenLastCalledWith({ + createdAt: { $gte: new Date('2026-09-01') } + }) + + await getMetadataCorrectionAuditLog({ endDate: '2026-09-03' }) + expect(collection.find).toHaveBeenLastCalledWith({ + createdAt: { $lte: new Date('2026-09-03') } + }) + }) + test('validates filters before querying DocumentDB', async () => { await expect(getMetadataCorrectionAuditLog({ status: 'unknown' })).rejects.toThrow('Invalid metadata correction audit status: unknown') @@ -153,6 +186,29 @@ describe('metadata correction audit queries', () => { startDate: 'not-a-date' })).rejects.toThrow('Invalid metadata correction audit startDate') + await expect(getMetadataCorrectionAuditLog({ + action: 'renamed' + })).rejects.toThrow('Invalid metadata correction audit action: renamed') + + await expect(getMetadataCorrectionAuditLog({ + scheme: 'not-a-scheme' + })).rejects.toThrow('Invalid metadata correction audit scheme: not-a-scheme') + + await expect(getMetadataCorrectionAuditLog({ + limit: '5000' + })).rejects.toThrow('Invalid metadata correction audit limit: expected an integer from 1 to 250') + + await expect(getMetadataCorrectionAuditLog({ + limit: '12records' + })).rejects.toThrow('Invalid metadata correction audit limit: expected an integer from 1 to 250') + + await expect(getMetadataCorrectionAuditLog({ + startDate: '2026-09-03', + endDate: '2026-09-01' + })).rejects.toThrow( + 'Invalid metadata correction audit date range: startDate must not be after endDate' + ) + await expect(getMetadataCorrectionAuditLog({ paginationToken: 'not-a-pagination-token' })).rejects.toThrow('Invalid metadata correction audit paginationToken') @@ -164,5 +220,7 @@ describe('metadata correction audit queries', () => { await expect(getMetadataCorrectionAuditLog({ paginationToken: invalidPaginationToken })).rejects.toThrow('Invalid metadata correction audit paginationToken') + + expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() }) }) diff --git a/serverless/src/shared/getMetadataCorrectionAuditLog.js b/serverless/src/shared/getMetadataCorrectionAuditLog.js index 68819a83..03e1416c 100644 --- a/serverless/src/shared/getMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/getMetadataCorrectionAuditLog.js @@ -1,27 +1,89 @@ +import { VALID_SCHEMES } from '@/shared/constants/validSchemes' import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' import { METADATA_CORRECTION_AUDIT_STATUSES } from '@/shared/persistMetadataCorrectionAuditLog' +import { CSV_FIELDS } from '@/shared/redis-path-store/helpers/constants' const DEFAULT_LIMIT = 100 const MAX_LIMIT = 250 +const VALID_AUDIT_ACTIONS = new Set([ + 'DELETED', + 'INSERTED', + 'MANUAL', + 'UPDATED' +]) +const VALID_AUDIT_SCHEMES = new Map([ + ...Object.entries(CSV_FIELDS) + .filter(([, fields]) => Array.isArray(fields)) + .map(([scheme]) => [scheme.toLowerCase(), scheme]), + ...VALID_SCHEMES.map((scheme) => [scheme.toLowerCase(), scheme]) +]) /** - * Converts the API limit to an integer within the supported page-size range. + * Validates the optional API page size. * * @example - * normalizeLimit('500') // 250 + * normalizeLimit('25') // 25 * normalizeLimit(undefined) // 100 * * @param {unknown} limit Requested page size. * @returns {number} A page size from 1 through 250. */ const normalizeLimit = (limit) => { - const parsedLimit = Number.parseInt(limit, 10) - - if (Number.isNaN(parsedLimit)) { + if (limit === undefined || limit === null) { return DEFAULT_LIMIT } - return Math.min(MAX_LIMIT, Math.max(1, parsedLimit)) + const parsedLimit = Number(limit) + + if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > MAX_LIMIT) { + throw new Error(`Invalid metadata correction audit limit: expected an integer from 1 to ${MAX_LIMIT}`) + } + + return parsedLimit +} + +/** + * Normalizes and validates a metadata-correction action filter. + * + * @example + * normalizeAction('updated') // 'UPDATED' + * + * @param {unknown} action Requested event action. + * @returns {string|undefined} Canonical action or undefined when omitted. + */ +const normalizeAction = (action) => { + if (action === undefined || action === null) return undefined + + const normalizedAction = String(action).trim().toUpperCase() + + if (!VALID_AUDIT_ACTIONS.has(normalizedAction)) { + throw new Error(`Invalid metadata correction audit action: ${action}`) + } + + return normalizedAction +} + +/** + * Validates a scheme and returns known stored spellings for case-insensitive API input. + * + * @example + * normalizeScheme('dataformat') // ['DataFormat', 'dataformat'] + * normalizeScheme('platforms') // ['platforms'] + * + * @param {unknown} scheme Requested KMS keyword scheme. + * @returns {string[]|undefined} Stored scheme spellings or undefined when omitted. + */ +const normalizeScheme = (scheme) => { + if (scheme === undefined || scheme === null) return undefined + + const normalizedScheme = String(scheme).trim().toLowerCase() + const canonicalScheme = VALID_AUDIT_SCHEMES.get(normalizedScheme) + + if (!canonicalScheme) { + throw new Error(`Invalid metadata correction audit scheme: ${scheme}`) + } + + return [...new Set([canonicalScheme, normalizedScheme])] } /** @@ -117,6 +179,7 @@ const encodePaginationToken = (document) => Buffer.from(JSON.stringify({ */ const buildAuditQuery = (filters) => { const query = {} + const matchClauses = [] const { action, collectionConceptId, @@ -131,15 +194,31 @@ const buildAuditQuery = (filters) => { } = filters if (collectionConceptId) query.collectionConceptId = collectionConceptId - if (action) query['trigger.eventType'] = action - if (keywordConceptUuid) query['corrections.keywordConceptUuid'] = keywordConceptUuid + const normalizedAction = normalizeAction(action) + if (normalizedAction) query['trigger.eventType'] = normalizedAction + if (keywordConceptUuid) { + matchClauses.push({ + $or: [ + { 'corrections.keywordConceptUuid': keywordConceptUuid }, + { 'trigger.keywordConceptUuid': keywordConceptUuid } + ] + }) + } + if (nativeFormat) query.nativeFormat = nativeFormat if (publishedVersionName) query.publishedVersionName = publishedVersionName - if (scheme) { - query.$or = [ - { 'corrections.scheme': scheme }, - { 'trigger.scheme': scheme } - ] + const normalizedSchemes = normalizeScheme(scheme) + if (normalizedSchemes) { + const schemeFilter = normalizedSchemes.length === 1 + ? normalizedSchemes[0] + : { $in: normalizedSchemes } + + matchClauses.push({ + $or: [ + { 'corrections.scheme': schemeFilter }, + { 'trigger.scheme': schemeFilter } + ] + }) } if (source) query.source = source @@ -155,6 +234,10 @@ const buildAuditQuery = (filters) => { const normalizedStartDate = normalizeDate(startDate, 'startDate') const normalizedEndDate = normalizeDate(endDate, 'endDate') + if (normalizedStartDate && normalizedEndDate && normalizedStartDate > normalizedEndDate) { + throw new Error('Invalid metadata correction audit date range: startDate must not be after endDate') + } + if (normalizedStartDate || normalizedEndDate) { query.createdAt = { ...(normalizedStartDate ? { $gte: normalizedStartDate } : {}), @@ -162,7 +245,15 @@ const buildAuditQuery = (filters) => { } } - return query + const queryClauses = [ + ...(Object.keys(query).length > 0 ? [query] : []), + ...matchClauses + ] + + if (queryClauses.length === 0) return {} + if (queryClauses.length === 1) return queryClauses[0] + + return { $and: queryClauses } } /** @@ -223,10 +314,9 @@ const normalizeAuditDocument = (document) => Object.fromEntries( * // { items: [{ runId: '...', status: 'applied', ... }], nextPaginationToken: '...' } */ export const getMetadataCorrectionAuditLog = async (filters = {}) => { - const collection = await getMetadataCorrectionAuditCollection() - const limit = normalizeLimit(filters.limit) const query = addPaginationTokenToQuery(buildAuditQuery(filters), filters.paginationToken) + const collection = await getMetadataCorrectionAuditCollection() const documents = await collection.find(query) .sort({ createdAt: -1, From 3b8dccb223f0fa208ae59dca1743c2be1c8f09ce Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 3 Sep 2026 19:11:25 -0400 Subject: [PATCH 03/17] KMS-703: Provision DocumentDB instance and retry audit index setup --- cdk/app/lib/MetadataCorrectionAuditStack.ts | 3 +- .../MetadataCorrectionAuditDatabaseSetup.ts | 12 +++- .../__tests__/handler.test.js | 69 +++++++++++++++++++ .../handler.js | 59 +++++++++++++++- 4 files changed, 139 insertions(+), 4 deletions(-) diff --git a/cdk/app/lib/MetadataCorrectionAuditStack.ts b/cdk/app/lib/MetadataCorrectionAuditStack.ts index 55eed05e..d6a418c3 100644 --- a/cdk/app/lib/MetadataCorrectionAuditStack.ts +++ b/cdk/app/lib/MetadataCorrectionAuditStack.ts @@ -83,6 +83,7 @@ export class MetadataCorrectionAuditStack extends cdk.Stack { && this.cluster && this.clientSecurityGroup && this.secret + && databaseSetup.dbInstance ) { // Define the deployment Lambda that connects to DocumentDB and creates the indexes. const indexInitializer = new NodejsFunction(this, 'IndexInitializer', { @@ -120,7 +121,7 @@ export class MetadataCorrectionAuditStack extends cdk.Stack { IndexDefinitions: METADATA_CORRECTION_AUDIT_INDEXES } }) - indexResource.node.addDependency(this.cluster) + indexResource.node.addDependency(databaseSetup.dbInstance) } } } diff --git a/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts index 3c9dae19..ce0d7756 100644 --- a/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts +++ b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts @@ -27,6 +27,8 @@ export class MetadataCorrectionAuditDatabaseSetup extends Construct { public readonly secret?: secretsmanager.ISecret + public readonly dbInstance?: docdb.DatabaseInstance + /** * Configures a local MongoDB URI for LocalStack, or provisions the deployed DocumentDB cluster, * secret, TLS environment, and paired database/client security groups. @@ -94,7 +96,7 @@ export class MetadataCorrectionAuditDatabaseSetup extends Construct { retention: cdk.Duration.days(7) }, deletionProtection: ['ops', 'prod'].includes(props.stage.toLowerCase()), - removalPolicy: cdk.RemovalPolicy.RETAIN, + removalPolicy: cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE, storageEncrypted: true, securityGroup: databaseSecurityGroup, vpc: props.vpc, @@ -103,6 +105,14 @@ export class MetadataCorrectionAuditDatabaseSetup extends Construct { } }) + // The scaling configuration defines capacity bounds; a db.serverless instance is still required. + this.dbInstance = new docdb.DatabaseInstance(this, 'DbInstance', { + cluster: this.cluster, + dbInstanceName: `${props.prefix}-${props.stage}-metadata-correction-audit-writer`, + instanceType: new ec2.InstanceType('serverless'), + removalPolicy: cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE + }) + this.secret = this.cluster.secret if (!this.secret) { diff --git a/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js index cda81208..d2dce3a6 100644 --- a/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js @@ -1,4 +1,5 @@ import { + afterEach, beforeEach, describe, expect, @@ -17,6 +18,7 @@ vi.mock('@/shared/documentDbClient', () => ({ describe('initializeMetadataCorrectionAudit', () => { const createIndexes = vi.fn() const consoleLog = vi.spyOn(console, 'log').mockImplementation(() => {}) + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => {}) const indexDefinitions = [ { key: { @@ -33,6 +35,10 @@ describe('initializeMetadataCorrectionAudit', () => { vi.mocked(getMetadataCorrectionAuditCollection).mockResolvedValue({ createIndexes }) }) + afterEach(() => { + vi.useRealTimers() + }) + test.each(['Create', 'Update'])('creates indexes for a %s deployment event', async (requestType) => { const result = await initializeMetadataCorrectionAudit({ RequestType: requestType, @@ -69,4 +75,67 @@ describe('initializeMetadataCorrectionAudit', () => { expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() }) + + test('retries while the new DocumentDB endpoint is not resolvable', async () => { + vi.useFakeTimers() + + vi.mocked(getMetadataCorrectionAuditCollection) + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND audit.cluster.example')) + .mockResolvedValue({ createIndexes }) + + const resultPromise = initializeMetadataCorrectionAudit({ + RequestType: 'Create', + ResourceProperties: { IndexDefinitions: indexDefinitions } + }) + + await vi.advanceTimersByTimeAsync(5_000) + + await expect(resultPromise).resolves.toEqual({ + PhysicalResourceId: 'metadata-correction-audit-indexes', + Data: { IndexCount: 1 } + }) + + expect(getMetadataCorrectionAuditCollection).toHaveBeenCalledTimes(2) + expect(consoleWarn).toHaveBeenCalledWith( + 'DocumentDB endpoint is not ready; retrying audit index creation', + { + attempt: 1, + error: 'Error: getaddrinfo ENOTFOUND audit.cluster.example' + } + ) + }) + + test('does not retry non-connection failures', async () => { + vi.mocked(getMetadataCorrectionAuditCollection) + .mockRejectedValue(new Error('DocumentDB secret is missing username')) + + await expect(initializeMetadataCorrectionAudit({ + RequestType: 'Create', + ResourceProperties: { IndexDefinitions: indexDefinitions } + })).rejects.toThrow('DocumentDB secret is missing username') + + expect(getMetadataCorrectionAuditCollection).toHaveBeenCalledTimes(1) + expect(consoleWarn).not.toHaveBeenCalled() + }) + + test('stops retrying when the DocumentDB readiness window expires', async () => { + vi.useFakeTimers() + + vi.mocked(getMetadataCorrectionAuditCollection) + .mockRejectedValue(new Error('connect ETIMEDOUT audit.cluster.example')) + + const resultPromise = initializeMetadataCorrectionAudit({ + RequestType: 'Create', + ResourceProperties: { IndexDefinitions: indexDefinitions } + }) + const rejection = expect(resultPromise).rejects.toThrow( + 'connect ETIMEDOUT audit.cluster.example' + ) + + await vi.runAllTimersAsync() + await rejection + + expect(getMetadataCorrectionAuditCollection).toHaveBeenCalledTimes(24) + expect(consoleWarn).toHaveBeenCalledTimes(23) + }) }) diff --git a/serverless/src/initializeMetadataCorrectionAudit/handler.js b/serverless/src/initializeMetadataCorrectionAudit/handler.js index e4f26dd8..4d8c0931 100644 --- a/serverless/src/initializeMetadataCorrectionAudit/handler.js +++ b/serverless/src/initializeMetadataCorrectionAudit/handler.js @@ -1,6 +1,62 @@ import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' const PHYSICAL_RESOURCE_ID = 'metadata-correction-audit-indexes' +const INDEX_CREATION_MAX_ATTEMPTS = 24 +const INDEX_CREATION_RETRY_DELAY_MS = 5_000 +const RETRYABLE_CONNECTION_ERROR_CODES = [ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ENOTFOUND', + 'ETIMEDOUT' +] + +/** + * Returns whether DocumentDB is still becoming reachable after cluster creation. + * + * @example + * isRetryableConnectionError(new Error('getaddrinfo ENOTFOUND cluster.example')) // true + * + * @param {unknown} error Connection error from the MongoDB driver. + * @returns {boolean} Whether retrying may succeed once the endpoint is ready. + */ +const isRetryableConnectionError = (error) => { + const errorMessage = String(error) + + return RETRYABLE_CONNECTION_ERROR_CODES.some((code) => errorMessage.includes(code)) +} + +/** + * Creates the configured indexes, retrying while a new DocumentDB endpoint becomes reachable. + * + * @param {Array} indexDefinitions MongoDB index definitions. + * @param {number} attempt Current connection attempt. + * @returns {Promise>} Names returned by MongoDB for the created indexes. + */ +const createAuditIndexes = async (indexDefinitions, attempt = 1) => { + try { + const collection = await getMetadataCorrectionAuditCollection() + + return await collection.createIndexes(indexDefinitions) + } catch (error) { + if ( + !isRetryableConnectionError(error) + || attempt >= INDEX_CREATION_MAX_ATTEMPTS + ) { + throw error + } + + console.warn('DocumentDB endpoint is not ready; retrying audit index creation', { + attempt, + error: String(error) + }) + + await new Promise((resolve) => { + setTimeout(resolve, INDEX_CREATION_RETRY_DELAY_MS) + }) + + return createAuditIndexes(indexDefinitions, attempt + 1) + } +} /** * Creates the metadata-correction audit indexes during CloudFormation deployment. @@ -33,8 +89,7 @@ export const initializeMetadataCorrectionAudit = async (event) => { throw new Error('Metadata correction audit index definitions are required') } - const collection = await getMetadataCorrectionAuditCollection() - const indexNames = await collection.createIndexes(indexDefinitions) + const indexNames = await createAuditIndexes(indexDefinitions) console.log('Metadata correction audit indexes are ready', { indexNames }) return { From d3b80eb582e4f73de3a4c4808f840ff5105b4a33 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 3 Sep 2026 20:02:52 -0400 Subject: [PATCH 04/17] KMS-703: Preserve numeric index directions during deployment --- cdk/app/lib/MetadataCorrectionAuditStack.ts | 3 ++- .../__tests__/handler.test.js | 3 ++- .../src/initializeMetadataCorrectionAudit/handler.js | 7 +++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cdk/app/lib/MetadataCorrectionAuditStack.ts b/cdk/app/lib/MetadataCorrectionAuditStack.ts index d6a418c3..83becdbe 100644 --- a/cdk/app/lib/MetadataCorrectionAuditStack.ts +++ b/cdk/app/lib/MetadataCorrectionAuditStack.ts @@ -118,7 +118,8 @@ export class MetadataCorrectionAuditStack extends cdk.Stack { serviceToken: indexProvider.serviceToken, properties: { ClusterEndpoint: this.cluster.clusterEndpoint.hostname, - IndexDefinitions: METADATA_CORRECTION_AUDIT_INDEXES + // Preserve numeric index directions across the string-based custom-resource boundary. + IndexDefinitions: JSON.stringify(METADATA_CORRECTION_AUDIT_INDEXES) } }) indexResource.node.addDependency(databaseSetup.dbInstance) diff --git a/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js index d2dce3a6..e54676bb 100644 --- a/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/initializeMetadataCorrectionAudit/__tests__/handler.test.js @@ -42,10 +42,11 @@ describe('initializeMetadataCorrectionAudit', () => { test.each(['Create', 'Update'])('creates indexes for a %s deployment event', async (requestType) => { const result = await initializeMetadataCorrectionAudit({ RequestType: requestType, - ResourceProperties: { IndexDefinitions: indexDefinitions } + ResourceProperties: { IndexDefinitions: JSON.stringify(indexDefinitions) } }) expect(createIndexes).toHaveBeenCalledWith(indexDefinitions) + expect(createIndexes.mock.calls[0][0][0].key.createdAt).toBeTypeOf('number') expect(consoleLog).toHaveBeenCalledWith( 'Metadata correction audit indexes are ready', { indexNames: ['createdAt_desc'] } diff --git a/serverless/src/initializeMetadataCorrectionAudit/handler.js b/serverless/src/initializeMetadataCorrectionAudit/handler.js index 4d8c0931..7e9cf562 100644 --- a/serverless/src/initializeMetadataCorrectionAudit/handler.js +++ b/serverless/src/initializeMetadataCorrectionAudit/handler.js @@ -66,7 +66,7 @@ const createAuditIndexes = async (indexDefinitions, attempt = 1) => { * await initializeMetadataCorrectionAudit({ * RequestType: 'Create', * ResourceProperties: { - * IndexDefinitions: [{ key: { createdAt: -1 }, name: 'createdAt_desc' }] + * IndexDefinitions: '[{"key":{"createdAt":-1},"name":"createdAt_desc"}]' * } * }) * // { PhysicalResourceId: 'metadata-correction-audit-indexes', Data: { IndexCount: 1 } } @@ -83,7 +83,10 @@ export const initializeMetadataCorrectionAudit = async (event) => { return { PhysicalResourceId: physicalResourceId } } - const indexDefinitions = event.ResourceProperties?.IndexDefinitions + const indexDefinitionsProperty = event.ResourceProperties?.IndexDefinitions + const indexDefinitions = typeof indexDefinitionsProperty === 'string' + ? JSON.parse(indexDefinitionsProperty) + : indexDefinitionsProperty if (!Array.isArray(indexDefinitions) || indexDefinitions.length === 0) { throw new Error('Metadata correction audit index definitions are required') From c5afef0ee8ca57191f2e18789672ca7198c4091e Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Mon, 7 Sep 2026 19:00:23 -0400 Subject: [PATCH 05/17] KMS-703: add native metadata diffs, concise audit responses, and writeback rate limiting --- README.md | 18 +- bin/deploy-bamboo.sh | 2 + cdk/app/lib/CmrEventProcessingStack.ts | 4 + cdk/app/lib/KmsStack.ts | 2 + cdk/app/lib/helper/KmsLambdaFunctions.ts | 14 ++ cdk/app/lib/helper/MetadataCorrectionSetup.ts | 31 ++- cdk/bin/main.ts | 4 + package-lock.json | 3 +- package.json | 1 + ...etadata_correction_applied_audit_smoke.mjs | 26 +- ...metadata_correction_failed_audit_smoke.mjs | 23 +- ...etadata_correction_request_delay_smoke.mjs | 25 +- .../run_metadata_correction_sync_smoke.mjs | 25 +- .../getCapabilities/__tests__/handler.test.js | 1 + serverless/src/getCapabilities/handler.js | 8 + .../__tests__/handler.test.js | 74 +++++- .../src/getMetadataCorrectionAudit/handler.js | 30 ++- .../__tests__/handler.test.js | 45 ++++ .../src/metadataCorrectionService/handler.js | 49 ++++ .../__tests__/buildNativeMetadataDiff.test.js | 63 +++++ .../shared/__tests__/cmrPutRequest.test.js | 51 +++- .../getMetadataCorrectionAuditLog.test.js | 227 +++++++++++++----- .../persistMetadataCorrectionAuditLog.test.js | 12 + .../runCollectionMetadataCorrection.test.js | 49 ++-- .../writeCorrectedMetadataToCmr.test.js | 30 ++- .../src/shared/buildNativeMetadataDiff.js | 81 +++++++ serverless/src/shared/cmrPutRequest.js | 27 ++- .../shared/getMetadataCorrectionAuditLog.js | 114 ++++++++- .../persistMetadataCorrectionAuditLog.js | 2 + .../shared/runCollectionMetadataCorrection.js | 15 ++ .../src/shared/writeCorrectedMetadataToCmr.js | 28 ++- 31 files changed, 962 insertions(+), 122 deletions(-) create mode 100644 serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js create mode 100644 serverless/src/shared/buildNativeMetadataDiff.js diff --git a/README.md b/README.md index d8d57a52..b00175fd 100644 --- a/README.md +++ b/README.md @@ -449,13 +449,19 @@ runtime correction and delegate flow works from normalized keyword objects. Each collection-correction run is stored as one audit document. Its `statusHistory` records the `checked`, `pending`, and terminal `applied` or `failed` transitions. The audit document also -links to the current CMR collection record and records the prior and resulting CMR revision IDs. +links to the current CMR collection record, records the prior and resulting CMR revision IDs, and +stores a bounded unified diff between the original native metadata and the corrected writeback +payload. The audit API is: - `GET /metadata_correction_audit` for newest-first, token-paginated audit searches. Supported filters include collection, keyword UUID, action, scheme, status, native format, KMS version, source, and date range. Supplied actions and schemes must be recognized KMS values, limits must - be integers from 1 through 250, and `startDate` must not be after `endDate`. + be integers from 1 through 250, and `startDate` must not be after `endDate`. List results contain + compact collection, status, and old-to-new keyword path summaries. +- `GET /metadata_correction_audit/{runId}` for the complete audit document. Add + `?includeDiff=true` when the native-metadata diff is needed; it is omitted by default to keep + routine responses small. Publisher events carry the published KMS version through the queue into this document. Manual correction endpoints look up the current published version before starting the run, so the @@ -536,6 +542,9 @@ export bamboo_CMR_WRITER_TOKEN=[optional complete bearer authorization value use export bamboo_CMR_WRITEBACK_PROVIDERS=[optional provider id, comma-separated list, or ALL] export bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS=[true|false; defaults to false] export bamboo_CMR_WRITEBACK_VALIDATE_UMM_C=[true|false; defaults to false] +export bamboo_CMR_WRITEBACK_TIMEOUT_MS=[optional timeout in milliseconds; defaults to 25000] +export bamboo_METADATA_CORRECTION_RUNS_PER_MINUTE=[optional positive integer correction run rate] +export bamboo_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY=[optional; defaults to 5] export bamboo_CORS_ORIGIN=[comma separated list of cors origins] export bamboo_RDF4J_CONTAINER_MEMORY_LIMIT=[7168 for sit|uat, 14336 for prod] export bamboo_RDF4J_INSTANCE_TYPE=["M5.LARGE" for sit|uat, "R5.LARGE" for prod] @@ -558,6 +567,11 @@ Notes: - Leave `bamboo_CMR_WRITEBACK_PROVIDERS` empty to disable provider rollout for CMR writeback. - Set `bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS` and `bamboo_CMR_WRITEBACK_VALIDATE_UMM_C` to `true` to reject writebacks that still fail CMR keyword or UMM-C validation. +- `bamboo_CMR_WRITEBACK_TIMEOUT_MS` is capped at 45000 milliseconds so the worker can record a + failed audit before its 60-second Lambda timeout. +- Setting `bamboo_METADATA_CORRECTION_RUNS_PER_MINUTE` enables queue pacing and forces the + metadata-correction worker concurrency to `1`. When it is unset, pacing is disabled and + `bamboo_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY` controls concurrency. - If you are not deploying into an existing API Gateway, set `bamboo_EXISTING_API_ID` and `bamboo_ROOT_RESOURCE_ID` to empty strings. - If `bamboo_RDF4J_BACKUP_VAULT_NAME` is set, `SnapshotStack` imports that existing backup vault. This is useful when `rdf4jSnapshotStack` is being recreated after an RDF4J recovery event and you need the new stack to reuse an existing vault instead of trying to create the same vault name again. - If `bamboo_RDF4J_BACKUP_VAULT_NAME` is not set, `SnapshotStack` creates the default `rdf4j-backup-vault`. diff --git a/bin/deploy-bamboo.sh b/bin/deploy-bamboo.sh index cf8924eb..fe1e3c81 100755 --- a/bin/deploy-bamboo.sh +++ b/bin/deploy-bamboo.sh @@ -70,6 +70,8 @@ dockerRun() { --env "CMR_WRITEBACK_PROVIDERS=${bamboo_CMR_WRITEBACK_PROVIDERS:-}" \ --env "CMR_WRITEBACK_VALIDATE_KEYWORDS=${bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS:-false}" \ --env "CMR_WRITEBACK_VALIDATE_UMM_C=${bamboo_CMR_WRITEBACK_VALIDATE_UMM_C:-false}" \ + --env "CMR_WRITEBACK_TIMEOUT_MS=${bamboo_CMR_WRITEBACK_TIMEOUT_MS:-25000}" \ + --env "METADATA_CORRECTION_RUNS_PER_MINUTE=${bamboo_METADATA_CORRECTION_RUNS_PER_MINUTE:-}" \ --env "METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY=${bamboo_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY:-5}" \ --env "BLOCK_PUBLISH_ON_KEYWORD_DIFF_FAILURE=${bamboo_BLOCK_PUBLISH_ON_KEYWORD_DIFF_FAILURE:-false}" \ --env "KEYWORD_SYNC_ALARM_EMAILS=${bamboo_KEYWORD_SYNC_ALARM_EMAILS:-}" \ diff --git a/cdk/app/lib/CmrEventProcessingStack.ts b/cdk/app/lib/CmrEventProcessingStack.ts index b830b55b..3168c557 100644 --- a/cdk/app/lib/CmrEventProcessingStack.ts +++ b/cdk/app/lib/CmrEventProcessingStack.ts @@ -15,7 +15,9 @@ import { VpcSetup } from './helper/VpcSetup' export interface CmrEventProcessingStackProps extends cdk.StackProps { cmrBaseUrl: string cmrSystemTokenParameterName?: string + cmrWritebackTimeoutMs?: string metadataCorrectionRequestDelayMs?: string + metadataCorrectionRunsPerMinute?: string metadataCorrectionServiceReservedConcurrency?: string cmrWriterToken?: string cmrWritebackProviders?: string @@ -67,7 +69,9 @@ export class CmrEventProcessingStack extends cdk.Stack { const metadataCorrectionSetup = new MetadataCorrectionSetup(this, 'MetadataCorrection', { cmrBaseUrl: props.cmrBaseUrl, cmrSystemTokenParameterName: props.cmrSystemTokenParameterName, + cmrWritebackTimeoutMs: props.cmrWritebackTimeoutMs, metadataCorrectionRequestDelayMs: props.metadataCorrectionRequestDelayMs, + metadataCorrectionRunsPerMinute: props.metadataCorrectionRunsPerMinute, metadataCorrectionServiceReservedConcurrency: props.metadataCorrectionServiceReservedConcurrency, cmrWriterToken: props.cmrWriterToken, diff --git a/cdk/app/lib/KmsStack.ts b/cdk/app/lib/KmsStack.ts index c2775c9e..c44d814c 100644 --- a/cdk/app/lib/KmsStack.ts +++ b/cdk/app/lib/KmsStack.ts @@ -20,6 +20,7 @@ import { VpcSetup } from './helper/VpcSetup' */ export interface KmsStackProps extends cdk.StackProps { cmrSystemTokenParameterName?: string + cmrWritebackTimeoutMs?: string cmrWritebackProviders?: string cmrWritebackValidateKeywords?: string cmrWritebackValidateUmmC?: string @@ -181,6 +182,7 @@ export class KmsStack extends cdk.Stack { lambdaRole: this.lambdaRole, metadataCorrectionEnvironment: { CMR_SYSTEM_TOKEN_PARAMETER_NAME: props.cmrSystemTokenParameterName || '', + CMR_WRITEBACK_TIMEOUT_MS: props.cmrWritebackTimeoutMs || '', CMR_WRITER_TOKEN: props.cmrWriterToken || '', CMR_WRITEBACK_PROVIDERS: props.cmrWritebackProviders || '', CMR_WRITEBACK_VALIDATE_KEYWORDS: props.cmrWritebackValidateKeywords || '', diff --git a/cdk/app/lib/helper/KmsLambdaFunctions.ts b/cdk/app/lib/helper/KmsLambdaFunctions.ts index d11682d2..923bba38 100644 --- a/cdk/app/lib/helper/KmsLambdaFunctions.ts +++ b/cdk/app/lib/helper/KmsLambdaFunctions.ts @@ -27,6 +27,7 @@ interface LambdaFunctionsProps { metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup; metadataCorrectionEnvironment?: { CMR_SYSTEM_TOKEN_PARAMETER_NAME?: string; + CMR_WRITEBACK_TIMEOUT_MS?: string; CMR_WRITER_TOKEN: string; CMR_WRITEBACK_PROVIDERS: string; CMR_WRITEBACK_VALIDATE_KEYWORDS: string; @@ -352,6 +353,19 @@ export class LambdaFunctions { this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables ) + this.createApiLambda( + scope, // CDK construct scope + 'getMetadataCorrectionAudit/handler.js', // Lambda handler path + 'get-metadata-correction-audit', // Reuse the audit Lambda + 'getMetadataCorrectionAudit', // Exported handler name + '/metadata_correction_audit/{runId}', // Detailed audit resource path + 'GET', // HTTP method + false, // Do not use the EDL authorizer + Duration.seconds(30), // Lambda timeout + 1024, // Lambda memory in MB + this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables + ) + this.createApiLambda( scope, 'getKeywordFullPathHistory/handler.js', diff --git a/cdk/app/lib/helper/MetadataCorrectionSetup.ts b/cdk/app/lib/helper/MetadataCorrectionSetup.ts index 39c56924..61f2d42c 100644 --- a/cdk/app/lib/helper/MetadataCorrectionSetup.ts +++ b/cdk/app/lib/helper/MetadataCorrectionSetup.ts @@ -23,7 +23,9 @@ import { NODE_LAMBDA_RUNTIME } from './NodeLambdaRuntime' interface MetadataCorrectionSetupProps { cmrBaseUrl: string cmrSystemTokenParameterName?: string + cmrWritebackTimeoutMs?: string metadataCorrectionRequestDelayMs?: string + metadataCorrectionRunsPerMinute?: string metadataCorrectionServiceReservedConcurrency?: string cmrWriterToken?: string cmrWritebackProviders?: string @@ -77,7 +79,9 @@ export class MetadataCorrectionSetup extends Construct { const { cmrBaseUrl, cmrSystemTokenParameterName, + cmrWritebackTimeoutMs, metadataCorrectionRequestDelayMs, + metadataCorrectionRunsPerMinute, metadataCorrectionServiceReservedConcurrency, cmrWriterToken, cmrWritebackProviders, @@ -99,12 +103,27 @@ export class MetadataCorrectionSetup extends Construct { const metadataCorrectionRequestsBaseName = `${prefix}-${stage}-metadata-correction-requests` const metadataCorrectionRequestsName = `${metadataCorrectionRequestsBaseName}.fifo` const projectRoot = path.join(__dirname, '../../../..') + const configuredRate = String(metadataCorrectionRunsPerMinute || '').trim() + const parsedRate = Number(configuredRate) + const hasRateLimit = configuredRate.length > 0 + + if (hasRateLimit && (!Number.isInteger(parsedRate) || parsedRate <= 0)) { + throw new Error('METADATA_CORRECTION_RUNS_PER_MINUTE must be a positive integer') + } + const parsedReservedConcurrency = Number(metadataCorrectionServiceReservedConcurrency) const hasValidReservedConcurrency = Number.isInteger(parsedReservedConcurrency) && parsedReservedConcurrency > 0 - const reservedConcurrency = hasValidReservedConcurrency - ? parsedReservedConcurrency - : MetadataCorrectionSetup.DEFAULT_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY + let reservedConcurrency = MetadataCorrectionSetup + .DEFAULT_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY + + if (hasValidReservedConcurrency) { + reservedConcurrency = parsedReservedConcurrency + } + + if (hasRateLimit) { + reservedConcurrency = 1 + } // TODO: Create a follow-up ticket for DLQ handling. This DLQ is only the // redrive target today; before adding a consumer, decide whether failures @@ -149,13 +168,14 @@ export class MetadataCorrectionSetup extends Construct { entry: path.join(projectRoot, 'serverless/src/metadataCorrectionService/handler.js'), handler: 'metadataCorrectionService', runtime: NODE_LAMBDA_RUNTIME, - timeout: cdk.Duration.seconds(30), + timeout: cdk.Duration.seconds(60), memorySize: 1024, // Broad keyword updates can fan out to hundreds of collections; cap concurrent // consumers so writebacks do not overwhelm downstream CMR ingest. reservedConcurrentExecutions: reservedConcurrency, environment: { CMR_BASE_URL: cmrBaseUrl, + ...(cmrWritebackTimeoutMs ? { CMR_WRITEBACK_TIMEOUT_MS: cmrWritebackTimeoutMs } : {}), ...(cmrSystemTokenParameterName ? { CMR_SYSTEM_TOKEN_PARAMETER_NAME: cmrSystemTokenParameterName } : {}), @@ -173,6 +193,9 @@ export class MetadataCorrectionSetup extends Construct { ...(metadataCorrectionRequestDelayMs ? { METADATA_CORRECTION_REQUEST_DELAY_MS: metadataCorrectionRequestDelayMs } : {}), + ...(hasRateLimit + ? { METADATA_CORRECTION_RUNS_PER_MINUTE: String(parsedRate) } + : {}), ...metadataCorrectionAuditEnvironment }, ...getDocumentDbCertificateBundling(metadataCorrectionAuditEnvironment), diff --git a/cdk/bin/main.ts b/cdk/bin/main.ts index b31de3d6..ddf9d408 100644 --- a/cdk/bin/main.ts +++ b/cdk/bin/main.ts @@ -203,6 +203,7 @@ async function main() { // Create KmsStack const kmsStackProps: KmsStackProps = { cmrSystemTokenParameterName: process.env.CMR_SYSTEM_TOKEN_PARAMETER_NAME || '', + cmrWritebackTimeoutMs: process.env.CMR_WRITEBACK_TIMEOUT_MS || '', cmrWriterToken: process.env.CMR_WRITER_TOKEN || '', cmrWritebackProviders: process.env.CMR_WRITEBACK_PROVIDERS || '', cmrWritebackValidateKeywords: process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS || '', @@ -258,7 +259,10 @@ async function main() { const cmrEventProcessingStack = new CmrEventProcessingStack(app, 'CmrEventProcessingStack', { cmrBaseUrl, cmrSystemTokenParameterName: process.env.CMR_SYSTEM_TOKEN_PARAMETER_NAME || '', + cmrWritebackTimeoutMs: process.env.CMR_WRITEBACK_TIMEOUT_MS || '', metadataCorrectionRequestDelayMs: process.env.METADATA_CORRECTION_REQUEST_DELAY_MS || '', + metadataCorrectionRunsPerMinute: + process.env.METADATA_CORRECTION_RUNS_PER_MINUTE || '', metadataCorrectionServiceReservedConcurrency: process.env.METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY || '', cmrWriterToken: process.env.CMR_WRITER_TOKEN || '', diff --git a/package-lock.json b/package-lock.json index a27400e2..763a964f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "compact-object-deep": "^1.0.0", "csv": "^6.3.11", "date-fns": "^4.1.0", + "diff": "^5.2.2", "fast-xml-parser": "^5.3.4", "html-entities": "^2.5.2", "html-escaper": "^3.0.3", @@ -7941,7 +7942,7 @@ "version": "5.2.2", "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", - "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } diff --git a/package.json b/package.json index c33d7a7c..8c1f4ce7 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "compact-object-deep": "^1.0.0", "csv": "^6.3.11", "date-fns": "^4.1.0", + "diff": "^5.2.2", "fast-xml-parser": "^5.3.4", "html-entities": "^2.5.2", "html-escaper": "^3.0.3", diff --git a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs index 75195d9b..2a7e3716 100644 --- a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs @@ -195,7 +195,10 @@ try { await clearAuditRowsForCollection() const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') - const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') + const { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog + } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') const { items: beforeRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, @@ -220,9 +223,16 @@ try { collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.flatMap((row) => ( - row.statusHistory?.map(({ status }) => status) || [row.status] - )))] + const appliedSummary = afterRows.find(({ status }) => status === 'applied') + const appliedRow = appliedSummary + ? await getMetadataCorrectionAuditByRunId({ + runId: appliedSummary.runId, + includeDiff: true + }) + : null + const statuses = [...new Set( + appliedRow?.statusHistory?.map(({ status }) => status) || [] + )] if (beforeRows.length !== 0) { throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) @@ -236,7 +246,6 @@ try { throw new Error(`Missing applied audit status for ${collectionConceptId}`) } - const appliedRow = afterRows.find(({ status }) => status === 'applied') if (appliedRow?.publishedVersionName !== 'local-published') { throw new Error(`Missing published KMS version for ${collectionConceptId}`) } @@ -250,6 +259,10 @@ try { throw new Error(`Missing the expected CMR collection URI for ${collectionConceptId}`) } + if (appliedRow.metadataDiff?.changed !== true || !appliedRow.metadataDiff.patch) { + throw new Error(`Missing the native metadata diff for ${collectionConceptId}`) + } + await fs.mkdir(outputDir, { recursive: true }) await fs.writeFile(outputPath, JSON.stringify({ collectionConceptId, @@ -258,7 +271,8 @@ try { beforeCount: beforeRows.length, afterCount: afterRows.length, statuses, - rows: afterRows + rows: afterRows, + appliedRow }, null, 2), 'utf8') console.log('[metadata-correction-audit-smoke] Completed successfully') diff --git a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs index 84a58f2f..e80c8344 100644 --- a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs @@ -203,7 +203,10 @@ try { await clearAuditRowsForCollection() const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') - const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') + const { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog + } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') const { items: beforeRows } = await getMetadataCorrectionAuditLog({ collectionConceptId, @@ -227,10 +230,16 @@ try { collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.flatMap((row) => ( - row.statusHistory?.map(({ status }) => status) || [row.status] - )))] - const failedRow = afterRows.find((row) => row.status === 'failed') + const failedSummary = afterRows.find(({ status }) => status === 'failed') + const failedRow = failedSummary + ? await getMetadataCorrectionAuditByRunId({ + runId: failedSummary.runId, + includeDiff: true + }) + : null + const statuses = [...new Set( + failedRow?.statusHistory?.map(({ status }) => status) || [] + )] if (beforeRows.length !== 0) { throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) @@ -268,6 +277,10 @@ try { ) } + if (failedRow.metadataDiff?.changed !== true || !failedRow.metadataDiff.patch) { + throw new Error(`Missing the native metadata diff for failed run ${collectionConceptId}`) + } + await fs.mkdir(outputDir, { recursive: true }) await fs.writeFile(outputPath, JSON.stringify({ collectionConceptId, diff --git a/scripts/local/run_metadata_correction_request_delay_smoke.mjs b/scripts/local/run_metadata_correction_request_delay_smoke.mjs index c5700933..a99e95f5 100644 --- a/scripts/local/run_metadata_correction_request_delay_smoke.mjs +++ b/scripts/local/run_metadata_correction_request_delay_smoke.mjs @@ -230,7 +230,10 @@ try { await clearAuditRowsForCollection() const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') - const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') + const { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog + } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') const { getCmrCollectionNativeMetadata } = await import('../../serverless/src/shared/getCmrCollectionNativeMetadata') const requestedAt = new Date().toISOString() @@ -294,9 +297,16 @@ try { collectionConceptId, limit: 20 }) - const statuses = [...new Set(auditRows.flatMap((row) => ( - row.statusHistory?.map(({ status }) => status) || [row.status] - )))] + const appliedSummary = auditRows.find(({ status }) => status === 'applied') + const appliedRow = appliedSummary + ? await getMetadataCorrectionAuditByRunId({ + runId: appliedSummary.runId, + includeDiff: true + }) + : null + const statuses = [...new Set( + appliedRow?.statusHistory?.map(({ status }) => status) || [] + )] if (!statuses.includes('pending')) { throw new Error(`Missing pending audit status for ${collectionConceptId}`) @@ -306,6 +316,10 @@ try { throw new Error(`Missing applied audit status for ${collectionConceptId}`) } + if (appliedRow?.metadataDiff?.changed !== true || !appliedRow.metadataDiff.patch) { + throw new Error(`Missing the native metadata diff for ${collectionConceptId}`) + } + await fs.mkdir(outputDir, { recursive: true }) await fs.writeFile(outputPath, JSON.stringify({ collectionConceptId, @@ -320,7 +334,8 @@ try { statuses, updatedPlatform, response, - rows: auditRows + rows: auditRows, + appliedRow }, null, 2), 'utf8') console.log('[metadata-correction-request-delay-smoke] Completed successfully') diff --git a/scripts/local/run_metadata_correction_sync_smoke.mjs b/scripts/local/run_metadata_correction_sync_smoke.mjs index 6d61a5e9..f10caff8 100644 --- a/scripts/local/run_metadata_correction_sync_smoke.mjs +++ b/scripts/local/run_metadata_correction_sync_smoke.mjs @@ -224,7 +224,10 @@ try { await clearAuditRowsForCollection() const { runMetadataCorrection } = await import('../../serverless/src/runMetadataCorrection/handler') - const { getMetadataCorrectionAuditLog } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') + const { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog + } = await import('../../serverless/src/shared/getMetadataCorrectionAuditLog') const { getCmrCollectionNativeMetadata } = await import('../../serverless/src/shared/getCmrCollectionNativeMetadata') const { items: beforeRows } = await getMetadataCorrectionAuditLog({ @@ -326,9 +329,16 @@ try { collectionConceptId, limit: 20 }) - const statuses = [...new Set(afterRows.flatMap((row) => ( - row.statusHistory?.map(({ status }) => status) || [row.status] - )))] + const appliedSummary = afterRows.find(({ status }) => status === 'applied') + const appliedRow = appliedSummary + ? await getMetadataCorrectionAuditByRunId({ + runId: appliedSummary.runId, + includeDiff: true + }) + : null + const statuses = [...new Set( + appliedRow?.statusHistory?.map(({ status }) => status) || [] + )] if (beforeRows.length !== 0) { throw new Error(`Expected no starting audit documents for ${collectionConceptId}, found ${beforeRows.length}`) @@ -342,6 +352,10 @@ try { throw new Error(`Missing applied audit status for ${collectionConceptId}`) } + if (appliedRow?.metadataDiff?.changed !== true || !appliedRow.metadataDiff.patch) { + throw new Error(`Missing the native metadata diff for ${collectionConceptId}`) + } + await fs.mkdir(outputDir, { recursive: true }) await fs.writeFile(outputPath, JSON.stringify({ collectionConceptId, @@ -352,7 +366,8 @@ try { beforeCount: beforeRows.length, afterCount: afterRows.length, statuses, - rows: afterRows + rows: afterRows, + appliedRow }, null, 2), 'utf8') console.log('[metadata-correction-sync-smoke] Completed successfully') diff --git a/serverless/src/getCapabilities/__tests__/handler.test.js b/serverless/src/getCapabilities/__tests__/handler.test.js index 25a2650a..9c2913be 100644 --- a/serverless/src/getCapabilities/__tests__/handler.test.js +++ b/serverless/src/getCapabilities/__tests__/handler.test.js @@ -53,6 +53,7 @@ describe('getCapabilities', () => { expect(result.body).toContain(' { action: 'GET' } }, + { + ':@': { + name: 'get_metadata_correction_audit_run', + href: '/metadata_correction_audit/{runId}', + params: 'includeDiff=', + action: 'GET' + } + }, { ':@': { name: 'get_concept_versions', diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index 38c348aa..bc9278d8 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -6,7 +6,10 @@ import { vi } from 'vitest' -import { getMetadataCorrectionAuditLog } from '@/shared/getMetadataCorrectionAuditLog' +import { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog +} from '@/shared/getMetadataCorrectionAuditLog' import { getMetadataCorrectionAudit } from '../handler' @@ -17,6 +20,7 @@ vi.mock('@/shared/getConfig', () => ({ })) vi.mock('@/shared/getMetadataCorrectionAuditLog', () => ({ + getMetadataCorrectionAuditByRunId: vi.fn(), getMetadataCorrectionAuditLog: vi.fn() })) @@ -42,9 +46,12 @@ describe('getMetadataCorrectionAudit', () => { collectionConceptId: 'C1234567890-LOCAL', collectionUri: 'https://cmr.example.com/search/concepts/C1234567890-LOCAL', status: 'applied', - trigger: { - eventType: 'UPDATED' - } + changes: [{ + scheme: 'platforms', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + hasMetadataDiff: true }], nextPaginationToken: null }) @@ -81,9 +88,12 @@ describe('getMetadataCorrectionAudit', () => { collectionConceptId: 'C1234567890-LOCAL', collectionUri: 'https://cmr.example.com/search/concepts/C1234567890-LOCAL', status: 'applied', - trigger: { - eventType: 'UPDATED' - } + changes: [{ + scheme: 'platforms', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + hasMetadataDiff: true } ], nextPaginationToken: null @@ -111,4 +121,54 @@ describe('getMetadataCorrectionAudit', () => { expect(result.statusCode).toBe(400) }) + + test('returns one detailed audit document with its native metadata diff', async () => { + vi.mocked(getMetadataCorrectionAuditByRunId).mockResolvedValue({ + runId: 'run-1', + status: 'applied', + metadataDiff: { + changed: true, + patch: '-old\n+new' + } + }) + + const result = await getMetadataCorrectionAudit({ + pathParameters: { runId: 'run-1' }, + queryStringParameters: { includeDiff: 'true' } + }) + + expect(getMetadataCorrectionAuditByRunId).toHaveBeenCalledWith({ + runId: 'run-1', + includeDiff: 'true' + }) + expect(getMetadataCorrectionAuditLog).not.toHaveBeenCalled() + expect(result.statusCode).toBe(200) + expect(JSON.parse(result.body).metadataDiff.patch).toBe('-old\n+new') + }) + + test('returns 404 when a detailed audit run does not exist', async () => { + vi.mocked(getMetadataCorrectionAuditByRunId).mockResolvedValue(null) + + const result = await getMetadataCorrectionAudit({ + pathParameters: { runId: 'missing-run' } + }) + + expect(result.statusCode).toBe(404) + expect(JSON.parse(result.body)).toEqual({ + error: 'Metadata correction audit run not found: missing-run' + }) + }) + + test('returns 400 when the detail diff flag is invalid', async () => { + vi.mocked(getMetadataCorrectionAuditByRunId).mockRejectedValue( + new Error('Invalid metadata correction audit includeDiff: expected true or false') + ) + + const result = await getMetadataCorrectionAudit({ + pathParameters: { runId: 'run-1' }, + queryStringParameters: { includeDiff: 'yes' } + }) + + expect(result.statusCode).toBe(400) + }) }) diff --git a/serverless/src/getMetadataCorrectionAudit/handler.js b/serverless/src/getMetadataCorrectionAudit/handler.js index e2b6e3f9..2a3cb07d 100644 --- a/serverless/src/getMetadataCorrectionAudit/handler.js +++ b/serverless/src/getMetadataCorrectionAudit/handler.js @@ -1,5 +1,8 @@ import { getApplicationConfig } from '@/shared/getConfig' -import { getMetadataCorrectionAuditLog } from '@/shared/getMetadataCorrectionAuditLog' +import { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog +} from '@/shared/getMetadataCorrectionAuditLog' import { logAnalyticsData } from '@/shared/logAnalyticsData' import { logger } from '@/shared/logger' @@ -31,6 +34,9 @@ import { logger } from '@/shared/logger' * - paginationToken * - limit * + * A `runId` path parameter returns one detailed audit document. Add `includeDiff=true` to that + * request to include its native-metadata patch. + * * @param {object} event - API Gateway event. * @param {object} context - Lambda context. * @returns {Promise} API Gateway response object. @@ -61,10 +67,32 @@ export const getMetadataCorrectionAudit = async (event, context) => { startDate, endDate, paginationToken, + includeDiff, limit } = event?.queryStringParameters || {} + const runId = event?.pathParameters?.runId try { + if (runId) { + const auditDocument = await getMetadataCorrectionAuditByRunId({ + runId, + includeDiff + }) + + return { + statusCode: auditDocument ? 200 : 404, + headers: { + ...defaultResponseHeaders, + 'Content-Type': 'application/json' + }, + body: JSON.stringify( + auditDocument || { error: `Metadata correction audit run not found: ${runId}` }, + null, + 2 + ) + } + } + const auditPage = await getMetadataCorrectionAuditLog({ collectionConceptId, keywordConceptUuid, diff --git a/serverless/src/metadataCorrectionService/__tests__/handler.test.js b/serverless/src/metadataCorrectionService/__tests__/handler.test.js index 3fb31850..ffac3441 100644 --- a/serverless/src/metadataCorrectionService/__tests__/handler.test.js +++ b/serverless/src/metadataCorrectionService/__tests__/handler.test.js @@ -124,6 +124,7 @@ describe('when the metadata correction service is invoked', () => { beforeEach(() => { vi.clearAllMocks() delete process.env.METADATA_CORRECTION_REQUEST_DELAY_MS + delete process.env.METADATA_CORRECTION_RUNS_PER_MINUTE vi.mocked(delay).mockResolvedValue(undefined) vi.mocked(emitConsumerMetricsSafely).mockResolvedValue(undefined) @@ -1239,6 +1240,50 @@ describe('when the metadata correction service is invoked', () => { })) }) + test('should pace correction runs when a per-minute rate is configured', async () => { + process.env.METADATA_CORRECTION_RUNS_PER_MINUTE = '10' + vi.mocked(getCmrCollectionUmmDetails).mockResolvedValue({ + collectionConceptId: 'C1234567890-PROV', + providerId: 'PROV', + nativeId: 'native-123', + revisionId: 7, + format: 'application/dif10+xml', + umm: {} + }) + vi.mocked(validateCmrCollectionUmm).mockResolvedValue({ + status: 200, + errors: [], + warnings: [], + responseBody: { + errors: [], + warnings: [] + } + }) + vi.mocked(extractKeywordValidationFailures).mockReturnValue([]) + + await metadataCorrectionService({ + Records: [{ + messageId: 'message-paced', + body: JSON.stringify({ + source: 'cmrKeywordEventsListener', + collectionConceptId: 'C1234567890-PROV' + }) + }] + }) + + expect(delay).toHaveBeenCalledOnce() + expect(delay).toHaveBeenCalledWith(6000) + expect(logger.info).toHaveBeenCalledWith( + '[metadata-correction] Pacing queued metadata correction request', + { + collectionConceptId: 'C1234567890-PROV', + messageId: 'message-paced', + pacingDelayMs: 6000, + runsPerMinute: 10 + } + ) + }) + test('should delay queued manual api requests when configured before running correction', async () => { const dateNowSpy = vi.spyOn(Date, 'now').mockReturnValue(10_000) process.env.METADATA_CORRECTION_REQUEST_DELAY_MS = '1500' diff --git a/serverless/src/metadataCorrectionService/handler.js b/serverless/src/metadataCorrectionService/handler.js index 89198096..4e1f938b 100644 --- a/serverless/src/metadataCorrectionService/handler.js +++ b/serverless/src/metadataCorrectionService/handler.js @@ -41,6 +41,50 @@ const buildBatchProcessingMetrics = ({ } const MAX_METADATA_CORRECTION_REQUEST_DELAY_MS = 20_000 +const MILLISECONDS_PER_MINUTE = 60_000 + +/** + * Converts the configured correction-run rate into a per-message pacing delay. + * + * @example + * // With METADATA_CORRECTION_RUNS_PER_MINUTE=10 + * getCorrectionRunPacingDelayMs() // 6000 + * + * @returns {number} Delay before each queued correction run, or zero when pacing is disabled. + */ +const getCorrectionRunPacingDelayMs = () => { + const runsPerMinute = Number(process.env.METADATA_CORRECTION_RUNS_PER_MINUTE) + + if (!Number.isInteger(runsPerMinute) || runsPerMinute <= 0) return 0 + + return Math.ceil(MILLISECONDS_PER_MINUTE / runsPerMinute) +} + +/** + * Holds the single configured consumer slot long enough to cap correction-run throughput. + * + * @param {Object} params Pacing log context. + * @param {string|undefined} params.collectionConceptId Collection being processed. + * @param {string|undefined} params.messageId SQS message identifier. + * @returns {Promise} + */ +const paceQueuedCorrectionRunIfNeeded = async ({ + collectionConceptId, + messageId +}) => { + const pacingDelayMs = getCorrectionRunPacingDelayMs() + + if (pacingDelayMs <= 0) return + + logger.info('[metadata-correction] Pacing queued metadata correction request', { + collectionConceptId, + messageId, + pacingDelayMs, + runsPerMinute: Number(process.env.METADATA_CORRECTION_RUNS_PER_MINUTE) + }) + + await delay(pacingDelayMs) +} /** * Reads the optional async correction request delay from environment configuration. @@ -130,6 +174,11 @@ export const metadataCorrectionService = async (event) => { metadataCorrectionRequest }) + await paceQueuedCorrectionRunIfNeeded({ + collectionConceptId: metadataCorrectionRequest.collectionConceptId, + messageId: record.messageId + }) + await delayQueuedManualRequestIfNeeded({ collectionConceptId: metadataCorrectionRequest.collectionConceptId, messageId: record.messageId, diff --git a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js new file mode 100644 index 00000000..4c72e35c --- /dev/null +++ b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js @@ -0,0 +1,63 @@ +import { describe, expect, test } from 'vitest' + +import { buildNativeMetadataDiff } from '../buildNativeMetadataDiff' + +describe('buildNativeMetadataDiff', () => { + test('creates a unified diff for native XML metadata', () => { + const result = buildNativeMetadataDiff({ + originalMetadata: 'GOSAT', + correctedMetadata: 'GOSAT - Test1', + priorRevisionId: 3 + }) + + expect(result).toEqual(expect.objectContaining({ + changed: true, + format: 'unified', + truncated: false, + originalBytes: 26, + correctedBytes: 34 + })) + expect(result.patch).toContain('--- cmr-revision-3') + expect(result.patch).toContain('+++ corrected-metadata') + expect(result.patch).toContain('-GOSAT') + expect(result.patch).toContain('+GOSAT - Test1') + }) + + test('serializes JSON metadata and reports identical payloads without a patch', () => { + const metadata = { Platforms: [{ ShortName: 'GOSAT' }] } + + expect(buildNativeMetadataDiff({ + originalMetadata: metadata, + correctedMetadata: structuredClone(metadata), + priorRevisionId: 4 + })).toEqual({ + changed: false, + format: 'unified', + patch: '', + truncated: false, + originalBytes: 63, + correctedBytes: 63 + }) + }) + + test('caps a large diff and marks it as truncated', () => { + const result = buildNativeMetadataDiff({ + originalMetadata: `old-${'a'.repeat(300_000)}`, + correctedMetadata: `new-${'b'.repeat(300_000)}` + }) + + expect(result.changed).toBe(true) + expect(result.truncated).toBe(true) + expect(result.patch).toHaveLength(250_000) + }) + + test('returns undefined when metadata cannot be serialized', () => { + const circularMetadata = {} + circularMetadata.self = circularMetadata + + expect(buildNativeMetadataDiff({ + originalMetadata: circularMetadata, + correctedMetadata: {} + })).toBeUndefined() + }) +}) diff --git a/serverless/src/shared/__tests__/cmrPutRequest.test.js b/serverless/src/shared/__tests__/cmrPutRequest.test.js index d98736c7..361dfcc8 100644 --- a/serverless/src/shared/__tests__/cmrPutRequest.test.js +++ b/serverless/src/shared/__tests__/cmrPutRequest.test.js @@ -63,6 +63,40 @@ describe('cmrPutRequest', () => { signal: expect.any(Object) }) ) + + expect(logger.info).toHaveBeenCalledWith( + '[cmr-put] CMR response received', + { + method: 'PUT', + path, + status: undefined, + durationMs: expect.any(Number), + requestId: undefined + } + ) + }) + + test('should log the CMR request id returned with a response', async () => { + global.fetch.mockResolvedValue({ + ok: true, + status: 200, + headers: { + get: vi.fn((name) => (name === 'cmr-request-id' ? 'request-123' : null)) + } + }) + + await cmrPutRequest({ + path: '/ingest/providers/KMS/collections/native-1' + }) + + expect(logger.info).toHaveBeenCalledWith( + '[cmr-put] CMR response received', + expect.objectContaining({ + status: 200, + durationMs: expect.any(Number), + requestId: 'request-123' + }) + ) }) test('should not include body in request if it is empty', async () => { @@ -175,6 +209,8 @@ describe('cmrPutRequest', () => { fullUrl: 'https://cmr-test.earthdata.nasa.gov/ingest/providers/KMS/collections/native-1', bodyLength: 2, timeoutMs: 25000, + durationMs: expect.any(Number), + timedOut: false, error: { name: 'TypeError', message: 'fetch failed', @@ -212,7 +248,9 @@ describe('cmrPutRequest', () => { path: '/ingest/providers/KMS/collections/native-1', fullUrl: 'https://cmr-test.earthdata.nasa.gov/ingest/providers/KMS/collections/native-1', bodyLength: 2, - timeoutMs: 25000 + timeoutMs: 25000, + durationMs: expect.any(Number), + timedOut: false }) expect(error.cmrCause).toBeUndefined() @@ -245,7 +283,9 @@ describe('cmrPutRequest', () => { path: '/ingest/providers/KMS/collections/native-1', fullUrl: 'https://cmr-test.earthdata.nasa.gov/ingest/providers/KMS/collections/native-1', bodyLength: undefined, - timeoutMs: 25000 + timeoutMs: 25000, + durationMs: expect.any(Number), + timedOut: false }) }) @@ -268,5 +308,12 @@ describe('cmrPutRequest', () => { await vi.advanceTimersByTimeAsync(25) await rejectionExpectation + + await expect(requestPromise).rejects.toMatchObject({ + cmrRequest: expect.objectContaining({ + timedOut: true, + durationMs: expect.any(Number) + }) + }) }) }) diff --git a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js index 94ba3dc9..6897784b 100644 --- a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js @@ -8,12 +8,31 @@ import { import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' -import { getMetadataCorrectionAuditLog } from '../getMetadataCorrectionAuditLog' +import { + getMetadataCorrectionAuditByRunId, + getMetadataCorrectionAuditLog +} from '../getMetadataCorrectionAuditLog' vi.mock('@/shared/documentDbClient', () => ({ getMetadataCorrectionAuditCollection: vi.fn() })) +const SUMMARY_PROJECTION = { + _id: 1, + runId: 1, + collectionConceptId: 1, + collectionUri: 1, + status: 1, + createdAt: 1, + updatedAt: 1, + 'corrections.scheme': 1, + 'corrections.action': 1, + 'corrections.oldKeywordPath': 1, + 'corrections.newKeywordPath': 1, + 'metadataDiff.changed': 1, + 'error.message': 1 +} + describe('metadata correction audit queries', () => { let collection let mongoCursor @@ -36,12 +55,23 @@ describe('metadata correction audit queries', () => { test('filters and returns newest-first audit documents', async () => { const createdAt = new Date('2026-09-02T12:00:00.000Z') + const updatedAt = new Date('2026-09-02T12:01:00.000Z') mongoCursor.toArray.mockResolvedValue([{ _id: 'run-1', runId: 'run-1', collectionConceptId: 'C123-PROV', + collectionUri: 'https://cmr.example.com/search/concepts/C123-PROV', createdAt, - status: 'applied' + updatedAt, + status: 'failed', + corrections: [{ + scheme: 'platforms', + action: 'replace', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + metadataDiff: { changed: true }, + error: { message: 'CMR writeback timed out' } }]) const result = await getMetadataCorrectionAuditLog({ @@ -58,34 +88,37 @@ describe('metadata correction audit queries', () => { status: 'applied' }) - expect(collection.find).toHaveBeenCalledWith({ - $and: [ - { - collectionConceptId: 'C123-PROV', - 'trigger.eventType': 'UPDATED', - nativeFormat: 'UMM', - publishedVersionName: '20.1', - source: 'cmrKeywordEventsListener', - status: 'applied', - createdAt: { - $gte: new Date('2026-09-01'), - $lte: new Date('2026-09-03') + expect(collection.find).toHaveBeenCalledWith( + { + $and: [ + { + collectionConceptId: 'C123-PROV', + 'trigger.eventType': 'UPDATED', + nativeFormat: 'UMM', + publishedVersionName: '20.1', + source: 'cmrKeywordEventsListener', + status: 'applied', + createdAt: { + $gte: new Date('2026-09-01'), + $lte: new Date('2026-09-03') + } + }, + { + $or: [ + { 'corrections.keywordConceptUuid': 'keyword-1' }, + { 'trigger.keywordConceptUuid': 'keyword-1' } + ] + }, + { + $or: [ + { 'corrections.scheme': { $in: ['DataFormat', 'dataformat'] } }, + { 'trigger.scheme': { $in: ['DataFormat', 'dataformat'] } } + ] } - }, - { - $or: [ - { 'corrections.keywordConceptUuid': 'keyword-1' }, - { 'trigger.keywordConceptUuid': 'keyword-1' } - ] - }, - { - $or: [ - { 'corrections.scheme': { $in: ['DataFormat', 'dataformat'] } }, - { 'trigger.scheme': { $in: ['DataFormat', 'dataformat'] } } - ] - } - ] - }) + ] + }, + { projection: SUMMARY_PROJECTION } + ) expect(mongoCursor.sort).toHaveBeenCalledWith({ createdAt: -1, @@ -97,8 +130,17 @@ describe('metadata correction audit queries', () => { items: [{ runId: 'run-1', collectionConceptId: 'C123-PROV', - createdAt, - status: 'applied' + collectionUri: 'https://cmr.example.com/search/concepts/C123-PROV', + status: 'failed', + updatedAt, + changes: [{ + scheme: 'platforms', + action: 'replace', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + hasMetadataDiff: true, + errorMessage: 'CMR writeback timed out' }], nextPaginationToken: null }) @@ -136,45 +178,56 @@ describe('metadata correction audit queries', () => { status: 'checked' }) - expect(collection.find).toHaveBeenLastCalledWith({ - $and: [ - { status: 'checked' }, - { - $or: [ - { createdAt: { $lt: new Date('2026-09-02') } }, - { - createdAt: new Date('2026-09-02'), - _id: { $lt: 'run-2' } - } - ] - } - ] - }) + expect(collection.find).toHaveBeenLastCalledWith( + { + $and: [ + { status: 'checked' }, + { + $or: [ + { createdAt: { $lt: new Date('2026-09-02') } }, + { + createdAt: new Date('2026-09-02'), + _id: { $lt: 'run-2' } + } + ] + } + ] + }, + { projection: SUMMARY_PROJECTION } + ) }) test('supports default filters, one-sided date ranges, and lowercase scheme storage', async () => { await getMetadataCorrectionAuditLog() - expect(collection.find).toHaveBeenLastCalledWith({}) + expect(collection.find).toHaveBeenLastCalledWith( + {}, + { projection: SUMMARY_PROJECTION } + ) expect(mongoCursor.limit).toHaveBeenLastCalledWith(101) await getMetadataCorrectionAuditLog({ scheme: 'PLATFORMS' }) - expect(collection.find).toHaveBeenLastCalledWith({ - $or: [ - { 'corrections.scheme': 'platforms' }, - { 'trigger.scheme': 'platforms' } - ] - }) + expect(collection.find).toHaveBeenLastCalledWith( + { + $or: [ + { 'corrections.scheme': 'platforms' }, + { 'trigger.scheme': 'platforms' } + ] + }, + { projection: SUMMARY_PROJECTION } + ) await getMetadataCorrectionAuditLog({ startDate: '2026-09-01' }) - expect(collection.find).toHaveBeenLastCalledWith({ - createdAt: { $gte: new Date('2026-09-01') } - }) + expect(collection.find).toHaveBeenLastCalledWith( + { createdAt: { $gte: new Date('2026-09-01') } }, + { projection: SUMMARY_PROJECTION } + ) await getMetadataCorrectionAuditLog({ endDate: '2026-09-03' }) - expect(collection.find).toHaveBeenLastCalledWith({ - createdAt: { $lte: new Date('2026-09-03') } - }) + expect(collection.find).toHaveBeenLastCalledWith( + { createdAt: { $lte: new Date('2026-09-03') } }, + { projection: SUMMARY_PROJECTION } + ) }) test('validates filters before querying DocumentDB', async () => { @@ -223,4 +276,62 @@ describe('metadata correction audit queries', () => { expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() }) + + test('returns one detailed audit run without the native metadata diff by default', async () => { + collection.findOne.mockResolvedValue({ + runId: 'run-1', + status: 'applied' + }) + + await expect(getMetadataCorrectionAuditByRunId({ + runId: 'run-1' + })).resolves.toEqual({ + runId: 'run-1', + status: 'applied' + }) + + expect(collection.findOne).toHaveBeenCalledWith( + { _id: 'run-1' }, + { projection: { _id: 0, metadataDiff: 0 } } + ) + }) + + test('includes the native metadata diff only when requested', async () => { + collection.findOne.mockResolvedValue({ + runId: 'run-1', + status: 'failed', + metadataDiff: { + changed: true, + patch: '-old\n+new' + } + }) + + const result = await getMetadataCorrectionAuditByRunId({ + runId: 'run-1', + includeDiff: 'true' + }) + + expect(result.metadataDiff.patch).toBe('-old\n+new') + expect(collection.findOne).toHaveBeenCalledWith( + { _id: 'run-1' }, + { projection: { _id: 0 } } + ) + }) + + test('returns null for an unknown run and validates detail parameters', async () => { + await expect(getMetadataCorrectionAuditByRunId({ + runId: 'missing-run' + })).resolves.toBeNull() + + await expect(getMetadataCorrectionAuditByRunId()).rejects.toThrow( + 'Invalid metadata correction audit runId' + ) + + await expect(getMetadataCorrectionAuditByRunId({ + runId: 'run-1', + includeDiff: 'yes' + })).rejects.toThrow( + 'Invalid metadata correction audit includeDiff: expected true or false' + ) + }) }) diff --git a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js index de42299d..969054ca 100644 --- a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js @@ -63,6 +63,12 @@ describe('persistMetadataCorrectionAuditLog', () => { scheme: 'platforms', uuid: 'platform-uuid' }, + metadataDiff: { + changed: true, + format: 'unified', + patch: '-GOSAT\n+GOSAT - Test1', + truncated: false + }, nativeFormat: 'UMM', priorRevisionId: 7, providerId: 'PROV', @@ -86,6 +92,12 @@ describe('persistMetadataCorrectionAuditLog', () => { publishedVersionName: '20.1', priorRevisionId: 7, status: 'checked', + metadataDiff: { + changed: true, + format: 'unified', + patch: '-GOSAT\n+GOSAT - Test1', + truncated: false + }, corrections: [expect.objectContaining({ keywordConceptUuid: 'platform-uuid', oldKeywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', diff --git a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js index 6591088f..128c790c 100644 --- a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js +++ b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js @@ -487,10 +487,22 @@ describe('runCollectionMetadataCorrection', () => { keywordEvent: { eventType: 'MANUAL' }, + metadataDiff: expect.objectContaining({ + changed: true, + format: 'unified', + truncated: false, + originalBytes: 6, + correctedBytes: 20 + }), status: 'pending' }) ) + expect(persistMetadataCorrectionAuditLog.mock.calls[1][0].metadataDiff.patch) + .toContain('-') + expect(persistMetadataCorrectionAuditLog.mock.calls[1][0].metadataDiff.patch) + .toContain('+corrected') + expect(persistMetadataCorrectionAuditLog).toHaveBeenNthCalledWith( 3, expect.objectContaining({ @@ -1282,18 +1294,21 @@ describe('runCollectionMetadataCorrection', () => { contentType: 'application/vnd.nasa.cmr.umm+json;version=1.16.2; charset=utf-8' }) - vi.mocked(invokeMetadataCorrectionDelegate).mockResolvedValue({ - delegateName: 'umm', - nativeFormat: 'UMM', - correctionCount: 1, - correctionsApplied: [ - { - scheme: 'sciencekeywords', - keywordConceptUuid: 'uuid-1' - } - ], - correctedMetadata: { - ShortName: 'TEST-UPDATED' + vi.mocked(invokeMetadataCorrectionDelegate).mockImplementation(async ({ metadataPayload }) => { + expect(metadataPayload.ShortName).toBe('TEST') + metadataPayload.ShortName = 'TEST-UPDATED' + + return { + delegateName: 'umm', + nativeFormat: 'UMM', + correctionCount: 1, + correctionsApplied: [ + { + scheme: 'sciencekeywords', + keywordConceptUuid: 'uuid-1' + } + ], + correctedMetadata: metadataPayload } }) @@ -1316,10 +1331,7 @@ describe('runCollectionMetadataCorrection', () => { }) expect(invokeMetadataCorrectionDelegate).toHaveBeenCalledWith(expect.objectContaining({ - nativeFormat: 'UMM', - metadataPayload: { - ShortName: 'TEST' - } + nativeFormat: 'UMM' })) expect(writeCorrectedMetadataToCmr).toHaveBeenCalledWith(expect.objectContaining({ @@ -1327,6 +1339,11 @@ describe('runCollectionMetadataCorrection', () => { nativeMetadataContentType: 'application/vnd.nasa.cmr.umm+json;version=1.16.2; charset=utf-8' })) + const pendingAudit = persistMetadataCorrectionAuditLog.mock.calls[1][0] + expect(pendingAudit.metadataDiff.changed).toBe(true) + expect(pendingAudit.metadataDiff.patch).toContain('- "ShortName": "TEST"') + expect(pendingAudit.metadataDiff.patch).toContain('+ "ShortName": "TEST-UPDATED"') + expect(emitConsumerMetricsSafely).toHaveBeenCalledWith(expect.objectContaining({ metrics: [ { diff --git a/serverless/src/shared/__tests__/writeCorrectedMetadataToCmr.test.js b/serverless/src/shared/__tests__/writeCorrectedMetadataToCmr.test.js index d9ef05aa..c13b9e6f 100644 --- a/serverless/src/shared/__tests__/writeCorrectedMetadataToCmr.test.js +++ b/serverless/src/shared/__tests__/writeCorrectedMetadataToCmr.test.js @@ -48,6 +48,7 @@ describe('when writing corrected metadata to cmr', () => { vi.clearAllMocks() process.env.CMR_WRITEBACK_PROVIDERS = 'KMS' process.env.CMR_WRITER_TOKEN = 'Bearer writer-token' + delete process.env.CMR_WRITEBACK_TIMEOUT_MS delete process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS delete process.env.CMR_WRITEBACK_VALIDATE_UMM_C @@ -97,7 +98,7 @@ describe('when writing corrected metadata to cmr', () => { body: '', contentType: 'application/dif10+xml', accept: 'application/json', - timeoutMs: 10000, + timeoutMs: 25000, headers: { Authorization: 'Bearer writer-token', 'Client-Id': 'kms-metadata-correction-service', @@ -285,7 +286,7 @@ describe('when writing corrected metadata to cmr', () => { body: JSON.stringify(correctedMetadata), contentType: 'application/vnd.nasa.cmr.umm+json;version=1.16.2', accept: 'application/json', - timeoutMs: 10000, + timeoutMs: 25000, headers: { Authorization: 'Bearer writer-token', 'Client-Id': 'kms-metadata-correction-service', @@ -543,4 +544,29 @@ describe('when writing corrected metadata to cmr', () => { correctedMetadata: '' })).rejects.toThrow('Unsupported native format for CMR writeback: UNKNOWN') }) + + test.each([ + ['30000', 30000], + ['60000', 45000], + ['not-a-number', 25000], + ['0', 25000] + ])('should resolve CMR writeback timeout %s to %i milliseconds', async ( + configuredTimeout, + expectedTimeout + ) => { + process.env.CMR_WRITEBACK_TIMEOUT_MS = configuredTimeout + + await writeCorrectedMetadataToCmr({ + collectionConceptId: 'C0000000000-KMS', + providerId: 'KMS', + nativeId: 'native-1', + nativeFormat: 'DIF10', + correctionCount: 1, + correctedMetadata: '' + }) + + expect(cmrPutRequest).toHaveBeenCalledWith(expect.objectContaining({ + timeoutMs: expectedTimeout + })) + }) }) diff --git a/serverless/src/shared/buildNativeMetadataDiff.js b/serverless/src/shared/buildNativeMetadataDiff.js new file mode 100644 index 00000000..20156f77 --- /dev/null +++ b/serverless/src/shared/buildNativeMetadataDiff.js @@ -0,0 +1,81 @@ +import { createTwoFilesPatch } from 'diff' + +const MAX_METADATA_DIFF_CHARACTERS = 250_000 + +/** + * Converts native XML or JSON metadata into comparable text without changing the write payload. + * + * @example + * snapshotNativeMetadataForDiff({ ShortName: 'GOSAT' }) + * // '{\n "ShortName": "GOSAT"\n}' + * + * @param {string|Object} metadata Native metadata payload. + * @returns {string|undefined} Text representation, or undefined when it cannot be serialized. + */ +export const snapshotNativeMetadataForDiff = (metadata) => { + if (typeof metadata === 'string') return metadata + + try { + return JSON.stringify(metadata, null, 2) + } catch { + return undefined + } +} + +/** + * Builds a bounded unified diff between the metadata fetched from CMR and the corrected payload. + * + * @example + * buildNativeMetadataDiff({ + * originalMetadata: 'GOSAT', + * correctedMetadata: 'GOSAT - Test1', + * priorRevisionId: 3 + * }) + * // { changed: true, format: 'unified', patch: '...', truncated: false, ... } + * + * @param {Object} params Diff inputs. + * @param {string|Object} params.originalMetadata Native metadata fetched from CMR. + * @param {string|Object} params.correctedMetadata Corrected metadata prepared for writeback. + * @param {number|string} [params.priorRevisionId] CMR revision represented by the original payload. + * @returns {Object|undefined} Bounded display diff, or undefined for unserializable metadata. + */ +export const buildNativeMetadataDiff = ({ + originalMetadata, + correctedMetadata, + priorRevisionId +}) => { + const originalText = snapshotNativeMetadataForDiff(originalMetadata) + const correctedText = snapshotNativeMetadataForDiff(correctedMetadata) + + if (typeof originalText !== 'string' || typeof correctedText !== 'string') { + return undefined + } + + const changed = originalText !== correctedText + const fullPatch = changed + ? createTwoFilesPatch( + `cmr-revision-${priorRevisionId ?? 'unknown'}`, + 'corrected-metadata', + originalText, + correctedText, + '', + '', + { context: 3 } + ) + : '' + const truncated = fullPatch.length > MAX_METADATA_DIFF_CHARACTERS + const patch = truncated + ? fullPatch.slice(0, MAX_METADATA_DIFF_CHARACTERS) + : fullPatch + + return { + changed, + format: 'unified', + patch, + truncated, + originalBytes: Buffer.byteLength(originalText, 'utf8'), + correctedBytes: Buffer.byteLength(correctedText, 'utf8') + } +} + +export default buildNativeMetadataDiff diff --git a/serverless/src/shared/cmrPutRequest.js b/serverless/src/shared/cmrPutRequest.js index 30324eda..cda9f4cc 100644 --- a/serverless/src/shared/cmrPutRequest.js +++ b/serverless/src/shared/cmrPutRequest.js @@ -42,6 +42,16 @@ const extractErrorDetails = (error) => { } } +/** + * Reads the request identifier returned by CMR for latency troubleshooting. + * + * @param {Response} response Fetch response from CMR. + * @returns {string|undefined} CMR request identifier when present. + */ +const getCmrRequestId = (response) => response.headers?.get?.('cmr-request-id') + || response.headers?.get?.('x-request-id') + || undefined + /** * Makes a PUT request to the CMR (Common Metadata Repository) API. * @@ -64,6 +74,7 @@ export const cmrPutRequest = async ({ }) => { const { endpoint } = getEndpointConfig() const fullUrl = `${endpoint}${path}` + const startedAt = Date.now() const fetchOptions = { method: 'PUT', @@ -103,7 +114,17 @@ export const cmrPutRequest = async ({ fetchOptions.signal = controller.signal } - return await fetch(fullUrl, fetchOptions) + const response = await fetch(fullUrl, fetchOptions) + + logger.info('[cmr-put] CMR response received', { + method: 'PUT', + path, + status: response.status, + durationMs: Date.now() - startedAt, + requestId: getCmrRequestId(response) + }) + + return response } catch (error) { const requestContext = { method: 'PUT', @@ -111,7 +132,9 @@ export const cmrPutRequest = async ({ path, fullUrl, bodyLength: typeof body === 'string' ? body.length : undefined, - timeoutMs + timeoutMs, + durationMs: Date.now() - startedAt, + timedOut: controller.signal.aborted } logger.error('[cmr-put] CMR write failed', { diff --git a/serverless/src/shared/getMetadataCorrectionAuditLog.js b/serverless/src/shared/getMetadataCorrectionAuditLog.js index 03e1416c..ddb62f45 100644 --- a/serverless/src/shared/getMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/getMetadataCorrectionAuditLog.js @@ -17,6 +17,21 @@ const VALID_AUDIT_SCHEMES = new Map([ .map(([scheme]) => [scheme.toLowerCase(), scheme]), ...VALID_SCHEMES.map((scheme) => [scheme.toLowerCase(), scheme]) ]) +const AUDIT_SUMMARY_PROJECTION = { + _id: 1, + runId: 1, + collectionConceptId: 1, + collectionUri: 1, + status: 1, + createdAt: 1, + updatedAt: 1, + 'corrections.scheme': 1, + 'corrections.action': 1, + 'corrections.oldKeywordPath': 1, + 'corrections.newKeywordPath': 1, + 'metadataDiff.changed': 1, + 'error.message': 1 +} /** * Validates the optional API page size. @@ -303,6 +318,65 @@ const normalizeAuditDocument = (document) => Object.fromEntries( Object.entries(document).filter(([key]) => key !== '_id') ) +/** + * Reduces a stored correction to the old-to-new path information needed in audit search results. + * + * @example + * normalizeAuditChange({ + * scheme: 'platforms', + * action: 'replace', + * oldKeywordPath: 'Platforms > GOSAT', + * newKeywordPath: 'Platforms > GOSAT - Test1' + * }) + * // { scheme: 'platforms', action: 'replace', oldKeywordPath: '...', newKeywordPath: '...' } + * + * @param {Object} correction Stored correction details. + * @returns {Object} Compact path change. + */ +const normalizeAuditChange = (correction = {}) => ({ + scheme: correction.scheme, + action: correction.action, + oldKeywordPath: correction.oldKeywordPath, + newKeywordPath: correction.newKeywordPath +}) + +/** + * Builds the compact representation returned by paginated audit searches. + * + * @param {Object} document Stored audit document. + * @returns {Object} Audit summary suitable for list views. + */ +const normalizeAuditSummary = (document) => ({ + runId: document.runId, + collectionConceptId: document.collectionConceptId, + collectionUri: document.collectionUri, + status: document.status, + updatedAt: document.updatedAt, + changes: Array.isArray(document.corrections) + ? document.corrections.map(normalizeAuditChange) + : [], + hasMetadataDiff: document.metadataDiff?.changed === true, + ...(document.error?.message ? { errorMessage: document.error.message } : {}) +}) + +/** + * Parses the optional detail flag used to include a potentially large native-metadata diff. + * + * @example + * normalizeIncludeDiff('true') // true + * normalizeIncludeDiff(undefined) // false + * + * @param {unknown} includeDiff Requested flag value. + * @returns {boolean} Whether the detailed response should include the diff. + */ +const normalizeIncludeDiff = (includeDiff) => { + if (includeDiff === undefined || includeDiff === null || includeDiff === false) return false + if (includeDiff === true || includeDiff === 'true') return true + if (includeDiff === 'false') return false + + throw new Error('Invalid metadata correction audit includeDiff: expected true or false') +} + /** * Returns metadata-correction audit runs using newest-first token pagination. * @@ -317,7 +391,9 @@ export const getMetadataCorrectionAuditLog = async (filters = {}) => { const limit = normalizeLimit(filters.limit) const query = addPaginationTokenToQuery(buildAuditQuery(filters), filters.paginationToken) const collection = await getMetadataCorrectionAuditCollection() - const documents = await collection.find(query) + const documents = await collection.find(query, { + projection: AUDIT_SUMMARY_PROJECTION + }) .sort({ createdAt: -1, _id: -1 @@ -328,11 +404,45 @@ export const getMetadataCorrectionAuditLog = async (filters = {}) => { const pageDocuments = hasNextPage ? documents.slice(0, limit) : documents return { - items: pageDocuments.map(normalizeAuditDocument), + items: pageDocuments.map(normalizeAuditSummary), nextPaginationToken: hasNextPage ? encodePaginationToken(pageDocuments[pageDocuments.length - 1]) : null } } +/** + * Retrieves one complete audit run, excluding the native diff unless explicitly requested. + * + * @example + * await getMetadataCorrectionAuditByRunId({ runId: 'run-1', includeDiff: 'true' }) + * // { runId: 'run-1', status: 'applied', metadataDiff: { ... } } + * + * @param {Object} params Detail lookup parameters. + * @param {string} params.runId Audit run identifier. + * @param {boolean|string} [params.includeDiff=false] Whether to include the stored metadata patch. + * @returns {Promise} Detailed audit document or null when it does not exist. + */ +export const getMetadataCorrectionAuditByRunId = async ({ + runId, + includeDiff = false +} = {}) => { + if (!runId) { + throw new Error('Invalid metadata correction audit runId') + } + + const shouldIncludeDiff = normalizeIncludeDiff(includeDiff) + const collection = await getMetadataCorrectionAuditCollection() + const document = await collection.findOne( + { _id: runId }, + { + projection: shouldIncludeDiff + ? { _id: 0 } + : { _id: 0, metadataDiff: 0 } + } + ) + + return document ? normalizeAuditDocument(document) : null +} + export default getMetadataCorrectionAuditLog diff --git a/serverless/src/shared/persistMetadataCorrectionAuditLog.js b/serverless/src/shared/persistMetadataCorrectionAuditLog.js index b70efe36..1d716811 100644 --- a/serverless/src/shared/persistMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/persistMetadataCorrectionAuditLog.js @@ -145,6 +145,7 @@ const buildAuditPatch = ({ keywordEvent, keywordValidationFailures, messageId, + metadataDiff, nativeFormat, outcome, priorRevisionId, @@ -178,6 +179,7 @@ const buildAuditPatch = ({ keywordValidationFailureCount: Array.isArray(keywordValidationFailures) ? keywordValidationFailures.length : undefined, + metadataDiff, outcome, error: error ? compactObject({ message: error.message || String(error), diff --git a/serverless/src/shared/runCollectionMetadataCorrection.js b/serverless/src/shared/runCollectionMetadataCorrection.js index 53c1d2f4..129eaa57 100644 --- a/serverless/src/shared/runCollectionMetadataCorrection.js +++ b/serverless/src/shared/runCollectionMetadataCorrection.js @@ -1,3 +1,7 @@ +import { + buildNativeMetadataDiff, + snapshotNativeMetadataForDiff +} from '@/shared/buildNativeMetadataDiff' import { detectNativeMetadataFormat } from '@/shared/detectNativeMetadataFormat' import { CONSUMER_METRIC_NAMES } from '@/shared/emitConsumerMetrics' import { emitConsumerMetricsSafely } from '@/shared/emitConsumerMetricsSafely' @@ -416,6 +420,7 @@ export const runCollectionMetadataCorrection = async ({ }) let nativeMetadataResponse + let originalMetadata let rawCorrectionResult try { @@ -428,6 +433,8 @@ export const runCollectionMetadataCorrection = async ({ const metadataPayload = nativeFormat === 'UMM' ? nativeMetadataResponse.metadataPayload : nativeMetadataResponse + // Snapshot JSON before invoking editors because some delegates mutate the payload in place. + originalMetadata = snapshotNativeMetadataForDiff(metadataPayload) rawCorrectionResult = await invokeMetadataCorrectionDelegate({ collectionConceptId: collectionDetails.collectionConceptId, @@ -467,6 +474,13 @@ export const runCollectionMetadataCorrection = async ({ : [] const normalizedCorrectionCount = Number(rawCorrectionResult.correctionCount || 0) const correctedMetadata = rawCorrectionResult.correctedMetadata ?? '' + const metadataDiff = correctionsApplied.length > 0 + ? buildNativeMetadataDiff({ + originalMetadata, + correctedMetadata, + priorRevisionId: collectionDetails.revisionId + }) + : undefined let pendingAuditResult = null let appliedAuditResult = null @@ -481,6 +495,7 @@ export const runCollectionMetadataCorrection = async ({ delegateName, corrections: correctionsApplied, keywordValidationFailures, + metadataDiff, priorRevisionId: collectionDetails.revisionId, outcome: 'writeback-pending', source, diff --git a/serverless/src/shared/writeCorrectedMetadataToCmr.js b/serverless/src/shared/writeCorrectedMetadataToCmr.js index 8324ca21..9c73a7d3 100644 --- a/serverless/src/shared/writeCorrectedMetadataToCmr.js +++ b/serverless/src/shared/writeCorrectedMetadataToCmr.js @@ -2,11 +2,31 @@ import { cmrPutRequest } from './cmrPutRequest' import { getCmrWriterToken } from './getCmrWriterToken' import { logger } from './logger' -// Keep the writeback timeout comfortably inside the metadataCorrectionService Lambda's -// 30s timeout so a stalled ingest request can still be recorded as a failed audit status. -const CMR_WRITEBACK_TIMEOUT_MS = 10_000 +// Keep the writeback timeout inside the metadataCorrectionService Lambda's 60s timeout so a +// stalled ingest request can still be recorded as a failed audit status. +const DEFAULT_CMR_WRITEBACK_TIMEOUT_MS = 25_000 +const MAX_CMR_WRITEBACK_TIMEOUT_MS = 45_000 const CMR_WRITEBACK_CLIENT_ID = 'kms-metadata-correction-service' +/** + * Reads the CMR ingest timeout while retaining enough Lambda time to persist a failure audit. + * + * @example + * // With CMR_WRITEBACK_TIMEOUT_MS=30000 + * getCmrWritebackTimeoutMs() // 30000 + * + * @returns {number} CMR write timeout between 1 and 45000 milliseconds. + */ +const getCmrWritebackTimeoutMs = () => { + const configuredTimeout = Number(process.env.CMR_WRITEBACK_TIMEOUT_MS) + + if (!Number.isInteger(configuredTimeout) || configuredTimeout <= 0) { + return DEFAULT_CMR_WRITEBACK_TIMEOUT_MS + } + + return Math.min(configuredTimeout, MAX_CMR_WRITEBACK_TIMEOUT_MS) +} + /** * Reads a CMR validation header flag from environment configuration. * Validation remains disabled unless the corresponding value is explicitly `true`. @@ -349,7 +369,7 @@ export const writeCorrectedMetadataToCmr = async ({ body: serializedMetadata, contentType, accept: 'application/json', - timeoutMs: CMR_WRITEBACK_TIMEOUT_MS, + timeoutMs: getCmrWritebackTimeoutMs(), headers: { Authorization: authorizationToken, 'Client-Id': CMR_WRITEBACK_CLIENT_ID, From 78dfabd1fa6652ebbe40474afaa8ba70419a4278 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Mon, 7 Sep 2026 19:29:55 -0400 Subject: [PATCH 06/17] KMS-703: optionally include native metadata diffs in audit list responses --- README.md | 3 +- serverless/src/getCapabilities/handler.js | 2 +- .../__tests__/handler.test.js | 23 ++++++++-- .../src/getMetadataCorrectionAudit/handler.js | 6 ++- .../__tests__/handler.test.js | 2 + .../__tests__/buildNativeMetadataDiff.test.js | 7 ++- .../getMetadataCorrectionAuditLog.test.js | 43 +++++++++++++++++-- .../runCollectionMetadataCorrection.test.js | 5 ++- .../shared/getMetadataCorrectionAuditLog.js | 33 +++++++++++--- 9 files changed, 104 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b00175fd..4b9868f1 100644 --- a/README.md +++ b/README.md @@ -458,7 +458,8 @@ The audit API is: filters include collection, keyword UUID, action, scheme, status, native format, KMS version, source, and date range. Supplied actions and schemes must be recognized KMS values, limits must be integers from 1 through 250, and `startDate` must not be after `endDate`. List results contain - compact collection, status, and old-to-new keyword path summaries. + compact collection, status, and old-to-new keyword path summaries. Add `?includeDiff=true` to + include each available native-metadata diff in the list results. - `GET /metadata_correction_audit/{runId}` for the complete audit document. Add `?includeDiff=true` when the native-metadata diff is needed; it is omitted by default to keep routine responses small. diff --git a/serverless/src/getCapabilities/handler.js b/serverless/src/getCapabilities/handler.js index 2a3ce0ad..5e1630cd 100644 --- a/serverless/src/getCapabilities/handler.js +++ b/serverless/src/getCapabilities/handler.js @@ -141,7 +141,7 @@ export const getCapabilities = async () => { ':@': { name: 'get_metadata_correction_audit', href: '/metadata_correction_audit', - params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&limit=', + params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&includeDiff=&limit=', action: 'GET' } }, diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index bc9278d8..ba84d592 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -50,8 +50,7 @@ describe('getMetadataCorrectionAudit', () => { scheme: 'platforms', oldKeywordPath: 'Platforms > GOSAT', newKeywordPath: 'Platforms > GOSAT - Test1' - }], - hasMetadataDiff: true + }] }], nextPaginationToken: null }) @@ -69,6 +68,7 @@ describe('getMetadataCorrectionAudit', () => { action: undefined, paginationToken: undefined, endDate: undefined, + includeDiff: undefined, scheme: undefined, status: undefined, nativeFormat: undefined, @@ -92,14 +92,28 @@ describe('getMetadataCorrectionAudit', () => { scheme: 'platforms', oldKeywordPath: 'Platforms > GOSAT', newKeywordPath: 'Platforms > GOSAT - Test1' - }], - hasMetadataDiff: true + }] } ], nextPaginationToken: null }) }) + test('requests native metadata diffs in list results when requested', async () => { + vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({ + items: [], + nextPaginationToken: null + }) + + await getMetadataCorrectionAudit({ + queryStringParameters: { includeDiff: 'true' } + }) + + expect(getMetadataCorrectionAuditLog).toHaveBeenCalledWith(expect.objectContaining({ + includeDiff: 'true' + })) + }) + test('returns 500 when the audit query fails', async () => { vi.mocked(getMetadataCorrectionAuditLog).mockRejectedValue(new Error('Audit query failed')) @@ -141,6 +155,7 @@ describe('getMetadataCorrectionAudit', () => { runId: 'run-1', includeDiff: 'true' }) + expect(getMetadataCorrectionAuditLog).not.toHaveBeenCalled() expect(result.statusCode).toBe(200) expect(JSON.parse(result.body).metadataDiff.patch).toBe('-old\n+new') diff --git a/serverless/src/getMetadataCorrectionAudit/handler.js b/serverless/src/getMetadataCorrectionAudit/handler.js index 2a3cb07d..df540771 100644 --- a/serverless/src/getMetadataCorrectionAudit/handler.js +++ b/serverless/src/getMetadataCorrectionAudit/handler.js @@ -32,10 +32,11 @@ import { logger } from '@/shared/logger' * - source * - startDate / endDate * - paginationToken + * - includeDiff * - limit * - * A `runId` path parameter returns one detailed audit document. Add `includeDiff=true` to that - * request to include its native-metadata patch. + * Add `includeDiff=true` to a list or detail request to include native-metadata patches. + * A `runId` path parameter returns one detailed audit document. * * @param {object} event - API Gateway event. * @param {object} context - Lambda context. @@ -105,6 +106,7 @@ export const getMetadataCorrectionAudit = async (event, context) => { startDate, endDate, paginationToken, + includeDiff, limit }) diff --git a/serverless/src/metadataCorrectionService/__tests__/handler.test.js b/serverless/src/metadataCorrectionService/__tests__/handler.test.js index ffac3441..2b20bc94 100644 --- a/serverless/src/metadataCorrectionService/__tests__/handler.test.js +++ b/serverless/src/metadataCorrectionService/__tests__/handler.test.js @@ -1250,6 +1250,7 @@ describe('when the metadata correction service is invoked', () => { format: 'application/dif10+xml', umm: {} }) + vi.mocked(validateCmrCollectionUmm).mockResolvedValue({ status: 200, errors: [], @@ -1259,6 +1260,7 @@ describe('when the metadata correction service is invoked', () => { warnings: [] } }) + vi.mocked(extractKeywordValidationFailures).mockReturnValue([]) await metadataCorrectionService({ diff --git a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js index 4c72e35c..54f71101 100644 --- a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js +++ b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js @@ -1,4 +1,8 @@ -import { describe, expect, test } from 'vitest' +import { + describe, + expect, + test +} from 'vitest' import { buildNativeMetadataDiff } from '../buildNativeMetadataDiff' @@ -17,6 +21,7 @@ describe('buildNativeMetadataDiff', () => { originalBytes: 26, correctedBytes: 34 })) + expect(result.patch).toContain('--- cmr-revision-3') expect(result.patch).toContain('+++ corrected-metadata') expect(result.patch).toContain('-GOSAT') diff --git a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js index 6897784b..99dbd6ff 100644 --- a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js @@ -29,7 +29,6 @@ const SUMMARY_PROJECTION = { 'corrections.action': 1, 'corrections.oldKeywordPath': 1, 'corrections.newKeywordPath': 1, - 'metadataDiff.changed': 1, 'error.message': 1 } @@ -139,7 +138,6 @@ describe('metadata correction audit queries', () => { oldKeywordPath: 'Platforms > GOSAT', newKeywordPath: 'Platforms > GOSAT - Test1' }], - hasMetadataDiff: true, errorMessage: 'CMR writeback timed out' }], nextPaginationToken: null @@ -197,6 +195,33 @@ describe('metadata correction audit queries', () => { ) }) + test('includes native metadata diffs in list results when requested', async () => { + const metadataDiff = { + changed: true, + format: 'unified', + patch: '-old\n+new' + } + mongoCursor.toArray.mockResolvedValue([{ + _id: 'run-1', + runId: 'run-1', + metadataDiff + }]) + + const result = await getMetadataCorrectionAuditLog({ includeDiff: 'true' }) + + expect(collection.find).toHaveBeenCalledWith({}, { + projection: { + ...SUMMARY_PROJECTION, + metadataDiff: 1 + } + }) + + expect(result.items[0]).toMatchObject({ + runId: 'run-1', + metadataDiff + }) + }) + test('supports default filters, one-sided date ranges, and lowercase scheme storage', async () => { await getMetadataCorrectionAuditLog() @@ -204,6 +229,7 @@ describe('metadata correction audit queries', () => { {}, { projection: SUMMARY_PROJECTION } ) + expect(mongoCursor.limit).toHaveBeenLastCalledWith(101) await getMetadataCorrectionAuditLog({ scheme: 'PLATFORMS' }) @@ -266,6 +292,12 @@ describe('metadata correction audit queries', () => { paginationToken: 'not-a-pagination-token' })).rejects.toThrow('Invalid metadata correction audit paginationToken') + await expect(getMetadataCorrectionAuditLog({ + includeDiff: 'yes' + })).rejects.toThrow( + 'Invalid metadata correction audit includeDiff: expected true or false' + ) + const invalidPaginationToken = Buffer.from(JSON.stringify({ createdAt: '2026-09-02T12:00:00.000Z', runId: '' @@ -292,7 +324,12 @@ describe('metadata correction audit queries', () => { expect(collection.findOne).toHaveBeenCalledWith( { _id: 'run-1' }, - { projection: { _id: 0, metadataDiff: 0 } } + { + projection: { + _id: 0, + metadataDiff: 0 + } + } ) }) diff --git a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js index 128c790c..01509657 100644 --- a/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js +++ b/serverless/src/shared/__tests__/runCollectionMetadataCorrection.test.js @@ -500,6 +500,7 @@ describe('runCollectionMetadataCorrection', () => { expect(persistMetadataCorrectionAuditLog.mock.calls[1][0].metadataDiff.patch) .toContain('-') + expect(persistMetadataCorrectionAuditLog.mock.calls[1][0].metadataDiff.patch) .toContain('+corrected') @@ -1294,7 +1295,9 @@ describe('runCollectionMetadataCorrection', () => { contentType: 'application/vnd.nasa.cmr.umm+json;version=1.16.2; charset=utf-8' }) - vi.mocked(invokeMetadataCorrectionDelegate).mockImplementation(async ({ metadataPayload }) => { + vi.mocked(invokeMetadataCorrectionDelegate).mockImplementation(async (input) => { + const { metadataPayload } = input + expect(metadataPayload.ShortName).toBe('TEST') metadataPayload.ShortName = 'TEST-UPDATED' diff --git a/serverless/src/shared/getMetadataCorrectionAuditLog.js b/serverless/src/shared/getMetadataCorrectionAuditLog.js index ddb62f45..11943e97 100644 --- a/serverless/src/shared/getMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/getMetadataCorrectionAuditLog.js @@ -29,10 +29,24 @@ const AUDIT_SUMMARY_PROJECTION = { 'corrections.action': 1, 'corrections.oldKeywordPath': 1, 'corrections.newKeywordPath': 1, - 'metadataDiff.changed': 1, 'error.message': 1 } +/** + * Adds the complete stored diff to list queries only when requested. + * + * @param {boolean} includeDiff Whether list results should include native-metadata patches. + * @returns {Object} MongoDB projection for the audit list query. + */ +const auditSummaryProjection = (includeDiff) => { + if (!includeDiff) return AUDIT_SUMMARY_PROJECTION + + return { + ...AUDIT_SUMMARY_PROJECTION, + metadataDiff: 1 + } +} + /** * Validates the optional API page size. * @@ -344,9 +358,10 @@ const normalizeAuditChange = (correction = {}) => ({ * Builds the compact representation returned by paginated audit searches. * * @param {Object} document Stored audit document. + * @param {boolean} includeDiff Whether to include the native-metadata diff. * @returns {Object} Audit summary suitable for list views. */ -const normalizeAuditSummary = (document) => ({ +const normalizeAuditSummary = (document, includeDiff = false) => ({ runId: document.runId, collectionConceptId: document.collectionConceptId, collectionUri: document.collectionUri, @@ -355,12 +370,12 @@ const normalizeAuditSummary = (document) => ({ changes: Array.isArray(document.corrections) ? document.corrections.map(normalizeAuditChange) : [], - hasMetadataDiff: document.metadataDiff?.changed === true, + ...(includeDiff && document.metadataDiff ? { metadataDiff: document.metadataDiff } : {}), ...(document.error?.message ? { errorMessage: document.error.message } : {}) }) /** - * Parses the optional detail flag used to include a potentially large native-metadata diff. + * Parses the optional flag used to include a potentially large native-metadata diff. * * @example * normalizeIncludeDiff('true') // true @@ -389,10 +404,11 @@ const normalizeIncludeDiff = (includeDiff) => { */ export const getMetadataCorrectionAuditLog = async (filters = {}) => { const limit = normalizeLimit(filters.limit) + const shouldIncludeDiff = normalizeIncludeDiff(filters.includeDiff) const query = addPaginationTokenToQuery(buildAuditQuery(filters), filters.paginationToken) const collection = await getMetadataCorrectionAuditCollection() const documents = await collection.find(query, { - projection: AUDIT_SUMMARY_PROJECTION + projection: auditSummaryProjection(shouldIncludeDiff) }) .sort({ createdAt: -1, @@ -404,7 +420,7 @@ export const getMetadataCorrectionAuditLog = async (filters = {}) => { const pageDocuments = hasNextPage ? documents.slice(0, limit) : documents return { - items: pageDocuments.map(normalizeAuditSummary), + items: pageDocuments.map((document) => normalizeAuditSummary(document, shouldIncludeDiff)), nextPaginationToken: hasNextPage ? encodePaginationToken(pageDocuments[pageDocuments.length - 1]) : null @@ -438,7 +454,10 @@ export const getMetadataCorrectionAuditByRunId = async ({ { projection: shouldIncludeDiff ? { _id: 0 } - : { _id: 0, metadataDiff: 0 } + : { + _id: 0, + metadataDiff: 0 + } } ) From 6b16bd316174a061e9df5f4f9bc9b4f32aca5e7d Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Mon, 7 Sep 2026 20:07:52 -0400 Subject: [PATCH 07/17] KMS-703: add browser-friendly metadata correction audit diffs --- README.md | 3 + cdk/app/lib/helper/DocumentDbLambdaConfig.ts | 3 + package-lock.json | 69 +++ package.json | 1 + serverless/src/getCapabilities/handler.js | 4 +- .../__tests__/handler.test.js | 97 +++++ .../src/getMetadataCorrectionAudit/handler.js | 108 ++++- .../renderMetadataCorrectionAuditHtml.test.js | 113 +++++ .../renderMetadataCorrectionAuditHtml.js | 399 ++++++++++++++++++ 9 files changed, 791 insertions(+), 6 deletions(-) create mode 100644 serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js create mode 100644 serverless/src/shared/renderMetadataCorrectionAuditHtml.js diff --git a/README.md b/README.md index 4b9868f1..2c322f36 100644 --- a/README.md +++ b/README.md @@ -463,6 +463,9 @@ The audit API is: - `GET /metadata_correction_audit/{runId}` for the complete audit document. Add `?includeDiff=true` when the native-metadata diff is needed; it is omitted by default to keep routine responses small. +- Add `?format=html` to either endpoint for a self-contained browser view with a keyword-change + table and colored side-by-side native metadata diff. HTML responses include diffs automatically + and default to 10 records per page to keep rendered responses bounded. Publisher events carry the published KMS version through the queue into this document. Manual correction endpoints look up the current published version before starting the run, so the diff --git a/cdk/app/lib/helper/DocumentDbLambdaConfig.ts b/cdk/app/lib/helper/DocumentDbLambdaConfig.ts index 50725cad..b2fe9801 100644 --- a/cdk/app/lib/helper/DocumentDbLambdaConfig.ts +++ b/cdk/app/lib/helper/DocumentDbLambdaConfig.ts @@ -27,6 +27,9 @@ export const getDocumentDbCertificateBundling = ( bundling: { // Node.js 24 SAM images do not supply AWS SDK packages to local Lambda containers. externalModules: [], + loader: { + '.css': 'text' + }, ...(environment.DOCUMENTDB_TLS_CA_FILE ? { commandHooks: { diff --git a/package-lock.json b/package-lock.json index 763a964f..2c33c22a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "csv": "^6.3.11", "date-fns": "^4.1.0", "diff": "^5.2.2", + "diff2html": "^3.4.56", "fast-xml-parser": "^5.3.4", "html-entities": "^2.5.2", "html-escaper": "^3.0.3", @@ -4081,6 +4082,18 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@profoundlogic/hogan": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@profoundlogic/hogan/-/hogan-3.0.4.tgz", + "integrity": "sha512-pmNVGuooS30Mm7YbZd5T7E5zYVO6D5Ct91sn4T39mUvMUc3sCGridcnhAufL1/Bz2QzAtzEn0agNrdk3+5yWzw==", + "license": "Apache-2.0", + "dependencies": { + "nopt": "1.0.10" + }, + "bin": { + "hulk": "bin/hulk" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -6267,6 +6280,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC" + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -7947,6 +7966,31 @@ "node": ">=0.3.1" } }, + "node_modules/diff2html": { + "version": "3.4.56", + "resolved": "https://registry.npmjs.org/diff2html/-/diff2html-3.4.56.tgz", + "integrity": "sha512-u9gfn+BlbHcyO7vItCIC4z49LJDUt31tODzOfAuJ5R1E7IdlRL6KjugcB9zOpejD+XiR+dDZbsnHSQ3g6A/u8A==", + "license": "MIT", + "dependencies": { + "@profoundlogic/hogan": "^3.0.4", + "diff": "^8.0.3" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "highlight.js": "11.11.1" + } + }, + "node_modules/diff2html/node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -9889,6 +9933,16 @@ "node": ">= 0.4" } }, + "node_modules/highlight.js": { + "version": "11.11.1", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz", + "integrity": "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/hosted-git-info": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", @@ -11763,6 +11817,21 @@ "dev": true, "license": "MIT" }, + "node_modules/nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==", + "license": "MIT", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "*" + } + }, "node_modules/normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", diff --git a/package.json b/package.json index 8c1f4ce7..e1ec65ec 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "csv": "^6.3.11", "date-fns": "^4.1.0", "diff": "^5.2.2", + "diff2html": "^3.4.56", "fast-xml-parser": "^5.3.4", "html-entities": "^2.5.2", "html-escaper": "^3.0.3", diff --git a/serverless/src/getCapabilities/handler.js b/serverless/src/getCapabilities/handler.js index 5e1630cd..886b0034 100644 --- a/serverless/src/getCapabilities/handler.js +++ b/serverless/src/getCapabilities/handler.js @@ -141,7 +141,7 @@ export const getCapabilities = async () => { ':@': { name: 'get_metadata_correction_audit', href: '/metadata_correction_audit', - params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&includeDiff=&limit=', + params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&includeDiff=&format=&limit=', action: 'GET' } }, @@ -149,7 +149,7 @@ export const getCapabilities = async () => { ':@': { name: 'get_metadata_correction_audit_run', href: '/metadata_correction_audit/{runId}', - params: 'includeDiff=', + params: 'includeDiff=&format=', action: 'GET' } }, diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index ba84d592..0dbca692 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -114,6 +114,103 @@ describe('getMetadataCorrectionAudit', () => { })) }) + test('renders an html change table and diff for browser requests', async () => { + vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({ + items: [{ + runId: 'run-html', + collectionConceptId: 'C1234567890-LOCAL', + status: 'applied', + changes: [{ + scheme: 'platforms', + action: 'UPDATED', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + metadataDiff: { + changed: true, + patch: '--- old\n+++ new\n@@ -1,1 +1,1 @@\n-GOSAT\n+GOSAT - Test1\n' + } + }], + nextPaginationToken: 'next-token' + }) + + const result = await getMetadataCorrectionAudit({ + queryStringParameters: { + collectionConceptId: 'C1234567890-LOCAL', + format: 'html' + } + }) + + expect(getMetadataCorrectionAuditLog).toHaveBeenCalledWith(expect.objectContaining({ + collectionConceptId: 'C1234567890-LOCAL', + includeDiff: true, + limit: '10' + })) + + expect(result.statusCode).toBe(200) + expect(result.headers['Content-Type']).toBe('text/html; charset=utf-8') + expect(result.headers['Cache-Control']).toBe('no-store') + expect(result.body).toContain('') + expect(result.body).toContain('class="d2h-ins d2h-change"') + expect(result.body).toContain('paginationToken=next-token') + }) + + test('renders one detailed audit document as html', async () => { + vi.mocked(getMetadataCorrectionAuditByRunId).mockResolvedValue({ + runId: 'run-1', + collectionConceptId: 'C123-PROV', + status: 'applied', + corrections: [] + }) + + const result = await getMetadataCorrectionAudit({ + pathParameters: { runId: 'run-1' }, + queryStringParameters: { format: 'html' } + }) + + expect(getMetadataCorrectionAuditByRunId).toHaveBeenCalledWith({ + runId: 'run-1', + includeDiff: true + }) + + expect(result.statusCode).toBe(200) + expect(result.headers['Content-Type']).toBe('text/html; charset=utf-8') + expect(result.body).toContain('Metadata correction audit detail') + }) + + test('renders missing detail and server errors as html', async () => { + vi.mocked(getMetadataCorrectionAuditByRunId).mockResolvedValue(null) + + const missingResult = await getMetadataCorrectionAudit({ + pathParameters: { runId: 'missing-run' }, + queryStringParameters: { format: 'html' } + }) + + expect(missingResult.statusCode).toBe(404) + expect(missingResult.body).toContain('Metadata correction audit run not found: missing-run') + + vi.mocked(getMetadataCorrectionAuditLog).mockRejectedValue(new Error('DocumentDB unavailable')) + + const errorResult = await getMetadataCorrectionAudit({ + queryStringParameters: { format: 'html' } + }) + + expect(errorResult.statusCode).toBe(500) + expect(errorResult.headers['Content-Type']).toBe('text/html; charset=utf-8') + expect(errorResult.body).toContain('Error: DocumentDB unavailable') + }) + + test('returns 400 for an unsupported response format', async () => { + const result = await getMetadataCorrectionAudit({ + queryStringParameters: { format: 'xml' } + }) + + expect(result.statusCode).toBe(400) + expect(JSON.parse(result.body)).toEqual({ + error: 'Error: Invalid metadata correction audit format: expected json or html' + }) + }) + test('returns 500 when the audit query fails', async () => { vi.mocked(getMetadataCorrectionAuditLog).mockRejectedValue(new Error('Audit query failed')) diff --git a/serverless/src/getMetadataCorrectionAudit/handler.js b/serverless/src/getMetadataCorrectionAudit/handler.js index df540771..cd81982f 100644 --- a/serverless/src/getMetadataCorrectionAudit/handler.js +++ b/serverless/src/getMetadataCorrectionAudit/handler.js @@ -5,6 +5,48 @@ import { } from '@/shared/getMetadataCorrectionAuditLog' import { logAnalyticsData } from '@/shared/logAnalyticsData' import { logger } from '@/shared/logger' +import { renderMetadataCorrectionAuditHtml } from '@/shared/renderMetadataCorrectionAuditHtml' + +/** + * Validates the requested audit response representation. + * + * @param {unknown} format Requested response format. + * @returns {'json'|'html'} Normalized response format. + */ +const normalizeResponseFormat = (format) => { + if (format === undefined || format === null || format === '' || format === 'json') return 'json' + if (format === 'html') return 'html' + + throw new Error('Invalid metadata correction audit format: expected json or html') +} + +/** + * Preserves the current filters while advancing an HTML audit search to its next page. + * + * @param {Object} queryStringParameters Current API query parameters. + * @param {string|null} paginationToken Opaque next-page token. + * @returns {string|undefined} Relative next-page URL when another page exists. + */ +const buildNextPageHref = (queryStringParameters, paginationToken) => { + if (!paginationToken) return undefined + + const parameters = new URLSearchParams() + Object.entries(queryStringParameters || {}).forEach(([key, value]) => { + if (value !== undefined && value !== null) parameters.set(key, String(value)) + }) + + parameters.set('format', 'html') + parameters.set('paginationToken', paginationToken) + + return `?${parameters.toString()}` +} + +const HTML_RESPONSE_HEADERS = { + 'Cache-Control': 'no-store', + 'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; frame-ancestors 'none'; form-action 'none'", + 'Content-Type': 'text/html; charset=utf-8', + 'X-Content-Type-Options': 'nosniff' +} /** * Read-side audit endpoint for metadata-correction activity. @@ -33,9 +75,11 @@ import { logger } from '@/shared/logger' * - startDate / endDate * - paginationToken * - includeDiff + * - format (`json` or `html`) * - limit * * Add `includeDiff=true` to a list or detail request to include native-metadata patches. + * HTML responses include native-metadata patches automatically. * A `runId` path parameter returns one detailed audit document. * * @param {object} event - API Gateway event. @@ -69,17 +113,40 @@ export const getMetadataCorrectionAudit = async (event, context) => { endDate, paginationToken, includeDiff, + format, limit } = event?.queryStringParameters || {} const runId = event?.pathParameters?.runId + let responseFormat = 'json' try { + responseFormat = normalizeResponseFormat(format) + const requestedIncludeDiff = responseFormat === 'html' ? true : includeDiff + const requestedLimit = responseFormat === 'html' && !limit ? '10' : limit + if (runId) { const auditDocument = await getMetadataCorrectionAuditByRunId({ runId, - includeDiff + includeDiff: requestedIncludeDiff }) + if (responseFormat === 'html') { + return { + statusCode: auditDocument ? 200 : 404, + headers: { + ...defaultResponseHeaders, + ...HTML_RESPONSE_HEADERS + }, + body: renderMetadataCorrectionAuditHtml({ + items: auditDocument ? [auditDocument] : [], + message: auditDocument + ? undefined + : `Metadata correction audit run not found: ${runId}`, + title: 'Metadata correction audit detail' + }) + } + } + return { statusCode: auditDocument ? 200 : 404, headers: { @@ -106,10 +173,27 @@ export const getMetadataCorrectionAudit = async (event, context) => { startDate, endDate, paginationToken, - includeDiff, - limit + includeDiff: requestedIncludeDiff, + limit: requestedLimit }) + if (responseFormat === 'html') { + return { + statusCode: 200, + headers: { + ...defaultResponseHeaders, + ...HTML_RESPONSE_HEADERS + }, + body: renderMetadataCorrectionAuditHtml({ + items: auditPage.items, + nextPageHref: buildNextPageHref( + event?.queryStringParameters, + auditPage.nextPaginationToken + ) + }) + } + } + return { statusCode: 200, headers: { @@ -124,12 +208,28 @@ export const getMetadataCorrectionAudit = async (event, context) => { const isClientError = String(error?.message || '') .startsWith('Invalid metadata correction audit') + const statusCode = isClientError ? 400 : 500 + + if (responseFormat === 'html') { + return { + headers: { + ...defaultResponseHeaders, + ...HTML_RESPONSE_HEADERS + }, + statusCode, + body: renderMetadataCorrectionAuditHtml({ + message: error.toString(), + title: 'Metadata correction audit error' + }) + } + } + return { headers: { ...defaultResponseHeaders, 'Content-Type': 'application/json' }, - statusCode: isClientError ? 400 : 500, + statusCode, body: JSON.stringify({ error: error.toString() }) diff --git a/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js new file mode 100644 index 00000000..74bd5632 --- /dev/null +++ b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js @@ -0,0 +1,113 @@ +import { + describe, + expect, + test +} from 'vitest' + +import { renderMetadataCorrectionAuditHtml } from '../renderMetadataCorrectionAuditHtml' + +const PATCH = `=================================================================== +--- cmr-revision-3 ++++ corrected-metadata +@@ -1,1 +1,1 @@ +-GOSAT ++GOSAT - Test1 +` + +describe('renderMetadataCorrectionAuditHtml', () => { + test('renders a safe change table and colored side-by-side native metadata diff', () => { + const view = renderMetadataCorrectionAuditHtml({ + items: [{ + runId: 'run-1', + collectionConceptId: 'C123-PROV', + collectionUri: 'https://cmr.example.com/search/concepts/C123-PROV', + status: 'applied', + updatedAt: new Date('2026-09-07T12:00:00.000Z'), + changes: [{ + scheme: 'platforms', + action: 'UPDATED', + oldKeywordPath: 'Platforms > GOSAT', + newKeywordPath: 'Platforms > GOSAT - Test1' + }], + metadataDiff: { + changed: true, + format: 'unified', + patch: PATCH, + truncated: false + } + }], + nextPageHref: '?format=html&paginationToken=next' + }) + + expect(view).toContain('') + expect(view).toContain('
') + expect(view).toContain('Platforms > GOSAT - Test1') + expect(view).toContain('class="d2h-del d2h-change"') + expect(view).toContain('class="d2h-ins d2h-change"') + expect(view).toContain(' - Test1') + expect(view).toContain('white-space: pre;') + expect(view).toContain('overflow-x: auto;') + expect(view).toContain('Next page') + }) + + test('escapes audit content and reports absent and truncated diffs', () => { + const view = renderMetadataCorrectionAuditHtml({ + items: [ + { + runId: '', + collectionConceptId: '', + collectionUri: 'mailto:not-a-web-link@example.com', + status: 'failed', + error: { message: '' } + }, + { + runId: 'run-2', + collectionConceptId: 'C456-PROV', + status: 'pending', + corrections: [], + metadataDiff: { + patch: PATCH, + truncated: true + } + }, + { + runId: 'run-3', + collectionConceptId: 'C789-PROV', + status: 'checked', + metadataDiff: { + patch: { invalid: true } + } + }, + { + runId: 'run-4', + collectionConceptId: 'C999-PROV', + status: 'applied', + metadataDiff: { + patch: PATCH.replace('GOSAT - Test1', '') + } + } + ] + }) + + expect(view).not.toContain('', diff --git a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js index aed56cd7..b4105719 100644 --- a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js +++ b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js @@ -75,6 +75,12 @@ const PAGE_STYLES = ` .audit-header a { color: var(--accent); } + .audit-actions { + display: flex; + align-items: center; + gap: 0.75rem; + } + .audit-meta { margin: 0; color: var(--muted); @@ -115,6 +121,34 @@ const PAGE_STYLES = ` text-transform: uppercase; } + .detail-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr)); + gap: 1rem; + margin: 0; + } + + .detail-grid div { + min-width: 0; + padding: 0.7rem 0.8rem; + border-left: 0.2rem solid var(--accent); + background: #f1f6f4; + } + + .detail-grid dt { + margin-bottom: 0.25rem; + color: var(--muted); + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.07em; + text-transform: uppercase; + } + + .detail-grid dd { + margin: 0; + overflow-wrap: anywhere; + } + .changes-table { width: 100%; table-layout: fixed; @@ -142,6 +176,27 @@ const PAGE_STYLES = ` .changes-table th:nth-child(1) { width: 14%; } .changes-table th:nth-child(2) { width: 11%; } + .history-table { + width: 100%; + border-collapse: collapse; + font-size: 0.86rem; + } + + .history-table th, + .history-table td { + padding: 0.6rem 0.7rem; + border-bottom: 1px solid var(--line); + text-align: left; + vertical-align: top; + } + + .history-table th { + color: var(--muted); + font-size: 0.7rem; + letter-spacing: 0.06em; + text-transform: uppercase; + } + .previous-path { background: #fff4f3; } .updated-path { background: #edf9f3; } @@ -196,6 +251,25 @@ const PAGE_STYLES = ` text-decoration: none; } + .view-details { + display: inline-block; + padding: 0.55rem 0.8rem; + border: 1px solid var(--accent); + border-radius: 0.35rem; + color: var(--accent); + font-size: 0.8rem; + font-weight: 700; + text-decoration: none; + } + + .diagnostic { + max-width: 100%; + margin: 0; + overflow-x: auto; + white-space: pre; + font: 0.78rem/1.5 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + } + @media (max-width: 700px) { main { width: min(94vw, 1600px); padding-top: 1.5rem; } .audit-header { display: block; } @@ -216,6 +290,33 @@ const displayValue = (value, fallback = 'Not available') => escape( String(value ?? fallback) ) +/** + * Formats a stored date consistently for detail fields and lifecycle history. + * + * @param {unknown} value Stored date value. + * @returns {string} HTML-safe ISO date or fallback text. + */ +const displayDate = (value) => displayValue(value instanceof Date ? value.toISOString() : value) + +/** + * Renders labeled audit values in a responsive detail grid. + * + * @param {Array<[string, unknown]>} entries Label/value pairs, including optional values. + * @returns {string} Detail grid containing only available values. + */ +const renderDetailGrid = (entries) => { + const availableEntries = entries.filter(([, value]) => value !== undefined && value !== null && value !== '') + + if (availableEntries.length === 0) return '' + + return `
${availableEntries.map(([label, value]) => ` +
+
${displayValue(label)}
+
${displayValue(value)}
+
+ `).join('')}
` +} + /** * Returns either a safe CMR collection link or a plain collection identifier. * @@ -307,13 +408,111 @@ const renderNativeMetadataDiff = (audit) => { ` } +/** + * Renders the fields available only on a complete audit document. + * + * @param {Object} audit Detailed audit document. + * @returns {string} Run overview and optional trigger detail sections. + */ +const renderRunDetails = (audit) => { + const overview = renderDetailGrid([ + ['Provider', audit.providerId], + ['Published KMS version', audit.publishedVersionName], + ['Native format', audit.nativeFormat], + ['Delegate', audit.delegateName], + ['Source', audit.source], + ['Outcome', audit.outcome], + ['Prior CMR revision', audit.priorRevisionId], + ['Resulting CMR revision', audit.resultingRevisionId], + ['Message ID', audit.messageId], + ['Created', audit.createdAt instanceof Date ? audit.createdAt.toISOString() : audit.createdAt], + ['Updated', audit.updatedAt instanceof Date ? audit.updatedAt.toISOString() : audit.updatedAt] + ]) + const trigger = audit.trigger && typeof audit.trigger === 'object' + ? renderDetailGrid([ + ['Event type', audit.trigger.eventType], + ['Scheme', audit.trigger.scheme], + ['Keyword UUID', audit.trigger.keywordConceptUuid], + ['Event timestamp', audit.trigger.timestamp] + ]) + : '' + + return ` + ${overview ? `

Run details

${overview}
` : ''} + ${trigger ? `

Trigger

${trigger}
` : ''} + ` +} + +/** + * Renders the status transitions recorded throughout one correction run. + * + * @param {Object} audit Detailed audit document. + * @returns {string} Lifecycle history section or an empty string. + */ +const renderStatusHistory = (audit) => { + if (!Array.isArray(audit.statusHistory) || audit.statusHistory.length === 0) return '' + + const rows = audit.statusHistory.map((entry) => ` + + + + + + + `).join('') + + return ` +
+

Lifecycle history

+
${displayValue(entry.status)}${displayDate(entry.timestamp)}${displayValue(entry.outcome, '')}${displayValue(entry.error, '')}
+ + ${rows} +
StatusTimestampOutcomeError
+ + ` +} + +/** + * Renders validation diagnostics retained on a detailed audit document. + * + * @param {Object} audit Detailed audit document. + * @returns {string} Escaped validation diagnostics or an empty string. + */ +const renderValidationFailures = (audit) => { + if (!Array.isArray(audit.keywordValidationFailures) + || audit.keywordValidationFailures.length === 0) return '' + + return ` +
+

Keyword validation failures

+ 
${displayValue(JSON.stringify(audit.keywordValidationFailures, null, 2))}
+
+ ` +} + +/** + * Builds a relative browser link from an audit summary to its complete record. + * + * @param {Object} audit Audit summary document. + * @returns {string} Detail link or an empty string when no run id exists. + */ +const renderDetailLink = (audit) => { + if (!audit.runId) return '' + + const runId = encodeURIComponent(String(audit.runId)) + + return `View details` +} + /** * Renders one audit run with its status, path changes, and native metadata diff. * * @param {Object} audit Audit summary or detail document. + * @param {Object} options Rendering options. + * @param {boolean} options.detail Whether this is the complete run view. * @returns {string} HTML audit card. */ -const renderAuditCard = (audit) => { +const renderAuditCard = (audit, { detail }) => { const status = String(audit.status || 'unknown').toLowerCase() const statusClass = status === 'failed' ? ' status-failed' : '' const updatedAt = audit.updatedAt || audit.createdAt @@ -332,17 +531,23 @@ const renderAuditCard = (audit) => {

${renderCollectionHeading(audit)}

Run ${displayValue(audit.runId)}${updatedText}

- ${displayValue(status)} +
+ ${displayValue(status)} + ${detail ? '' : renderDetailLink(audit)} +
+ ${detail ? renderRunDetails(audit) : ''} ${errorSection}

Keyword changes

${renderChangesTable(audit)}
-
+ ${detail ? renderValidationFailures(audit) : ''} + ${detail ? renderStatusHistory(audit) : ''} + ${detail || audit.metadataDiff ? `

Native metadata diff

${renderNativeMetadataDiff(audit)} -
+
` : ''} ` } @@ -357,6 +562,7 @@ const renderAuditCard = (audit) => { * // '...' * * @param {Object} params Page data. + * @param {boolean} [params.detail=false] Whether to render complete run information. * @param {Array} [params.items=[]] Audit records to render. * @param {string} [params.message] Optional empty-state or error message. * @param {string} [params.nextPageHref] Link to the next result page. @@ -364,13 +570,14 @@ const renderAuditCard = (audit) => { * @returns {string} Complete HTML document. */ export const renderMetadataCorrectionAuditHtml = ({ + detail = false, items = [], message, nextPageHref, title = 'Metadata correction audit' } = {}) => { const cards = items.length > 0 - ? items.map(renderAuditCard).join('') + ? items.map((audit) => renderAuditCard(audit, { detail })).join('') : `

${displayValue(message, 'No matching audit records were found.')}

` const nextPageLink = nextPageHref ? `Next page` From 8a889ed117b41ed0b3fbb4d28879d4fb90001684 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 8 Sep 2026 16:42:30 -0400 Subject: [PATCH 09/17] KMS-703: link audit summary run IDs to detail views --- .../__tests__/handler.test.js | 4 +- .../renderMetadataCorrectionAuditHtml.test.js | 7 ++-- .../renderMetadataCorrectionAuditHtml.js | 42 +++++++------------ 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index c59cbfe1..1a3b3cf1 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -147,7 +147,7 @@ describe('getMetadataCorrectionAudit', () => { expect(result.headers['Content-Type']).toBe('text/html; charset=utf-8') expect(result.headers['Cache-Control']).toBe('no-store') expect(result.body).toContain('') - expect(result.body).toContain('metadata_correction_audit/run-html?format=html') + expect(result.body).toContain('run-html') expect(result.body).not.toContain('Native metadata diff') expect(result.body).toContain('paginationToken=next-token') }) @@ -184,7 +184,7 @@ describe('getMetadataCorrectionAudit', () => { expect(result.body).toContain('Prior CMR revision') expect(result.body).toContain('Lifecycle history') expect(result.body).toContain('Native metadata diff') - expect(result.body).not.toContain('View details') + expect(result.body).not.toContain('metadata_correction_audit/run-1?format=html') }) test('renders missing detail and server errors as html', async () => { diff --git a/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js index 2034fefb..e554f728 100644 --- a/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js +++ b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js @@ -48,7 +48,7 @@ describe('renderMetadataCorrectionAuditHtml', () => { expect(view).toContain('white-space: pre;') expect(view).toContain('overflow-x: auto;') expect(view).toContain('Next page') - expect(view).toContain('metadata_correction_audit/run-1?format=html') + expect(view).toContain('run-1') }) test('keeps summaries compact when a native metadata diff was not requested', () => { @@ -67,7 +67,7 @@ describe('renderMetadataCorrectionAuditHtml', () => { }) expect(view).toContain('Keyword changes') - expect(view).toContain('metadata_correction_audit/run%2Fsummary?format=html') + expect(view).toContain('run/summary') expect(view).not.toContain('Native metadata diff') expect(view).not.toContain('Run details') }) @@ -122,7 +122,8 @@ describe('renderMetadataCorrectionAuditHtml', () => { expect(view).toContain('corrections-resolved') expect(view).toContain('CMR failed') expect(view).toContain('No native metadata diff was recorded for this run.') - expect(view).not.toContain('View details') + expect(view).toContain('

Run run-detail') + expect(view).not.toContain('metadata_correction_audit/run-detail?format=html') }) test('escapes audit content and reports absent and truncated diffs', () => { diff --git a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js index b4105719..521c5952 100644 --- a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js +++ b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js @@ -75,12 +75,6 @@ const PAGE_STYLES = ` .audit-header a { color: var(--accent); } - .audit-actions { - display: flex; - align-items: center; - gap: 0.75rem; - } - .audit-meta { margin: 0; color: var(--muted); @@ -89,6 +83,11 @@ const PAGE_STYLES = ` overflow-wrap: anywhere; } + .audit-meta a { + color: var(--accent); + font-weight: 700; + } + .status { flex: none; padding: 0.35rem 0.65rem; @@ -251,17 +250,6 @@ const PAGE_STYLES = ` text-decoration: none; } - .view-details { - display: inline-block; - padding: 0.55rem 0.8rem; - border: 1px solid var(--accent); - border-radius: 0.35rem; - color: var(--accent); - font-size: 0.8rem; - font-weight: 700; - text-decoration: none; - } - .diagnostic { max-width: 100%; margin: 0; @@ -491,17 +479,20 @@ const renderValidationFailures = (audit) => { } /** - * Builds a relative browser link from an audit summary to its complete record. + * Renders a run id as a detail link in summary mode and plain text in detail mode. * * @param {Object} audit Audit summary document. - * @returns {string} Detail link or an empty string when no run id exists. + * @param {boolean} detail Whether this is already the complete run view. + * @returns {string} Linked or plain-text run id. */ -const renderDetailLink = (audit) => { - if (!audit.runId) return '' +const renderRunId = (audit, detail) => { + const displayRunId = displayValue(audit.runId) + + if (detail || !audit.runId) return displayRunId const runId = encodeURIComponent(String(audit.runId)) - return `View details` + return `${displayRunId}` } /** @@ -529,12 +520,9 @@ const renderAuditCard = (audit, { detail }) => {

${renderCollectionHeading(audit)}

-

Run ${displayValue(audit.runId)}${updatedText}

-
-
- ${displayValue(status)} - ${detail ? '' : renderDetailLink(audit)} +

Run ${renderRunId(audit, detail)}${updatedText}

+ ${displayValue(status)}
${detail ? renderRunDetails(audit) : ''} ${errorSection} From a48b1956aad1efc0d1338d3027618986c55ce13b Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 8 Sep 2026 17:13:37 -0400 Subject: [PATCH 10/17] KMS-703: format XML metadata for readable audit diffs --- .../__tests__/buildNativeMetadataDiff.test.js | 34 +++++++++++++ .../src/shared/buildNativeMetadataDiff.js | 48 +++++++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js index 54f71101..b105c0d7 100644 --- a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js +++ b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js @@ -28,6 +28,40 @@ describe('buildNativeMetadataDiff', () => { expect(result.patch).toContain('+GOSAT - Test1') }) + test('formats minified XML into a readable line-level diff', () => { + const result = buildNativeMetadataDiff({ + originalMetadata: 'TESTGOSATGreenhouse Gases Observing Satellite', + correctedMetadata: 'TESTGOSAT - Test1Greenhouse Gases Observing Satellite', + priorRevisionId: 5 + }) + + expect(result.patch).toContain(' ') + expect(result.patch).toContain('- GOSAT') + expect(result.patch).toContain('+ GOSAT - Test1') + expect(result.patch).toContain(' Greenhouse Gases Observing Satellite') + expect(result.patch).not.toContain('-') + }) + + test('ignores XML formatting differences', () => { + const result = buildNativeMetadataDiff({ + originalMetadata: '\n TEST\n', + correctedMetadata: 'TEST' + }) + + expect(result.changed).toBe(false) + expect(result.patch).toBe('') + }) + + test('leaves invalid XML text unchanged for diffing', () => { + const result = buildNativeMetadataDiff({ + originalMetadata: 'OLD', + correctedMetadata: 'NEW' + }) + + expect(result.patch).toContain('-OLD') + expect(result.patch).toContain('+NEW') + }) + test('serializes JSON metadata and reports identical payloads without a patch', () => { const metadata = { Platforms: [{ ShortName: 'GOSAT' }] } diff --git a/serverless/src/shared/buildNativeMetadataDiff.js b/serverless/src/shared/buildNativeMetadataDiff.js index 20156f77..eae10dcf 100644 --- a/serverless/src/shared/buildNativeMetadataDiff.js +++ b/serverless/src/shared/buildNativeMetadataDiff.js @@ -1,6 +1,46 @@ import { createTwoFilesPatch } from 'diff' +import { + XMLBuilder, + XMLParser, + XMLValidator +} from 'fast-xml-parser' const MAX_METADATA_DIFF_CHARACTERS = 250_000 +const XML_FORMAT_OPTIONS = { + cdataPropName: '#cdata', + ignoreAttributes: false, + preserveOrder: true, + processEntities: false, + trimValues: true +} +const xmlParser = new XMLParser(XML_FORMAT_OPTIONS) +const xmlBuilder = new XMLBuilder({ + ...XML_FORMAT_OPTIONS, + format: true, + indentBy: ' ', + suppressEmptyNode: true +}) + +/** + * Pretty-prints valid XML for a readable line-level diff without changing the CMR write payload. + * + * @example + * formatNativeMetadataTextForDiff('A') + * // '\n A\n' + * + * @param {string} metadataText Serialized native metadata. + * @returns {string} Formatted XML, or the original text for JSON, plain text, or invalid XML. + */ +const formatNativeMetadataTextForDiff = (metadataText) => { + if (!metadataText.trimStart().startsWith('<') + || XMLValidator.validate(metadataText) !== true) return metadataText + + try { + return `${xmlBuilder.build(xmlParser.parse(metadataText)).trim()}\n` + } catch { + return metadataText + } +} /** * Converts native XML or JSON metadata into comparable text without changing the write payload. @@ -51,13 +91,15 @@ export const buildNativeMetadataDiff = ({ return undefined } - const changed = originalText !== correctedText + const formattedOriginalText = formatNativeMetadataTextForDiff(originalText) + const formattedCorrectedText = formatNativeMetadataTextForDiff(correctedText) + const changed = formattedOriginalText !== formattedCorrectedText const fullPatch = changed ? createTwoFilesPatch( `cmr-revision-${priorRevisionId ?? 'unknown'}`, 'corrected-metadata', - originalText, - correctedText, + formattedOriginalText, + formattedCorrectedText, '', '', { context: 3 } From 9d815b8434991cd61e3db59c0fcdf331c3b69e00 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 8 Sep 2026 18:46:52 -0400 Subject: [PATCH 11/17] KMS-703: Added missed coverage. --- .../__tests__/buildNativeMetadataDiff.test.js | 17 ++++++++++++++++- .../shared/__tests__/documentDbClient.test.js | 9 +++++++++ .../persistMetadataCorrectionAuditLog.test.js | 9 +++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js index b105c0d7..d7085fef 100644 --- a/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js +++ b/serverless/src/shared/__tests__/buildNativeMetadataDiff.test.js @@ -1,7 +1,9 @@ +import { XMLBuilder } from 'fast-xml-parser' import { describe, expect, - test + test, + vi } from 'vitest' import { buildNativeMetadataDiff } from '../buildNativeMetadataDiff' @@ -62,6 +64,19 @@ describe('buildNativeMetadataDiff', () => { expect(result.patch).toContain('+NEW') }) + test('uses the original XML when formatting unexpectedly fails', () => { + vi.spyOn(XMLBuilder.prototype, 'build') + .mockImplementationOnce(() => { throw new Error('formatting failed') }) + + const result = buildNativeMetadataDiff({ + originalMetadata: 'GOSAT', + correctedMetadata: 'GOSAT - Test1' + }) + + expect(result.patch).toContain('-GOSAT') + expect(result.patch).toContain('+GOSAT - Test1') + }) + test('serializes JSON metadata and reports identical payloads without a patch', () => { const metadata = { Platforms: [{ ShortName: 'GOSAT' }] } diff --git a/serverless/src/shared/__tests__/documentDbClient.test.js b/serverless/src/shared/__tests__/documentDbClient.test.js index 5b5595bd..a4f25e1d 100644 --- a/serverless/src/shared/__tests__/documentDbClient.test.js +++ b/serverless/src/shared/__tests__/documentDbClient.test.js @@ -121,6 +121,15 @@ describe('documentDbClient', () => { expect(mongoClientConstructor).toHaveBeenCalledTimes(2) }) + test('uses the default database and audit collection names', async () => { + process.env.DOCUMENTDB_URI = 'mongodb://localhost:27018' + const { getMetadataCorrectionAuditCollection } = await import('../documentDbClient') + + await expect(getMetadataCorrectionAuditCollection()).resolves.toBe(collection) + + expect(db).toHaveBeenCalledWith('kms') + }) + test('builds the deployed TLS connection from Secrets Manager credentials', async () => { process.env.DOCUMENTDB_HOST = 'audit.cluster.docdb.amazonaws.com' process.env.DOCUMENTDB_PORT = '27017' diff --git a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js index 969054ca..4413ad7c 100644 --- a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js @@ -63,6 +63,10 @@ describe('persistMetadataCorrectionAuditLog', () => { scheme: 'platforms', uuid: 'platform-uuid' }, + keywordValidationFailures: [{ + keywordConceptUuid: 'invalid-keyword-uuid', + reason: 'Keyword was not found in the published version' + }], metadataDiff: { changed: true, format: 'unified', @@ -89,6 +93,11 @@ describe('persistMetadataCorrectionAuditLog', () => { $set: expect.objectContaining({ collectionConceptId: 'C123-PROV', collectionUri: 'https://cmr.example.com/search/concepts/C123-PROV', + keywordValidationFailureCount: 1, + keywordValidationFailures: [{ + keywordConceptUuid: 'invalid-keyword-uuid', + reason: 'Keyword was not found in the published version' + }], publishedVersionName: '20.1', priorRevisionId: 7, status: 'checked', From 2097e7f8afc92dd1bf6ce6907fa06ade55d1a2d7 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 8 Sep 2026 18:55:06 -0400 Subject: [PATCH 12/17] KMS-703: Ran npm audit fix --- package-lock.json | 155 +++++++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 72 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2c33c22a..1e576c5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4777,20 +4777,14 @@ } }, "node_modules/@smithy/middleware-compression": { - "version": "4.3.46", - "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.3.46.tgz", - "integrity": "sha512-9f4AZ5dKqKRmO49MPhOoxFoQBLfBgxE9YKG8bQ6lsW9xk+Bn8rkfGlpW8OYlvhuarN+8mja9PjhEudFiR8wGFQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.6.2.tgz", + "integrity": "sha512-Q9d+luiRjyHT6kCL/9NyGpdZJgodh4vtvfHC6H8SqoVKrV2k9RoyI9/IloVfdCYks3/2DIi7BsYYLcda5KZS0A==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.17", - "@smithy/is-array-buffer": "^4.2.2", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", - "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-middleware": "^4.2.14", - "@smithy/util-utf8": "^4.2.2", - "fflate": "0.8.1", + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "fflate": "0.8.3", "tslib": "^2.6.2" }, "engines": { @@ -6264,9 +6258,9 @@ } }, "node_modules/@xmldom/xmldom": { - "version": "0.8.13", - "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", - "integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==", + "version": "0.8.15", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.15.tgz", + "integrity": "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -7190,6 +7184,19 @@ "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.21", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.21.tgz", + "integrity": "sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -7239,9 +7246,9 @@ } }, "node_modules/browserslist": { - "version": "4.25.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.3.tgz", - "integrity": "sha512-cDGv1kkDI4/0e5yON9yM5G/0A5u8sf5TnmdX5C9qHzI9PPu++sQ9zjm1k9NiOrf3riY4OkK0zSGqfvJyJsgCBQ==", + "version": "4.28.9", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.9.tgz", + "integrity": "sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg==", "dev": true, "funding": [ { @@ -7259,10 +7266,11 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001735", - "electron-to-chromium": "^1.5.204", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.3" + "baseline-browser-mapping": "^2.11.20", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.420", + "node-releases": "^2.0.54", + "update-browserslist-db": "^1.3.2" }, "bin": { "browserslist": "cli.js" @@ -7396,9 +7404,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001736", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001736.tgz", - "integrity": "sha512-ImpN5gLEY8gWeqfLUyEF4b7mYWcYoR2Si1VhnrbM4JizRFmfGaAQ12PhNykq6nvI4XvKLrsp8Xde74D5phJOSw==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -7495,9 +7503,9 @@ "license": "MIT" }, "node_modules/colord": { - "version": "2.9.3", - "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", - "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.10.0.tgz", + "integrity": "sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==", "dev": true, "license": "MIT" }, @@ -7692,36 +7700,36 @@ "peer": true }, "node_modules/csv": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/csv/-/csv-6.4.1.tgz", - "integrity": "sha512-ajGosmTGnTwYyGl8STqZDu7R6LkDf3xL39XiOmliV/GufQeVUxHzTKIm4NOBCwmEuujK7B6isxs4Uqt9GcRCvA==", + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.6.3.tgz", + "integrity": "sha512-X2AnOgcxqV+OdLm1M2FOl+bPQrM1LK1jwo4FhEoq8hi8JTR0tQ8ZWv3x3N2gVMIuUmpb4CLrV08rTP49IdFQ8w==", "license": "MIT", "dependencies": { - "csv-generate": "^4.5.0", - "csv-parse": "^6.1.0", - "csv-stringify": "^6.6.0", - "stream-transform": "^3.4.0" + "csv-generate": "^4.6.1", + "csv-parse": "^7.0.2", + "csv-stringify": "^6.8.3", + "stream-transform": "^3.5.1" }, "engines": { "node": ">= 0.1.90" } }, "node_modules/csv-generate": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.5.0.tgz", - "integrity": "sha512-aQr/vmOKyBSBHNwYhAoXw1+kUsPnMSwmYgpNoo36rIXoG1ecWILnvPGZeQ6oUjzrWknZAD3+jfpqYOBAl4x15A==", + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.6.1.tgz", + "integrity": "sha512-eELl9K716LSSeP2/YcCjch525JztnnERe3jEARWw2v1FN9ukUYfZTNYZ4Rq2Jj/MFKMauffOy9VCaqQTpFThDQ==", "license": "MIT" }, "node_modules/csv-parse": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz", - "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz", + "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==", "license": "MIT" }, "node_modules/csv-stringify": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz", - "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==", + "version": "6.8.3", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.8.3.tgz", + "integrity": "sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==", "license": "MIT" }, "node_modules/damerau-levenshtein": { @@ -8061,9 +8069,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.207", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.207.tgz", - "integrity": "sha512-mryFrrL/GXDTmAtIVMVf+eIXM09BBPlO5IQ7lUyKmK8d+A4VpRGG+M3ofoVef6qyF8s60rJei8ymlJxjUA8Faw==", + "version": "1.5.425", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.425.tgz", + "integrity": "sha512-QvPtl41EUOnuT1HBvMKgxXRIaHNcagBPs50u7VULzhZXaGfqTbZyE16LQsctZ/RQHlGu+FOWeDTR4mY6YbeF1g==", "dev": true, "license": "ISC" }, @@ -9271,9 +9279,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "dev": true, "funding": [ { @@ -9345,9 +9353,9 @@ } }, "node_modules/fflate": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz", - "integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==", + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", "license": "MIT" }, "node_modules/file-entry-cache": { @@ -10724,9 +10732,9 @@ } }, "node_modules/joi": { - "version": "17.13.4", - "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.4.tgz", - "integrity": "sha512-1RuuER6kmt8K8I3nIWvPZKi5RQCb568ZPyY4Pwjlua+yo+63ZTmIwxLZH0heBmiKN4uxjvCiarDrjaeH84xicQ==", + "version": "17.13.7", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.7.tgz", + "integrity": "sha512-MF80Dm5Y2veNy8QWVx9Bj3ui4mo7+VPSPsR1M+oaHXV0Gx6zGX9a2F+OZG3Blby9tOlzU9Rs5FUimlEhbKtfnQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -10762,9 +10770,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -11811,11 +11819,14 @@ } }, "node_modules/node-releases": { - "version": "2.0.19", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", - "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/nopt": { "version": "1.0.10", @@ -12393,9 +12404,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -13462,9 +13473,9 @@ } }, "node_modules/stream-transform": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.4.0.tgz", - "integrity": "sha512-QO3OGhKyeIV8p6eRQdG+W6WounFw519zk690hHCNfhgfP9bylVS+NTXsuBc7n+RsGn31UgFPGrWYIgoAbArKEw==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.5.1.tgz", + "integrity": "sha512-TTDX+qKFr7GGRXATn66rprmlFPx08W0UBIccE/rMPgW2sT7GovduZYP4xcdJ7Nu2YigF17U+CNFxYY11+W1oPw==", "license": "MIT" }, "node_modules/string-width": { @@ -14469,9 +14480,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", - "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", "dev": true, "funding": [ { From 92a4adfa2fa15cc3d52ef032a75d5555fa52a20a Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Wed, 9 Sep 2026 17:21:21 -0400 Subject: [PATCH 13/17] KMS-703: Add collection ID filter to audit HTML view --- .../__tests__/handler.test.js | 4 + .../src/getMetadataCorrectionAudit/handler.js | 3 +- .../renderMetadataCorrectionAuditHtml.test.js | 5 ++ .../renderMetadataCorrectionAuditHtml.js | 84 ++++++++++++++++++- 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js index 1a3b3cf1..e79c7386 100644 --- a/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js +++ b/serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js @@ -146,6 +146,9 @@ describe('getMetadataCorrectionAudit', () => { expect(result.statusCode).toBe(200) expect(result.headers['Content-Type']).toBe('text/html; charset=utf-8') expect(result.headers['Cache-Control']).toBe('no-store') + expect(result.headers['Content-Security-Policy']).toContain("form-action 'self'") + expect(result.body).toContain('
') + expect(result.body).toContain('name="collectionConceptId" value="C1234567890-LOCAL"') expect(result.body).toContain('
') expect(result.body).toContain('run-html') expect(result.body).not.toContain('Native metadata diff') @@ -184,6 +187,7 @@ describe('getMetadataCorrectionAudit', () => { expect(result.body).toContain('Prior CMR revision') expect(result.body).toContain('Lifecycle history') expect(result.body).toContain('Native metadata diff') + expect(result.body).not.toContain('class="audit-filter"') expect(result.body).not.toContain('metadata_correction_audit/run-1?format=html') }) diff --git a/serverless/src/getMetadataCorrectionAudit/handler.js b/serverless/src/getMetadataCorrectionAudit/handler.js index aaddd4e8..1833bf28 100644 --- a/serverless/src/getMetadataCorrectionAudit/handler.js +++ b/serverless/src/getMetadataCorrectionAudit/handler.js @@ -43,7 +43,7 @@ const buildNextPageHref = (queryStringParameters, paginationToken) => { const HTML_RESPONSE_HEADERS = { 'Cache-Control': 'no-store', - 'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; frame-ancestors 'none'; form-action 'none'", + 'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'", 'Content-Type': 'text/html; charset=utf-8', 'X-Content-Type-Options': 'nosniff' } @@ -186,6 +186,7 @@ export const getMetadataCorrectionAudit = async (event, context) => { ...HTML_RESPONSE_HEADERS }, body: renderMetadataCorrectionAuditHtml({ + collectionConceptId, items: auditPage.items, nextPageHref: buildNextPageHref( event?.queryStringParameters, diff --git a/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js index e554f728..9acabec4 100644 --- a/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js +++ b/serverless/src/shared/__tests__/renderMetadataCorrectionAuditHtml.test.js @@ -17,6 +17,7 @@ const PATCH = `================================================================= describe('renderMetadataCorrectionAuditHtml', () => { test('renders a safe change table and colored side-by-side native metadata diff', () => { const view = renderMetadataCorrectionAuditHtml({ + collectionConceptId: 'C123-PROV', items: [{ runId: 'run-1', collectionConceptId: 'C123-PROV', @@ -40,6 +41,10 @@ describe('renderMetadataCorrectionAuditHtml', () => { }) expect(view).toContain('') + expect(view).toContain('') + expect(view).toContain('name="format" value="html"') + expect(view).toContain('name="collectionConceptId" value="C123-PROV"') + expect(view).toContain('Clear') expect(view).toContain('
') expect(view).toContain('Platforms > GOSAT - Test1') expect(view).toContain('class="d2h-del d2h-change"') diff --git a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js index 521c5952..3b4da59a 100644 --- a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js +++ b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js @@ -59,6 +59,60 @@ const PAGE_STYLES = ` box-shadow: 0 0.7rem 2rem rgba(20, 47, 60, 0.08); } + .audit-filter { + display: flex; + align-items: end; + gap: 0.75rem; + margin-bottom: 1.5rem; + padding: 1rem; + border: 1px solid var(--line); + border-radius: 0.75rem; + background: var(--paper); + box-shadow: 0 0.7rem 2rem rgba(20, 47, 60, 0.06); + } + + .audit-filter label { + display: grid; + flex: 1; + gap: 0.35rem; + color: var(--muted); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + } + + .audit-filter input { + width: 100%; + padding: 0.7rem 0.8rem; + border: 1px solid #9db6b6; + border-radius: 0.4rem; + background: #ffffff; + color: var(--ink); + font: 0.9rem ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + } + + .audit-filter button, + .audit-filter a { + padding: 0.7rem 1rem; + border: 1px solid var(--accent); + border-radius: 0.4rem; + font: inherit; + font-weight: 700; + text-decoration: none; + } + + .audit-filter button { + background: var(--accent); + color: #ffffff; + cursor: pointer; + } + + .audit-filter a { + background: transparent; + color: var(--accent); + } + .audit-header { display: flex; align-items: flex-start; @@ -260,6 +314,7 @@ const PAGE_STYLES = ` @media (max-width: 700px) { main { width: min(94vw, 1600px); padding-top: 1.5rem; } + .audit-filter { align-items: stretch; flex-direction: column; } .audit-header { display: block; } .status { display: inline-block; margin-top: 0.8rem; } .changes-table th:nth-child(1), @@ -286,6 +341,30 @@ const displayValue = (value, fallback = 'Not available') => escape( */ const displayDate = (value) => displayValue(value instanceof Date ? value.toISOString() : value) +/** + * Renders an HTML-only collection concept ID filter for the audit summary. + * + * @param {unknown} collectionConceptId Current exact-match filter value. + * @returns {string} GET form that retains the HTML response format. + */ +const renderCollectionFilter = (collectionConceptId) => { + const clearLink = collectionConceptId + ? 'Clear' + : '' + + return ` + + + + + ${clearLink} + + ` +} + /** * Renders labeled audit values in a responsive detail grid. * @@ -550,6 +629,7 @@ const renderAuditCard = (audit, { detail }) => { * // '...' * * @param {Object} params Page data. + * @param {string} [params.collectionConceptId] Current collection ID filter. * @param {boolean} [params.detail=false] Whether to render complete run information. * @param {Array} [params.items=[]] Audit records to render. * @param {string} [params.message] Optional empty-state or error message. @@ -558,6 +638,7 @@ const renderAuditCard = (audit, { detail }) => { * @returns {string} Complete HTML document. */ export const renderMetadataCorrectionAuditHtml = ({ + collectionConceptId, detail = false, items = [], message, @@ -576,7 +657,7 @@ export const renderMetadataCorrectionAuditHtml = ({ - + ${displayValue(title)} @@ -584,6 +665,7 @@ export const renderMetadataCorrectionAuditHtml = ({

${displayValue(title)}

${items.length} audit ${items.length === 1 ? 'record' : 'records'} on this page

+ ${detail ? '' : renderCollectionFilter(collectionConceptId)} ${cards} ${nextPageLink}
From 4d5cb37551627503dbca3bd0b703e19a224f2d42 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 10 Sep 2026 10:25:42 -0400 Subject: [PATCH 14/17] MMT-703: Added comment to the html view --- serverless/src/shared/renderMetadataCorrectionAuditHtml.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js index 3b4da59a..32caf5c8 100644 --- a/serverless/src/shared/renderMetadataCorrectionAuditHtml.js +++ b/serverless/src/shared/renderMetadataCorrectionAuditHtml.js @@ -622,6 +622,9 @@ const renderAuditCard = (audit, { detail }) => { /** * Builds a self-contained browser view of metadata-correction audit records. * + * This HTML view is a temporary stopgap until MMT provides an audit interface backed by the JSON + * API and its filters. Once that MMT interface is available, this renderer can likely be removed. + * * @example * renderMetadataCorrectionAuditHtml({ * items: [{ runId: 'run-1', collectionConceptId: 'C123-PROV', status: 'applied' }] From 3eaebe092b2a5a78c231a6e7fbfd7bca663fe740 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 10 Sep 2026 21:10:01 -0400 Subject: [PATCH 15/17] KMS-703: upgrade xmldom and fix invalid XML test namespaces --- .../shared/__tests__/metadataCorrectionDelegateStubs.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serverless/src/shared/__tests__/metadataCorrectionDelegateStubs.test.js b/serverless/src/shared/__tests__/metadataCorrectionDelegateStubs.test.js index fcddadc2..359e9c6d 100644 --- a/serverless/src/shared/__tests__/metadataCorrectionDelegateStubs.test.js +++ b/serverless/src/shared/__tests__/metadataCorrectionDelegateStubs.test.js @@ -179,7 +179,7 @@ describe('metadata correction delegate stubs', () => { test('returns the expected ISO19115 payload shape when corrections are provided', async () => { const mockPayload = ` - + From 6a8ff5e65ce27e8ddf9fcaea5f35209c25a98897 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 10 Sep 2026 21:10:15 -0400 Subject: [PATCH 16/17] KMS-703: upgrade xmldom and fix invalid XML test namespaces --- package-lock.json | 10 +++++----- package.json | 2 +- serverless/src/shared/__mocks__/iso-smap.xml | 5 +++-- .../__tests__/applyIso19115MetadataCorrections.test.js | 4 ++-- .../__tests__/applyIsoSmapMetadataCorrections.test.js | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1e576c5b..a1c8fbfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@aws-sdk/client-sqs": "^3.997.0", "@aws-sdk/client-ssm": "^3.1096.0", "@aws-sdk/s3-request-presigner": "3.981.0", - "@xmldom/xmldom": "^0.8.10", + "@xmldom/xmldom": "^0.9.12", "compact-object-deep": "^1.0.0", "csv": "^6.3.11", "date-fns": "^4.1.0", @@ -6258,12 +6258,12 @@ } }, "node_modules/@xmldom/xmldom": { - "version": "0.8.15", - "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.15.tgz", - "integrity": "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==", + "version": "0.9.12", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.9.12.tgz", + "integrity": "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A==", "license": "MIT", "engines": { - "node": ">=10.0.0" + "node": ">=14.6" } }, "node_modules/abab": { diff --git a/package.json b/package.json index e1ec65ec..74eaa3da 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sqs": "^3.997.0", "@aws-sdk/client-ssm": "^3.1096.0", "@aws-sdk/s3-request-presigner": "3.981.0", - "@xmldom/xmldom": "^0.8.10", + "@xmldom/xmldom": "^0.9.12", "compact-object-deep": "^1.0.0", "csv": "^6.3.11", "date-fns": "^4.1.0", diff --git a/serverless/src/shared/__mocks__/iso-smap.xml b/serverless/src/shared/__mocks__/iso-smap.xml index 950f9bc4..4980db24 100644 --- a/serverless/src/shared/__mocks__/iso-smap.xml +++ b/serverless/src/shared/__mocks__/iso-smap.xml @@ -1,7 +1,8 @@ + xmlns:gco="http://www.isotc211.org/2005/gco" + xmlns:xlink="http://www.w3.org/1999/xlink"> @@ -297,4 +298,4 @@ - \ No newline at end of file + diff --git a/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js b/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js index ddd62770..6ed76ced 100644 --- a/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js @@ -61,7 +61,7 @@ const mockIso19115WithOneScienceKeyword = ` describe('applyIso19115MetadataCorrections', () => { test('should handle missing corrections array gracefully', async () => { const params = { - metadataPayload: '' + metadataPayload: '' } const result = await applyIso19115MetadataCorrections(params) @@ -89,7 +89,7 @@ describe('applyIso19115MetadataCorrections', () => { test('should skip corrections with unknown schemes', async () => { const params = { - metadataPayload: '', + metadataPayload: '', corrections: [ { scheme: 'invalid-scheme', diff --git a/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js b/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js index ec082e49..002db482 100644 --- a/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js @@ -406,7 +406,7 @@ describe('when applying dataformat ISO-19115 corrections', () => { describe('applyIsoSmapMetadataCorrections coverage', () => { test('should gracefully handle an unknown scheme in corrections', async () => { const params = { - metadataPayload: '', + metadataPayload: '', corrections: [ { scheme: 'unknownScheme', // This will trigger the !delegate check on line 38 From 55e6172c4612e1f0dadbd56795f457cd1e7df3ad Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 15 Sep 2026 16:17:27 -0400 Subject: [PATCH 17/17] KMS-703: harden audit deployment, validation, and test helpers --- .gitignore | 1 + README.md | 5 +- bin/deploy-bamboo.sh | 8 ++ .../MetadataCorrectionAuditDatabaseSetup.ts | 2 +- cdk/app/lib/helper/MetadataCorrectionSetup.ts | 42 ++++++---- .../local/metadataCorrectionSmokeHelpers.mjs | 19 +++++ ...etadata_correction_applied_audit_smoke.mjs | 20 +---- ...orrection_consumer_metrics_async_smoke.mjs | 20 +---- ...ion_consumer_metrics_manual_sync_smoke.mjs | 21 +---- ...metadata_correction_failed_audit_smoke.mjs | 20 +---- ...etadata_correction_request_delay_smoke.mjs | 20 +---- .../run_metadata_correction_sync_smoke.mjs | 20 +---- serverless/certs/us-east-1-bundle.pem | 76 ------------------- .../getMetadataCorrectionAuditLog.test.js | 34 ++++++--- .../shared/getMetadataCorrectionAuditLog.js | 4 +- 15 files changed, 106 insertions(+), 206 deletions(-) create mode 100644 scripts/local/metadataCorrectionSmokeHelpers.mjs delete mode 100644 serverless/certs/us-east-1-bundle.pem diff --git a/.gitignore b/.gitignore index 4f56f9ab..84bfec65 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ notes/ archive-processor/downloaded-rdf archive-processor/local-kms-csv extern +serverless/certs/us-east-1-bundle.pem diff --git a/README.md b/README.md index e8f65b35..85a6b50d 100644 --- a/README.md +++ b/README.md @@ -473,8 +473,9 @@ correction endpoints look up the current published version before starting the r metadata-correction consumer and audit API do not query RDF4J. Deployed Lambdas use the public AWS `us-east-1` CA bundle to validate DocumentDB TLS connections. -The checked-in `serverless/certs/us-east-1-bundle.pem` was downloaded from the -[AWS certificate trust store](https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem): +The Bamboo deployment downloads the current bundle from the +[AWS certificate trust store](https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem) +before building the deployment image. For an AWS deployment outside Bamboo, download it manually: ```bash curl --fail --location \ diff --git a/bin/deploy-bamboo.sh b/bin/deploy-bamboo.sh index fe1e3c81..4504cdc1 100755 --- a/bin/deploy-bamboo.sh +++ b/bin/deploy-bamboo.sh @@ -18,6 +18,14 @@ config="`jq '.edl.uid = $newValue' --arg newValue $bamboo_EDL_UID <<< $config`" # overwrite static.config.json with new values echo $config > tmp.$$.json && mv tmp.$$.json static.config.json +# Download the current public AWS CA bundle before it is packaged with the Lambdas. +documentDbCaBundleUrl='https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem' +documentDbCaBundlePath='serverless/certs/us-east-1-bundle.pem' +mkdir -p "$(dirname "$documentDbCaBundlePath")" +curl --fail --silent --show-error --location \ + "$documentDbCaBundleUrl" \ + --output "$documentDbCaBundlePath" + # Set up Docker image ##################### diff --git a/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts index ce0d7756..7a17cf14 100644 --- a/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts +++ b/cdk/app/lib/helper/MetadataCorrectionAuditDatabaseSetup.ts @@ -95,7 +95,7 @@ export class MetadataCorrectionAuditDatabaseSetup extends Construct { backup: { retention: cdk.Duration.days(7) }, - deletionProtection: ['ops', 'prod'].includes(props.stage.toLowerCase()), + deletionProtection: true, removalPolicy: cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE, storageEncrypted: true, securityGroup: databaseSecurityGroup, diff --git a/cdk/app/lib/helper/MetadataCorrectionSetup.ts b/cdk/app/lib/helper/MetadataCorrectionSetup.ts index 61f2d42c..0bc464fe 100644 --- a/cdk/app/lib/helper/MetadataCorrectionSetup.ts +++ b/cdk/app/lib/helper/MetadataCorrectionSetup.ts @@ -50,6 +50,31 @@ interface MetadataCorrectionSetupProps { export class MetadataCorrectionSetup extends Construct { private static readonly DEFAULT_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY = 5 + /** + * Resolves the Lambda concurrency limit, forcing one worker when rate limiting is enabled. + * + * @example + * MetadataCorrectionSetup.resolveReservedConcurrency(true, '5') // 1 + * MetadataCorrectionSetup.resolveReservedConcurrency(false, '3') // 3 + * + * @param hasRateLimit Whether request rate limiting is enabled. + * @param configuredReservedConcurrency Configured Lambda concurrency limit. + * @returns The concurrency limit to apply to the metadata-correction Lambda. + */ + private static resolveReservedConcurrency( + hasRateLimit: boolean, + configuredReservedConcurrency?: string + ): number { + if (hasRateLimit) return 1 + + const parsed = Number(configuredReservedConcurrency) + const isValid = Number.isInteger(parsed) && parsed > 0 + + return isValid + ? parsed + : MetadataCorrectionSetup.DEFAULT_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY + } + public readonly metadataCorrectionRequestsTopic: sns.Topic public readonly metadataCorrectionRequestsQueue: sqs.Queue @@ -111,19 +136,10 @@ export class MetadataCorrectionSetup extends Construct { throw new Error('METADATA_CORRECTION_RUNS_PER_MINUTE must be a positive integer') } - const parsedReservedConcurrency = Number(metadataCorrectionServiceReservedConcurrency) - const hasValidReservedConcurrency = Number.isInteger(parsedReservedConcurrency) - && parsedReservedConcurrency > 0 - let reservedConcurrency = MetadataCorrectionSetup - .DEFAULT_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY - - if (hasValidReservedConcurrency) { - reservedConcurrency = parsedReservedConcurrency - } - - if (hasRateLimit) { - reservedConcurrency = 1 - } + const reservedConcurrency = MetadataCorrectionSetup.resolveReservedConcurrency( + hasRateLimit, + metadataCorrectionServiceReservedConcurrency + ) // TODO: Create a follow-up ticket for DLQ handling. This DLQ is only the // redrive target today; before adding a consumer, decide whether failures diff --git a/scripts/local/metadataCorrectionSmokeHelpers.mjs b/scripts/local/metadataCorrectionSmokeHelpers.mjs new file mode 100644 index 00000000..395fe350 --- /dev/null +++ b/scripts/local/metadataCorrectionSmokeHelpers.mjs @@ -0,0 +1,19 @@ +import { getMetadataCorrectionAuditCollection } from '../../serverless/src/shared/documentDbClient' + +/** + * Removes prior audit documents for a smoke collection so assertions start from a clean state. + * + * @example + * await clearAuditDocumentsForCollection('C1234567890-LOCAL') + * + * @param {string} collectionConceptId Collection whose local audit documents should be removed. + * @returns {Promise} Resolves after matching audit documents are deleted. + */ +export const clearAuditDocumentsForCollection = async (collectionConceptId) => { + process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI + || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` + + const auditCollection = await getMetadataCorrectionAuditCollection() + + await auditCollection.deleteMany({ collectionConceptId }) +} diff --git a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs index 2a7e3716..854502b2 100644 --- a/scripts/local/run_metadata_correction_applied_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_applied_audit_smoke.mjs @@ -6,6 +6,8 @@ import path from 'node:path' import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end audit smoke for metadata correction. * @@ -150,22 +152,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes prior audit documents for the smoke collection so assertions start from a clean state. - * - * @returns {Promise} Resolves after matching local audit documents are deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - let mockServerProcess let redisClient @@ -192,7 +178,7 @@ try { process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { diff --git a/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs b/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs index d61cd7c1..d2e8c3be 100644 --- a/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs +++ b/scripts/local/run_metadata_correction_consumer_metrics_async_smoke.mjs @@ -10,6 +10,8 @@ import { CONSUMER_METRIC_NAMESPACE } from '../../serverless/src/shared/emitConsumerMetrics' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end smoke for async consumer metrics. * @@ -200,22 +202,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes any existing audit documents for the smoke collection. - * - * @returns {Promise} Resolves once prior audit documents are deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - /** * Creates an XML parser for CloudWatch Query API responses. * @@ -531,7 +517,7 @@ try { process.env.AWS_ENDPOINT_URL = cloudWatchEndpoint redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { getCmrCollectionNativeMetadata } = await import('../../serverless/src/shared/getCmrCollectionNativeMetadata') diff --git a/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs b/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs index 9efa28b5..3cf048d2 100644 --- a/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs +++ b/scripts/local/run_metadata_correction_consumer_metrics_manual_sync_smoke.mjs @@ -5,12 +5,13 @@ import fs from 'node:fs/promises' import path from 'node:path' import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' - import { CONSUMER_METRIC_NAMES, CONSUMER_METRIC_NAMESPACE } from '../../serverless/src/shared/emitConsumerMetrics' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end smoke for manual sync consumer metrics. * @@ -188,22 +189,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes any existing audit documents for the smoke collection. - * - * @returns {Promise} Resolves once prior audit documents are deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - /** * Creates an XML parser for CloudWatch Query API responses. * @@ -519,7 +504,7 @@ try { process.env.AWS_ENDPOINT_URL = cloudWatchEndpoint redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { runMetadataCorrection } = await import('../../serverless/src/runMetadataCorrection/handler') const { getCmrCollectionNativeMetadata } = await import('../../serverless/src/shared/getCmrCollectionNativeMetadata') diff --git a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs index e80c8344..3ac7089f 100644 --- a/scripts/local/run_metadata_correction_failed_audit_smoke.mjs +++ b/scripts/local/run_metadata_correction_failed_audit_smoke.mjs @@ -6,6 +6,8 @@ import path from 'node:path' import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end audit smoke for failed metadata-correction writeback. * @@ -156,22 +158,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes prior audit documents for the smoke collection so assertions start from a clean state. - * - * @returns {Promise} Resolves after matching local audit documents are deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - let mockServerProcess let redisClient @@ -200,7 +186,7 @@ try { process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { diff --git a/scripts/local/run_metadata_correction_request_delay_smoke.mjs b/scripts/local/run_metadata_correction_request_delay_smoke.mjs index a99e95f5..2275cc93 100644 --- a/scripts/local/run_metadata_correction_request_delay_smoke.mjs +++ b/scripts/local/run_metadata_correction_request_delay_smoke.mjs @@ -6,6 +6,8 @@ import path from 'node:path' import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end smoke for the queued manual-request delay path. * @@ -183,22 +185,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes any existing audit documents for the smoke collection so assertions start clean. - * - * @returns {Promise} Resolves once prior audit documents have been deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - let mockServerProcess let redisClient @@ -227,7 +213,7 @@ try { process.env.AWS_ENDPOINT_URL = process.env.AWS_ENDPOINT_URL || 'http://127.0.0.1:4566' redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { metadataCorrectionService } = await import('../../serverless/src/metadataCorrectionService/handler') const { diff --git a/scripts/local/run_metadata_correction_sync_smoke.mjs b/scripts/local/run_metadata_correction_sync_smoke.mjs index f10caff8..afc28fd4 100644 --- a/scripts/local/run_metadata_correction_sync_smoke.mjs +++ b/scripts/local/run_metadata_correction_sync_smoke.mjs @@ -6,6 +6,8 @@ import path from 'node:path' import { closeDocumentDbClient } from '../../serverless/src/shared/documentDbClient' +import { clearAuditDocumentsForCollection } from './metadataCorrectionSmokeHelpers.mjs' + /** * Local end-to-end smoke for the synchronous metadata-correction endpoint. * @@ -179,22 +181,6 @@ const seedKeywordCaches = async () => { return redisClient } -/** - * Removes any existing audit documents for the smoke collection so assertions start clean. - * - * @returns {Promise} Resolves once prior audit documents have been deleted. - */ -const clearAuditRowsForCollection = async () => { - process.env.DOCUMENTDB_URI = process.env.DOCUMENTDB_URI - || `mongodb://localhost:${process.env.DOCUMENTDB_HOST_PORT || 27018}` - const { getMetadataCorrectionAuditCollection } = await import( - '../../serverless/src/shared/documentDbClient' - ) - const auditCollection = await getMetadataCorrectionAuditCollection() - - await auditCollection.deleteMany({ collectionConceptId }) -} - let mockServerProcess let redisClient @@ -221,7 +207,7 @@ try { process.env.CMR_WRITER_TOKEN = process.env.CMR_WRITER_TOKEN || 'Bearer local-writer-token' redisClient = await seedKeywordCaches() - await clearAuditRowsForCollection() + await clearAuditDocumentsForCollection(collectionConceptId) const { runMetadataCorrection } = await import('../../serverless/src/runMetadataCorrection/handler') const { diff --git a/serverless/certs/us-east-1-bundle.pem b/serverless/certs/us-east-1-bundle.pem deleted file mode 100644 index f8fb3755..00000000 --- a/serverless/certs/us-east-1-bundle.pem +++ /dev/null @@ -1,76 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID/zCCAuegAwIBAgIRAPVSMfFitmM5PhmbaOFoGfUwDQYJKoZIhvcNAQELBQAw -gZcxCzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJ -bmMuMRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEwMC4GA1UEAwwn -QW1hem9uIFJEUyB1cy1lYXN0LTEgUm9vdCBDQSBSU0EyMDQ4IEcxMRAwDgYDVQQH -DAdTZWF0dGxlMCAXDTIxMDUyNTIyMzQ1N1oYDzIwNjEwNTI1MjMzNDU3WjCBlzEL -MAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIEluYy4x -EzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdBbWF6 -b24gUkRTIHVzLWVhc3QtMSBSb290IENBIFJTQTIwNDggRzExEDAOBgNVBAcMB1Nl -YXR0bGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDu9H7TBeGoDzMr -dxN6H8COntJX4IR6dbyhnj5qMD4xl/IWvp50lt0VpmMd+z2PNZzx8RazeGC5IniV -5nrLg0AKWRQ2A/lGGXbUrGXCSe09brMQCxWBSIYe1WZZ1iU1IJ/6Bp4D2YEHpXrW -bPkOq5x3YPcsoitgm1Xh8ygz6vb7PsvJvPbvRMnkDg5IqEThapPjmKb8ZJWyEFEE -QRrkCIRueB1EqQtJw0fvP4PKDlCJAKBEs/y049FoOqYpT3pRy0WKqPhWve+hScMd -6obq8kxTFy1IHACjHc51nrGII5Bt76/MpTWhnJIJrCnq1/Uc3Qs8IVeb+sLaFC8K -DI69Sw6bAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFE7PCopt -lyOgtXX0Y1lObBUxuKaCMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOC -AQEAFj+bX8gLmMNefr5jRJfHjrL3iuZCjf7YEZgn89pS4z8408mjj9z6Q5D1H7yS -jNETVV8QaJip1qyhh5gRzRaArgGAYvi2/r0zPsy+Tgf7v1KGL5Lh8NT8iCEGGXwF -g3Ir+Nl3e+9XUp0eyyzBIjHtjLBm6yy8rGk9p6OtFDQnKF5OxwbAgip42CD75r/q -p421maEDDvvRFR4D+99JZxgAYDBGqRRceUoe16qDzbMvlz0A9paCZFclxeftAxv6 -QlR5rItMz/XdzpBJUpYhdzM0gCzAzdQuVO5tjJxmXhkSMcDP+8Q+Uv6FA9k2VpUV -E/O5jgpqUJJ2Hc/5rs9VkAPXeA== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIF/jCCA+agAwIBAgIQaRHaEqqacXN20e8zZJtmDDANBgkqhkiG9w0BAQwFADCB -lzELMAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIElu -Yy4xEzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdB -bWF6b24gUkRTIHVzLWVhc3QtMSBSb290IENBIFJTQTQwOTYgRzExEDAOBgNVBAcM -B1NlYXR0bGUwIBcNMjEwNTI1MjIzODM1WhgPMjEyMTA1MjUyMzM4MzVaMIGXMQsw -CQYDVQQGEwJVUzEiMCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjET -MBEGA1UECwwKQW1hem9uIFJEUzELMAkGA1UECAwCV0ExMDAuBgNVBAMMJ0FtYXpv -biBSRFMgdXMtZWFzdC0xIFJvb3QgQ0EgUlNBNDA5NiBHMTEQMA4GA1UEBwwHU2Vh -dHRsZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAInfBCaHuvj6Rb5c -L5Wmn1jv2PHtEGMHm+7Z8dYosdwouG8VG2A+BCYCZfij9lIGszrTXkY4O7vnXgru -JUNdxh0Q3M83p4X+bg+gODUs3jf+Z3Oeq7nTOk/2UYvQLcxP4FEXILxDInbQFcIx -yen1ESHggGrjEodgn6nbKQNRfIhjhW+TKYaewfsVWH7EF2pfj+cjbJ6njjgZ0/M9 -VZifJFBgat6XUTOf3jwHwkCBh7T6rDpgy19A61laImJCQhdTnHKvzTpxcxiLRh69 -ZObypR7W04OAUmFS88V7IotlPmCL8xf7kwxG+gQfvx31+A9IDMsiTqJ1Cc4fYEKg -bL+Vo+2Ii4W2esCTGVYmHm73drznfeKwL+kmIC/Bq+DrZ+veTqKFYwSkpHRyJCEe -U4Zym6POqQ/4LBSKwDUhWLJIlq99bjKX+hNTJykB+Lbcx0ScOP4IAZQoxmDxGWxN -S+lQj+Cx2pwU3S/7+OxlRndZAX/FKgk7xSMkg88HykUZaZ/ozIiqJqSnGpgXCtED -oQ4OJw5ozAr+/wudOawaMwUWQl5asD8fuy/hl5S1nv9XxIc842QJOtJFxhyeMIXt -LVECVw/dPekhMjS3Zo3wwRgYbnKG7YXXT5WMxJEnHu8+cYpMiRClzq2BEP6/MtI2 -AZQQUFu2yFjRGL2OZA6IYjxnXYiRAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8w -HQYDVR0OBBYEFADCcQCPX2HmkqQcmuHfiQ2jjqnrMA4GA1UdDwEB/wQEAwIBhjAN -BgkqhkiG9w0BAQwFAAOCAgEASXkGQ2eUmudIKPeOIF7RBryCoPmMOsqP0+1qxF8l -pGkwmrgNDGpmd9s0ArfIVBTc1jmpgB3oiRW9c6n2OmwBKL4UPuQ8O3KwSP0iD2sZ -KMXoMEyphCEzW1I2GRvYDugL3Z9MWrnHkoaoH2l8YyTYvszTvdgxBPpM2x4pSkp+ -76d4/eRpJ5mVuQ93nC+YG0wXCxSq63hX4kyZgPxgCdAA+qgFfKIGyNqUIqWgeyTP -n5OgKaboYk2141Rf2hGMD3/hsGm0rrJh7g3C0ZirPws3eeJfulvAOIy2IZzqHUSY -jkFzraz6LEH3IlArT3jUPvWKqvh2lJWnnp56aqxBR7qHH5voD49UpJWY1K0BjGnS -OHcurpp0Yt/BIs4VZeWdCZwI7JaSeDcPMaMDBvND3Ia5Fga0thgYQTG6dE+N5fgF -z+hRaujXO2nb0LmddVyvE8prYlWRMuYFv+Co8hcMdJ0lEZlfVNu0jbm9/GmwAZ+l -9umeYO9yz/uC7edC8XJBglMAKUmVK9wNtOckUWAcCfnPWYLbYa/PqtXBYcxrso5j -iaS/A7iEW51uteHBGrViCy1afGG+hiUWwFlesli+Rq4dNstX3h6h2baWABaAxEVJ -y1RnTQSz6mROT1VmZSgSVO37rgIyY0Hf0872ogcTS+FfvXgBxCxsNWEbiQ/XXva4 -0Ws= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIICrjCCAjSgAwIBAgIRAPAlEk8VJPmEzVRRaWvTh2AwCgYIKoZIzj0EAwMwgZYx -CzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJbmMu -MRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEvMC0GA1UEAwwmQW1h -em9uIFJEUyB1cy1lYXN0LTEgUm9vdCBDQSBFQ0MzODQgRzExEDAOBgNVBAcMB1Nl -YXR0bGUwIBcNMjEwNTI1MjI0MTU1WhgPMjEyMTA1MjUyMzQxNTVaMIGWMQswCQYD -VQQGEwJVUzEiMCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEG -A1UECwwKQW1hem9uIFJEUzELMAkGA1UECAwCV0ExLzAtBgNVBAMMJkFtYXpvbiBS -RFMgdXMtZWFzdC0xIFJvb3QgQ0EgRUNDMzg0IEcxMRAwDgYDVQQHDAdTZWF0dGxl -MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEx5xjrup8II4HOJw15NTnS3H5yMrQGlbj -EDA5MMGnE9DmHp5dACIxmPXPMe/99nO7wNdl7G71OYPCgEvWm0FhdvVUeTb3LVnV -BnaXt32Ek7/oxGk1T+Df03C+W0vmuJ+wo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0G -A1UdDgQWBBTGXmqBWN/1tkSea4pNw0oHrjk2UDAOBgNVHQ8BAf8EBAMCAYYwCgYI -KoZIzj0EAwMDaAAwZQIxAIqqZWCSrIkZ7zsv/FygtAusW6yvlL935YAWYPVXU30m -jkMFLM+/RJ9GMvnO8jHfCgIwB+whlkcItzE9CRQ6CsMo/d5cEHDUu/QW6jSIh9BR -OGh9pTYPVkUbBiKPA7lVVhre ------END CERTIFICATE----- diff --git a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js index 99dbd6ff..d0d1f35f 100644 --- a/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/getMetadataCorrectionAuditLog.test.js @@ -31,6 +31,8 @@ const SUMMARY_PROJECTION = { 'corrections.newKeywordPath': 1, 'error.message': 1 } +const DETAIL_RUN_ID = 'f3351653-dfc3-47d8-9176-294ea90bc118' +const UNKNOWN_RUN_ID = '11111111-1111-4111-8111-111111111111' describe('metadata correction audit queries', () => { let collection @@ -311,19 +313,19 @@ describe('metadata correction audit queries', () => { test('returns one detailed audit run without the native metadata diff by default', async () => { collection.findOne.mockResolvedValue({ - runId: 'run-1', + runId: DETAIL_RUN_ID, status: 'applied' }) await expect(getMetadataCorrectionAuditByRunId({ - runId: 'run-1' + runId: DETAIL_RUN_ID })).resolves.toEqual({ - runId: 'run-1', + runId: DETAIL_RUN_ID, status: 'applied' }) expect(collection.findOne).toHaveBeenCalledWith( - { _id: 'run-1' }, + { _id: DETAIL_RUN_ID }, { projection: { _id: 0, @@ -335,7 +337,7 @@ describe('metadata correction audit queries', () => { test('includes the native metadata diff only when requested', async () => { collection.findOne.mockResolvedValue({ - runId: 'run-1', + runId: DETAIL_RUN_ID, status: 'failed', metadataDiff: { changed: true, @@ -344,31 +346,43 @@ describe('metadata correction audit queries', () => { }) const result = await getMetadataCorrectionAuditByRunId({ - runId: 'run-1', + runId: DETAIL_RUN_ID, includeDiff: 'true' }) expect(result.metadataDiff.patch).toBe('-old\n+new') expect(collection.findOne).toHaveBeenCalledWith( - { _id: 'run-1' }, + { _id: DETAIL_RUN_ID }, { projection: { _id: 0 } } ) }) - test('returns null for an unknown run and validates detail parameters', async () => { + test('returns null for an unknown run', async () => { await expect(getMetadataCorrectionAuditByRunId({ - runId: 'missing-run' + runId: UNKNOWN_RUN_ID })).resolves.toBeNull() + }) + test('validates detail parameters before querying the audit collection', async () => { await expect(getMetadataCorrectionAuditByRunId()).rejects.toThrow( 'Invalid metadata correction audit runId' ) await expect(getMetadataCorrectionAuditByRunId({ - runId: 'run-1', + runId: 123 + })).rejects.toThrow('Invalid metadata correction audit runId') + + await expect(getMetadataCorrectionAuditByRunId({ + runId: 'run-1' + })).rejects.toThrow('Invalid metadata correction audit runId') + + await expect(getMetadataCorrectionAuditByRunId({ + runId: DETAIL_RUN_ID, includeDiff: 'yes' })).rejects.toThrow( 'Invalid metadata correction audit includeDiff: expected true or false' ) + + expect(getMetadataCorrectionAuditCollection).not.toHaveBeenCalled() }) }) diff --git a/serverless/src/shared/getMetadataCorrectionAuditLog.js b/serverless/src/shared/getMetadataCorrectionAuditLog.js index 11943e97..91ec3e93 100644 --- a/serverless/src/shared/getMetadataCorrectionAuditLog.js +++ b/serverless/src/shared/getMetadataCorrectionAuditLog.js @@ -1,3 +1,5 @@ +import { validate as isUuid } from 'uuid' + import { VALID_SCHEMES } from '@/shared/constants/validSchemes' import { getMetadataCorrectionAuditCollection } from '@/shared/documentDbClient' import { METADATA_CORRECTION_AUDIT_STATUSES } from '@/shared/persistMetadataCorrectionAuditLog' @@ -443,7 +445,7 @@ export const getMetadataCorrectionAuditByRunId = async ({ runId, includeDiff = false } = {}) => { - if (!runId) { + if (typeof runId !== 'string' || !isUuid(runId)) { throw new Error('Invalid metadata correction audit runId') }