Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -452,14 +452,22 @@ Each collection-correction run is stored as one audit document. Its `statusHisto
links to the current CMR collection record, records the prior and resulting CMR revision IDs, and
stores a bounded unified diff between the original native metadata and the corrected writeback
payload.
When a publisher event finds no matching collections, the listener stores a no-op audit document
with the published KMS version, original keyword change, `collectionCount: 0`, and a
`no-collections-found` outcome. This keeps published-version audit queries complete without
changing the existing per-collection audit workflow.
The audit API is:

- `GET /metadata_correction_audit` for newest-first, token-paginated audit searches. Supported
filters include collection, keyword UUID, action, scheme, status, native format, KMS version,
source, and date range. Supplied actions and schemes must be recognized KMS values, limits must
filters include collection, keyword UUID, action, scheme, status, native format, source, and
date range. Supplied actions and schemes must be recognized KMS values, limits must
be integers from 1 through 250, and `startDate` must not be after `endDate`. List results contain
compact collection, status, and old-to-new keyword path summaries. Add `?includeDiff=true` to
include each available native-metadata diff in the list results.
- `GET /metadata_correction_audit/published` for audit documents across all published KMS
versions, or `GET /metadata_correction_audit/published/{versionName}` for one published version.
These are the official published-version reporting endpoints and use the same pagination and
response formats as the general audit search.
- `GET /metadata_correction_audit/{runId}` for the complete audit document. Add
`?includeDiff=true` when the native-metadata diff is needed; it is omitted by default to keep
routine responses small.
Expand Down
4 changes: 4 additions & 0 deletions cdk/app/lib/CmrEventProcessingStack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ export class CmrEventProcessingStack extends cdk.Stack {
prefix: props.prefix,
stage: props.stage,
keywordEventsTopic: topic,
metadataCorrectionAuditClientSecurityGroup:
props.metadataCorrectionAuditClientSecurityGroup,
metadataCorrectionAuditEnvironment: props.metadataCorrectionAuditEnvironment,
metadataCorrectionAuditSecret: props.metadataCorrectionAuditSecret,
metadataCorrectionRequestsTopic: metadataCorrectionSetup.metadataCorrectionRequestsTopic,
securityGroup: this.securityGroup,
useLocalstack,
Expand Down
20 changes: 19 additions & 1 deletion cdk/app/lib/helper/CmrKeywordEventsListenerSetup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,16 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2'
import * as iam from 'aws-cdk-lib/aws-iam'
import * as eventsources from 'aws-cdk-lib/aws-lambda-event-sources'
import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'
import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'
import * as sns from 'aws-cdk-lib/aws-sns'
import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions'
import * as sqs from 'aws-cdk-lib/aws-sqs'
import { Construct } from 'constructs'

import {
getDocumentDbCertificateBundling,
getDocumentDbLambdaSecurityGroups
} from './DocumentDbLambdaConfig'
import { NODE_LAMBDA_RUNTIME } from './NodeLambdaRuntime'

/**
Expand All @@ -22,6 +27,9 @@ interface CmrKeywordEventsListenerSetupProps {
securityGroup: ec2.SecurityGroup
stage: string
keywordEventsTopic: sns.ITopic
metadataCorrectionAuditClientSecurityGroup?: ec2.ISecurityGroup
metadataCorrectionAuditEnvironment: Record<string, string>
metadataCorrectionAuditSecret?: secretsmanager.ISecret
metadataCorrectionRequestsTopic: sns.ITopic
useLocalstack: boolean
vpc: ec2.IVpc
Expand Down Expand Up @@ -49,6 +57,9 @@ export class CmrKeywordEventsListenerSetup extends Construct {
cmrBaseUrl,
cmrSystemTokenParameterName,
keywordEventsTopic,
metadataCorrectionAuditClientSecurityGroup,
metadataCorrectionAuditEnvironment,
metadataCorrectionAuditSecret,
metadataCorrectionRequestsTopic,
prefix,
securityGroup,
Expand Down Expand Up @@ -78,16 +89,22 @@ export class CmrKeywordEventsListenerSetup extends Construct {
...(cmrSystemTokenParameterName
? { CMR_SYSTEM_TOKEN_PARAMETER_NAME: cmrSystemTokenParameterName }
: {}),
...metadataCorrectionAuditEnvironment,
METADATA_CORRECTION_REQUESTS_TOPIC_ARN: metadataCorrectionRequestsTopic.topicArn
},
depsLockFilePath: path.join(projectRoot, 'package-lock.json'),
projectRoot,
...getDocumentDbCertificateBundling(metadataCorrectionAuditEnvironment),
...(useLocalstack ? {} : {
vpc,
vpcSubnets: {
subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS
},
securityGroups: [securityGroup]
securityGroups: getDocumentDbLambdaSecurityGroups({
clientSecurityGroup: metadataCorrectionAuditClientSecurityGroup,
environment: metadataCorrectionAuditEnvironment,
securityGroup
})
})
})

Expand All @@ -97,6 +114,7 @@ export class CmrKeywordEventsListenerSetup extends Construct {

this.queue.grantConsumeMessages(this.listenerLambda)
metadataCorrectionRequestsTopic.grantPublish(this.listenerLambda)
metadataCorrectionAuditSecret?.grantRead(this.listenerLambda)

if (cmrSystemTokenParameterName) {
const systemTokenParameterArn = cdk.Stack.of(this).formatArn({
Expand Down
26 changes: 26 additions & 0 deletions cdk/app/lib/helper/KmsLambdaFunctions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,32 @@ export class LambdaFunctions {
this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables
)

this.createApiLambda(
scope, // CDK construct scope
'getMetadataCorrectionAudit/handler.js', // Lambda handler path
'get-metadata-correction-audit', // Reuse the audit Lambda
'getMetadataCorrectionAudit', // Exported handler name
'/metadata_correction_audit/published', // All published versions
'GET', // HTTP method
false, // Do not use the EDL authorizer
Duration.seconds(30), // Lambda timeout
1024, // Lambda memory in MB
this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables
)

this.createApiLambda(
scope, // CDK construct scope
'getMetadataCorrectionAudit/handler.js', // Lambda handler path
'get-metadata-correction-audit', // Reuse the audit Lambda
'getMetadataCorrectionAudit', // Exported handler name
'/metadata_correction_audit/published/{versionName}', // One published KMS version
'GET', // HTTP method
false, // Do not use the EDL authorizer
Duration.seconds(30), // Lambda timeout
1024, // Lambda memory in MB
this.props.metadataCorrectionEnvironment || {} // Additional Lambda environment variables
)

this.createApiLambda(
scope, // CDK construct scope
'getMetadataCorrectionAudit/handler.js', // Lambda handler path
Expand Down
27 changes: 27 additions & 0 deletions serverless/src/cmrKeywordEventsListener/__tests__/handler.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import {

import { getCmrCollectionConceptIds } from '@/shared/getCmrCollectionConceptIds'
import { logger } from '@/shared/logger'
import {
persistMetadataCorrectionNoOpAuditLog
} from '@/shared/persistMetadataCorrectionNoOpAuditLog'
import { publishMetadataCorrectionRequest } from '@/shared/publishMetadataCorrectionRequest'

import { cmrKeywordEventsListener } from '../handler'
Expand Down Expand Up @@ -58,6 +61,10 @@ vi.mock('@/shared/publishMetadataCorrectionRequest', () => ({
publishMetadataCorrectionRequest: vi.fn()
}))

vi.mock('@/shared/persistMetadataCorrectionNoOpAuditLog', () => ({
persistMetadataCorrectionNoOpAuditLog: vi.fn()
}))

describe('when the CMR keyword events processor is invoked', () => {
beforeEach(() => {
vi.clearAllMocks()
Expand All @@ -71,6 +78,12 @@ describe('when the CMR keyword events processor is invoked', () => {
message: '{}',
topicArn: 'arn:aws:sns:us-east-1:000000000000:kms-dev-metadata-correction-requests.fifo'
})

vi.mocked(persistMetadataCorrectionNoOpAuditLog).mockResolvedValue({
runId: 'f3351653-dfc3-47d8-9176-294ea90bc118',
status: 'checked',
created: true
})
})

describe('when the invocation is successful', () => {
Expand Down Expand Up @@ -225,6 +238,16 @@ describe('when the CMR keyword events processor is invoked', () => {
)

expect(publishMetadataCorrectionRequest).not.toHaveBeenCalled()
expect(persistMetadataCorrectionNoOpAuditLog).toHaveBeenCalledWith({
keywordEvent: {
EventType: 'UPDATED',
Scheme: 'sciencekeywords',
UUID: '1234'
},
messageId: 'message-123',
publisherMessageId: undefined
})

expect(result).toEqual({
batchItemFailures: []
})
Expand Down Expand Up @@ -262,6 +285,7 @@ describe('when the CMR keyword events processor is invoked', () => {

expect(getCmrCollectionConceptIds).not.toHaveBeenCalled()
expect(publishMetadataCorrectionRequest).not.toHaveBeenCalled()
expect(persistMetadataCorrectionNoOpAuditLog).not.toHaveBeenCalled()
expect(result).toEqual({
batchItemFailures: []
})
Expand Down Expand Up @@ -636,6 +660,8 @@ describe('when the CMR keyword events processor is invoked', () => {
}
})
})

expect(persistMetadataCorrectionNoOpAuditLog).not.toHaveBeenCalled()
})
})

Expand All @@ -660,6 +686,7 @@ describe('when the CMR keyword events processor is invoked', () => {
})).rejects.toThrow('SNS unavailable')

expect(logger.error).toHaveBeenCalled()
expect(persistMetadataCorrectionNoOpAuditLog).not.toHaveBeenCalled()
})
})
})
Expand Down
9 changes: 9 additions & 0 deletions serverless/src/cmrKeywordEventsListener/handler.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ import { formatKeywordObjectForLog } from '@/shared/formatKeywordObjectForLog'
import { getCmrCollectionConceptIds } from '@/shared/getCmrCollectionConceptIds'
import { hasMeaningfulKeywordObject } from '@/shared/hasMeaningfulKeywordObject'
import { logger } from '@/shared/logger'
import {
persistMetadataCorrectionNoOpAuditLog
} from '@/shared/persistMetadataCorrectionNoOpAuditLog'
import { publishMetadataCorrectionRequest } from '@/shared/publishMetadataCorrectionRequest'

/**
Expand Down Expand Up @@ -192,6 +195,12 @@ export const cmrKeywordEventsListener = async (event) => {
)

if (collectionConceptIds.length === 0) {
await persistMetadataCorrectionNoOpAuditLog({
keywordEvent,
messageId,
publisherMessageId: snsEnvelope.MessageId
})

logger.info(
'[consumer] No affected collection concept ids found for keyword event '
+ `scheme=${scheme} `
Expand Down
2 changes: 2 additions & 0 deletions serverless/src/getCapabilities/__tests__/handler.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ describe('getCapabilities', () => {
expect(result.body).toContain('<a name="get_concepts_all"')
expect(result.body).toContain('<a name="get_metadata_correction_audit"')
expect(result.body).toContain('<a name="get_metadata_correction_audit_run"')
expect(result.body).toContain('<a name="get_metadata_correction_audit_published"')
expect(result.body).toContain('<a name="get_metadata_correction_audit_published_version"')
expect(result.body).toContain('<a name="get_historical_concept_by_full_path"')
expect(result.body).toContain('<a name="get_historical_concept_by_short_name"')
expect(result.body).not.toContain('/cache/rebuild')
Expand Down
18 changes: 17 additions & 1 deletion serverless/src/getCapabilities/handler.js
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ export const getCapabilities = async () => {
':@': {
name: 'get_metadata_correction_audit',
href: '/metadata_correction_audit',
params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&publishedVersionName=&source=&startDate=&endDate=&paginationToken=&includeDiff=&format=&limit=',
params: 'collectionConceptId=&keywordConceptUuid=&action=&scheme=&status=&nativeFormat=&source=&startDate=&endDate=&paginationToken=&includeDiff=&format=&limit=',
action: 'GET'
}
},
Expand All @@ -153,6 +153,22 @@ export const getCapabilities = async () => {
action: 'GET'
}
},
{
':@': {
name: 'get_metadata_correction_audit_published',
href: '/metadata_correction_audit/published',
params: 'paginationToken=&includeDiff=&format=&limit=',
action: 'GET'
}
},
{
':@': {
name: 'get_metadata_correction_audit_published_version',
href: '/metadata_correction_audit/published/{versionName}',
params: 'paginationToken=&includeDiff=&format=&limit=',
action: 'GET'
}
},
{
':@': {
name: 'get_concept_versions',
Expand Down
108 changes: 108 additions & 0 deletions serverless/src/getMetadataCorrectionAudit/__tests__/handler.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ describe('getMetadataCorrectionAudit', () => {
status: undefined,
nativeFormat: undefined,
publishedVersionName: undefined,
publishedOnly: false,
source: undefined,
startDate: undefined,
limit: '10'
Expand Down Expand Up @@ -114,6 +115,113 @@ describe('getMetadataCorrectionAudit', () => {
}))
})

test('filters audit documents using the published version path', async () => {
vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({
items: [],
nextPaginationToken: null
})

await getMetadataCorrectionAudit({
resource: '/metadata_correction_audit/published/{versionName}',
pathParameters: {
versionName: 'published-42'
}
})

expect(getMetadataCorrectionAuditLog).toHaveBeenCalledWith(expect.objectContaining({
publishedVersionName: 'published-42',
publishedOnly: true
}))
})

test('returns all published versions when the published path omits a version name', async () => {
vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({
items: [],
nextPaginationToken: null
})

await getMetadataCorrectionAudit({
resource: '/metadata_correction_audit/published',
pathParameters: {},
queryStringParameters: {}
})

expect(getMetadataCorrectionAuditLog).toHaveBeenCalledWith(expect.objectContaining({
publishedVersionName: undefined,
publishedOnly: true
}))
})

test('renders all published versions as grouped html tables', async () => {
vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({
items: [{
runId: 'run-published',
publishedVersionName: 'published-42',
status: 'applied'
}],
nextPaginationToken: null
})

const result = await getMetadataCorrectionAudit({
resource: '/metadata_correction_audit/published',
pathParameters: {},
queryStringParameters: {
format: 'html'
}
})

expect(result.statusCode).toBe(200)
expect(result.body).toContain('<title>Published metadata correction audit</title>')
expect(result.body).toContain('Published Version: published-42')
expect(result.body).toContain('<table class="published-table">')
expect(result.body).not.toContain('<h1>Published metadata correction audit</h1>')
expect(result.body).not.toContain('<form class="audit-filter"')
})

test('renders a selected published version as a table', async () => {
vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({
items: [{
runId: 'run-published',
collectionConceptId: 'C123-PROV',
publishedVersionName: 'published-42',
status: 'applied'
}],
nextPaginationToken: null
})

const result = await getMetadataCorrectionAudit({
resource: '/metadata_correction_audit/published/{versionName}',
pathParameters: {
versionName: 'published-42'
},
queryStringParameters: {
format: 'html'
}
})

expect(result.statusCode).toBe(200)
expect(result.body).toContain('Published Version: published-42')
expect(result.body).toContain('<table class="published-table">')
expect(result.body).toContain('<th scope="col">Collection ID</th>')
expect(result.body).toContain('<th scope="col">Outcome</th>')
})

test('directs published version searches to the official published route', async () => {
const result = await getMetadataCorrectionAudit({
queryStringParameters: {
publishedVersionName: 'published-42'
}
})

expect(result.statusCode).toBe(400)
expect(JSON.parse(result.body)).toEqual({
error: 'Error: Invalid metadata correction audit publishedVersionName: '
+ 'use /metadata_correction_audit/published/{versionName}'
})

expect(getMetadataCorrectionAuditLog).not.toHaveBeenCalled()
})

test('renders a compact html summary with detail links for browser requests', async () => {
vi.mocked(getMetadataCorrectionAuditLog).mockResolvedValue({
items: [{
Expand Down
Loading