From 1957387e0eaecea78b1a60374a9ef35f00384ee3 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Wed, 19 Aug 2026 12:35:38 -0400 Subject: [PATCH 1/5] KMS-705: parse keyword CSVs by named columns and preserve auxiliary metadata --- ...n_platform_keyword_event_mapping_smoke.mjs | 92 ++++ .../__tests__/createCsvForScheme.test.js | 96 +++-- .../__tests__/getPublishKeywordEvents.test.js | 82 +++- .../redis-path-store/createCsvForScheme.js | 157 +------ .../getPublishKeywordEvents.js | 213 +++++---- .../helpers/__tests__/keywordCsv.test.js | 239 +++++++++++ .../helpers/__tests__/parseCsv.test.js | 29 -- .../__tests__/parseFullPathCsvRecords.test.js | 33 -- .../parseShortNameCsvRecords.test.js | 59 --- .../redis-path-store/helpers/constants.js | 18 + .../redis-path-store/helpers/keywordCsv.js | 403 ++++++++++++++++++ .../redis-path-store/helpers/parseCsv.js | 13 - .../helpers/parseFullPathCsvRecords.js | 23 - .../helpers/parseShortNameCsvRecords.js | 61 --- .../rebuildHistoricalConceptCache.js | 8 +- .../writePublishedConceptCaches.js | 8 +- 16 files changed, 1014 insertions(+), 520 deletions(-) create mode 100644 scripts/local/run_platform_keyword_event_mapping_smoke.mjs create mode 100644 serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/__tests__/parseCsv.test.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/__tests__/parseFullPathCsvRecords.test.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/__tests__/parseShortNameCsvRecords.test.js create mode 100644 serverless/src/shared/redis-path-store/helpers/keywordCsv.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/parseCsv.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/parseFullPathCsvRecords.js delete mode 100644 serverless/src/shared/redis-path-store/helpers/parseShortNameCsvRecords.js diff --git a/scripts/local/run_platform_keyword_event_mapping_smoke.mjs b/scripts/local/run_platform_keyword_event_mapping_smoke.mjs new file mode 100644 index 00000000..eb864896 --- /dev/null +++ b/scripts/local/run_platform_keyword_event_mapping_smoke.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict' + +import { + compareKeywordCsvContent, + createKeywordEvents +} from '../../serverless/src/shared/redis-path-store/getPublishKeywordEvents' + +/** + * Verifies platform publisher events keep Short_Name and Long_Name in their correct object fields + * for both a forward keyword change and its revert. + * + * Run with: + * npx vite-node --config vite.config.js scripts/local/run_platform_keyword_event_mapping_smoke.mjs + */ + +const keywordUuid = 'f1ccca0a-30c2-4d22-8b78-a1a36b8901b3' +const defaultLongName = 'Greenhouse Gases Observing Satellite' + +const createPlatformsCsv = (shortName, longName = defaultLongName) => [ + '"Keyword Version: smoke"', + '"Category","Class","Type","Short_Name","Long_Name","UUID"', + `"Platforms","Space-based Platforms","Earth Observation Satellites","${shortName}","${longName}","${keywordUuid}"` +].join('\n') + +const buildExpectedKeywordObject = (shortName, longName = defaultLongName) => ({ + Category: 'Platforms', + Class: 'Space-based Platforms', + Type: 'Earth Observation Satellites', + ShortName: shortName, + LongName: longName +}) + +const transitions = [ + { + versionName: 'local-amazonia-regression', + oldShortName: 'Amazonia-1', + newShortName: 'Amazonia-1-Test', + longName: 'Amazonia-1' + }, + { + versionName: 'local-keyword-republish-v1', + oldShortName: 'GOSAT', + newShortName: 'GOSAT - Test1' + }, + { + versionName: 'local-keyword-republish-v2', + oldShortName: 'GOSAT - Test1', + newShortName: 'GOSAT' + } +] + +const results = transitions.map((transition) => { + const comparison = compareKeywordCsvContent({ + oldCsvContent: createPlatformsCsv(transition.oldShortName, transition.longName), + newCsvContent: createPlatformsCsv(transition.newShortName, transition.longName), + scheme: 'platforms' + }) + const events = createKeywordEvents(new Map([ + ['platforms', comparison] + ])) + + assert.equal(events.length, 1) + + const [keywordEvent] = events + const expectedOldKeywordObject = buildExpectedKeywordObject( + transition.oldShortName, + transition.longName + ) + const expectedNewKeywordObject = buildExpectedKeywordObject( + transition.newShortName, + transition.longName + ) + + assert.equal(keywordEvent.EventType, 'UPDATED') + assert.equal(keywordEvent.UUID, keywordUuid) + assert.deepEqual(keywordEvent.OldKeywordObject, expectedOldKeywordObject) + assert.deepEqual(keywordEvent.NewKeywordObject, expectedNewKeywordObject) + + return { + ...transition, + oldKeywordObject: keywordEvent.OldKeywordObject, + newKeywordObject: keywordEvent.NewKeywordObject + } +}) + +console.log('[platform-keyword-event-mapping-smoke] Completed successfully') +console.log(JSON.stringify({ + keywordUuid, + results +}, null, 2)) diff --git a/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js b/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js index 3f2d471e..3158f223 100644 --- a/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js @@ -10,7 +10,8 @@ import { getProviderUrlsMap } from '../../getProviderUrlsMap' import { getRootConceptForScheme } from '../../getRootConceptForScheme' import { isCsvLongNameFlag } from '../../isCsvLongNameFlag' import { isCsvProviderUrlFlag } from '../../isCsvProviderUrlFlag' -import { createCsvForScheme, formatKeywordCsvPath } from '../createCsvForScheme' +import { createCsvForScheme } from '../createCsvForScheme' +import { formatKeywordCsvPath } from '../helpers/keywordCsv' vi.mock('../../createCsv', () => ({ createCsv: vi.fn() @@ -74,82 +75,105 @@ describe('createCsvForScheme', () => { test('formatKeywordCsvPath pads platforms and related short-name schemes correctly', () => { expect(formatKeywordCsvPath({ scheme: 'platforms', - csvHeadersCount: 5, - path: ['a', 'b', 'c'], + path: ['Platforms', 'Space-based Platforms', 'Earth Observation Satellites'], isLeaf: false - })).toEqual(['a', 'b', 'c']) + })).toEqual(['Platforms', 'Space-based Platforms', 'Earth Observation Satellites', '']) expect(formatKeywordCsvPath({ scheme: 'instruments', - csvHeadersCount: 6, - path: ['Instrument1'], + path: ['Instruments'], isLeaf: false - })).toEqual(['Instrument1', '', '', '']) + })).toEqual(['Instruments', '', '', '']) expect(formatKeywordCsvPath({ scheme: 'projects', - csvHeadersCount: 6, - path: ['Project Category', 'ShortName'], + path: ['Projects', 'EOSDIS'], isLeaf: true - })).toEqual(['Project Category', '', '', 'ShortName']) + })).toEqual(['Projects', 'EOSDIS']) }) test('formatKeywordCsvPath pads full-path and provider schemes correctly', () => { expect(formatKeywordCsvPath({ scheme: 'sciencekeywords', - csvHeadersCount: 3, - path: ['a', 'b'], + path: [ + 'EARTH SCIENCE', + 'LAND SURFACE', + 'LAND USE/LAND COVER', + 'LAND USE/LAND COVER CLASSIFICATION', + 'VEGETATION INDEX', + 'NORMALIZED DIFFERENCE VEGETATION INDEX (NDVI)', + 'VEGETATION INDEX 1 (VI1)' + ], isLeaf: false - })).toEqual(['a', 'b']) + })).toEqual([ + 'EARTH SCIENCE', + 'LAND SURFACE', + 'LAND USE/LAND COVER', + 'LAND USE/LAND COVER CLASSIFICATION', + 'VEGETATION INDEX', + 'NORMALIZED DIFFERENCE VEGETATION INDEX (NDVI)', + 'VEGETATION INDEX 1 (VI1)' + ]) expect(formatKeywordCsvPath({ scheme: 'sciencekeywords', - csvHeadersCount: 5, - path: ['a', 'b'], + path: ['EARTH SCIENCE', 'ATMOSPHERE'], isLeaf: false - })).toEqual(['a', 'b', '', '']) + })).toEqual(['EARTH SCIENCE', 'ATMOSPHERE', '', '', '', '', '']) expect(formatKeywordCsvPath({ scheme: 'providers', - csvHeadersCount: 5, - path: ['a', 'b'], + path: ['NASA', 'GSFC', 'EOSDIS', 'GHRC', 'GHRC_DAAC'], isLeaf: false - })).toEqual(['a', 'b']) + })).toEqual(['NASA', 'GSFC', 'EOSDIS', 'GHRC', 'GHRC_DAAC']) expect(formatKeywordCsvPath({ scheme: 'providers', - csvHeadersCount: 6, - path: ['a', 'b'], + path: ['NASA', 'GSFC'], isLeaf: false - })).toEqual(['a', 'b', '']) + })).toEqual(['NASA', 'GSFC', '', '', '']) expect(formatKeywordCsvPath({ scheme: 'providers', - csvHeadersCount: 6, - path: ['a', 'b'], + path: ['NASA', 'GHRC_DAAC'], isLeaf: true - })).toEqual(['a', '', 'b']) + })).toEqual(['NASA', '', '', '', 'GHRC_DAAC']) expect(formatKeywordCsvPath({ scheme: 'sciencekeywords', - csvHeadersCount: 5, - path: ['a', 'b', 'c', 'd', 'e'], + path: [ + 'EARTH SCIENCE', + 'LAND SURFACE', + 'LAND USE/LAND COVER', + 'LAND USE/LAND COVER CLASSIFICATION', + 'VEGETATION INDEX', + 'NORMALIZED DIFFERENCE VEGETATION INDEX (NDVI)', + 'VEGETATION INDEX 1 (VI1)', + 'EXTRA DETAIL' + ], isLeaf: false - })).toEqual(['a', 'b', 'c', 'd', 'e']) + })).toEqual([ + 'EARTH SCIENCE', + 'LAND SURFACE', + 'LAND USE/LAND COVER', + 'LAND USE/LAND COVER CLASSIFICATION', + 'VEGETATION INDEX', + 'NORMALIZED DIFFERENCE VEGETATION INDEX (NDVI)', + 'VEGETATION INDEX 1 (VI1)', + 'EXTRA DETAIL' + ]) expect(formatKeywordCsvPath({ scheme: 'providers', - csvHeadersCount: 6, - path: ['a', 'b', 'c', 'd'], + path: ['NASA', 'GSFC', 'EOSDIS', 'GHRC', 'DAAC', 'GHRC_DAAC'], isLeaf: true - })).toEqual(['a', 'b', 'c', 'd']) + })).toEqual(['NASA', 'GSFC', 'EOSDIS', 'GHRC', 'DAAC', 'GHRC_DAAC']) expect(formatKeywordCsvPath({ scheme: 'unknown', - csvHeadersCount: 5, - path: ['a', 'b', 'c'], + path: ['EARTH SCIENCE', 'ATMOSPHERE', 'AIR QUALITY'], isLeaf: false - })).toEqual(['a', 'b', 'c']) + })).toEqual(['EARTH SCIENCE', 'ATMOSPHERE', 'AIR QUALITY']) }) test('writes sorted csv rows for a scheme through the centralized hierarchy walk', async () => { @@ -394,7 +418,7 @@ describe('createCsvForScheme', () => { ['mocked metadata'], ['Bucket_Level_0', 'Bucket_Level_1', 'Bucket_Level_2', 'Bucket_Level_3', 'Short_Name', 'Data_Center_URL', 'UUID'], [ - ['', '', '', 'KPDC', 'https://example.com/provider', 'provider'] + ['', '', '', '', 'KPDC', 'https://example.com/provider', 'provider'] ] ) }) @@ -443,7 +467,7 @@ describe('createCsvForScheme', () => { ['mocked metadata'], ['Bucket_Level_0', 'Bucket_Level_1', 'Bucket_Level_2', 'Bucket_Level_3', 'Short_Name', 'Data_Center_URL', 'UUID'], [ - ['', '', '', 'KPDC', '', 'provider'] + ['', '', '', '', 'KPDC', '', 'provider'] ] ) }) diff --git a/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js b/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js index 42de100c..2f257115 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js @@ -7,6 +7,7 @@ import { logger } from '@/shared/logger' import { compareKeywordCsvContent, + createKeywordEvents, getKeywordChangeSummary, getPublishKeywordEvents, parseCsvContent, @@ -39,20 +40,22 @@ describe('getPublishKeywordEvents', () => { vi.useRealTimers() }) - test('parseCsvContent creates a uuid-to-path map from keyword csv content', () => { + test('parseCsvContent creates keyword records by uuid from keyword csv content', () => { const publishedCsv = readFileSync( join(__dirname, '../../__mocks__/sciencekeywords-published.csv'), 'utf-8' ) - const result = parseCsvContent(publishedCsv) + const result = parseCsvContent(publishedCsv, { + scheme: 'sciencekeywords' + }) expect(result).toBeInstanceOf(Map) - expect(result.get('91697b7d-8f2b-4954-850e-61d5f61c867d')).toBe( + expect(result.get('91697b7d-8f2b-4954-850e-61d5f61c867d').path).toBe( 'EARTH SCIENCE > OCEANS > > > > > ' ) - expect(result.get('f6c057c9-c789-4cd5-ba22-e9b08aae152b')).toBe( + expect(result.get('f6c057c9-c789-4cd5-ba22-e9b08aae152b').path).toBe( 'EARTH SCIENCE > OCEANS > AQUATIC SCIENCES > AQUACULTURE > > > ' ) @@ -81,20 +84,21 @@ describe('getPublishKeywordEvents', () => { const comparison = compareKeywordCsvContent({ oldCsvContent: publishedCsv, - newCsvContent: draftCsv + newCsvContent: draftCsv, + scheme: 'sciencekeywords' }) - expect(comparison.addedKeywords.get('3472f70b-874f-6dc5-87db-4b3ebc4b9a73')).toEqual({ + expect(comparison.addedKeywords.get('3472f70b-874f-6dc5-87db-4b3ebc4b9a73')).toMatchObject({ oldPath: undefined, newPath: 'EARTH SCIENCE > OCEANS > COASTAL PROCESSES > SHORELINES > NEW SHORELINES > > ' }) - expect(comparison.removedKeywords.get('488f4df2-712e-4fac-98d1-46ab134b84ee')).toEqual({ + expect(comparison.removedKeywords.get('488f4df2-712e-4fac-98d1-46ab134b84ee')).toMatchObject({ oldPath: 'EARTH SCIENCE > OCEANS > COASTAL PROCESSES > ROCKY COASTS > > > ', newPath: undefined }) - expect(comparison.changedKeywords.get('7863ce31-0e06-42a5-bcf8-25981c44dec8')).toEqual({ + expect(comparison.changedKeywords.get('7863ce31-0e06-42a5-bcf8-25981c44dec8')).toMatchObject({ oldPath: 'EARTH SCIENCE > OCEANS > MARINE GEOPHYSICS > MARINE MAGNETICS > > > ', newPath: 'EARTH SCIENCE > OCEANS > MARINE GEOPHYSICS > MARINE MAGNETICS MODIFIED > > > ' }) @@ -112,6 +116,68 @@ describe('getPublishKeywordEvents', () => { }) }) + test('maps Short_Name to ShortName and Long_Name to LongName in platform update events', () => { + const createPlatformsCsv = ( + shortName, + longName = 'Greenhouse Gases Observing Satellite' + ) => [ + '"Keyword Version: test"', + '"Category","Class","Type","Short_Name","Long_Name","UUID"', + `"Platforms","Space-based Platforms","Earth Observation Satellites","${shortName}","${longName}","uuid-1"` + ].join('\n') + const comparison = compareKeywordCsvContent({ + oldCsvContent: createPlatformsCsv('GOSAT'), + newCsvContent: createPlatformsCsv('GOSAT - Test1'), + scheme: 'platforms' + }) + + expect(comparison.changedKeywords.get('uuid-1')).toMatchObject({ + oldPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', + newPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT - Test1' + }) + + const [keywordEvent] = createKeywordEvents(new Map([ + ['platforms', comparison] + ])) + + expect(keywordEvent).toMatchObject({ + EventType: 'UPDATED', + UUID: 'uuid-1' + }) + + expect(keywordEvent.OldKeywordObject).toEqual({ + Category: 'Platforms', + Class: 'Space-based Platforms', + Type: 'Earth Observation Satellites', + ShortName: 'GOSAT', + LongName: 'Greenhouse Gases Observing Satellite' + }) + + expect(keywordEvent.NewKeywordObject).toEqual({ + Category: 'Platforms', + Class: 'Space-based Platforms', + Type: 'Earth Observation Satellites', + ShortName: 'GOSAT - Test1', + LongName: 'Greenhouse Gases Observing Satellite' + }) + + const longNameComparison = compareKeywordCsvContent({ + oldCsvContent: createPlatformsCsv('GOSAT'), + newCsvContent: createPlatformsCsv('GOSAT', 'Updated GOSAT Satellite'), + scheme: 'platforms' + }) + const [longNameEvent] = createKeywordEvents(new Map([ + ['platforms', longNameComparison] + ])) + + expect(longNameComparison.changedKeywords.get('uuid-1')).toMatchObject({ + oldPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', + newPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT' + }) + + expect(longNameEvent.NewKeywordObject.LongName).toBe('Updated GOSAT Satellite') + }) + test('returns no publish keyword events when neither draft nor published has schemes', async () => { vi.mocked(getConceptSchemeDetails).mockResolvedValue([]) diff --git a/serverless/src/shared/redis-path-store/createCsvForScheme.js b/serverless/src/shared/redis-path-store/createCsvForScheme.js index 1e9e1f51..65b01860 100644 --- a/serverless/src/shared/redis-path-store/createCsvForScheme.js +++ b/serverless/src/shared/redis-path-store/createCsvForScheme.js @@ -8,116 +8,11 @@ import { getNarrowers } from '../getNarrowers' import { getNarrowersMap } from '../getNarrowersMap' import { getProviderUrlsMap } from '../getProviderUrlsMap' import { getRootConceptForScheme } from '../getRootConceptForScheme' -import { isCsvLongNameFlag } from '../isCsvLongNameFlag' -import { isCsvProviderUrlFlag } from '../isCsvProviderUrlFlag' +import { getCsvRowValues, prepareCsvRows } from './helpers/keywordCsv' import { normalizeKeywordScheme } from './helpers/normalizeKeywordScheme' -/** - * Pads or repositions keyword path columns so a hierarchy row fits the scheme's CSV layout. - * - * This is exported because callers and tests sometimes need the exact row-shaping behavior - * without generating an entire CSV payload. - * - * @param {object} params - CSV path formatting inputs. - * @param {string} params.scheme - KMS keyword scheme. - * @param {number} params.csvHeadersCount - Number of CSV header columns before UUID and extras are appended. - * @param {string[]} params.path - Current hierarchy path segments. - * @param {boolean} params.isLeaf - Whether the row represents a leaf concept. - * @returns {string[]} The same `path` array after any required padding/repositioning. - * - * @example - * // Request - * const rowPath = formatKeywordCsvPath({ - * scheme: 'providers', - * csvHeadersCount: 6, - * path: ['NASA', 'GHRC'], - * isLeaf: true - * }) - * - * // Response - * // ['NASA', '', 'GHRC'] - */ -export const formatKeywordCsvPath = ({ - scheme, - csvHeadersCount, - path, - isLeaf -}) => { - const normalizedScheme = normalizeKeywordScheme(scheme) - - if (['platforms', 'instruments', 'projects'].includes(normalizedScheme)) { - const maxLevel = csvHeadersCount - 2 - - if (maxLevel === path.length) { - return path - } - - while (maxLevel > path.length) { - if (!isLeaf) { - path.push('') - } else { - path.splice(path.length - 1, 0, '') - } - } - - return path - } - - if ( - [ - 'sciencekeywords', - 'chronounits', - 'locations', - 'discipline', - 'rucontenttype', - 'measurementname' - ].includes(normalizedScheme) - ) { - const maxLevel = csvHeadersCount - 1 - - if (maxLevel === path.length) { - return path - } - - if (maxLevel > path.length) { - while (maxLevel > path.length) { - path.push('') - } - - return path - } - } - - if (normalizedScheme === 'providers') { - const maxLevel = csvHeadersCount - 3 - - if (maxLevel === path.length) { - return path - } - - if ((maxLevel > path.length) && !isLeaf) { - while (maxLevel > path.length) { - path.push('') - } - - return path - } - - if ((maxLevel > path.length) && isLeaf) { - while (maxLevel > path.length) { - path.splice(path.length - 1, 0, '') - } - - return path - } - } - - return path -} - const appendHierarchicalCsvRows = async ({ - csvHeadersCount, providerUrlsMap, longNamesMap, scheme, @@ -138,7 +33,6 @@ const appendHierarchicalCsvRows = async ({ const isLeaf = narrowers.length === 0 await Promise.all(narrowers.map((obj) => appendHierarchicalCsvRows({ - csvHeadersCount, providerUrlsMap, longNamesMap, scheme, @@ -151,29 +45,18 @@ const appendHierarchicalCsvRows = async ({ if (currentPath.length > 1) { currentPath.shift() - formatKeywordCsvPath({ - scheme, - csvHeadersCount, + paths.push({ path: currentPath, - isLeaf + isLeaf, + uuid, + longName: longNameValue || '', + dataCenterUrl: providerUrlsValue ? providerUrlsValue[0] : '' }) - - if (isCsvLongNameFlag(scheme)) { - currentPath.push(longNameValue || '') - } - - if (isCsvProviderUrlFlag(scheme)) { - currentPath.push(providerUrlsValue ? providerUrlsValue[0] : '') - } - - currentPath.push(uuid) - paths.push(currentPath) } } const getCsvRowsForScheme = async ({ scheme, - csvHeadersCount, version }) => { const csvRows = [] @@ -194,7 +77,6 @@ const getCsvRowsForScheme = async ({ } await appendHierarchicalCsvRows({ - csvHeadersCount, providerUrlsMap, longNamesMap, scheme, @@ -208,18 +90,6 @@ const getCsvRowsForScheme = async ({ return csvRows.reverse() } -const sortCsvRows = (paths) => { - paths.sort((line1, line2) => { - for (let index = 0; index < Math.min(line1.length, line2.length); index += 1) { - if (line1[index] !== line2[index]) { - return line1[index].localeCompare(line2[index]) - } - } - - return line1.length - line2.length - }) -} - /** * Builds the full CSV export payload for one concept scheme/version. * @@ -258,19 +128,22 @@ export const createCsvForScheme = async ({ scheme }) let csvHeaders = await getCsvHeaders(scheme, version) - const csvHeadersCount = csvHeaders.length const csvRows = await getCsvRowsForScheme({ scheme, - csvHeadersCount, version }) if (csvHeaders.length === 0) { - const maxColumns = getMaxLengthOfSubArray(csvRows) + const maxColumns = getMaxLengthOfSubArray(csvRows.map((row) => getCsvRowValues({ + ...row, + scheme + }))) csvHeaders = await generateCsvHeaders(scheme, version, maxColumns) } - sortCsvRows(csvRows) - - return createCsv(csvMetadata, csvHeaders, csvRows) + return createCsv(csvMetadata, csvHeaders, prepareCsvRows({ + csvHeaders, + csvRows, + scheme + })) } diff --git a/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js b/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js index 2fe40ff5..036d22ee 100644 --- a/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js +++ b/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js @@ -1,81 +1,51 @@ -import { parse } from 'csv/sync' - import { downloadConcepts } from '@/shared/downloadConcepts' import { getConceptSchemeDetails } from '@/shared/getConceptSchemeDetails' import { logger } from '@/shared/logger' -import { buildKeywordObjectFromPath } from './helpers/buildKeywordObjectFromPath' import { KEYWORD_DIFF_SKIP_HEADER_ROWS, KEYWORD_PATH_SEPARATOR } from './helpers/constants' import { delay } from './helpers/delay' -import { hasKeywordObjectValue } from './helpers/hasKeywordObjectValue' - -/** - * Parses a keyword CSV payload into a uuid-to-path map using the canonical path separator. - * - * @param {string} csvContent - Raw CSV content to parse. - * @param {object} [options={}] - Parse options. - * @param {number} [options.skipHeaderRows=KEYWORD_DIFF_SKIP_HEADER_ROWS] - Number of leading rows to ignore. - * @param {string} [options.pathSeparator=KEYWORD_PATH_SEPARATOR] - Separator used to flatten path columns. - * @returns {Map} Map of keyword uuid to flattened path string. - * - * @example - * // Request - * const records = parseCsvContent( - * '"Version"\n"Category","Topic","Term","UUID"\n"EARTH SCIENCE","ATMOSPHERE","AEROSOLS","uuid-1"' - * ) - * - * // Response - * // Map([['uuid-1', 'EARTH SCIENCE > ATMOSPHERE > AEROSOLS']]) - */ -export const parseCsvContent = ( - csvContent, - { - skipHeaderRows = KEYWORD_DIFF_SKIP_HEADER_ROWS, - pathSeparator = KEYWORD_PATH_SEPARATOR - } = {} -) => { - const rows = parse(csvContent || '', { - skip_empty_lines: true, - relax_quotes: true, - relax_column_count: true - }) - - const dataRows = rows.slice(skipHeaderRows).filter((row) => row && row.length >= 2) +import { parseKeywordCsvContent as parseCsvContent } from './helpers/keywordCsv' - return new Map( - dataRows - .map((row) => { - const uuid = String(row[row.length - 1] || '').trim() - const path = row - .slice(0, -1) - .map((column) => String(column || '').trim()) - .join(pathSeparator) - - return [uuid, path] - }) - .filter(([uuid]) => uuid.length > 0) - ) -} +export { parseCsvContent } /** * Compares two keyword CSV payloads and groups differences into added, removed, and changed maps. + * Keyword objects contain the scheme's named hierarchy fields. They also include `LongName` for + * schemes that support it and `DataCenterUrl` for providers when those values are present. * * @param {object} params - Keyword CSV comparison inputs. * @param {string} params.oldCsvContent - Baseline CSV content, typically `published`. * @param {string} params.newCsvContent - Candidate CSV content, typically `draft`. + * @param {string} params.scheme - Scheme used to normalize CSV column order. * @param {number} [params.skipHeaderRows=KEYWORD_DIFF_SKIP_HEADER_ROWS] - Number of leading rows to ignore. * @param {string} [params.pathSeparator=KEYWORD_PATH_SEPARATOR] - Separator used to flatten path columns. * @returns {{ - * addedKeywords: Map, - * removedKeywords: Map, - * changedKeywords: Map + * addedKeywords: Map|undefined + * }>, + * removedKeywords: Map|undefined + * }>, + * changedKeywords: Map|undefined, + * newKeywordObject: Record|undefined + * }> * }} Keyword path comparison result. * * @example * // Request * const comparison = compareKeywordCsvContent({ - * oldCsvContent: '"Version"\n"Category","Topic","Term","UUID"\n"EARTH SCIENCE","ATMOSPHERE","LEGACY AEROSOLS","uuid-1"', - * newCsvContent: '"Version"\n"Category","Topic","Term","UUID"\n"EARTH SCIENCE","ATMOSPHERE","AEROSOLS","uuid-1"' + * oldCsvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' + * + '"Platforms","Space-based Platforms","Satellite","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', + * newCsvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' + * + '"Platforms","Space-based Platforms","Satellite","GOSAT - Test1","Greenhouse Gases Observing Satellite","uuid-1"', + * scheme: 'platforms' * }) * * // Response @@ -84,8 +54,22 @@ export const parseCsvContent = ( * // removedKeywords: Map(0) {}, * // changedKeywords: Map([ * // ['uuid-1', { - * // oldPath: 'EARTH SCIENCE > ATMOSPHERE > LEGACY AEROSOLS', - * // newPath: 'EARTH SCIENCE > ATMOSPHERE > AEROSOLS' + * // oldPath: 'Platforms > Space-based Platforms > Satellite > GOSAT', + * // newPath: 'Platforms > Space-based Platforms > Satellite > GOSAT - Test1', + * // oldKeywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT', + * // LongName: 'Greenhouse Gases Observing Satellite' + * // }, + * // newKeywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT - Test1', + * // LongName: 'Greenhouse Gases Observing Satellite' + * // } * // }] * // ]) * // } @@ -93,14 +77,17 @@ export const parseCsvContent = ( export const compareKeywordCsvContent = ({ oldCsvContent, newCsvContent, + scheme, skipHeaderRows = KEYWORD_DIFF_SKIP_HEADER_ROWS, pathSeparator = KEYWORD_PATH_SEPARATOR }) => { const oldRecords = parseCsvContent(oldCsvContent, { + scheme, skipHeaderRows, pathSeparator }) const newRecords = parseCsvContent(newCsvContent, { + scheme, skipHeaderRows, pathSeparator }) @@ -108,31 +95,35 @@ export const compareKeywordCsvContent = ({ const removedKeywords = new Map() const changedKeywords = new Map() - Array.from(newRecords.entries()).forEach(([uuid, newPath]) => { - const oldPath = oldRecords.get(uuid) + Array.from(newRecords.entries()).forEach(([uuid, newRecord]) => { + const oldRecord = oldRecords.get(uuid) - if (oldPath === undefined) { + if (oldRecord === undefined) { addedKeywords.set(uuid, { oldPath: undefined, - newPath + newPath: newRecord.path, + newKeywordObject: newRecord.keywordObject }) return } - if (oldPath !== newPath) { + if (JSON.stringify(oldRecord) !== JSON.stringify(newRecord)) { changedKeywords.set(uuid, { - oldPath, - newPath + oldPath: oldRecord.path, + newPath: newRecord.path, + oldKeywordObject: oldRecord.keywordObject, + newKeywordObject: newRecord.keywordObject }) } }) - Array.from(oldRecords.entries()).forEach(([uuid, oldPath]) => { + Array.from(oldRecords.entries()).forEach(([uuid, oldRecord]) => { if (!newRecords.has(uuid)) { removedKeywords.set(uuid, { - oldPath, - newPath: undefined + oldPath: oldRecord.path, + newPath: undefined, + oldKeywordObject: oldRecord.keywordObject }) } }) @@ -203,21 +194,7 @@ export const toJSON = (comparison) => ({ changedKeywords: Object.fromEntries(comparison.changedKeywords) }) -const buildKeywordEventObject = ({ - scheme, - keywordPath -}) => { - const keywordObject = buildKeywordObjectFromPath({ - scheme, - keywordPath - }) - - return hasKeywordObjectValue(keywordObject) - ? keywordObject - : undefined -} - -const createKeywordEvents = (keywordChangesMap) => { +export const createKeywordEvents = (keywordChangesMap) => { const timestamp = new Date().toISOString() const metadataSpecification = { URL: 'https://cdn.earthdata.nasa.gov/kms-keyword-event/v1.0', @@ -235,10 +212,7 @@ const createKeywordEvents = (keywordChangesMap) => { EventType: 'INSERTED', Scheme: scheme, UUID: uuid, - NewKeywordObject: buildKeywordEventObject({ - scheme, - keywordPath: pathInfo.newPath - }), + NewKeywordObject: pathInfo.newKeywordObject, Timestamp: timestamp, MetadataSpecification: metadataSpecification }) @@ -249,10 +223,7 @@ const createKeywordEvents = (keywordChangesMap) => { EventType: 'DELETED', Scheme: scheme, UUID: uuid, - OldKeywordObject: buildKeywordEventObject({ - scheme, - keywordPath: pathInfo.oldPath - }), + OldKeywordObject: pathInfo.oldKeywordObject, Timestamp: timestamp, MetadataSpecification: metadataSpecification }) @@ -263,14 +234,8 @@ const createKeywordEvents = (keywordChangesMap) => { EventType: 'UPDATED', Scheme: scheme, UUID: uuid, - OldKeywordObject: buildKeywordEventObject({ - scheme, - keywordPath: pathInfo.oldPath - }), - NewKeywordObject: buildKeywordEventObject({ - scheme, - keywordPath: pathInfo.newPath - }), + OldKeywordObject: pathInfo.oldKeywordObject, + NewKeywordObject: pathInfo.newKeywordObject, Timestamp: timestamp, MetadataSpecification: metadataSpecification }) @@ -303,7 +268,8 @@ const getKeywordChangesForScheme = async ({ return compareKeywordCsvContent({ oldCsvContent: publishedCsv, - newCsvContent: draftCsv + newCsvContent: draftCsv, + scheme: notation }) } @@ -317,7 +283,8 @@ const getKeywordChangesForScheme = async ({ return compareKeywordCsvContent({ oldCsvContent: publishedCsv, - newCsvContent: '' + newCsvContent: '', + scheme: notation }) } @@ -330,7 +297,8 @@ const getKeywordChangesForScheme = async ({ return compareKeywordCsvContent({ oldCsvContent: '', - newCsvContent: draftCsv + newCsvContent: draftCsv, + scheme: notation }) } @@ -380,9 +348,22 @@ const getKeywordChangesForSchemeWithRetry = async ({ * @param {boolean} [params.blockOnFailure=false] - Whether a single scheme failure should abort the whole workflow. * @returns {Promise<{ * keywordChangesMap: Map, - * removedKeywords: Map, - * changedKeywords: Map + * addedKeywords: Map|undefined + * }>, + * removedKeywords: Map|undefined + * }>, + * changedKeywords: Map|undefined, + * newKeywordObject: Record|undefined + * }> * }>, * keywordEvents: Array, * keywordChangeSummary: { addedCount: number, removedCount: number, changedCount: number }, @@ -400,14 +381,26 @@ const getKeywordChangesForSchemeWithRetry = async ({ * * // Response * // { - * // keywordChangesMap: Map([['sciencekeywords', comparison]]), + * // keywordChangesMap: Map([['platforms', comparison]]), * // keywordEvents: [ * // { * // EventType: 'UPDATED', - * // Scheme: 'sciencekeywords', - * // UUID: '2e5a401b-1507-4f57-82b8-36557c13b154', - * // OldKeywordObject: { Category: 'EARTH SCIENCE', Topic: 'ATMOSPHERE', Term: 'LEGACY AEROSOLS' }, - * // NewKeywordObject: { Category: 'EARTH SCIENCE', Topic: 'ATMOSPHERE', Term: 'AEROSOLS' }, + * // Scheme: 'platforms', + * // UUID: 'uuid-1', + * // OldKeywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT', + * // LongName: 'Greenhouse Gases Observing Satellite' + * // }, + * // NewKeywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT - Test1', + * // LongName: 'Greenhouse Gases Observing Satellite' + * // }, * // Timestamp: '2026-06-04T15:20:00.000Z', * // MetadataSpecification: { * // URL: 'https://cdn.earthdata.nasa.gov/kms-keyword-event/v1.0', diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js new file mode 100644 index 00000000..8b92f920 --- /dev/null +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js @@ -0,0 +1,239 @@ +import { + parseCsv, + parseFullPathCsvRecords, + parseKeywordCsvContent, + parseShortNameCsvRecords, + prepareCsvRows +} from '../keywordCsv' + +describe('when preparing CSV rows by header name', () => { + test('places GOSAT platform values under reordered headers', () => { + expect(prepareCsvRows({ + csvHeaders: ['Long_Name', 'UUID', 'Type', 'Short_Name', 'Category', 'Class'], + csvRows: [{ + path: [ + 'Platforms', + 'Space-based Platforms', + 'Earth Observation Satellites', + 'GOSAT' + ], + isLeaf: true, + uuid: 'uuid-gosat', + longName: 'Greenhouse Gases Observing Satellite', + dataCenterUrl: '' + }], + scheme: 'platforms' + })).toEqual([[ + 'Greenhouse Gases Observing Satellite', + 'uuid-gosat', + 'Earth Observation Satellites', + 'GOSAT', + 'Platforms', + 'Space-based Platforms' + ]]) + }) +}) + +describe('when parsing CSV rows by header name', () => { + test('returns normalized fields when optional column values are missing', () => { + expect(parseCsv([ + '"Keyword Version: test"', + '"Short_Name","Long_Name","UUID"', + '', + '"GOSAT","Greenhouse Gases Observing Satellite","uuid-gosat"', + '"AQUA"' + ].join('\n'), { + scheme: 'platforms' + })).toEqual([ + { + uuid: 'uuid-gosat', + shortName: 'GOSAT', + longName: 'Greenhouse Gases Observing Satellite', + providerUrl: '', + keywordPath: ' > > > GOSAT' + }, + { + uuid: '', + shortName: 'AQUA', + longName: '', + providerUrl: '', + keywordPath: ' > > > AQUA' + } + ]) + }) + + test('accepts relaxed quotes in keyword values', () => { + expect(parseCsv([ + '"Keyword Version: test"', + '"Short_Name"', + '"GOSAT "Test""' + ].join('\n'), { + scheme: 'platforms' + })).toEqual([ + { + uuid: '', + shortName: '"GOSAT "Test""', + longName: '', + providerUrl: '', + keywordPath: ' > > > "GOSAT "Test""' + } + ]) + }) +}) + +describe('when parsing keyword CSV content for publication', () => { + test('uses generated fields for a scheme without configured path fields', () => { + const csvContent = [ + '"Keyword Version: test"', + '"ProductFlag","UUID"', + '"SCIENCE_QUALITY","uuid-science-quality"' + ].join('\n') + + expect(parseKeywordCsvContent(csvContent, { + scheme: 'productflag' + })).toEqual(new Map([ + ['uuid-science-quality', { + path: 'SCIENCE_QUALITY', + keywordObject: { + Value: 'SCIENCE_QUALITY' + } + }] + ])) + }) + + test('includes the GHRC_DAAC provider URL in its keyword object', () => { + const csvContent = [ + '"Keyword Version: test"', + '"Data_Center_URL","Short_Name","Bucket_Level_3","UUID","Bucket_Level_0","Bucket_Level_2","Bucket_Level_1"', + '"https://ghrc.nsstc.nasa.gov","GHRC_DAAC","GHRC","uuid-ghrc","NASA","EOSDIS","GSFC"' + ].join('\n') + + expect(parseKeywordCsvContent(csvContent, { + scheme: 'providers' + })).toEqual(new Map([ + ['uuid-ghrc', { + path: 'NASA > GSFC > EOSDIS > GHRC > GHRC_DAAC', + keywordObject: { + BucketLevel0: 'NASA', + BucketLevel1: 'GSFC', + BucketLevel2: 'EOSDIS', + BucketLevel3: 'GHRC', + ShortName: 'GHRC_DAAC', + DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' + } + }] + ])) + }) +}) + +describe('when parsing full-path keyword CSV content', () => { + test('builds canonical science keyword paths when columns are reordered', () => { + const csvContent = [ + '"Science_Keywords_v99.0.0"', + '"UUID","Term","Category","Variable_Level_3","Topic","Detailed_Variable","Variable_Level_1","Variable_Level_2"', + '"uuid-aerosols","AEROSOLS","EARTH SCIENCE","","ATMOSPHERE","","",""', + '"uuid-cryosphere","","EARTH SCIENCE","","CRYOSPHERE","","",""' + ].join('\n') + + expect(parseFullPathCsvRecords({ + csvContent, + scheme: 'sciencekeywords' + })).toEqual(new Map([ + ['EARTH SCIENCE > ATMOSPHERE > AEROSOLS > > > > ', 'uuid-aerosols'], + ['EARTH SCIENCE > CRYOSPHERE > > > > > ', 'uuid-cryosphere'] + ])) + }) + + test('skips science keyword rows without a UUID', () => { + const csvContent = [ + '"Science_Keywords_v99.0.0"', + '"Category","Topic","Term","UUID"', + '"EARTH SCIENCE"' + ].join('\n') + + expect(parseFullPathCsvRecords({ + csvContent, + scheme: 'sciencekeywords' + })).toEqual(new Map()) + }) + + test('builds an ISO topic category path from its named column', () => { + const csvContent = [ + '"Keyword Version: test"', + '"UUID","ISO_Topic_Category"', + '"uuid-oceans","oceans"' + ].join('\n') + + expect(parseFullPathCsvRecords({ + csvContent, + scheme: 'isotopiccategory' + })).toEqual(new Map([ + ['oceans', 'uuid-oceans'] + ])) + }) +}) + +describe('when parsing short-name keyword CSV content', () => { + test('returns no records for empty content', () => { + expect(parseShortNameCsvRecords({ + csvContent: '', + scheme: 'platforms' + })).toEqual(new Map()) + }) + + test('builds a GOSAT platform record with its long name', () => { + const csvContent = [ + '"Keyword Version: test"', + '"Long_Name","UUID","Type","Short_Name","Category","Class"', + '"Greenhouse Gases Observing Satellite","uuid-gosat","Earth Observation Satellites","GOSAT","Platforms","Space-based Platforms"' + ].join('\n') + + expect(parseShortNameCsvRecords({ + csvContent, + scheme: 'platforms' + })).toEqual(new Map([ + ['GOSAT', { + uuid: 'uuid-gosat', + fullPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', + longName: 'Greenhouse Gases Observing Satellite', + providerUrl: '', + keywordObject: { + Category: 'Platforms', + Class: 'Space-based Platforms', + Type: 'Earth Observation Satellites', + ShortName: 'GOSAT', + LongName: 'Greenhouse Gases Observing Satellite' + } + }] + ])) + }) + + test('builds a GHRC_DAAC provider record and skips the row without a short name', () => { + const csvContent = [ + '"Keyword Version: test"', + '"Data_Center_URL","Short_Name","Bucket_Level_3","UUID","Bucket_Level_0","Bucket_Level_2","Bucket_Level_1"', + '"https://ghrc.nsstc.nasa.gov","GHRC_DAAC","GHRC","uuid-ghrc","NASA","EOSDIS","GSFC"', + '"https://ghrc.nsstc.nasa.gov","","GHRC","uuid-missing-short-name","NASA","EOSDIS","GSFC"' + ].join('\n') + + expect(parseShortNameCsvRecords({ + csvContent, + scheme: 'providers' + })).toEqual(new Map([ + ['GHRC_DAAC', { + uuid: 'uuid-ghrc', + fullPath: 'NASA > GSFC > EOSDIS > GHRC > GHRC_DAAC', + longName: '', + providerUrl: 'https://ghrc.nsstc.nasa.gov', + keywordObject: { + BucketLevel0: 'NASA', + BucketLevel1: 'GSFC', + BucketLevel2: 'EOSDIS', + BucketLevel3: 'GHRC', + ShortName: 'GHRC_DAAC', + DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' + } + }] + ])) + }) +}) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/parseCsv.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/parseCsv.test.js deleted file mode 100644 index 020c64a8..00000000 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/parseCsv.test.js +++ /dev/null @@ -1,29 +0,0 @@ -import { - describe, - expect, - test -} from 'vitest' - -import { parseCsv } from '../parseCsv' - -describe('parseCsv', () => { - test('parses csv content while skipping empty lines and tolerating uneven columns', () => { - expect(parseCsv([ - '"Header1","Header2"', - '', - '"A","B"', - '"C"' - ].join('\n'))).toEqual([ - ['Header1', 'Header2'], - ['A', 'B'], - ['C'] - ]) - }) - - test('tolerates relaxed quotes used by existing keyword csv exports', () => { - expect(parseCsv('"Header"\n"value with "quotes""')).toEqual([ - ['Header'], - ['"value with "quotes""'] - ]) - }) -}) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/parseFullPathCsvRecords.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/parseFullPathCsvRecords.test.js deleted file mode 100644 index de33aacf..00000000 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/parseFullPathCsvRecords.test.js +++ /dev/null @@ -1,33 +0,0 @@ -import { - describe, - expect, - test -} from 'vitest' - -import { parseFullPathCsvRecords } from '../parseFullPathCsvRecords' - -describe('parseFullPathCsvRecords', () => { - test('builds a full-path-to-uuid map from exported full-path csv rows', () => { - const csvContent = [ - '"Science_Keywords_v99.0.0"', - '"Category","Topic","Term","UUID"', - '"EARTH SCIENCE","ATMOSPHERE","AEROSOLS","uuid-1"', - '"EARTH SCIENCE","CRYOSPHERE","","uuid-2"' - ].join('\n') - - expect(parseFullPathCsvRecords(csvContent)).toEqual(new Map([ - ['EARTH SCIENCE > ATMOSPHERE > AEROSOLS', 'uuid-1'], - ['EARTH SCIENCE > CRYOSPHERE > ', 'uuid-2'] - ])) - }) - - test('skips rows that do not contain at least a path and uuid column', () => { - const csvContent = [ - '"Science_Keywords_v99.0.0"', - '"Category","Topic","Term","UUID"', - '"only-one-column"' - ].join('\n') - - expect(parseFullPathCsvRecords(csvContent)).toEqual(new Map()) - }) -}) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/parseShortNameCsvRecords.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/parseShortNameCsvRecords.test.js deleted file mode 100644 index 090ea309..00000000 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/parseShortNameCsvRecords.test.js +++ /dev/null @@ -1,59 +0,0 @@ -import { parseShortNameCsvRecords } from '../parseShortNameCsvRecords' - -describe('parseShortNameCsvRecords', () => { - test('parses short-name rows with long-name metadata', () => { - const csvContent = [ - 'Category,Class,Type,ShortName,LongName,UUID', - 'Header,Header,Header,Header,Header,Header', - 'Space-based Platforms,Earth Observation Satellites,,Aqua,Aqua Satellite,uuid-1' - ].join('\n') - - expect(parseShortNameCsvRecords({ - csvContent, - scheme: 'platforms' - })).toEqual(new Map([ - ['Aqua', { - uuid: 'uuid-1', - fullPath: 'Space-based Platforms > Earth Observation Satellites > > Aqua', - longName: 'Aqua Satellite', - providerUrl: '', - keywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', - ShortName: 'Aqua', - LongName: 'Aqua Satellite' - } - }] - ])) - }) - - test('parses provider rows with provider urls and skips rows without short names', () => { - const csvContent = [ - 'BucketLevel0,BucketLevel1,BucketLevel2,BucketLevel3,ShortName,DataCenterUrl,UUID', - 'Header,Header,Header,Header,Header,Header,Header', - 'NASA,GSFC,EOSDIS,GHRC,GHRC_DAAC,https://ghrc.nsstc.nasa.gov,uuid-2', - 'NASA,GSFC,EOSDIS,GHRC,,https://ghrc.nsstc.nasa.gov,uuid-3' - ].join('\n') - - expect(parseShortNameCsvRecords({ - csvContent, - scheme: 'providers' - })).toEqual(new Map([ - ['GHRC', { - uuid: 'uuid-3', - fullPath: 'NASA > GSFC > EOSDIS > GHRC', - longName: '', - providerUrl: 'https://ghrc.nsstc.nasa.gov', - keywordObject: { - BucketLevel0: 'NASA', - BucketLevel1: 'GSFC', - BucketLevel2: 'EOSDIS', - BucketLevel3: '', - ShortName: 'GHRC', - DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' - } - }] - ])) - }) -}) diff --git a/serverless/src/shared/redis-path-store/helpers/constants.js b/serverless/src/shared/redis-path-store/helpers/constants.js index 27a6c284..a55bb2f5 100644 --- a/serverless/src/shared/redis-path-store/helpers/constants.js +++ b/serverless/src/shared/redis-path-store/helpers/constants.js @@ -90,6 +90,24 @@ export const SHORT_NAME_OBJECT_FIELDS = Object.freeze({ ] }) +/** + * CSV hierarchy fields in canonical keyword-path order. + * UUID, Long_Name, and Data_Center_URL are auxiliary fields and are intentionally omitted. + * + * @type {Readonly>} + */ +export const CSV_PATH_FIELDS = Object.freeze({ + ...FULL_PATH_VALUE_FIELDS, + ...SHORT_NAME_OBJECT_FIELDS, + discipline: ['DisciplineName', 'Subdiscipline'], + isotopiccategory: ['ISOTopicCategory'], + temporalresolutionrange: ['TemporalResolutionRange'], + verticalresolutionrange: ['VerticalResolutionRange'], + horizontalresolutionrange: ['HorizontalResolutionRange'], + productlevelid: ['ProductLevelId'], + measurementname: ['ContextMedium', 'Object', 'Quantity'] +}) + /** * Schemes that resolve lookups by canonical full path. * diff --git a/serverless/src/shared/redis-path-store/helpers/keywordCsv.js b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js new file mode 100644 index 00000000..da843d54 --- /dev/null +++ b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js @@ -0,0 +1,403 @@ +import { parse } from 'csv/sync' + +import { isCsvLongNameFlag } from '@/shared/isCsvLongNameFlag' +import { isCsvProviderUrlFlag } from '@/shared/isCsvProviderUrlFlag' + +import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' +import { + CSV_PATH_FIELDS, + KEYWORD_DIFF_SKIP_HEADER_ROWS, + KEYWORD_PATH_SEPARATOR +} from './constants' +import { hasKeywordObjectValue } from './hasKeywordObjectValue' +import { isLookupShortNameScheme } from './isLookupShortNameScheme' +import { normalizeKeywordScheme } from './normalizeKeywordScheme' +import { trimKeywordPathSegment } from './trimKeywordPathSegment' + +/** + * Normalizes CSV and scheme field names to the same lookup key. + * + * @example + * normalizeCsvHeader('Short_Name') + * // Returns: 'shortname' + * + * normalizeCsvHeader('Data_Center_URL') + * // Returns: 'datacenterurl' + */ +export const normalizeCsvHeader = (header) => String(header) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]/g, '') + +const AUXILIARY_CSV_HEADERS = new Set(['uuid', 'longname', 'datacenterurl']) + +/** + * Pads a hierarchy path to the number of fields configured for its scheme. Leaf values for + * short-name schemes stay in the final field. + * + * @param {object} params CSV path formatting input. + * @param {string} params.scheme Keyword scheme. + * @param {string[]} params.path Hierarchy path segments. + * @param {boolean} params.isLeaf Whether the row represents a leaf concept. + * @returns {string[]} Padded hierarchy path. + * + * @example + * formatKeywordCsvPath({ + * scheme: 'providers', + * path: ['ACADEMIC', 'GHRC_DAAC'], + * isLeaf: true + * }) + * // Returns: ['ACADEMIC', '', '', '', 'GHRC_DAAC'] + */ +export const formatKeywordCsvPath = ({ + scheme, + path, + isLeaf +}) => { + const normalizedScheme = normalizeKeywordScheme(scheme) + const pathFieldsCount = CSV_PATH_FIELDS[normalizedScheme]?.length ?? path.length + + while (pathFieldsCount > path.length) { + if (isLookupShortNameScheme(normalizedScheme) && isLeaf) { + path.splice(path.length - 1, 0, '') + } else { + path.push('') + } + } + + return path +} + +/** + * Returns a hierarchy row's values in canonical order for column counting and sorting. + * + * @param {object} row CSV hierarchy row. + * @returns {string[]} Canonically ordered row values. + * + * @example + * getCsvRowValues({ + * path: ['Platforms', 'GOSAT'], + * longName: 'Greenhouse Gases Observing Satellite', + * dataCenterUrl: '', + * uuid: 'uuid-1', + * scheme: 'platforms' + * }) + * // Returns: ['Platforms', 'GOSAT', 'Greenhouse Gases Observing Satellite', 'uuid-1'] + */ +export const getCsvRowValues = ({ + dataCenterUrl, + longName, + path, + scheme, + uuid +}) => [ + ...path, + ...(isCsvLongNameFlag(scheme) ? [longName] : []), + ...(isCsvProviderUrlFlag(scheme) ? [dataCenterUrl] : []), + uuid +] + +/** + * Sorts hierarchy rows and places each value under its named CSV header. + * + * @param {object} params CSV row preparation input. + * @param {string[]} params.csvHeaders CSV headers in output order. + * @param {object[]} params.csvRows Hierarchy rows to prepare. + * @param {string} params.scheme Keyword scheme. + * @returns {string[][]} Sorted CSV values in output header order. + * + * @example + * prepareCsvRows({ + * csvHeaders: ['UUID', 'Short_Name', 'Long_Name'], + * csvRows: [{ + * path: ['GOSAT'], + * isLeaf: true, + * uuid: 'uuid-1', + * longName: 'Greenhouse Gases Observing Satellite', + * dataCenterUrl: '' + * }], + * scheme: 'platforms' + * }) + * // Returns: [['uuid-1', 'GOSAT', 'Greenhouse Gases Observing Satellite']] + */ +export const prepareCsvRows = ({ + csvHeaders, + csvRows, + scheme +}) => { + const configuredFields = CSV_PATH_FIELDS[normalizeKeywordScheme(scheme)] + const sortedRows = [...csvRows].sort((first, second) => { + const firstValues = getCsvRowValues({ + ...first, + scheme + }) + const secondValues = getCsvRowValues({ + ...second, + scheme + }) + + for (let index = 0; index < Math.min(firstValues.length, secondValues.length); index += 1) { + if (firstValues[index] !== secondValues[index]) { + return firstValues[index].localeCompare(secondValues[index]) + } + } + + return firstValues.length - secondValues.length + }) + + if (!configuredFields) { + return sortedRows.map((row) => getCsvRowValues({ + ...row, + scheme + })) + } + + const pathHeaders = configuredFields.map(normalizeCsvHeader) + + return sortedRows.map((row) => { + const path = formatKeywordCsvPath({ + scheme, + path: [...row.path], + isLeaf: row.isLeaf + }) + const record = Object.fromEntries( + pathHeaders.map((header, index) => [header, path[index] || '']) + ) + + record.uuid = row.uuid + + if (isCsvLongNameFlag(scheme)) record.longname = row.longName + if (isCsvProviderUrlFlag(scheme)) record.datacenterurl = row.dataCenterUrl + + return csvHeaders.map((header) => record[normalizeCsvHeader(header)] || '') + }) +} + +/** + * Parses keyword CSV rows using named columns and the scheme's canonical path-field order. + * + * @param {string} csvContent CSV text content. + * @param {object} [options={}] Parse options. + * @param {string} options.scheme Keyword scheme. + * @param {number} [options.headerRow=2] One-based line containing the CSV headers. + * @param {string} [options.pathSeparator=KEYWORD_PATH_SEPARATOR] Keyword-path separator. + * @returns {{uuid: string, shortName: string, longName: string, providerUrl: string, keywordPath: string}[]} + * Parsed keyword records. + * + * @example + * parseCsv( + * '"Version"\n"Long_Name","UUID","Type","Short_Name","Category","Class"\n' + * + '"Greenhouse Gases Observing Satellite","uuid-1","Earth Observation Satellites","GOSAT","Platforms","Space-based Platforms"', + * { scheme: 'platforms' } + * ) + * // Returns: [{ + * // uuid: 'uuid-1', + * // shortName: 'GOSAT', + * // longName: 'Greenhouse Gases Observing Satellite', + * // providerUrl: '', + * // keywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT' + * // }] + */ +export const parseCsv = ( + csvContent, + { + scheme, + headerRow = 2, + pathSeparator = KEYWORD_PATH_SEPARATOR + } = {} +) => { + const normalizedScheme = normalizeKeywordScheme(scheme) + const configuredFields = CSV_PATH_FIELDS[normalizedScheme] + let pathHeaders = [] + const rows = parse(csvContent || '', { + columns: (headers) => { + const normalizedHeaders = headers.map(normalizeCsvHeader) + + pathHeaders = (configuredFields || normalizedHeaders.filter( + (header) => !AUXILIARY_CSV_HEADERS.has(header) + )).map(normalizeCsvHeader) + + return normalizedHeaders + }, + from_line: headerRow, + skip_empty_lines: true, + relax_quotes: true, + relax_column_count: true + }) + + return rows.map((row) => ({ + uuid: trimKeywordPathSegment(row.uuid), + shortName: trimKeywordPathSegment(row.shortname), + longName: trimKeywordPathSegment(row.longname), + providerUrl: trimKeywordPathSegment(row.datacenterurl), + keywordPath: pathHeaders + .map((header) => trimKeywordPathSegment(row[header])) + .join(pathSeparator) + })) +} + +/** + * Parses keyword CSV content into records used to compare rows and create event objects. + * + * @param {string} csvContent - Raw CSV content to parse. + * @param {object} [options={}] - Parse options. + * @param {string} options.scheme - Scheme used to place named columns in canonical path order. + * @param {number} [options.skipHeaderRows=KEYWORD_DIFF_SKIP_HEADER_ROWS] - Number of leading rows to ignore. + * @param {string} [options.pathSeparator=KEYWORD_PATH_SEPARATOR] - Separator used to flatten path columns. + * @returns {Map} Records by UUID. + * + * @example + * parseKeywordCsvContent( + * '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' + * + '"Platforms","Space-based Platforms","Satellite","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', + * { scheme: 'platforms' } + * ) + * // Returns: Map([['uuid-1', { + * // path: 'Platforms > Space-based Platforms > Satellite > GOSAT', + * // keywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT', + * // LongName: 'Greenhouse Gases Observing Satellite' + * // } + * // }]]) + */ +export const parseKeywordCsvContent = ( + csvContent, + { + scheme, + skipHeaderRows = KEYWORD_DIFF_SKIP_HEADER_ROWS, + pathSeparator = KEYWORD_PATH_SEPARATOR + } = {} +) => { + const normalizedScheme = normalizeKeywordScheme(scheme) + const records = parseCsv(csvContent, { + scheme: normalizedScheme, + headerRow: skipHeaderRows, + pathSeparator + }).filter(({ uuid }) => uuid) + + return new Map( + records + .map(({ + keywordPath, + longName, + providerUrl, + uuid + }) => { + const keywordObject = buildKeywordObjectFromPath({ + scheme: normalizedScheme, + keywordPath + }) + + if (isCsvLongNameFlag(normalizedScheme) && longName) { + keywordObject.LongName = longName + } + + if (isCsvProviderUrlFlag(normalizedScheme) && providerUrl) { + keywordObject.DataCenterUrl = providerUrl + } + + return [uuid, { + path: keywordPath, + keywordObject: hasKeywordObjectValue(keywordObject) ? keywordObject : undefined + }] + }) + ) +} + +/** + * Parses full-path CSV content into a map of canonical full path to UUID. + * + * @param {object} params Full-path CSV parse input. + * @param {string} params.csvContent CSV export text for a full-path scheme. + * @param {string} params.scheme Keyword scheme name. + * @returns {Map} Map of canonical full path to concept UUID. + * + * @example + * parseFullPathCsvRecords({ + * csvContent: '"Version"\n"Category","Topic","Term","UUID"\n' + * + '"EARTH SCIENCE","ATMOSPHERE","AEROSOLS","uuid-1"', + * scheme: 'sciencekeywords' + * }) + * // Returns: Map([ + * // ['EARTH SCIENCE > ATMOSPHERE > AEROSOLS > > > > ', 'uuid-1'] + * // ]) + */ +export const parseFullPathCsvRecords = ({ + csvContent, + scheme +}) => new Map(parseCsv(csvContent, { + scheme +}) + .filter(({ uuid }) => uuid) + .map(({ keywordPath, uuid }) => [keywordPath, uuid])) + +/** + * Parses short-name CSV content into lookup records with full-path and keyword-object metadata. + * + * @param {Object} params The short-name CSV parse input. + * @param {string} params.csvContent CSV export text for a short-name scheme. + * @param {string} params.scheme Keyword scheme name. + * @returns {Map} Map of short name to cached lookup metadata. + * + * @example + * parseShortNameCsvRecords({ + * csvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' + * + '"Platforms","Space-based Platforms","Satellite","GOSAT","GOSAT Satellite","uuid-1"', + * scheme: 'platforms' + * }) + * // Returns: Map([['GOSAT', { + * // uuid: 'uuid-1', + * // fullPath: 'Platforms > Space-based Platforms > Satellite > GOSAT', + * // longName: 'GOSAT Satellite', + * // providerUrl: '', + * // keywordObject: { + * // Category: 'Platforms', + * // Class: 'Space-based Platforms', + * // Type: 'Satellite', + * // ShortName: 'GOSAT', + * // LongName: 'GOSAT Satellite' + * // } + * // }]]) + */ +export const parseShortNameCsvRecords = ({ + csvContent, + scheme +}) => { + const normalizedScheme = normalizeKeywordScheme(scheme) + + return new Map(parseCsv(csvContent, { + scheme: normalizedScheme + }) + .filter(({ shortName }) => shortName) + .map(({ + keywordPath, + longName, + providerUrl, + shortName, + uuid + }) => { + const keywordObject = buildKeywordObjectFromPath({ + scheme: normalizedScheme, + keywordPath + }) + + if (longName) { + keywordObject.LongName = longName + } + + if (providerUrl) { + keywordObject.DataCenterUrl = providerUrl + } + + return [shortName, { + uuid, + fullPath: keywordPath, + longName, + providerUrl, + keywordObject + }] + })) +} diff --git a/serverless/src/shared/redis-path-store/helpers/parseCsv.js b/serverless/src/shared/redis-path-store/helpers/parseCsv.js deleted file mode 100644 index 3f8aef04..00000000 --- a/serverless/src/shared/redis-path-store/helpers/parseCsv.js +++ /dev/null @@ -1,13 +0,0 @@ -import { parse } from 'csv/sync' - -/** - * Parses CSV content using the relaxed options shared across redis-path-store helpers. - * - * @param {string} csvContent CSV text content. - * @returns {string[][]} Parsed CSV rows. - */ -export const parseCsv = (csvContent) => parse(csvContent, { - skip_empty_lines: true, - relax_quotes: true, - relax_column_count: true -}) diff --git a/serverless/src/shared/redis-path-store/helpers/parseFullPathCsvRecords.js b/serverless/src/shared/redis-path-store/helpers/parseFullPathCsvRecords.js deleted file mode 100644 index 24474c32..00000000 --- a/serverless/src/shared/redis-path-store/helpers/parseFullPathCsvRecords.js +++ /dev/null @@ -1,23 +0,0 @@ -import { parseCsv } from './parseCsv' -import { trimKeywordPathSegment } from './trimKeywordPathSegment' - -/** - * Parses full-path CSV content into a map of canonical full path to UUID. - * - * @param {string} csvContent CSV export text for a full-path scheme. - * @returns {Map} Map of canonical full path to concept UUID. - */ -export const parseFullPathCsvRecords = (csvContent) => { - const rows = parseCsv(csvContent) - const dataRows = rows.slice(2).filter((row) => row && row.length >= 2) - - return new Map(dataRows.map((row) => { - const uuid = trimKeywordPathSegment(row[row.length - 1]) - const fullPath = row - .slice(0, -1) - .map((column) => trimKeywordPathSegment(column)) - .join(' > ') - - return [fullPath, uuid] - })) -} diff --git a/serverless/src/shared/redis-path-store/helpers/parseShortNameCsvRecords.js b/serverless/src/shared/redis-path-store/helpers/parseShortNameCsvRecords.js deleted file mode 100644 index 1c3373f8..00000000 --- a/serverless/src/shared/redis-path-store/helpers/parseShortNameCsvRecords.js +++ /dev/null @@ -1,61 +0,0 @@ -import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' -import { normalizeKeywordScheme } from './normalizeKeywordScheme' -import { parseCsv } from './parseCsv' -import { trimKeywordPathSegment } from './trimKeywordPathSegment' - -/** - * Parses short-name CSV content into lookup records with full-path and keyword-object metadata. - * - * @param {Object} params The short-name CSV parse input. - * @param {string} params.csvContent CSV export text for a short-name scheme. - * @param {string} params.scheme Keyword scheme name. - * @returns {Map} Map of short name to cached lookup metadata. - */ -export const parseShortNameCsvRecords = ({ - csvContent, - scheme -}) => { - const rows = parseCsv(csvContent) - const normalizedScheme = normalizeKeywordScheme(scheme) - const shortNameColumn = normalizedScheme === 'providers' ? -4 : -3 - const longNameColumn = normalizedScheme === 'providers' ? -3 : -2 - const providerUrlColumn = normalizedScheme === 'providers' ? -2 : null - const minColumns = normalizedScheme === 'providers' ? 4 : 3 - const dataRows = rows.slice(2).filter((row) => row && row.length >= minColumns) - - return new Map(dataRows - .map((row) => { - const uuid = trimKeywordPathSegment(row[row.length - 1]) - const shortName = trimKeywordPathSegment(row[row.length + shortNameColumn]) - const longName = trimKeywordPathSegment(row[row.length + longNameColumn]) - const providerUrl = providerUrlColumn === null - ? '' - : trimKeywordPathSegment(row[row.length + providerUrlColumn]) - const pathEndIndex = normalizedScheme === 'providers' ? -3 : -2 - const fullPath = row - .slice(0, pathEndIndex) - .map((column) => trimKeywordPathSegment(column)) - .join(' > ') - const keywordObject = buildKeywordObjectFromPath({ - scheme: normalizedScheme, - keywordPath: fullPath - }) - - if (longName) { - keywordObject.LongName = longName - } - - if (providerUrl) { - keywordObject.DataCenterUrl = providerUrl - } - - return [shortName, { - uuid, - fullPath, - longName, - providerUrl, - keywordObject - }] - }) - .filter(([shortName]) => shortName)) -} diff --git a/serverless/src/shared/redis-path-store/rebuildHistoricalConceptCache.js b/serverless/src/shared/redis-path-store/rebuildHistoricalConceptCache.js index ed0e757a..50f36a18 100644 --- a/serverless/src/shared/redis-path-store/rebuildHistoricalConceptCache.js +++ b/serverless/src/shared/redis-path-store/rebuildHistoricalConceptCache.js @@ -19,9 +19,8 @@ import { HISTORICAL_CACHE_SHORT_NAME_SCHEME_SET } from './helpers/constants' import { createShortNameConceptResponseBody } from './helpers/createShortNameConceptResponseBody' +import { parseFullPathCsvRecords, parseShortNameCsvRecords } from './helpers/keywordCsv' import { normalizeKeywordScheme } from './helpers/normalizeKeywordScheme' -import { parseFullPathCsvRecords } from './helpers/parseFullPathCsvRecords' -import { parseShortNameCsvRecords } from './helpers/parseShortNameCsvRecords' import { writeCacheEntries } from './helpers/writeCacheEntries' const writeHistoricalConceptCacheFromCsv = async ({ @@ -32,7 +31,10 @@ const writeHistoricalConceptCacheFromCsv = async ({ const normalizedScheme = normalizeKeywordScheme(scheme) const cacheEntries = HISTORICAL_CACHE_FULL_PATH_SCHEME_SET.has(normalizedScheme) ? buildCacheEntries({ - records: parseFullPathCsvRecords(csvContent), + records: parseFullPathCsvRecords({ + csvContent, + scheme: normalizedScheme + }), createCacheKey: (fullPath) => createConceptResponseCacheKeyByFullPath({ fullPath: fullPath.toLowerCase(), scheme: normalizedScheme diff --git a/serverless/src/shared/redis-path-store/writePublishedConceptCaches.js b/serverless/src/shared/redis-path-store/writePublishedConceptCaches.js index 89036b6e..a0f6e3e8 100644 --- a/serverless/src/shared/redis-path-store/writePublishedConceptCaches.js +++ b/serverless/src/shared/redis-path-store/writePublishedConceptCaches.js @@ -21,9 +21,8 @@ import { } from './helpers/constants' import { createShortNameConceptResponseBody } from './helpers/createShortNameConceptResponseBody' import { delay } from './helpers/delay' +import { parseFullPathCsvRecords, parseShortNameCsvRecords } from './helpers/keywordCsv' import { normalizeKeywordScheme } from './helpers/normalizeKeywordScheme' -import { parseFullPathCsvRecords } from './helpers/parseFullPathCsvRecords' -import { parseShortNameCsvRecords } from './helpers/parseShortNameCsvRecords' import { writeCacheEntries } from './helpers/writeCacheEntries' const defaultContext = { @@ -64,7 +63,10 @@ const writePublishedConceptCacheFromCsv = async ( if (PUBLISHED_CACHE_FULL_PATH_SCHEME_SET.has(normalizedScheme)) { return { cacheEntries: buildCacheEntries({ - records: parseFullPathCsvRecords(publishedCsvContent), + records: parseFullPathCsvRecords({ + csvContent: publishedCsvContent, + scheme: normalizedScheme + }), createCacheKey: (fullPath) => createPublishedConceptResponseCacheKeyByFullPath({ fullPath: fullPath.toLowerCase(), scheme: normalizedScheme From 0f38c8757445c8121491c09945d43265e077de05 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 20 Aug 2026 17:29:52 -0400 Subject: [PATCH 2/5] KMS-705: Correct publisher keyword-event mapping so published and draft keyword CSVs produce complete, CSV-shaped correction objects for every supported scheme. --- .../getKeywordsTree/__tests__/handler.test.js | 2 - serverless/src/shared/Iso19115DomEditor.js | 120 +-- .../deletions/dif10.corrections.json | 31 +- .../deletions/echo10.corrections.json | 15 +- .../deletions/iso19115.corrections.json | 32 +- .../deletions/isosmap.corrections.json | 31 +- .../deletions/ummc.corrections.json | 26 +- .../generated/chronounits/draft.csv | 3 + .../generated/chronounits/published.csv | 4 + .../end_to_end/generated/dataformat/draft.csv | 6 + .../generated/dataformat/published.csv | 9 + .../generated/granuledataformat/draft.csv | 3 + .../generated/granuledataformat/published.csv | 3 + .../horizontalresolutionrange/draft.csv | 3 + .../horizontalresolutionrange/published.csv | 4 + .../end_to_end/generated/idnnode/draft.csv | 4 + .../generated/idnnode/published.csv | 5 + .../generated/instruments/draft.csv | 9 + .../generated/instruments/published.csv | 14 + .../generated/isotopiccategory/draft.csv | 5 + .../generated/isotopiccategory/published.csv | 8 + .../end_to_end/generated/locations/draft.csv | 6 + .../generated/locations/published.csv | 9 + .../end_to_end/generated/platforms/draft.csv | 9 + .../generated/platforms/published.csv | 14 + .../generated/productlevelid/draft.csv | 5 + .../generated/productlevelid/published.csv | 8 + .../end_to_end/generated/projects/draft.csv | 8 + .../generated/projects/published.csv | 13 + .../end_to_end/generated/providers/draft.csv | 9 + .../generated/providers/published.csv | 15 + .../generated/rucontenttype/draft.csv | 5 + .../generated/rucontenttype/published.csv | 8 + .../generated/sciencekeywords/draft.csv | 6 + .../generated/sciencekeywords/published.csv | 11 + .../generated/short-name-events.json | 119 +++ .../verticalresolutionrange/draft.csv | 3 + .../verticalresolutionrange/published.csv | 4 + .../end_to_end/ummc.before.json | 4 + .../end_to_end/updates/dif10.corrections.json | 74 +- .../end_to_end/updates/echo10.after.xml | 4 +- .../updates/echo10.corrections.json | 50 +- .../updates/iso19115.corrections.json | 64 +- .../updates/isosmap.corrections.json | 64 +- .../end_to_end/updates/ummc.after.json | 4 + .../end_to_end/updates/ummc.corrections.json | 92 ++- .../applyDif10MetadataCorrections.test.js | 681 ++++++++++-------- .../applyEcho10MetadataCorrections.test.js | 297 +++++--- .../applyIso19115MetadataCorrections.test.js | 48 +- .../applyIsoSmapMetadataCorrections.test.js | 48 +- ...lyNativeMetadataCorrectionMappings.test.js | 319 +++++++- .../applyUmmcMetadataCorrections.test.js | 188 ++--- .../persistMetadataCorrectionAuditLog.test.js | 12 +- .../resolveOldKeywordConceptUuid.test.js | 8 +- serverless/src/shared/dif10DomEditor.js | 189 ++--- serverless/src/shared/echo10DomEditor.js | 190 ++--- serverless/src/shared/isCsvLongNameFlag.js | 12 +- .../__tests__/createCsvForScheme.test.js | 2 +- .../getHistoricalConceptByKeyword.test.js | 10 +- .../getHistoricalConceptByShortName.test.js | 22 +- .../getKeywordPathFromKeywordObject.test.js | 23 +- .../__tests__/getPublishKeywordEvents.test.js | 55 +- .../getPublishedConceptByKeyword.test.js | 11 +- .../rebuildHistoricalConceptCache.test.js | 12 +- .../writePublishedConceptCaches.test.js | 4 +- .../getHistoricalConceptByKeyword.js | 5 +- .../getHistoricalConceptByShortName.js | 6 +- .../getKeywordPathFromKeywordObject.js | 37 +- .../getPublishKeywordEvents.js | 38 +- .../buildKeywordObjectFromPath.test.js | 202 +++++- .../buildKeywordPathFromObject.test.js | 35 +- ...buildKeywordPathObjectFromSegments.test.js | 11 +- .../buildKeywordPathObjectFromValue.test.js | 9 +- .../helpers/__tests__/keywordCsv.test.js | 211 +++++- .../parseCachedConceptResponse.test.js | 2 +- .../resolveLookupKeywordObject.test.js | 13 +- .../helpers/buildFullPathLookupValue.js | 6 +- .../helpers/buildKeywordObjectFromPath.js | 115 +-- .../helpers/buildKeywordPathFromObject.js | 29 +- .../buildKeywordPathObjectFromSegments.js | 4 +- .../buildKeywordPathObjectFromValue.js | 4 +- .../redis-path-store/helpers/constants.js | 187 +++-- .../redis-path-store/helpers/keywordCsv.js | 188 ++--- .../helpers/parseCachedConceptResponse.js | 4 +- .../helpers/resolveLookupKeywordObject.js | 8 +- serverless/src/shared/ummcDomEditor.js | 121 ++-- 86 files changed, 2727 insertions(+), 1584 deletions(-) create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/published.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/short-name-events.json create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/draft.csv create mode 100644 serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/published.csv diff --git a/serverless/src/getKeywordsTree/__tests__/handler.test.js b/serverless/src/getKeywordsTree/__tests__/handler.test.js index 28f41c5c..332300a2 100644 --- a/serverless/src/getKeywordsTree/__tests__/handler.test.js +++ b/serverless/src/getKeywordsTree/__tests__/handler.test.js @@ -27,12 +27,10 @@ vi.mock('@/shared/buildKeywordsTree') vi.mock('@/shared/filterKeywordTree') vi.mock('@/shared/filterScienceKeywordsTree') vi.mock('@/shared/getConceptSchemeDetails') -vi.mock('@/shared/getConfig') vi.mock('@/shared/getNarrowersMap') vi.mock('@/shared/getRootConceptForScheme') vi.mock('@/shared/getRootConceptsForAllSchemes') vi.mock('@/shared/sortKeywordNodes') -vi.mock('@/shared/sortKeywordSchemes') vi.mock('@/shared/toTitleCase') vi.mock('@/shared/getVersionMetadata') vi.mock('@/shared/redisCacheStore', async () => { diff --git a/serverless/src/shared/Iso19115DomEditor.js b/serverless/src/shared/Iso19115DomEditor.js index 8d798f37..77e526d6 100644 --- a/serverless/src/shared/Iso19115DomEditor.js +++ b/serverless/src/shared/Iso19115DomEditor.js @@ -1,5 +1,5 @@ import Iso19115MetadataPathEditor from './Iso19115MetadataPathEditor' -import { FULL_PATH_VALUE_FIELDS } from './redis-path-store/helpers/constants' +import { CSV_FIELDS, ISO19115_FIELDS } from './redis-path-store/helpers/constants' /** * Helper factory function to create a block editor configuration. @@ -55,25 +55,32 @@ const createKeywordBlock = (type, { ]`.replace(/\s+/g, ' '), find: { - fieldPaths: ['gmx:Anchor', 'gco:CharacterString'], + fieldPaths: ISO19115_FIELDS.keywordValue, valueKeys: fieldKeys, matchKeys, getNodeValueObject: ({ node, editor }) => { - const anchorNode = editor.selectNodes('./gmx:Anchor', node)[0] - const charStringNode = editor.selectNodes('./gco:CharacterString', node)[0] + const anchorNode = editor.selectNodes(`./${ISO19115_FIELDS.keywordValue[0]}`, node)[0] + const charStringNode = editor.selectNodes(`./${ISO19115_FIELDS.keywordValue[1]}`, node)[0] const fullString = (anchorNode || charStringNode)?.textContent || '' const parts = fullString.split(' > ').map((s) => s.trim()) + // Example: "EARTH SCIENCE > ATMOSPHERE > AEROSOLS" with fieldKeys + // ["Category", "Topic", "Term"] becomes + // { Category: "EARTH SCIENCE", Topic: "ATMOSPHERE", Term: "AEROSOLS" }. return fieldKeys.reduce((acc, key, index) => { acc[key] = parts[index] || '' return acc - }, { Value: fullString.trim() }) + }, {}) } }, replace: [ { - fieldPath: ({ node, editor }) => (editor.selectNodes('./gmx:Anchor', node).length > 0 ? 'gmx:Anchor' : 'gco:CharacterString'), + fieldPath: ({ node, editor }) => ( + editor.selectNodes(`./${ISO19115_FIELDS.keywordValue[0]}`, node).length > 0 + ? ISO19115_FIELDS.keywordValue[0] + : ISO19115_FIELDS.keywordValue[1] + ), source: { type: 'computed', getValue: getValue || (({ correction }) => buildKeywordPath( @@ -99,23 +106,29 @@ const createKeywordBlock = (type, { const createIsoTopicCategoryEditor = () => leafScheme({ nodeXPath: '//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:topicCategory', find: { - getNodeValueObject: ({ node }) => ({ Value: node.textContent?.trim() || '' }) + getNodeValueObject: ({ node }) => ({ + [CSV_FIELDS.isoTopicCategory]: node.textContent?.trim() || '' + }) }, replace: [ { // 1. Update the visible text content - fieldPath: 'gmd:MD_TopicCategoryCode', + fieldPath: ISO19115_FIELDS.isoTopicCategory, source: { type: 'computed', - getValue: ({ correction }) => correction.newKeywordObject.Value + getValue: ({ correction }) => ( + correction.newKeywordObject[CSV_FIELDS.isoTopicCategory] + ) } }, { // 2. Update the codeListValue attribute - fieldPath: 'gmd:MD_TopicCategoryCode/@codeListValue', + fieldPath: ISO19115_FIELDS.isoTopicCategoryCodeListValue, source: { type: 'computed', - getValue: ({ correction }) => correction.newKeywordObject.Value + getValue: ({ correction }) => ( + correction.newKeywordObject[CSV_FIELDS.isoTopicCategory] + ) } } ] @@ -128,7 +141,8 @@ const createProductLevelIdEditor = () => leafScheme({ nodeXPath: '//gmd:processingLevel/gmd:MD_Identifier[gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]', find: { getNodeValueObject: ({ node, editor }) => ({ - Value: editor.getNestedText(node, 'gmd:code/gco:CharacterString')?.trim() || '' + [CSV_FIELDS.productLevelId]: editor + .getNestedText(node, ISO19115_FIELDS.productLevelId)?.trim() || '' }) }, delete: [ @@ -145,10 +159,10 @@ const createProductLevelIdEditor = () => leafScheme({ ], replace: [ { - fieldPath: 'gmd:code/gco:CharacterString', + fieldPath: ISO19115_FIELDS.productLevelId, source: { type: 'computed', - getValue: ({ correction }) => correction.newKeywordObject.Value + getValue: ({ correction }) => correction.newKeywordObject[CSV_FIELDS.productLevelId] } }, { @@ -156,7 +170,7 @@ const createProductLevelIdEditor = () => leafScheme({ fieldPath: '//gmd:contentInfo/gmd:MD_ImageDescription/gmd:processingLevelCode/gmd:MD_Identifier[gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]/gmd:code/gco:CharacterString', source: { type: 'computed', - getValue: ({ correction }) => correction.newKeywordObject.Value + getValue: ({ correction }) => correction.newKeywordObject[CSV_FIELDS.productLevelId] } } ] @@ -171,27 +185,27 @@ export const ISO_19115_SCHEME_EDITORS = { sciencekeywords: createKeywordBlock( 'theme', { - fieldKeys: FULL_PATH_VALUE_FIELDS.sciencekeywords, - matchKeys: FULL_PATH_VALUE_FIELDS.sciencekeywords + fieldKeys: CSV_FIELDS.sciencekeywords, + matchKeys: CSV_FIELDS.sciencekeywords } ), locations: createKeywordBlock( 'place', { - fieldKeys: FULL_PATH_VALUE_FIELDS.locations, - matchKeys: FULL_PATH_VALUE_FIELDS.locations + fieldKeys: CSV_FIELDS.locations, + matchKeys: CSV_FIELDS.locations } ), platforms: createKeywordBlock('platform', { - fieldKeys: ['ShortName', 'LongName'], - matchKeys: ['ShortName'], + fieldKeys: [CSV_FIELDS.shortName, ISO19115_FIELDS.keywordLongName], + matchKeys: [CSV_FIELDS.shortName], getValue: ({ correction }) => { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName }, // Sync with acquisition information section // Smart detection: if gmd:code contains ' > ', use combined format (CWIC style) @@ -203,14 +217,14 @@ export const ISO_19115_SCHEME_EDITORS = { const existingValue = node?.textContent || '' // If existing value contains ' > ', keep combined format if (existingValue.includes(' > ')) { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } // Otherwise use ShortName only (NSIDC/NOAA format) - return correction.newKeywordObject.ShortName + return correction.newKeywordObject[CSV_FIELDS.shortName] } }, { @@ -218,13 +232,13 @@ export const ISO_19115_SCHEME_EDITORS = { getValue: ({ correction, node }) => { const existingValue = node?.textContent || '' if (existingValue.includes(' > ')) { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } - return correction.newKeywordObject.ShortName + return correction.newKeywordObject[CSV_FIELDS.shortName] } }, { @@ -257,13 +271,13 @@ export const ISO_19115_SCHEME_EDITORS = { }), instruments: createKeywordBlock('instrument', { - fieldKeys: ['ShortName', 'LongName'], - matchKeys: ['ShortName'], + fieldKeys: [CSV_FIELDS.shortName, ISO19115_FIELDS.keywordLongName], + matchKeys: [CSV_FIELDS.shortName], getValue: ({ correction }) => { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName }, // Sync with acquisition information section // Smart detection: if gmd:code contains ' > ', use combined format (CWIC style) @@ -274,13 +288,13 @@ export const ISO_19115_SCHEME_EDITORS = { getValue: ({ correction, node }) => { const existingValue = node?.textContent || '' if (existingValue.includes(' > ')) { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } - return correction.newKeywordObject.ShortName + return correction.newKeywordObject[CSV_FIELDS.shortName] } }, { @@ -288,13 +302,13 @@ export const ISO_19115_SCHEME_EDITORS = { getValue: ({ correction, node }) => { const existingValue = node?.textContent || '' if (existingValue.includes(' > ')) { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } - return correction.newKeywordObject.ShortName + return correction.newKeywordObject[CSV_FIELDS.shortName] } }, { @@ -325,23 +339,23 @@ export const ISO_19115_SCHEME_EDITORS = { }), projects: createKeywordBlock('project', { - fieldKeys: ['ShortName', 'LongName'], - matchKeys: ['ShortName'], + fieldKeys: [CSV_FIELDS.shortName, ISO19115_FIELDS.keywordLongName], + matchKeys: [CSV_FIELDS.shortName], getValue: ({ correction }) => { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName }, // Sync with acquisition information section (gmi:MI_Operation) additionalPaths: [ { path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:operation/gmi:MI_Operation/gmi:identifier/gmd:MD_Identifier[gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.projectshortname"]/gmd:code/gco:CharacterString', getValue: ({ correction }) => { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } }, { @@ -352,13 +366,13 @@ export const ISO_19115_SCHEME_EDITORS = { }), providers: createKeywordBlock('dataCentre', { - fieldKeys: ['ShortName', 'LongName'], - matchKeys: ['ShortName'], + fieldKeys: [CSV_FIELDS.shortName, ISO19115_FIELDS.keywordLongName], + matchKeys: [CSV_FIELDS.shortName], getValue: ({ correction }) => { - const { ShortName } = correction.newKeywordObject + const shortName = correction.newKeywordObject[CSV_FIELDS.shortName] const LongName = correction.newLongName || '' - return LongName ? `${ShortName} > ${LongName}` : ShortName + return LongName ? `${shortName} > ${LongName}` : shortName } }), @@ -369,15 +383,17 @@ export const ISO_19115_SCHEME_EDITORS = { getNodeValueObject: ({ node, editor }) => { const charStringNode = editor.selectNodes('.//gmd:name/gco:CharacterString', node)[0] - return { ShortName: charStringNode?.textContent?.trim() || '' } + return { + [CSV_FIELDS.shortName]: charStringNode?.textContent?.trim() || '' + } } }, replace: [ { - fieldPath: 'gmd:MD_Format/gmd:name/gco:CharacterString', + fieldPath: ISO19115_FIELDS.dataformat, source: { type: 'computed', - getValue: ({ correction }) => correction.newKeywordObject.ShortName + getValue: ({ correction }) => correction.newKeywordObject[CSV_FIELDS.shortName] } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/dif10.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/dif10.corrections.json index ffe6c781..8e10ee26 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/dif10.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/dif10.corrections.json @@ -7,12 +7,12 @@ "scheme": "locations", "action": "delete", "oldKeywordObject": { - "Category": "CONTINENT", - "Type": "ANTARCTICA", - "Subregion1": "", - "Subregion2": "", - "Subregion3": "", - "DetailedLocation": "MCMURDO SOUND_SHOULD_DELETE" + "LocationCategory": "CONTINENT", + "LocationType": "ANTARCTICA", + "LocationSubregion1": "", + "LocationSubregion2": "", + "LocationSubregion3": "", + "LocationSubregion4": "MCMURDO SOUND_SHOULD_DELETE" } }, { @@ -21,7 +21,8 @@ "oldKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "GEOPHONES_SHOULD_DELETE" } }, @@ -36,9 +37,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "SPOT-4_SHOULD_DELETE" } }, @@ -71,7 +72,7 @@ "scheme": "projects", "action": "delete", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "ALIENS_SHOULD_DELETE" } }, @@ -99,28 +100,28 @@ "scheme": "isotopiccategory", "action": "delete", "oldKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION_SHOULD_DELETE" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION_SHOULD_DELETE" } }, { "scheme": "verticalresolutionrange", "action": "delete", "oldKeywordObject": { - "Value": "15 meters_SHOULD_DELETE" + "VerticalResolutionRange": "15 meters_SHOULD_DELETE" } }, { "scheme": "horizontalresolutionrange", "action": "delete", "oldKeywordObject": { - "Value": "20 meters_SHOULD_DELETE" + "HorizontalResolutionRange": "20 meters_SHOULD_DELETE" } }, { "scheme": "productlevelid", "action": "delete", "oldKeywordObject": { - "Value": "NA_SHOULD_DELETE" + "ProductLevelId": "NA_SHOULD_DELETE" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/echo10.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/echo10.corrections.json index 50835818..70fa55eb 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/echo10.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/echo10.corrections.json @@ -9,7 +9,8 @@ "oldKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "AMSR-E_SHOULD_DELETE" } }, @@ -17,9 +18,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "Terra_SHOULD_DELETE" } }, @@ -40,7 +41,7 @@ "scheme": "projects", "action": "delete", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "AMSRICE06_SHOULD_DELETE" } }, @@ -79,14 +80,14 @@ "scheme": "dataformat", "action": "delete", "oldKeywordObject": { - "Value": "netCDF-4_SHOULD_DELETE" + "ShortName": "netCDF-4_SHOULD_DELETE" } }, { "scheme": "productlevelid", "action": "delete", "oldKeywordObject": { - "Value": "3_SHOULD_DELETE" + "ProductLevelId": "3_SHOULD_DELETE" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/iso19115.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/iso19115.corrections.json index d2a1f6af..b54b1330 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/iso19115.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/iso19115.corrections.json @@ -7,6 +7,10 @@ "scheme": "instruments", "action": "delete", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS_SHOULD_DELETE" } }, @@ -14,6 +18,10 @@ "scheme": "instruments", "action": "delete", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "ATLAS_SHOULD_DELETE" } }, @@ -21,6 +29,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA_SHOULD_DELETE" } }, @@ -28,6 +39,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "TERRA_SHOULD_DELETE" } }, @@ -48,19 +62,19 @@ "scheme": "locations", "action": "delete", "oldKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA_SHOULD_DELETE", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA_SHOULD_DELETE", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { "scheme": "projects", "action": "delete", "oldKeywordObject": { - "Category": "M - N", + "Bucket": "M - N", "ShortName": "MEASURES_SHOULD_DELETE" } }, @@ -79,7 +93,7 @@ "scheme": "isotopiccategory", "action": "delete", "oldKeywordObject": { - "Value": "FARMING_SHOULD_DELETE" + "ISOTopicCategory": "FARMING_SHOULD_DELETE" } }, { @@ -93,7 +107,7 @@ "scheme": "productlevelid", "action": "delete", "oldKeywordObject": { - "Value": "3_SHOULD_DELETE" + "ProductLevelId": "3_SHOULD_DELETE" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/isosmap.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/isosmap.corrections.json index c39eb96a..75072a78 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/isosmap.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/isosmap.corrections.json @@ -20,6 +20,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "TERRA_SHOULD_DELETE" } }, @@ -27,6 +30,10 @@ "scheme": "instruments", "action": "delete", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS_SHOULD_DELETE" } }, @@ -34,6 +41,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA_SHOULD_DELETE" } }, @@ -54,18 +64,19 @@ "scheme": "locations", "action": "delete", "oldKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA_SHOULD_DELETE", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA_SHOULD_DELETE", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { "scheme": "projects", "action": "delete", "oldKeywordObject": { + "Bucket": "", "ShortName": "MEASURES_SHOULD_DELETE" } }, @@ -73,7 +84,7 @@ "scheme": "isotopiccategory", "action": "delete", "oldKeywordObject": { - "Value": "climatology_SHOULD_DELETE" + "ISOTopicCategory": "climatology_SHOULD_DELETE" } }, { @@ -87,6 +98,10 @@ "scheme": "providers", "action": "delete", "oldKeywordObject": { + "BucketLevel0": "", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", "ShortName": "SMAP_PROVIDER_SHOULD_DELETE" } }, @@ -94,7 +109,7 @@ "scheme": "productlevelid", "action": "delete", "oldKeywordObject": { - "Value": "3_SHOULD_DELETE" + "ProductLevelId": "3_SHOULD_DELETE" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/ummc.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/ummc.corrections.json index 5aa8b1d1..a52f5866 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/ummc.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/deletions/ummc.corrections.json @@ -7,12 +7,12 @@ "scheme": "locations", "action": "delete", "oldKeywordObject": { - "Category": "OCEAN", - "Type": "ATLANTIC OCEAN", - "Subregion1": "SOUTH ATLANTIC OCEAN", - "Subregion2": "SOUTH GEORGIA ISLAND_SHOULD_DELETE", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "OCEAN", + "LocationType": "ATLANTIC OCEAN", + "LocationSubregion1": "SOUTH ATLANTIC OCEAN", + "LocationSubregion2": "SOUTH GEORGIA ISLAND_SHOULD_DELETE", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { @@ -21,7 +21,8 @@ "oldKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "SSALT_SHOULD_DELETE" } }, @@ -36,8 +37,9 @@ "scheme": "platforms", "action": "delete", "oldKeywordObject": { - "Class": "Air-based Platforms", - "Type": "Rotorcraft/Helicopter", + "Basis": "Air-based Platforms", + "Category": "Rotorcraft/Helicopter", + "SubCategory": "", "ShortName": "AS350-B3_SHOULD_DELETE" } }, @@ -70,7 +72,7 @@ "scheme": "projects", "action": "delete", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "CASERTZ_SHOULD_DELETE" } }, @@ -98,7 +100,7 @@ "scheme": "isotopiccategory", "action": "delete", "oldKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION_SHOULD_DELETE" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION_SHOULD_DELETE" } }, { @@ -112,7 +114,7 @@ "scheme": "productlevelid", "action": "delete", "oldKeywordObject": { - "Value": "2_SHOULD_DELETE" + "ProductLevelId": "2_SHOULD_DELETE" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/draft.csv new file mode 100644 index 00000000..ae62a3bf --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/draft.csv @@ -0,0 +1,3 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/chronounits/?format=xml" +"Eon","Era","Period","Epoch","Age","Sub-Age","UUID" +"PHANEROZOIC","CENOZOIC","QUATERNARY","HOLOCENE_CHANGED",,,"811cf9f2-398e-446a-875a-4c125c40772a" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/published.csv new file mode 100644 index 00000000..3d0d2db1 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/chronounits/published.csv @@ -0,0 +1,4 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/chronounits/?format=xml" +"Eon","Era","Period","Epoch","Age","Sub-Age","UUID" +"PHANEROZOIC","CENOZOIC","QUATERNARY","HOLOCENE_SHOULD_DELETE",,,"a08eda21-330f-480a-ae95-0dc2d8bb2623" +"PHANEROZOIC","CENOZOIC","QUATERNARY","HOLOCENE",,,"811cf9f2-398e-446a-875a-4c125c40772a" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/draft.csv new file mode 100644 index 00000000..c3eb5c20 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/draft.csv @@ -0,0 +1,6 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/dataformat/?format=xml" +"Short_Name","Long_Name","UUID" +"GeoTIFF - Test1","Georeferenced Tagged Image File Format","464bd0da-8af4-4976-8ad1-5e3201c08016" +"COG_CHANGED",,"4c1c7d32-53d7-49d0-a1bd-0dd1ff74d965" +"HDF5_CHANGED",,"3f761734-f80b-4fee-b5d3-8efb67d45c56" +"netCDF-4_CHANGED",,"366f34a6-489a-4a57-a204-0f2490795173" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/published.csv new file mode 100644 index 00000000..155f2a11 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/dataformat/published.csv @@ -0,0 +1,9 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/dataformat/?format=xml" +"Short_Name","Long_Name","UUID" +"GeoTIFF","Georeferenced Tagged Image File Format","464bd0da-8af4-4976-8ad1-5e3201c08016" +"COG_SHOULD_DELETE",,"9b9cb967-d271-4dc6-8a48-bb035ef86f41" +"COG",,"4c1c7d32-53d7-49d0-a1bd-0dd1ff74d965" +"HDF5_SHOULD_DELETE",,"8eb05695-eb85-48df-bc8d-ee360a3da06a" +"HDF5",,"3f761734-f80b-4fee-b5d3-8efb67d45c56" +"netCDF-4_SHOULD_DELETE",,"c1693001-4e6c-4ed4-8df3-1d0a6958f71f" +"netCDF-4",,"366f34a6-489a-4a57-a204-0f2490795173" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/draft.csv new file mode 100644 index 00000000..9851f29d --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/draft.csv @@ -0,0 +1,3 @@ +"Keyword Version: 24.7","Revision: 2026-08-21T00:00:00.000Z","Timestamp: 2026-08-21 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/granuledataformat/?format=xml" +"Short_Name","Long_Name","UUID" +"GeoTIFF - Test1","Georeferenced Tagged Image File Format","3139eabd-24d2-4d4a-99aa-8e955938ac38" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/published.csv new file mode 100644 index 00000000..5a518dcc --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/granuledataformat/published.csv @@ -0,0 +1,3 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/granuledataformat/?format=xml" +"Short_Name","Long_Name","UUID" +"GeoTIFF","Georeferenced Tagged Image File Format","3139eabd-24d2-4d4a-99aa-8e955938ac38" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/draft.csv new file mode 100644 index 00000000..6bac0601 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/draft.csv @@ -0,0 +1,3 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/horizontalresolutionrange/?format=xml" +"Horizontal_Resolution_Range","UUID" +"10 meters_CHANGED","cfc17cdf-8bcb-41d8-9d76-b1d8d99ea1f5" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/published.csv new file mode 100644 index 00000000..f0266c7d --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/horizontalresolutionrange/published.csv @@ -0,0 +1,4 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/horizontalresolutionrange/?format=xml" +"Horizontal_Resolution_Range","UUID" +"10 meters","cfc17cdf-8bcb-41d8-9d76-b1d8d99ea1f5" +"20 meters_SHOULD_DELETE","ecaed425-4f97-4cb4-8e7d-47dbddc05040" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/draft.csv new file mode 100644 index 00000000..014cced7 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/draft.csv @@ -0,0 +1,4 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/idnnode/?format=xml" +"Short_Name","Long_Name","UUID" +"IDN_TEST - Test1","International Directory Network Test Node","47f0cffe-4e1a-4716-a82c-7f026ef4fb63" +"AMD/NZ_CHANGED","Antarctic Master Directory/New Zealand_CHANGED","56f16e77-84e0-4941-81bf-32d9b3394123" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/published.csv new file mode 100644 index 00000000..6cb3e883 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/idnnode/published.csv @@ -0,0 +1,5 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/idnnode/?format=xml" +"Short_Name","Long_Name","UUID" +"IDN_TEST","International Directory Network Test Node","47f0cffe-4e1a-4716-a82c-7f026ef4fb63" +"AMD/NZ",,"56f16e77-84e0-4941-81bf-32d9b3394123" +"CEOS_SHOULD_DELETE",,"d788eb97-2a79-41bd-9ca6-d04b26d66178" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/draft.csv new file mode 100644 index 00000000..d38c5345 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/draft.csv @@ -0,0 +1,9 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/instruments/?format=xml" +"Category","Class","Type","Subtype","Short_Name","Long_Name","UUID" +"Earth Remote Sensing Instruments","Active Remote Sensing","Altimeters",,"ALT-TEST - Test1","Altitude Test Instrument","3ef0c50f-9935-4480-9ab8-0a9fe5c20211" +,,,,"ATLAS_CHANGED","Advanced Topographic Laser Altimeter System_CHANGED","5579c70e-ad7a-4296-ac46-f872f751933d" +"Imaging Spectrometers/Radiometers",,,,"AMSR-E_CHANGED","Advanced Microwave Scanning Radiometer-EOS_CHANGED","c8f2f76d-9b14-466d-a0bd-62017ccf143c" +"Imaging Spectrometers/Radiometers",,,,"MABEL_CHANGED","Multiple Altimeter Beam Experimental Lidar_CHANGED","13d8e9e9-8113-4d32-8bd6-391576fb4498" +"In Situ/Laboratory Instruments",,,,"SEISMIC REFLECTION PROFILERS_CHANGED","Seismic Reflection Profilers_CHANGED","974689c3-afb9-413b-88bb-d0db092e88ea" +,,,,"MODIS_CHANGED","Moderate-Resolution Imaging Spectroradiometer_CHANGED","ff03ac3d-12bc-475a-be57-673f0bad93f7" +"Synthetic Aperture Radar",,,,"TSX-1_CHANGED","Synthetic Aperture Radar_CHANGED","55b17bd8-637e-4f8f-8cda-71c6b690ad43" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/published.csv new file mode 100644 index 00000000..231d20a4 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/instruments/published.csv @@ -0,0 +1,14 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/instruments/?format=xml" +"Category","Class","Type","Subtype","Short_Name","Long_Name","UUID" +"Earth Remote Sensing Instruments","Active Remote Sensing","Altimeters",,"ALT-TEST","Altitude Test Instrument","3ef0c50f-9935-4480-9ab8-0a9fe5c20211" +,,,,"ATLAS_SHOULD_DELETE",,"d97a2a0f-6fe3-4e10-86ea-1d892237b773" +,,,,"ATLAS",,"5579c70e-ad7a-4296-ac46-f872f751933d" +"Imaging Spectrometers/Radiometers",,,,"AMSR-E_SHOULD_DELETE",,"f1794f36-baa3-4661-8172-7a8796db4f94" +"Imaging Spectrometers/Radiometers",,,,"AMSR-E",,"c8f2f76d-9b14-466d-a0bd-62017ccf143c" +"Imaging Spectrometers/Radiometers",,,,"GEOPHONES_SHOULD_DELETE",,"8903f3c3-9487-490d-9749-67b6f6a17db2" +"Imaging Spectrometers/Radiometers",,,,"MABEL",,"13d8e9e9-8113-4d32-8bd6-391576fb4498" +"Imaging Spectrometers/Radiometers",,,,"SSALT_SHOULD_DELETE",,"99d30ac4-8804-434a-8ac5-bfd169944215" +"In Situ/Laboratory Instruments",,,,"SEISMIC REFLECTION PROFILERS",,"974689c3-afb9-413b-88bb-d0db092e88ea" +,,,,"MODIS_SHOULD_DELETE",,"abd98dcb-d277-41fc-bf42-9116074f5fa8" +,,,,"MODIS",,"ff03ac3d-12bc-475a-be57-673f0bad93f7" +"Synthetic Aperture Radar",,,,"TSX-1",,"55b17bd8-637e-4f8f-8cda-71c6b690ad43" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/draft.csv new file mode 100644 index 00000000..ecd90e33 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/draft.csv @@ -0,0 +1,5 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/isotopiccategory/?format=xml" +"ISO_Topic_Category","UUID" +"climatology_CHANGED","557b587a-4fd8-44fd-aee7-e79f4cbfe036" +"FARMING_CHANGED","6ea35b3e-82c3-4b04-a417-79d1a9cf8c01" +"GEOSCIENTIFIC INFORMATION_CHANGED","81d089df-b0a1-4f5f-919c-065d12c872ac" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/published.csv new file mode 100644 index 00000000..04fcdc27 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/isotopiccategory/published.csv @@ -0,0 +1,8 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/isotopiccategory/?format=xml" +"ISO_Topic_Category","UUID" +"climatology_SHOULD_DELETE","d6c271e7-ca8d-4901-8975-499d4c58c55d" +"climatology","557b587a-4fd8-44fd-aee7-e79f4cbfe036" +"FARMING_SHOULD_DELETE","098329e0-047e-4a9d-ab79-17659cd9dbaf" +"FARMING","6ea35b3e-82c3-4b04-a417-79d1a9cf8c01" +"GEOSCIENTIFIC INFORMATION_SHOULD_DELETE","01b84a42-9d47-4c62-ba42-c306f8119a33" +"GEOSCIENTIFIC INFORMATION","81d089df-b0a1-4f5f-919c-065d12c872ac" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/draft.csv new file mode 100644 index 00000000..5ac2b7e6 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/draft.csv @@ -0,0 +1,6 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/locations/?format=xml" +"Location_Category","Location_Type","Location_Subregion1","Location_Subregion2","Location_Subregion3","Location_Subregion4","UUID" +"CONTINENT","NORTH AMERICA","CANADA","ALBERTA_CHANGED",,,"d8bea9da-dd25-4dc7-a277-92c0ecbf27ef" +"GEOGRAPHIC REGION","GLOBAL LAND_CHANGED",,,,,"17579498-9488-413f-93f5-1221125bb886" +"GEOGRAPHIC REGION","POLAR_CHANGED",,,,,"ed4bdf1b-826f-40f0-b5cd-25d4b3fe94ee" +"OCEAN","ATLANTIC OCEAN","NORTH ATLANTIC OCEAN","CARIBBEAN SEA","BARBADOS_CHANGED",,"5b7698fc-ba7e-4af4-bff0-11a68496538b" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/published.csv new file mode 100644 index 00000000..f5af1097 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/locations/published.csv @@ -0,0 +1,9 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/locations/?format=xml" +"Location_Category","Location_Type","Location_Subregion1","Location_Subregion2","Location_Subregion3","Location_Subregion4","UUID" +"CONTINENT","ANTARCTICA",,,,"MCMURDO SOUND_SHOULD_DELETE","675c99e7-73a9-450a-9804-a55e2122c412" +"CONTINENT","NORTH AMERICA","CANADA","ALBERTA_SHOULD_DELETE",,,"17fea13a-fad0-42e1-b7c0-4a75dd434608" +"CONTINENT","NORTH AMERICA","CANADA","ALBERTA",,,"d8bea9da-dd25-4dc7-a277-92c0ecbf27ef" +"GEOGRAPHIC REGION","GLOBAL LAND",,,,,"17579498-9488-413f-93f5-1221125bb886" +"GEOGRAPHIC REGION","POLAR",,,,,"ed4bdf1b-826f-40f0-b5cd-25d4b3fe94ee" +"OCEAN","ATLANTIC OCEAN","NORTH ATLANTIC OCEAN","CARIBBEAN SEA","BARBADOS",,"5b7698fc-ba7e-4af4-bff0-11a68496538b" +"OCEAN","ATLANTIC OCEAN","SOUTH ATLANTIC OCEAN","SOUTH GEORGIA ISLAND_SHOULD_DELETE",,,"41a94784-5a0c-4872-991c-c47e2c44d385" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/draft.csv new file mode 100644 index 00000000..52f50a99 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/draft.csv @@ -0,0 +1,9 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/platforms/?format=xml" +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites",,"GOSAT - Test1","Greenhouse Gases Observing Satellite","918b8740-4d85-4b75-90fd-315bea76747a" +"Air-based Platforms","Rotorcraft/Helicopter",,"AS350-B3_CHANGED","Helicopter AS350-B3_CHANGED","e0c2f0bd-cca2-4f48-baef-b9789bcf90a6" +,,,"AQUA_CHANGED","Earth Observing System_CHANGED","b5d32419-15dc-41d1-9bf4-0fd4fe897f43" +"Space-based Platforms","Earth Observation Satellites",,"SPOT-4_CHANGED","Systeme Probatoire d'Observation de la Terre-4_CHANGED","513a30a4-cd9c-41a6-b81f-fc0c042a783b" +"Space-based Platforms","Earth Observation Satellites",,"Amazonia-1-Chris-Test_CHANGED",,"f0ddf64c-019d-43ba-8c83-a9db783e9392" +"Space-based Platforms","Earth Observation Satellites",,"Terra_CHANGED","Earth Observing System, Terra (AM-1)_CHANGED","27c48216-19d0-4b55-b6f6-1095fd315234" +,,,"TERRA_CHANGED","Earth Observing System, TERRA (AM-1)_CHANGED","a8e27d4a-3e27-4e35-b9c5-a8fc80f3ded2" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/published.csv new file mode 100644 index 00000000..8b72fdd5 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/platforms/published.csv @@ -0,0 +1,14 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/platforms/?format=xml" +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites",,"GOSAT","Greenhouse Gases Observing Satellite","918b8740-4d85-4b75-90fd-315bea76747a" +"Air-based Platforms","Rotorcraft/Helicopter",,"AS350-B3_SHOULD_DELETE",,"ca1059da-b637-4e8c-99b8-602773c39adb" +"Air-based Platforms","Rotorcraft/Helicopter",,"AS350-B3",,"e0c2f0bd-cca2-4f48-baef-b9789bcf90a6" +,,,"AQUA_SHOULD_DELETE",,"b82f73c1-d292-411a-8d15-3b2c6ae51be1" +,,,"AQUA",,"b5d32419-15dc-41d1-9bf4-0fd4fe897f43" +"Space-based Platforms","Earth Observation Satellites",,"SPOT-4_SHOULD_DELETE",,"9e00bcf6-7200-4ce2-8cb0-5cc6ad73aac5" +"Space-based Platforms","Earth Observation Satellites",,"SPOT-4",,"513a30a4-cd9c-41a6-b81f-fc0c042a783b" +"Space-based Platforms","Earth Observation Satellites",,"Amazonia-1",,"f0ddf64c-019d-43ba-8c83-a9db783e9392" +"Space-based Platforms","Earth Observation Satellites",,"Terra_SHOULD_DELETE",,"59af64b4-1d71-4de4-9984-2ba5f4719268" +"Space-based Platforms","Earth Observation Satellites",,"Terra",,"27c48216-19d0-4b55-b6f6-1095fd315234" +,,,"TERRA_SHOULD_DELETE",,"5d285826-449a-4900-b4b7-7a0286099ea0" +,,,"TERRA",,"a8e27d4a-3e27-4e35-b9c5-a8fc80f3ded2" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/draft.csv new file mode 100644 index 00000000..14ba2306 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/draft.csv @@ -0,0 +1,5 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/productlevelid/?format=xml" +"ProductLevelId","UUID" +"2_CHANGED","3157dce5-f876-45eb-8fd1-fa582c84dead" +"3_CHANGED","a43e104d-7af1-48ea-ad5a-012c8485de9a" +"NA_CHANGED","f02a4097-1f39-4680-852b-cdb05e558abb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/published.csv new file mode 100644 index 00000000..38bad974 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/productlevelid/published.csv @@ -0,0 +1,8 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/productlevelid/?format=xml" +"ProductLevelId","UUID" +"2_SHOULD_DELETE","d8536319-1ed3-40e2-bf7f-2fed40580575" +"2","3157dce5-f876-45eb-8fd1-fa582c84dead" +"3_SHOULD_DELETE","352c4881-3f41-4558-a3fe-dc6951dfc1c1" +"3","a43e104d-7af1-48ea-ad5a-012c8485de9a" +"NA_SHOULD_DELETE","963b3d00-21a4-412a-8365-9b9511f852c0" +"NA","f02a4097-1f39-4680-852b-cdb05e558abb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/draft.csv new file mode 100644 index 00000000..f6f9a76e --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/draft.csv @@ -0,0 +1,8 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/projects/?format=xml" +"Bucket","Short_Name","Long_Name","UUID" +"G - I","GPM - Test1","Global Precipitation Measurement","297f51be-14cb-49e5-9711-5286d9953ade" +"A - C","ALIENS_CHANGED","Aliens in Antarctica_CHANGED","9f83dac5-ae71-451a-8b85-1193c8b0ff38" +"A - C","AMSRICE06_CHANGED","Advanced Microwave Scanning Radiometer Sea Ice Product Validation_CHANGED","d85b2560-8dc8-4682-a3fc-276e2ad3607e" +"A - C","CASERTZ_CHANGED","Corridor Aerogeophysics of the South East Ross Transect Zone_CHANGED","8607d64a-06f9-4d90-9e91-4e9d9ebf8a10" +"M - N","MEASURES_CHANGED","Making Earth System Data Records for Use in Research Environments_CHANGED","20d8e2a6-769b-40cb-a524-34760914927f" +,"MEASURES_CHANGED","Making Earth System Data Records for Use in Research Environments_CHANGED","70f75059-e00a-4e66-be0e-75e958d378bb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/published.csv new file mode 100644 index 00000000..41184d2c --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/projects/published.csv @@ -0,0 +1,13 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/projects/?format=xml" +"Bucket","Short_Name","Long_Name","UUID" +"G - I","GPM","Global Precipitation Measurement","297f51be-14cb-49e5-9711-5286d9953ade" +"A - C","ALIENS_SHOULD_DELETE",,"eb77c150-e875-4a1e-ae08-3459d796763a" +"A - C","ALIENS",,"9f83dac5-ae71-451a-8b85-1193c8b0ff38" +"A - C","AMSRICE06_SHOULD_DELETE",,"b15111fd-5f3c-4b9e-8fa8-b72752246823" +"A - C","AMSRICE06",,"d85b2560-8dc8-4682-a3fc-276e2ad3607e" +"A - C","CASERTZ_SHOULD_DELETE",,"d81f01de-b035-4fab-847b-b8fd1a4e8b92" +"A - C","CASERTZ",,"8607d64a-06f9-4d90-9e91-4e9d9ebf8a10" +"M - N","MEASURES_SHOULD_DELETE",,"0f322031-4cc8-48f7-a88e-2d893caa7c98" +"M - N","MEASURES",,"20d8e2a6-769b-40cb-a524-34760914927f" +,"MEASURES_SHOULD_DELETE",,"9403624e-4f3c-4f51-9f46-6692b4c0fe05" +,"MEASURES",,"70f75059-e00a-4e66-be0e-75e958d378bb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/draft.csv new file mode 100644 index 00000000..9fe395e5 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/draft.csv @@ -0,0 +1,9 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/providers/?format=xml" +"Bucket_Level0","Bucket_Level1","Bucket_Level2","Bucket_Level3","Short_Name","Long_Name","Data_Center_URL","UUID" +"ARCHIVER",,,,"TEST_DAAC - Test1","Test Distributed Active Archive Center","https://example.earthdata.nasa.gov","9d230f00-3b1b-48df-a70c-a46a9bedca7c" +"ARCHIVER",,,,"DOC/NOAA/NESDIS/NCEI_CHANGED","National Centers for Environmental Information, NESDIS, NOAA, U.S. Department of Commerce_CHANGED",,"dffa7c82-fc38-4aa8-914b-6de443439490" +"ARCHIVER",,,,"NSIDC_CHANGED",,,"3dae37f3-f768-4942-bf5e-09de63a188e4" +"ARCHIVER",,,,"NZ/NZAI/ANZ_CHANGED","Antarctica New Zealand_CHANGED",,"1299b2dc-8a97-4856-aaf0-986efefa597f" +"PROCESSOR",,,,"DRSC_CHANGED","Dekker Remote Sensing Corporation_CHANGED",,"27eee966-6e25-41af-aed0-76d81de2f3af" +"PROCESSOR",,,,"NASA/MSFC/AMSR-E SIPS_CHANGED",,,"5def0da1-5836-441b-ba1e-f90e464eb7ec" +,,,,"SMAP_PROVIDER_CHANGED","SMAP Provider Changed",,"164213c7-5e77-4f5f-b2a7-aa9bc14fc5bb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/published.csv new file mode 100644 index 00000000..b9f47e1b --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/providers/published.csv @@ -0,0 +1,15 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/providers/?format=xml" +"Bucket_Level0","Bucket_Level1","Bucket_Level2","Bucket_Level3","Short_Name","Long_Name","Data_Center_URL","UUID" +"ARCHIVER",,,,"TEST_DAAC","Test Distributed Active Archive Center","https://example.earthdata.nasa.gov","9d230f00-3b1b-48df-a70c-a46a9bedca7c" +"ARCHIVER",,,,"DOC/NOAA/NESDIS/NCEI_SHOULD_DELETE",,,"cc63aaad-d050-4083-8502-065ad7dedabb" +"ARCHIVER",,,,"DOC/NOAA/NESDIS/NCEI",,,"dffa7c82-fc38-4aa8-914b-6de443439490" +"ARCHIVER",,,,"DRSC_SHOULD_DELETE",,,"07146fe3-ebb1-42ca-a592-c9229c2fe032" +"ARCHIVER",,,,"NSIDC_SHOULD_DELETE",,,"35cf0506-7d49-4bae-9538-40bd8d2936e5" +"ARCHIVER",,,,"NSIDC",,,"3dae37f3-f768-4942-bf5e-09de63a188e4" +"ARCHIVER",,,,"NZ/NZAI/ANZ_SHOULD_DELETE",,,"06392d19-aa5d-4533-9e1a-ae9795c9560f" +"ARCHIVER",,,,"NZ/NZAI/ANZ",,,"1299b2dc-8a97-4856-aaf0-986efefa597f" +"PROCESSOR",,,,"DRSC",,,"27eee966-6e25-41af-aed0-76d81de2f3af" +"PROCESSOR",,,,"NASA/MSFC/AMSR-E SIPS_SHOULD_DELETE",,,"53940073-0ef8-4233-bff7-e2fbf88c1464" +"PROCESSOR",,,,"NASA/MSFC/AMSR-E SIPS",,,"5def0da1-5836-441b-ba1e-f90e464eb7ec" +,,,,"SMAP_PROVIDER_SHOULD_DELETE",,,"209d195b-bfe1-4ac1-a4b3-c1075c5ca3fc" +,,,,"SMAP_PROVIDER",,,"164213c7-5e77-4f5f-b2a7-aa9bc14fc5bb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/draft.csv new file mode 100644 index 00000000..3c330d71 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/draft.csv @@ -0,0 +1,5 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/rucontenttype/?format=xml" +"URLContentType","Type","Subtype","UUID" +"DistributionURL","GET DATA","VIRTUAL COLLECTION_CHANGED","cc9c6d5e-63d1-4c00-8392-bf341679082c" +"DistributionURL","VIEW RELATED INFORMATION","OpenSearch_CHANGED","47ee2b6f-3c1f-4dc2-b476-381a1805be5a" +"PublicationURL_CHANGED","VIEW RELATED INFORMATION_CHANGED","Subtype_CHANGED","40a75b8a-ab53-410f-b36b-0dad0be95d61" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/published.csv new file mode 100644 index 00000000..fa1d1f84 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/rucontenttype/published.csv @@ -0,0 +1,8 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/rucontenttype/?format=xml" +"URLContentType","Type","Subtype","UUID" +"DistributionURL","GET DATA","VIRTUAL COLLECTION_SHOULD_DELETE","858f16f3-7a6f-446c-935f-2a6e17729a87" +"DistributionURL","GET DATA","VIRTUAL COLLECTION","cc9c6d5e-63d1-4c00-8392-bf341679082c" +"DistributionURL","VIEW RELATED INFORMATION","OpenSearch_SHOULD_DELETE","1610e98f-e7ce-4d70-9cef-de603d0e04c7" +"DistributionURL","VIEW RELATED INFORMATION","OpenSearch","47ee2b6f-3c1f-4dc2-b476-381a1805be5a" +"PublicationURL_SHOULD_DELETE","VIEW RELATED INFORMATION_SHOULD_DELETE","Subtype_SHOULD_DELETE","4c261b52-4351-4407-80eb-a95b51e71e20" +"PublicationURL","VIEW RELATED INFORMATION",,"40a75b8a-ab53-410f-b36b-0dad0be95d61" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/draft.csv new file mode 100644 index 00000000..cee5e62c --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/draft.csv @@ -0,0 +1,6 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords/?format=xml" +"Category","Topic","Term","Variable_Level_1","Variable_Level_2","Variable_Level_3","Detailed_Variable","UUID" +"EARTH SCIENCE_CHANGED","TERRESTRIAL HYDROSPHERE_CHANGED","SNOW/ICE_CHANGED","SNOW WATER EQUIVALENT_CHANGED","SNOW ICE TEST_CHANGED","WATER ICE TEST_CHANGED","RAIN_CHANGED","be0d7a14-e1d5-4b44-b5c1-059274fc2dfb" +"EARTH SCIENCE","ATMOSPHERE","AEROSOLS_CHANGED",,,,,"d568c349-8559-4657-a57f-7c4b014c6acb" +"EARTH SCIENCE","ATMOSPHERE","ATMOSPHERIC TEMPERATURE","UPPER AIR TEMPERATURE","BOUNDARY LAYER TEMPERATURE","BOUNDARY LAYER TEMPERATURE PROFILES_CHANGED",,"f0af0cc9-2f22-4563-8a91-06b969c61d2c" +"EARTH SCIENCE","OCEANS","MARINE SEDIMENTS_CHANGED",,,,,"b9bb5978-2e0c-48a4-97d8-b484e3ef9511" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/published.csv new file mode 100644 index 00000000..0f32209f --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/sciencekeywords/published.csv @@ -0,0 +1,11 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords/?format=xml" +"Category","Topic","Term","Variable_Level_1","Variable_Level_2","Variable_Level_3","Detailed_Variable","UUID" +"EARTH SCIENCE_SHOULD_DELETE","TERRESTRIAL HYDROSPHERE_SHOULD_DELETE","SNOW/ICE_SHOULD_DELETE","SNOW WATER EQUIVALENT_SHOULD_DELETE","SNOW ICE TEST_SHOULD_DELETE","WATER ICE TEST_SHOULD_DELETE","RAIN_SHOULD_DELETE","b1bd7116-941c-4de1-9f0f-e1cab58ef586" +"EARTH SCIENCE","ATMOSPHERE","AEROSOLS_SHOULD_DELETE",,,,,"66bbd8f8-8703-40ff-bbbe-bda176ad30a5" +"EARTH SCIENCE","ATMOSPHERE","AEROSOLS",,,,,"d568c349-8559-4657-a57f-7c4b014c6acb" +"EARTH SCIENCE","ATMOSPHERE","ATMOSPHERIC TEMPERATURE","UPPER AIR TEMPERATURE","BOUNDARY LAYER TEMPERATURE","BOUNDARY LAYER TEMPERATURE PROFILES_SHOULD_DELETE",,"d1aab677-1cd3-4918-9bfb-14a9eba831ba" +"EARTH SCIENCE","ATMOSPHERE","ATMOSPHERIC TEMPERATURE","UPPER AIR TEMPERATURE","BOUNDARY LAYER TEMPERATURE","BOUNDARY LAYER TEMPERATURE PROFILES",,"f0af0cc9-2f22-4563-8a91-06b969c61d2c" +"EARTH SCIENCE","CRYOSPHERE","GLACIERS/ICE SHEETS","Glacier Topography/Ice Sheet Topography","Surface Morphology_SHOULD_DELETE",,,"25ca0fb7-4d5f-4350-8170-6f7708056b0c" +"EARTH SCIENCE","OCEANS","MARINE SEDIMENTS_SHOULD_DELETE",,,,,"256f88a0-592a-4b5e-ac3c-1f3d00221128" +"EARTH SCIENCE","OCEANS","MARINE SEDIMENTS",,,,,"b9bb5978-2e0c-48a4-97d8-b484e3ef9511" +"EARTH SCIENCE","TERRESTRIAL HYDROSPHERE","SNOW/ICE","SNOW WATER EQUIVALENT","SNOW ICE TEST","WATER ICE TEST","RAIN","be0d7a14-e1d5-4b44-b5c1-059274fc2dfb" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/short-name-events.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/short-name-events.json new file mode 100644 index 00000000..1507c52b --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/short-name-events.json @@ -0,0 +1,119 @@ +[ + { + "EventType": "UPDATED", + "Scheme": "dataformat", + "UUID": "464bd0da-8af4-4976-8ad1-5e3201c08016", + "OldKeywordObject": { + "ShortName": "GeoTIFF", + "LongName": "Georeferenced Tagged Image File Format" + }, + "NewKeywordObject": { + "ShortName": "GeoTIFF - Test1", + "LongName": "Georeferenced Tagged Image File Format" + } + }, + { + "EventType": "UPDATED", + "Scheme": "granuledataformat", + "UUID": "3139eabd-24d2-4d4a-99aa-8e955938ac38", + "OldKeywordObject": { + "ShortName": "GeoTIFF", + "LongName": "Georeferenced Tagged Image File Format" + }, + "NewKeywordObject": { + "ShortName": "GeoTIFF - Test1", + "LongName": "Georeferenced Tagged Image File Format" + } + }, + { + "EventType": "UPDATED", + "Scheme": "idnnode", + "UUID": "47f0cffe-4e1a-4716-a82c-7f026ef4fb63", + "OldKeywordObject": { + "ShortName": "IDN_TEST", + "LongName": "International Directory Network Test Node" + }, + "NewKeywordObject": { + "ShortName": "IDN_TEST - Test1", + "LongName": "International Directory Network Test Node" + } + }, + { + "EventType": "UPDATED", + "Scheme": "instruments", + "UUID": "3ef0c50f-9935-4480-9ab8-0a9fe5c20211", + "OldKeywordObject": { + "Category": "Earth Remote Sensing Instruments", + "Class": "Active Remote Sensing", + "Type": "Altimeters", + "Subtype": "", + "ShortName": "ALT-TEST", + "LongName": "Altitude Test Instrument" + }, + "NewKeywordObject": { + "Category": "Earth Remote Sensing Instruments", + "Class": "Active Remote Sensing", + "Type": "Altimeters", + "Subtype": "", + "ShortName": "ALT-TEST - Test1", + "LongName": "Altitude Test Instrument" + } + }, + { + "EventType": "UPDATED", + "Scheme": "platforms", + "UUID": "918b8740-4d85-4b75-90fd-315bea76747a", + "OldKeywordObject": { + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", + "ShortName": "GOSAT", + "LongName": "Greenhouse Gases Observing Satellite" + }, + "NewKeywordObject": { + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", + "ShortName": "GOSAT - Test1", + "LongName": "Greenhouse Gases Observing Satellite" + } + }, + { + "EventType": "UPDATED", + "Scheme": "projects", + "UUID": "297f51be-14cb-49e5-9711-5286d9953ade", + "OldKeywordObject": { + "Bucket": "G - I", + "ShortName": "GPM", + "LongName": "Global Precipitation Measurement" + }, + "NewKeywordObject": { + "Bucket": "G - I", + "ShortName": "GPM - Test1", + "LongName": "Global Precipitation Measurement" + } + }, + { + "EventType": "UPDATED", + "Scheme": "providers", + "UUID": "9d230f00-3b1b-48df-a70c-a46a9bedca7c", + "OldKeywordObject": { + "BucketLevel0": "ARCHIVER", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", + "ShortName": "TEST_DAAC", + "LongName": "Test Distributed Active Archive Center", + "DataCenterURL": "https://example.earthdata.nasa.gov" + }, + "NewKeywordObject": { + "BucketLevel0": "ARCHIVER", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", + "ShortName": "TEST_DAAC - Test1", + "LongName": "Test Distributed Active Archive Center", + "DataCenterURL": "https://example.earthdata.nasa.gov" + } + } +] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/draft.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/draft.csv new file mode 100644 index 00000000..c228a7e9 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/draft.csv @@ -0,0 +1,3 @@ +"Keyword Version: Draft","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/verticalresolutionrange/?format=xml" +"Vertical_Resolution_Range","UUID" +"5 meters_CHANGED","eaa23b5a-2643-4300-9193-5e73d00c438f" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/published.csv b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/published.csv new file mode 100644 index 00000000..9f7ea619 --- /dev/null +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/generated/verticalresolutionrange/published.csv @@ -0,0 +1,4 @@ +"Keyword Version: 24.6","Revision: 2026-08-20T00:00:00.000Z","Timestamp: 2026-08-20 00:00:00","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/verticalresolutionrange/?format=xml" +"Vertical_Resolution_Range","UUID" +"15 meters_SHOULD_DELETE","2adbe188-ee5d-4760-9c0a-2f8eef3fd55e" +"5 meters","eaa23b5a-2643-4300-9193-5e73d00c438f" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/ummc.before.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/ummc.before.json index 8431ee17..bd4a18c3 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/ummc.before.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/ummc.before.json @@ -118,6 +118,10 @@ } ] }, + { + "Type": "Earth Observation Satellites", + "ShortName": "Amazonia-1" + }, { "Type": "Rotorcraft/Helicopter", "ShortName": "AS350-B3_SHOULD_DELETE", diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/dif10.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/dif10.corrections.json index bc053c4d..e5595ec1 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/dif10.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/dif10.corrections.json @@ -29,20 +29,20 @@ "scheme": "locations", "action": "replace", "oldKeywordObject": { - "Category": "GEOGRAPHIC REGION", - "Type": "POLAR", - "Subregion1": "", - "Subregion2": "", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "GEOGRAPHIC REGION", + "LocationType": "POLAR", + "LocationSubregion1": "", + "LocationSubregion2": "", + "LocationSubregion3": "", + "LocationSubregion4": "" }, "newKeywordObject": { - "Category": "GEOGRAPHIC REGION", - "Type": "POLAR_CHANGED", - "Subregion1": "", - "Subregion2": "", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "GEOGRAPHIC REGION", + "LocationType": "POLAR_CHANGED", + "LocationSubregion1": "", + "LocationSubregion2": "", + "LocationSubregion3": "", + "LocationSubregion4": "" }, "newLongName": null }, @@ -71,15 +71,15 @@ "action": "replace", "newLongName": "Systeme Probatoire d'Observation de la Terre-4_CHANGED", "oldKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "SPOT-4" }, "newKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "SPOT-4_CHANGED" } }, @@ -89,13 +89,15 @@ "oldKeywordObject": { "Category": "Synthetic Aperture Radar", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "TSX-1" }, "newKeywordObject": { "Category": "Synthetic Aperture Radar", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "TSX-1_CHANGED" }, "newLongName": "Synthetic Aperture Radar_CHANGED" @@ -107,13 +109,15 @@ "oldKeywordObject": { "Category": "In Situ/Laboratory Instruments", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "SEISMIC REFLECTION PROFILERS" }, "newKeywordObject": { "Category": "In Situ/Laboratory Instruments", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "SEISMIC REFLECTION PROFILERS_CHANGED" } }, @@ -124,13 +128,15 @@ "oldKeywordObject": { "Category": "In Situ/Laboratory Instruments", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "SEISMIC REFLECTION PROFILERS" }, "newKeywordObject": { "Category": "In Situ/Laboratory Instruments", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "SEISMIC REFLECTION PROFILERS_CHANGED" } }, @@ -139,11 +145,11 @@ "action": "replace", "newLongName": "Aliens in Antarctica_CHANGED", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "ALIENS" }, "newKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "ALIENS_CHANGED" } }, @@ -195,40 +201,40 @@ "scheme": "isotopiccategory", "action": "replace", "oldKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION" }, "newKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION_CHANGED" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION_CHANGED" } }, { "scheme": "verticalresolutionrange", "action": "replace", "oldKeywordObject": { - "Value": "5 meters" + "VerticalResolutionRange": "5 meters" }, "newKeywordObject": { - "Value": "5 meters_CHANGED" + "VerticalResolutionRange": "5 meters_CHANGED" } }, { "scheme": "horizontalresolutionrange", "action": "replace", "oldKeywordObject": { - "Value": "10 meters" + "HorizontalResolutionRange": "10 meters" }, "newKeywordObject": { - "Value": "10 meters_CHANGED" + "HorizontalResolutionRange": "10 meters_CHANGED" } }, { "scheme": "productlevelid", "action": "replace", "oldKeywordObject": { - "Value": "NA" + "ProductLevelId": "NA" }, "newKeywordObject": { - "Value": "NA_CHANGED" + "ProductLevelId": "NA_CHANGED" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.after.xml b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.after.xml index 5035c576..d34f1135 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.after.xml +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.after.xml @@ -279,8 +279,8 @@ 1 - AMSR-E_CHANGED - Advanced Microwave Scanning Radiometer-EOS_CHANGED + AMSR-E + Advanced Microwave Scanning Radiometer-EOS Broadband scanning radiometry 1 diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.corrections.json index 24ce5ae0..75613abf 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/echo10.corrections.json @@ -30,15 +30,15 @@ "action": "replace", "newLongName": "Earth Observing System, Terra (AM-1)_CHANGED", "oldKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "Terra" }, "newKeywordObject": { - "Category": "Platforms", - "Class": "Space-based Platforms", - "Type": "Earth Observation Satellites", + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", "ShortName": "Terra_CHANGED" } }, @@ -48,44 +48,29 @@ "oldKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "AMSR-E" }, "newKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "AMSR-E_CHANGED" }, "newLongName": "Advanced Microwave Scanning Radiometer-EOS_CHANGED" }, - { - "scheme": "instruments", - "action": "replace", - "newLongName": "Advanced Microwave Scanning Radiometer-EOS_CHANGED", - "oldKeywordObject": { - "Category": "Imaging Spectrometers/Radiometers", - "Class": "", - "Subclass": "", - "ShortName": "AMSR-E" - }, - "newKeywordObject": { - "Category": "Imaging Spectrometers/Radiometers", - "Class": "", - "Subclass": "", - "ShortName": "AMSR-E_CHANGED" - } - }, { "scheme": "projects", "action": "replace", "newLongName": "Advanced Microwave Scanning Radiometer Sea Ice Product Validation_CHANGED", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "AMSRICE06" }, "newKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "AMSRICE06_CHANGED" } }, @@ -130,7 +115,8 @@ "action": "replace", "oldKeywordObject": { "URLContentType": "PublicationURL", - "Type": "VIEW RELATED INFORMATION" + "Type": "VIEW RELATED INFORMATION", + "Subtype": "" }, "newKeywordObject": { "URLContentType": "PublicationURL_CHANGED", @@ -142,20 +128,20 @@ "scheme": "dataformat", "action": "replace", "oldKeywordObject": { - "Value": "netCDF-4" + "ShortName": "netCDF-4" }, "newKeywordObject": { - "Value": "netCDF-4_CHANGED" + "ShortName": "netCDF-4_CHANGED" } }, { "scheme": "productlevelid", "action": "replace", "oldKeywordObject": { - "Value": "3" + "ProductLevelId": "3" }, "newKeywordObject": { - "Value": "3_CHANGED" + "ProductLevelId": "3_CHANGED" } } ] diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/iso19115.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/iso19115.corrections.json index b627ca25..4ccdbf9e 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/iso19115.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/iso19115.corrections.json @@ -29,20 +29,20 @@ "scheme": "locations", "action": "replace", "oldKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA", + "LocationSubregion3": "", + "LocationSubregion4": "" }, "newKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA_CHANGED", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA_CHANGED", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { @@ -50,9 +50,15 @@ "action": "replace", "newLongName": "Earth Observing System_CHANGED", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA" }, "newKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA_CHANGED" } }, @@ -61,9 +67,15 @@ "action": "replace", "newLongName": "Earth Observing System, TERRA (AM-1)_CHANGED", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "TERRA" }, "newKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "TERRA_CHANGED" } }, @@ -72,9 +84,17 @@ "action": "replace", "newLongName": "Moderate-Resolution Imaging Spectroradiometer_CHANGED", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS" }, "newKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS_CHANGED" } }, @@ -83,9 +103,17 @@ "action": "replace", "newLongName": "Advanced Topographic Laser Altimeter System_CHANGED", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "ATLAS" }, "newKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "ATLAS_CHANGED" } }, @@ -94,11 +122,11 @@ "action": "replace", "newLongName": "Making Earth System Data Records for Use in Research Environments_CHANGED", "oldKeywordObject": { - "Category": "M - N", + "Bucket": "M - N", "ShortName": "MEASURES" }, "newKeywordObject": { - "Category": "M - N", + "Bucket": "M - N", "ShortName": "MEASURES_CHANGED" } }, @@ -125,20 +153,20 @@ "scheme": "isotopiccategory", "action": "replace", "oldKeywordObject": { - "Value": "FARMING" + "ISOTopicCategory": "FARMING" }, "newKeywordObject": { - "Value": "FARMING_CHANGED" + "ISOTopicCategory": "FARMING_CHANGED" } }, { "scheme": "productlevelid", "action": "replace", "oldKeywordObject": { - "Value": "3" + "ProductLevelId": "3" }, "newKeywordObject": { - "Value": "3_CHANGED" + "ProductLevelId": "3_CHANGED" } }, { diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/isosmap.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/isosmap.corrections.json index 79aaf415..a5191cc9 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/isosmap.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/isosmap.corrections.json @@ -29,29 +29,35 @@ "scheme": "locations", "action": "replace", "oldKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA", + "LocationSubregion3": "", + "LocationSubregion4": "" }, "newKeywordObject": { - "Category": "CONTINENT", - "Type": "NORTH AMERICA", - "Subregion1": "CANADA", - "Subregion2": "ALBERTA_CHANGED", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "CONTINENT", + "LocationType": "NORTH AMERICA", + "LocationSubregion1": "CANADA", + "LocationSubregion2": "ALBERTA_CHANGED", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { "scheme": "platforms", "action": "replace", "oldKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA" }, "newKeywordObject": { + "Basis": "", + "Category": "", + "SubCategory": "", "ShortName": "AQUA_CHANGED" }, "newLongName": "Earth Observing System_CHANGED" @@ -60,9 +66,17 @@ "scheme": "instruments", "action": "replace", "oldKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS" }, "newKeywordObject": { + "Category": "", + "Class": "", + "Type": "", + "Subtype": "", "ShortName": "MODIS_CHANGED" }, "newLongName": "Moderate-Resolution Imaging Spectroradiometer_CHANGED" @@ -71,9 +85,11 @@ "scheme": "projects", "action": "replace", "oldKeywordObject": { + "Bucket": "", "ShortName": "MEASURES" }, "newKeywordObject": { + "Bucket": "", "ShortName": "MEASURES_CHANGED" }, "newLongName": "Making Earth System Data Records for Use in Research Environments_CHANGED" @@ -82,20 +98,20 @@ "scheme": "isotopiccategory", "action": "replace", "newKeywordObject": { - "Value": "climatology_CHANGED" + "ISOTopicCategory": "climatology_CHANGED" }, "oldKeywordObject": { - "Value": "climatology" + "ISOTopicCategory": "climatology" } }, { "scheme": "productlevelid", "action": "replace", "oldKeywordObject": { - "Value": "3" + "ProductLevelId": "3" }, "newKeywordObject": { - "Value": "3_CHANGED" + "ProductLevelId": "3_CHANGED" } }, { @@ -112,9 +128,17 @@ "scheme": "providers", "action": "replace", "oldKeywordObject": { + "BucketLevel0": "", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", "ShortName": "SMAP_PROVIDER" }, "newKeywordObject": { + "BucketLevel0": "", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", "ShortName": "SMAP_PROVIDER_CHANGED" }, "newLongName": "SMAP Provider Changed" @@ -123,9 +147,17 @@ "scheme": "providers", "action": "replace", "oldKeywordObject": { + "BucketLevel0": "", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", "ShortName": "SMAP_PROVIDER" }, "newKeywordObject": { + "BucketLevel0": "", + "BucketLevel1": "", + "BucketLevel2": "", + "BucketLevel3": "", "ShortName": "SMAP_PROVIDER_CHANGED" }, "newLongName": "SMAP Provider Changed" diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.after.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.after.json index 993dc05f..ef4247ed 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.after.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.after.json @@ -118,6 +118,10 @@ } ] }, + { + "Type": "Earth Observation Satellites", + "ShortName": "Amazonia-1-Chris-Test_CHANGED" + }, { "Type": "Rotorcraft/Helicopter", "ShortName": "AS350-B3_SHOULD_DELETE", diff --git a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.corrections.json b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.corrections.json index 84af608f..cb2d4bdf 100644 --- a/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.corrections.json +++ b/serverless/src/shared/__mocks__/native_metadata_correction_mappings/end_to_end/updates/ummc.corrections.json @@ -29,20 +29,20 @@ "scheme": "locations", "action": "replace", "oldKeywordObject": { - "Category": "OCEAN", - "Type": "ATLANTIC OCEAN", - "Subregion1": "NORTH ATLANTIC OCEAN", - "Subregion2": "CARIBBEAN SEA", - "Subregion3": "BARBADOS", - "DetailedLocation": "" + "LocationCategory": "OCEAN", + "LocationType": "ATLANTIC OCEAN", + "LocationSubregion1": "NORTH ATLANTIC OCEAN", + "LocationSubregion2": "CARIBBEAN SEA", + "LocationSubregion3": "BARBADOS", + "LocationSubregion4": "" }, "newKeywordObject": { - "Category": "OCEAN", - "Type": "ATLANTIC OCEAN", - "Subregion1": "NORTH ATLANTIC OCEAN", - "Subregion2": "CARIBBEAN SEA", - "Subregion3": "BARBADOS_CHANGED", - "DetailedLocation": "" + "LocationCategory": "OCEAN", + "LocationType": "ATLANTIC OCEAN", + "LocationSubregion1": "NORTH ATLANTIC OCEAN", + "LocationSubregion2": "CARIBBEAN SEA", + "LocationSubregion3": "BARBADOS_CHANGED", + "LocationSubregion4": "" }, "newLongName": null }, @@ -50,20 +50,20 @@ "scheme": "locations", "action": "replace", "oldKeywordObject": { - "Category": "GEOGRAPHIC REGION", - "Type": "GLOBAL LAND", - "Subregion1": "", - "Subregion2": "", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "GEOGRAPHIC REGION", + "LocationType": "GLOBAL LAND", + "LocationSubregion1": "", + "LocationSubregion2": "", + "LocationSubregion3": "", + "LocationSubregion4": "" }, "newKeywordObject": { - "Category": "GEOGRAPHIC REGION", - "Type": "GLOBAL LAND_CHANGED", - "Subregion1": "", - "Subregion2": "", - "Subregion3": "", - "DetailedLocation": "" + "LocationCategory": "GEOGRAPHIC REGION", + "LocationType": "GLOBAL LAND_CHANGED", + "LocationSubregion1": "", + "LocationSubregion2": "", + "LocationSubregion3": "", + "LocationSubregion4": "" } }, { @@ -91,29 +91,49 @@ "action": "replace", "newLongName": "Helicopter AS350-B3_CHANGED", "oldKeywordObject": { - "Class": "Air-based Platforms", - "Type": "Rotorcraft/Helicopter", + "Basis": "Air-based Platforms", + "Category": "Rotorcraft/Helicopter", + "SubCategory": "", "ShortName": "AS350-B3" }, "newKeywordObject": { - "Class": "Air-based Platforms", - "Type": "Rotorcraft/Helicopter", + "Basis": "Air-based Platforms", + "Category": "Rotorcraft/Helicopter", + "SubCategory": "", "ShortName": "AS350-B3_CHANGED" } }, + { + "scheme": "platforms", + "action": "replace", + "oldKeywordObject": { + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", + "ShortName": "Amazonia-1" + }, + "newKeywordObject": { + "Basis": "Space-based Platforms", + "Category": "Earth Observation Satellites", + "SubCategory": "", + "ShortName": "Amazonia-1-Chris-Test_CHANGED" + } + }, { "scheme": "instruments", "action": "replace", "oldKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "MABEL" }, "newKeywordObject": { "Category": "Imaging Spectrometers/Radiometers", "Class": "", - "Subclass": "", + "Type": "", + "Subtype": "", "ShortName": "MABEL_CHANGED" }, "newLongName": "Multiple Altimeter Beam Experimental Lidar_CHANGED" @@ -123,11 +143,11 @@ "action": "replace", "newLongName": "Corridor Aerogeophysics of the South East Ross Transect Zone_CHANGED", "oldKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "CASERTZ" }, "newKeywordObject": { - "Category": "A - C", + "Bucket": "A - C", "ShortName": "CASERTZ_CHANGED" } }, @@ -179,20 +199,20 @@ "scheme": "isotopiccategory", "action": "replace", "oldKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION" }, "newKeywordObject": { - "Value": "GEOSCIENTIFIC INFORMATION_CHANGED" + "ISOTopicCategory": "GEOSCIENTIFIC INFORMATION_CHANGED" } }, { "scheme": "productlevelid", "action": "replace", "oldKeywordObject": { - "Value": "2" + "ProductLevelId": "2" }, "newKeywordObject": { - "Value": "2_CHANGED" + "ProductLevelId": "2_CHANGED" } }, { diff --git a/serverless/src/shared/__tests__/applyDif10MetadataCorrections.test.js b/serverless/src/shared/__tests__/applyDif10MetadataCorrections.test.js index 6be4bf0f..04721051 100644 --- a/serverless/src/shared/__tests__/applyDif10MetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyDif10MetadataCorrections.test.js @@ -210,15 +210,15 @@ describe('when applying DIF10 metadata corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'In Situ Land-based Platforms', Category: '', - Class: 'In Situ Land-based Platforms', - Type: '', ShortName: 'GROUND STATIONS' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'C-130' } } @@ -286,12 +286,12 @@ describe('when applying DIF10 metadata corrections', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'ARCTIC', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'ARCTIC', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -348,15 +348,15 @@ describe('when correcting a DIF10 record', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4-UPDATED' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -367,13 +367,15 @@ describe('when correcting a DIF10 record', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES-UPDATED' }, newLongName: 'Updated Geophone Array' @@ -382,30 +384,30 @@ describe('when correcting a DIF10 record', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'ANTARCTICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'ANTARCTICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'SOUTH AMERICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'SOUTH AMERICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }, { scheme: 'projects', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS-UPDATED' }, newLongName: 'Aliens in Antarctica Updated' @@ -469,10 +471,10 @@ describe('when correcting a DIF10 record', () => { scheme: 'ProductLevelId', action: 'replace', oldKeywordObject: { - Value: 'NA' + ProductLevelId: 'NA' }, newKeywordObject: { - Value: '1A' + ProductLevelId: '1A' } } ] @@ -580,12 +582,12 @@ describe('when verifying DIF10 corrections do not remove unrelated metadata', () scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'ANTARCTICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: 'MCMURDO SOUND' + LocationCategory: 'CONTINENT', + LocationType: 'ANTARCTICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: 'MCMURDO SOUND' } }, { @@ -612,15 +614,15 @@ describe('when verifying DIF10 corrections do not remove unrelated metadata', () scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4-UPDATED' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -631,7 +633,8 @@ describe('when verifying DIF10 corrections do not remove unrelated metadata', () oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES' } }, @@ -639,11 +642,11 @@ describe('when verifying DIF10 corrections do not remove unrelated metadata', () scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS-UPDATED' }, newLongName: 'Aliens in Antarctica Updated' @@ -692,40 +695,40 @@ describe('when verifying DIF10 corrections do not remove unrelated metadata', () scheme: 'isotopiccategory', action: 'replace', oldKeywordObject: { - Value: 'GEOSCIENTIFIC INFORMATION' + ISOTopicCategory: 'GEOSCIENTIFIC INFORMATION' }, newKeywordObject: { - Value: 'OCEANS' + ISOTopicCategory: 'OCEANS' } }, { scheme: 'verticalresolutionrange', action: 'replace', oldKeywordObject: { - Value: '5 meters' + VerticalResolutionRange: '5 meters' }, newKeywordObject: { - Value: '50 meters' + VerticalResolutionRange: '50 meters' } }, { scheme: 'horizontalresolutionrange', action: 'replace', oldKeywordObject: { - Value: '10 meters' + HorizontalResolutionRange: '10 meters' }, newKeywordObject: { - Value: '100 meters' + HorizontalResolutionRange: '100 meters' } }, { scheme: 'productlevelid', action: 'replace', oldKeywordObject: { - Value: 'NA' + ProductLevelId: 'NA' }, newKeywordObject: { - Value: '1A' + ProductLevelId: '1A' } } ] @@ -1277,10 +1280,10 @@ describe('when applying horizontal resolution DIF10 corrections', () => { action: 'replace', ummPath: ['HorizontalResolutionRanges', 1], oldKeywordObject: { - Value: '1 - 10 meters' + HorizontalResolutionRange: '1 - 10 meters' }, newKeywordObject: { - Value: 'Updated Range' + HorizontalResolutionRange: 'Updated Range' } }] }) @@ -1299,10 +1302,10 @@ describe('when applying horizontal resolution DIF10 corrections', () => { action: 'replace', ummPath: ['HorizontalResolutionRanges', 0], oldKeywordObject: { - Value: 'Old' + HorizontalResolutionRange: 'Old' }, newKeywordObject: { - Value: 'New' + HorizontalResolutionRange: 'New' } }] }) @@ -1321,7 +1324,7 @@ describe('when applying horizontal resolution DIF10 corrections', () => { action: 'delete', ummPath: ['HorizontalResolutionRanges', 0], oldKeywordObject: { - Value: '0 - 1 meter' + HorizontalResolutionRange: '0 - 1 meter' } }] }) @@ -1345,7 +1348,7 @@ describe('when applying horizontal resolution DIF10 corrections', () => { scheme: 'horizontalresolutionrange', action: 'delete', oldKeywordObject: { - Value: '1 - 10 meters' + HorizontalResolutionRange: '1 - 10 meters' }, newKeywordObject: {} }] @@ -1366,7 +1369,7 @@ describe('when applying horizontal resolution DIF10 corrections', () => { scheme: 'horizontalresolutionrange', action: 'delete', oldKeywordObject: { - Value: 'Only Range' + HorizontalResolutionRange: 'Only Range' } }] }) @@ -1394,7 +1397,7 @@ describe('when applying horizontal resolution DIF10 corrections', () => { action: 'delete', ummPath: ['HorizontalResolutionRanges', 0], oldKeywordObject: { - Value: 'Range 1' + HorizontalResolutionRange: 'Range 1' } }, { @@ -1402,7 +1405,7 @@ describe('when applying horizontal resolution DIF10 corrections', () => { action: 'delete', ummPath: ['HorizontalResolutionRanges', 0], oldKeywordObject: { - Value: 'Range 2' + HorizontalResolutionRange: 'Range 2' } } ] @@ -1437,10 +1440,10 @@ describe('when applying horizontal resolution DIF10 corrections', () => { scheme: 'horizontalresolutionrange', action: 'invalid_action_name', // This bypasses both 'delete' and 'replace' blocks oldKeywordObject: { - Value: '0 - 1 meter' + HorizontalResolutionRange: '0 - 1 meter' }, newKeywordObject: { - Value: 'New Value' + HorizontalResolutionRange: 'New Value' } }] }) @@ -1685,13 +1688,15 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newLongName: 'Updated Infrared Multispectral Scanner' @@ -1725,13 +1730,15 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'LISS-II' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'LISSUPDATE-II' }, newLongName: 'Linear Imaging Self Scanning Sensor II Updated' @@ -1763,7 +1770,8 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: {} @@ -1798,7 +1806,8 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I1' } }, @@ -1808,7 +1817,8 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I2' } } @@ -1837,7 +1847,8 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I1' } } @@ -1868,13 +1879,15 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'OLD-SHORT' }, newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' }, newLongName: '' // Triggers delete target['Long_Name'] @@ -1911,13 +1924,15 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS1' } } @@ -1942,13 +1957,15 @@ describe('when applying instrument DIF10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS1' }, newLongName: 'Infrared Multispectral Scanner Updated' @@ -1971,7 +1988,8 @@ describe('when instrument guard clauses prevent a correction', () => { newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -1991,7 +2009,8 @@ describe('when instrument guard clauses prevent a correction', () => { newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -2009,13 +2028,15 @@ describe('when instrument guard clauses prevent a correction', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'NOT-REAL' }, newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -2051,10 +2072,10 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'replace', oldKeywordObject: { - Value: 'CLIMATOLOGY/METEOROLOGY/ATMOSPHERE' + ISOTopicCategory: 'CLIMATOLOGY/METEOROLOGY/ATMOSPHERE' }, newKeywordObject: { - Value: 'FARMING' + ISOTopicCategory: 'FARMING' } }] }) @@ -2073,10 +2094,10 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'replace', oldKeywordObject: { - Value: 'OLD_CAT' + ISOTopicCategory: 'OLD_CAT' }, newKeywordObject: { - Value: 'NEW_CAT' + ISOTopicCategory: 'NEW_CAT' } }] }) @@ -2094,7 +2115,7 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'delete', oldKeywordObject: { - Value: 'BIOTA' + ISOTopicCategory: 'BIOTA' } }] }) @@ -2112,7 +2133,7 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'delete', oldKeywordObject: { - Value: 'LAST_ONE' + ISOTopicCategory: 'LAST_ONE' } }] }) @@ -2129,7 +2150,7 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'delete', oldKeywordObject: { - Value: 'SINGLE_STRING' + ISOTopicCategory: 'SINGLE_STRING' } }] }) @@ -2147,14 +2168,14 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'delete', oldKeywordObject: { - Value: 'BIOTA' + ISOTopicCategory: 'BIOTA' } }, { scheme: 'isotopiccategory', action: 'delete', oldKeywordObject: { - Value: 'CLIMATOLOGY/METEOROLOGY/ATMOSPHERE' + ISOTopicCategory: 'CLIMATOLOGY/METEOROLOGY/ATMOSPHERE' } } ] @@ -2208,10 +2229,10 @@ describe('when applying ISO topic category DIF10 corrections', () => { scheme: 'isotopiccategory', action: 'replace', oldKeywordObject: { - Value: 'NOT_REAL' + ISOTopicCategory: 'NOT_REAL' }, newKeywordObject: { - Value: 'FAIL' + ISOTopicCategory: 'FAIL' } }] }) @@ -2254,20 +2275,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 0], oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'SOUTH AMERICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'SOUTH AMERICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2299,20 +2320,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 2], oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'CANADA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'CANADA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2337,20 +2358,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 2], oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: 'CALIFORNIA', - Subregion3: 'LOS ANGELES', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: 'CALIFORNIA', + LocationSubregion3: 'LOS ANGELES', + LocationSubregion4: '' } } ] @@ -2375,20 +2396,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 2], oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'EUROPE', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'EUROPE', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2412,12 +2433,12 @@ describe('when applying location DIF10 corrections', () => { action: 'delete', ummPath: ['Locations', 1], oldKeywordObject: { - Category: 'OCEAN', - Type: 'PACIFIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'PACIFIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: {} } @@ -2450,12 +2471,12 @@ describe('when applying location DIF10 corrections', () => { action: 'delete', ummPath: ['Locations', 0], oldKeywordObject: { - Category: 'A', - Type: '', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'A', + LocationType: '', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }, { @@ -2463,12 +2484,12 @@ describe('when applying location DIF10 corrections', () => { action: 'delete', ummPath: ['Locations', 0], oldKeywordObject: { - Category: 'B', - Type: '', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'B', + LocationType: '', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2500,20 +2521,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 0], oldKeywordObject: { - Category: 'OCEAN', - Type: 'ATLANTIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'ATLANTIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'OCEAN', - Type: 'INDIAN OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'INDIAN OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2542,20 +2563,20 @@ describe('when applying location DIF10 corrections', () => { action: 'replace', ummPath: ['Locations', 0], oldKeywordObject: { - Category: 'OCEAN', - Type: 'ATLANTIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'ATLANTIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'OCEAN', - Type: 'PACIFIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'PACIFIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2575,12 +2596,12 @@ describe('when location guard clauses prevent a correction', () => { action: 'replace', ummPath: ['Locations', 'first'], // String instead of Number newKeywordObject: { - Category: 'CONTINENT', - Type: 'EUROPE', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'EUROPE', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }] }) @@ -2596,12 +2617,12 @@ describe('when location guard clauses prevent a correction', () => { action: 'replace', ummPath: ['Locations', 99], // Index does not exist newKeywordObject: { - Category: 'CONTINENT', - Type: 'EUROPE', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'EUROPE', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }] }) @@ -2617,12 +2638,12 @@ describe('when location guard clauses prevent a correction', () => { action: 'invalid_action', // Not replace or delete ummPath: ['Locations', 0], newKeywordObject: { - Category: 'CONTINENT', - Type: 'EUROPE', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'EUROPE', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }] }) @@ -2645,20 +2666,20 @@ describe('when location guard clauses prevent a correction', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'GLOBAL', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'GLOBAL', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'CONTINENT', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'CONTINENT', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2683,12 +2704,12 @@ describe('when location guard clauses prevent a correction', () => { scheme: 'locations', action: 'unsupported_action_type', oldKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'GLOBAL', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'GLOBAL', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2719,12 +2740,12 @@ describe('when location guard clauses prevent a correction', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'GLOBAL', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'GLOBAL', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -2767,6 +2788,19 @@ const mockDif10WithPlatforms = ` ` +const mockDif10WithPlatformBasis = ` + + Platforms_Basis_Test + 001 + + + Other + Aircraft + SPOT-4 + Systeme Observation de la Terre-4 Baseline + +` + describe('when applying platform DIF10 corrections', () => { test('should apply long name correction to first Platform', async () => { const result = await applyDif10MetadataCorrections({ @@ -2779,15 +2813,15 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -2823,15 +2857,15 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-5' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-7-New' }, newLongName: 'Systeme Observation de la Terre-5 Updated' @@ -2853,6 +2887,43 @@ describe('when applying platform DIF10 corrections', () => { expect(result.correctedMetadata).not.toContain('Earth Observation planes') }) + test('should preserve platform Basis while updating short and long name', async () => { + const result = await applyDif10MetadataCorrections({ + collectionConceptId: 'C1', + providerId: 'PROV', + nativeId: 'native-1', + metadataPayload: mockDif10WithPlatformBasis, + corrections: [ + { + scheme: 'platforms', + action: 'replace', + oldKeywordObject: { + Basis: 'Other', + SubCategory: '', + Category: 'Aircraft', + ShortName: 'SPOT-4' + }, + newKeywordObject: { + Basis: 'Other-UPDATED', + SubCategory: '', + Category: 'Aircraft', + ShortName: 'SPOT-4-UPDATED' + }, + newLongName: 'Systeme Observation de la Terre-4 Updated' + } + ] + }) + + expect(result.correctionCount).toBe(1) + expect(result.correctionsApplied).toHaveLength(1) + + expect(result.correctedMetadata).toContain('Other') + expect(result.correctedMetadata).not.toContain('Other-UPDATED') + expect(result.correctedMetadata).toContain('SPOT-4-UPDATED') + expect(result.correctedMetadata).toContain('Systeme Observation de la Terre-4 Updated') + expect(result.correctedMetadata).not.toContain('Systeme Observation de la Terre-4 Baseline') + }) + test('should apply platform correction when there is only a single platform (object branch)', async () => { const singlePlatformXml = ` @@ -2873,15 +2944,15 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'In Situ Land-based Platforms', Category: '', - Class: 'In Situ Land-based Platforms', - Type: '', ShortName: 'GROUND STATIONS' }, newKeywordObject: { - Category: '', - Class: 'In Situ Land-based Platforms', - Type: 'Aircraft', + SubCategory: '', + Basis: 'In Situ Land-based Platforms', + Category: 'Aircraft', ShortName: 'C-130' }, newLongName: 'Lockheed C-130 Hercules' @@ -2918,9 +2989,9 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A1' } }, @@ -2928,9 +2999,9 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A2' } } @@ -2954,9 +3025,9 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: {} @@ -2983,9 +3054,9 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'not_an_action', // Triggers the final return false newKeywordObject: { - Category: '', - Class: 'A', - Type: 'B', + SubCategory: '', + Basis: 'A', + Category: 'B', ShortName: 'C' } } @@ -3013,9 +3084,9 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A1' } } @@ -3044,16 +3115,16 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'OLD-SHORT' }, // Providing only one segment forces the Long_Name field to hit the 'else { delete }' branch newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'NEW-SHORT' }, newLongName: '' @@ -3088,15 +3159,15 @@ describe('when applying platform DIF10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'Earth Observation planes', Category: '', - Class: 'Earth Observation planes', - Type: '', ShortName: 'SPOT-5' }, newKeywordObject: { + SubCategory: '', + Basis: 'Earth Observation rockets', Category: '', - Class: 'Earth Observation rockets', - Type: '', ShortName: 'SPOT-7' } } @@ -3119,15 +3190,15 @@ describe('when applying platform DIF10 corrections', () => { action: 'replace', ummPath: ['Platform', 10], oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-5' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-7' } } @@ -3152,7 +3223,7 @@ describe('when applying Product_Level_Id DIF10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 2' + ProductLevelId: 'Level 2' } }] }) @@ -3168,7 +3239,7 @@ describe('when applying Product_Level_Id DIF10 corrections', () => { corrections: [{ scheme: 'productlevelid', newKeywordObject: { - Value: 'Level 3' + ProductLevelId: 'Level 3' } }] }) @@ -3228,7 +3299,7 @@ describe('when applying Product_Level_Id DIF10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 4' + ProductLevelId: 'Level 4' } }] }) @@ -3245,7 +3316,7 @@ describe('when applying Product_Level_Id DIF10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 4' + ProductLevelId: 'Level 4' } }] }) @@ -3260,7 +3331,7 @@ describe('when applying Product_Level_Id DIF10 corrections', () => { scheme: 'productlevelid', action: 'invalid_action_type', newKeywordObject: { - Value: 'Level 2' + ProductLevelId: 'Level 2' } }] }) @@ -3299,11 +3370,11 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newLongName: 'Updated Earth Science Information Partners Program' @@ -3333,11 +3404,11 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS UP' }, newLongName: 'Aliens research in Antarctica' @@ -3367,7 +3438,7 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: {} @@ -3393,7 +3464,7 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' } }, @@ -3401,7 +3472,7 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' } } @@ -3430,7 +3501,7 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'SINGLE-PROJ' } } @@ -3460,7 +3531,7 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'SINGLE-PROJ' } } @@ -3491,11 +3562,11 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP-7' } } @@ -3514,13 +3585,13 @@ describe('when applying project DIF10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, // Providing a single segment and NO newLongName // results in normalizedSegments[1] (Long_Name) being undefined newKeywordObject: { - Category: 'M - O', + Bucket: 'M - O', ShortName: 'ONLY_SHORT' } } @@ -3551,11 +3622,11 @@ describe('when applying project DIF10 corrections', () => { action: 'replace', ummPath: ['Project', 10], oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP-7' }, newLongName: 'Updated Earth Science Information Partners Program' @@ -3597,7 +3668,7 @@ describe('when applying project DIF10 corrections', () => { corrections: [{ scheme: 'projects', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'NOT-REAL' } }] @@ -5209,7 +5280,7 @@ describe('when applying temporal resolution DIF10 corrections', () => { scheme: 'temporalresolutionrange', action: 'delete', oldKeywordObject: { - Value: 'Daily' + TemporalResolutionRange: 'Daily' } }] }) @@ -5238,10 +5309,10 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'replace', ummPath: ['VerticalResolutionRanges', 1], newKeywordObject: { - Value: 'Updated Range' + VerticalResolutionRange: 'Updated Range' }, oldKeywordObject: { - Value: '10 - 50 meters' + VerticalResolutionRange: '10 - 50 meters' } }] }) @@ -5260,10 +5331,10 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'replace', ummPath: ['VerticalResolutionRanges', 0], newKeywordObject: { - Value: 'New Range' + VerticalResolutionRange: 'New Range' }, oldKeywordObject: { - Value: 'Old Range' + VerticalResolutionRange: 'Old Range' } }] }) @@ -5282,7 +5353,7 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'delete', ummPath: ['VerticalResolutionRanges', 0], oldKeywordObject: { - Value: '1 - 10 meters' + VerticalResolutionRange: '1 - 10 meters' } }] }) @@ -5301,7 +5372,7 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'delete', ummPath: ['VerticalResolutionRanges', 0], oldKeywordObject: { - Value: 'Final Range' + VerticalResolutionRange: 'Final Range' } }] }) @@ -5330,7 +5401,7 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'delete', ummPath: ['VerticalResolutionRanges', 0], oldKeywordObject: { - Value: 'Range 1' + VerticalResolutionRange: 'Range 1' } }, { @@ -5338,7 +5409,7 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'delete', ummPath: ['VerticalResolutionRanges', 0], oldKeywordObject: { - Value: 'Range 2' + VerticalResolutionRange: 'Range 2' } } ] @@ -5398,7 +5469,7 @@ describe('when applying vertical resolution DIF10 corrections', () => { action: 'unsupported_action', ummPath: ['VerticalResolutionRanges', 0], oldKeywordObject: { - Value: '1 - 10 meters' + VerticalResolutionRange: '1 - 10 meters' } }] }) diff --git a/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js b/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js index 4c67a0c1..fe87c325 100644 --- a/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js @@ -182,15 +182,15 @@ describe('when applying ECHO10 metadata corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: '', - Type: 'In Situ Land-based Platforms', + SubCategory: '', + Basis: '', + Category: 'In Situ Land-based Platforms', ShortName: 'GROUND STATIONS' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'C-130' } } @@ -275,15 +275,15 @@ describe('when correcting a ECHO10 record', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4-UPDATED' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -294,13 +294,15 @@ describe('when correcting a ECHO10 record', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES-UPDATED' }, newLongName: 'Updated Geophone Array' @@ -308,11 +310,11 @@ describe('when correcting a ECHO10 record', () => { { scheme: 'projects', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS-UPDATED' }, newLongName: 'Aliens in Antarctica Updated' @@ -386,10 +388,10 @@ describe('when correcting a ECHO10 record', () => { scheme: 'ProductLevelId', action: 'replace', oldKeywordObject: { - Value: 'NA' + ProductLevelId: 'NA' }, newKeywordObject: { - Value: '1A' + ProductLevelId: '1A' } } ] @@ -490,15 +492,15 @@ describe('when verifying ECHO10 corrections do not remove unrelated metadata', ( scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4-UPDATED' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -509,7 +511,8 @@ describe('when verifying ECHO10 corrections do not remove unrelated metadata', ( oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'GEOPHONES' } }, @@ -517,11 +520,11 @@ describe('when verifying ECHO10 corrections do not remove unrelated metadata', ( scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS-UPDATED' }, newLongName: 'Aliens in Antarctica Updated' @@ -573,10 +576,10 @@ describe('when verifying ECHO10 corrections do not remove unrelated metadata', ( scheme: 'productlevelid', action: 'replace', oldKeywordObject: { - Value: 'NA' + ProductLevelId: 'NA' }, newKeywordObject: { - Value: '1A' + ProductLevelId: '1A' } } ] @@ -668,13 +671,15 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newLongName: 'Updated Infrared Multispectral Scanner' @@ -708,13 +713,15 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'LISS-II' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'LISSUPDATE-II' }, newLongName: 'Linear Imaging Self Scanning Sensor II Updated' @@ -746,7 +753,8 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: {} @@ -785,7 +793,8 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I1' } }, @@ -795,7 +804,8 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I2' } } @@ -830,7 +840,8 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'I1' } } @@ -865,13 +876,15 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Instrument', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'OLD-SHORT' }, newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' }, newLongName: '' // Triggers delete target['LongName'] @@ -908,13 +921,15 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS1' } } @@ -939,13 +954,15 @@ describe('when applying instrument ECHO10 corrections', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS' }, newKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'IRMSS1' }, newLongName: 'Infrared Multispectral Scanner Updated' @@ -968,7 +985,8 @@ describe('when instrument guard clauses prevent a correction', () => { newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -988,7 +1006,8 @@ describe('when instrument guard clauses prevent a correction', () => { newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -1006,13 +1025,15 @@ describe('when instrument guard clauses prevent a correction', () => { oldKeywordObject: { Category: 'Imaging Spectrometers/Radiometers', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'NOT-REAL' }, newKeywordObject: { Category: 'Category', Class: 'Topic', - Subclass: 'Term', + Type: 'Term', + Subtype: '', ShortName: 'NEW-SHORT' } }] @@ -1063,6 +1084,19 @@ const mockEcho10WithPlatforms = ` ` +const mockEcho10WithPlatformBasis = ` + Platforms_Basis_Test + 001 + + + Other + Aircraft + SPOT-4 + Systeme Observation de la Terre-4 Baseline + + +` + describe('when applying platform ECHO10 corrections', () => { test('should apply long name correction to first Platform', async () => { const result = await applyEcho10MetadataCorrections({ @@ -1075,15 +1109,15 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -1119,15 +1153,15 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-5' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-7-New' }, newLongName: 'Systeme Observation de la Terre-5 Updated' @@ -1149,6 +1183,43 @@ describe('when applying platform ECHO10 corrections', () => { expect(result.correctedMetadata).not.toContain('Earth Observation planes') }) + test('should preserve platform Basis while updating short and long name', async () => { + const result = await applyEcho10MetadataCorrections({ + collectionConceptId: 'C1', + providerId: 'PROV', + nativeId: 'native-1', + metadataPayload: mockEcho10WithPlatformBasis, + corrections: [ + { + scheme: 'platforms', + action: 'replace', + oldKeywordObject: { + Basis: 'Other', + SubCategory: '', + Category: 'Aircraft', + ShortName: 'SPOT-4' + }, + newKeywordObject: { + Basis: 'Other-UPDATED', + SubCategory: '', + Category: 'Aircraft', + ShortName: 'SPOT-4-UPDATED' + }, + newLongName: 'Systeme Observation de la Terre-4 Updated' + } + ] + }) + + expect(result.correctionCount).toBe(1) + expect(result.correctionsApplied).toHaveLength(1) + + expect(result.correctedMetadata).toContain('Other') + expect(result.correctedMetadata).not.toContain('Other-UPDATED') + expect(result.correctedMetadata).toContain('SPOT-4-UPDATED') + expect(result.correctedMetadata).toContain('Systeme Observation de la Terre-4 Updated') + expect(result.correctedMetadata).not.toContain('Systeme Observation de la Terre-4 Baseline') + }) + test('should apply platform correction when there is only a single platform (object branch)', async () => { const singlePlatformXml = ` SINGLE_PLAT_TEST @@ -1169,15 +1240,15 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'In Situ Land-based Platforms', Category: '', - Class: 'In Situ Land-based Platforms', - Type: '', ShortName: 'GROUND STATIONS' }, newKeywordObject: { - Category: '', - Class: 'In Situ Land-based Platforms', - Type: 'Aircraft', + SubCategory: '', + Basis: 'In Situ Land-based Platforms', + Category: 'Aircraft', ShortName: 'C-130' }, newLongName: 'Lockheed C-130 Hercules' @@ -1216,9 +1287,9 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A1' } }, @@ -1226,9 +1297,9 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A2' } } @@ -1252,9 +1323,9 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: {} @@ -1281,9 +1352,9 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'not_an_action', // Triggers the final return false newKeywordObject: { - Category: '', - Class: 'A', - Type: 'B', + SubCategory: '', + Basis: 'A', + Category: 'B', ShortName: 'C' } } @@ -1311,9 +1382,9 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'delete', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'A1' } } @@ -1344,16 +1415,16 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'Aircraft', Category: '', - Class: 'Aircraft', - Type: '', ShortName: 'OLD-SHORT' }, // Providing only one segment forces the LongName field to hit the 'else { delete }' branch newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'NEW-SHORT' }, newLongName: '' @@ -1386,15 +1457,15 @@ describe('when applying platform ECHO10 corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { + SubCategory: '', + Basis: 'Earth Observation planes', Category: '', - Class: 'Earth Observation planes', - Type: '', ShortName: 'SPOT-5' }, newKeywordObject: { + SubCategory: '', + Basis: 'Earth Observation rockets', Category: '', - Class: 'Earth Observation rockets', - Type: '', ShortName: 'SPOT-7' } } @@ -1417,15 +1488,15 @@ describe('when applying platform ECHO10 corrections', () => { action: 'replace', ummPath: ['Platform', 10], oldKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-5' }, newKeywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-7' } } @@ -1600,7 +1671,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 2' + ProductLevelId: 'Level 2' } }] }) @@ -1616,7 +1687,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { corrections: [{ scheme: 'productlevelid', newKeywordObject: { - Value: 'Level 3' + ProductLevelId: 'Level 3' } }] }) @@ -1676,7 +1747,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 4' + ProductLevelId: 'Level 4' } }] }) @@ -1693,7 +1764,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 4' + ProductLevelId: 'Level 4' } }] }) @@ -1709,7 +1780,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { scheme: 'productlevelid', action: 'replace', newKeywordObject: { - Value: 'Level 4' + ProductLevelId: 'Level 4' } }] }) @@ -1726,7 +1797,7 @@ describe('when applying ProcessingLevelId ECHO10 corrections', () => { scheme: 'productlevelid', action: 'invalid_action_type', newKeywordObject: { - Value: 'Level 2' + ProductLevelId: 'Level 2' } }] }) @@ -1764,11 +1835,11 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newLongName: 'Updated Earth Science Information Partners Program' @@ -1798,11 +1869,11 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' }, newKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS UP' }, newLongName: 'Aliens research in Antarctica' @@ -1832,7 +1903,7 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: {} @@ -1858,7 +1929,7 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' } }, @@ -1866,7 +1937,7 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' } } @@ -1897,7 +1968,7 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'SINGLE-PROJ' } } @@ -1929,7 +2000,7 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'delete', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'SINGLE-PROJ' } } @@ -1958,11 +2029,11 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP-7' } } @@ -1981,13 +2052,13 @@ describe('when applying project ECHO10 corrections', () => { scheme: 'projects', action: 'replace', oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, // Providing a single segment and NO newLongName // results in normalizedSegments[1] (LongName) being undefined newKeywordObject: { - Category: 'M - O', + Bucket: 'M - O', ShortName: 'ONLY_SHORT' } } @@ -2018,11 +2089,11 @@ describe('when applying project ECHO10 corrections', () => { action: 'replace', ummPath: ['Project', 10], oldKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP' }, newKeywordObject: { - Category: 'D - F', + Bucket: 'D - F', ShortName: 'ESIP-7' }, newLongName: 'Updated Earth Science Information Partners Program' @@ -2064,7 +2135,7 @@ describe('when applying project ECHO10 corrections', () => { corrections: [{ scheme: 'projects', oldKeywordObject: { - Category: 'S - U', + Bucket: 'S - U', ShortName: 'NOT-REAL' } }] diff --git a/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js b/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js index 88d3abe1..ddd62770 100644 --- a/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyIso19115MetadataCorrections.test.js @@ -208,20 +208,20 @@ describe('when applying locations ISO-19115 corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'CANADA', - Subregion2: 'ALBERTA', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'CANADA', + LocationSubregion2: 'ALBERTA', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'MEXICO', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'MEXICO', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } @@ -242,12 +242,12 @@ describe('when applying locations ISO-19115 corrections', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'Continent', - Type: 'North America', - Subregion1: 'Greenland', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'Continent', + LocationType: 'North America', + LocationSubregion1: 'Greenland', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } @@ -679,8 +679,8 @@ describe('when applying isotopiccategory ISO-19115 corrections', () => { const correction = { scheme: 'isotopiccategory', action: 'replace', - oldKeywordObject: { Value: 'FARMING' }, - newKeywordObject: { Value: 'BIOTA' } + oldKeywordObject: { ISOTopicCategory: 'FARMING' }, + newKeywordObject: { ISOTopicCategory: 'BIOTA' } } const config = ISO_19115_SCHEME_EDITORS.isotopiccategory @@ -700,7 +700,7 @@ describe('when applying isotopiccategory ISO-19115 corrections', () => { const correction = { scheme: 'isotopiccategory', action: 'delete', - oldKeywordObject: { Value: 'LOCATION' } + oldKeywordObject: { ISOTopicCategory: 'LOCATION' } } const config = ISO_19115_SCHEME_EDITORS.isotopiccategory @@ -723,8 +723,8 @@ describe('when applying productlevelid ISO-19115 corrections', () => { const correction = { scheme: 'productlevelid', action: 'replace', - oldKeywordObject: { Value: '3' }, - newKeywordObject: { Value: '5' } + oldKeywordObject: { ProductLevelId: '3' }, + newKeywordObject: { ProductLevelId: '5' } } const config = ISO_19115_SCHEME_EDITORS.productlevelid @@ -746,7 +746,7 @@ describe('when applying productlevelid ISO-19115 corrections', () => { const correction = { scheme: 'productlevelid', action: 'delete', - oldKeywordObject: { Value: '3' } + oldKeywordObject: { ProductLevelId: '3' } } const config = ISO_19115_SCHEME_EDITORS.productlevelid diff --git a/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js b/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js index 78e499ed..ec082e49 100644 --- a/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyIsoSmapMetadataCorrections.test.js @@ -19,8 +19,8 @@ describe('when applying isotopiccategory ISO-SMAP corrections', () => { const correction = { scheme: 'isotopiccategory', action: 'replace', - newKeywordObject: { Value: 'oceans' }, - oldKeywordObject: { Value: 'climatology' } + newKeywordObject: { ISOTopicCategory: 'oceans' }, + oldKeywordObject: { ISOTopicCategory: 'climatology' } } const config = ISO_19115_SCHEME_EDITORS.isotopiccategory @@ -38,7 +38,7 @@ describe('when applying isotopiccategory ISO-SMAP corrections', () => { const correction = { scheme: 'isotopiccategory', action: 'delete', - oldKeywordObject: { Value: 'water' } + oldKeywordObject: { ISOTopicCategory: 'water' } } const config = ISO_19115_SCHEME_EDITORS.isotopiccategory @@ -58,8 +58,8 @@ describe('when applying productlevelid ISO-SMAP corrections', () => { const correction = { scheme: 'productlevelid', action: 'replace', - oldKeywordObject: { Value: '3' }, - newKeywordObject: { Value: '5' } + oldKeywordObject: { ProductLevelId: '3' }, + newKeywordObject: { ProductLevelId: '5' } } const config = ISO_19115_SCHEME_EDITORS.productlevelid @@ -81,7 +81,7 @@ describe('when applying productlevelid ISO-SMAP corrections', () => { const correction = { scheme: 'productlevelid', action: 'delete', - oldKeywordObject: { Value: '3' } + oldKeywordObject: { ProductLevelId: '3' } } const config = ISO_19115_SCHEME_EDITORS.productlevelid @@ -262,20 +262,20 @@ describe('when applying locations ISO-SMAP corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'CANADA', - Subregion2: 'ALBERTA', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'CANADA', + LocationSubregion2: 'ALBERTA', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'MEXICO', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'MEXICO', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } @@ -296,12 +296,12 @@ describe('when applying locations ISO-SMAP corrections', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'Continent', - Type: 'North America', - Subregion1: 'Greenland', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'Continent', + LocationType: 'North America', + LocationSubregion1: 'Greenland', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } diff --git a/serverless/src/shared/__tests__/applyNativeMetadataCorrectionMappings.test.js b/serverless/src/shared/__tests__/applyNativeMetadataCorrectionMappings.test.js index f8eabba4..6f4f8985 100644 --- a/serverless/src/shared/__tests__/applyNativeMetadataCorrectionMappings.test.js +++ b/serverless/src/shared/__tests__/applyNativeMetadataCorrectionMappings.test.js @@ -2,9 +2,11 @@ import { existsSync, readFileSync } from 'fs' import { join } from 'path' import { + afterEach, describe, expect, - test + test, + vi } from 'vitest' import { applyDif10MetadataCorrections } from '../applyDif10MetadataCorrections' @@ -15,8 +17,23 @@ import { applyUmmcMetadataCorrections } from '../applyUmmcMetadataCorrections' import { DIF10_SCHEME_EDITORS } from '../dif10DomEditor' import { ECHO10_SCHEME_EDITORS } from '../echo10DomEditor' import { ISO_19115_SCHEME_EDITORS } from '../Iso19115DomEditor' +import { getHistoricalConceptByKeyword } from '../redis-path-store/getHistoricalConceptByKeyword' +import { getPublishedConceptByUuid } from '../redis-path-store/getPublishedConceptByUuid' +import { + compareKeywordCsvContent, + createKeywordEvents +} from '../redis-path-store/getPublishKeywordEvents' +import { resolveOldKeywordConceptUuid } from '../resolveOldKeywordConceptUuid' import { UMMC_SCHEME_EDITORS } from '../ummcDomEditor' +vi.mock('../redis-path-store/getHistoricalConceptByKeyword', () => ({ + getHistoricalConceptByKeyword: vi.fn() +})) + +vi.mock('../redis-path-store/getPublishedConceptByUuid', () => ({ + getPublishedConceptByUuid: vi.fn() +})) + const fixtureDirectory = join( __dirname, '../__mocks__/native_metadata_correction_mappings/end_to_end' @@ -38,6 +55,202 @@ const readFixture = (filename) => readFileSync(join(fixtureDirectory, filename), */ const readJsonFixture = (filename) => JSON.parse(readFixture(filename)) +/** + * Reads one production-shaped generated CSV fixture for a keyword scheme. + * + * @param {Object} options CSV selection options. + * @param {string} options.scheme Keyword scheme. + * @param {'published'|'draft'} options.version Keyword version. + * @returns {string} KMS-style CSV containing metadata, headers, and keyword records. + */ +const readGeneratedCsv = ({ + scheme, + version +}) => readFixture(`generated/${scheme}/${version}.csv`) + +/** + * Removes CSV-only auxiliary values from a correction keyword object. + * + * @param {Object} [keywordObject={}] Keyword object parsed from CSV. + * @returns {Object} Keyword object containing only the scheme hierarchy fields. + */ +const normalizeCorrectionKeywordObject = (keywordObject = {}) => Object.fromEntries( + Object.entries(keywordObject).filter(([key]) => ![ + 'DataCenterURL', + 'LongName' + ].includes(key)) +) + +/** + * Returns the stable logical portion of one generated or checked correction. + * + * @param {Object} correction Correction descriptor. + * @returns {Object} Comparable correction descriptor. + */ +const normalizeCorrection = (correction) => { + const normalizedCorrection = { + scheme: correction.scheme, + action: correction.action, + oldKeywordObject: normalizeCorrectionKeywordObject(correction.oldKeywordObject) + } + + if (correction.action === 'replace') { + normalizedCorrection.newKeywordObject = normalizeCorrectionKeywordObject( + correction.newKeywordObject + ) + } + + if (correction.newLongName) { + normalizedCorrection.newLongName = correction.newLongName + } + + return normalizedCorrection +} + +/** + * Recursively sorts object keys so equivalent corrections have the same signature. + * + * @param {unknown} value Value to normalize. + * @returns {unknown} Value with stable object-key order. + */ +const sortObjectKeys = (value) => { + if (Array.isArray(value)) { + return value.map(sortObjectKeys) + } + + if (value && typeof value === 'object') { + return Object.fromEntries(Object.keys(value) + .sort() + .map((key) => [key, sortObjectKeys(value[key])])) + } + + return value +} + +/** + * Normalizes, de-duplicates, and sorts logical corrections for comparison. + * + * Checked native fixtures may repeat a logical correction to update duplicate XML nodes, while + * a published/draft CSV comparison emits one event per keyword UUID. + * + * @param {Object[]} corrections Corrections to normalize. + * @returns {Object[]} Stable logical corrections. + */ +const normalizeLogicalCorrections = (corrections) => { + const correctionsBySignature = new Map() + + corrections.forEach((correction) => { + const normalizedCorrection = sortObjectKeys(normalizeCorrection(correction)) + + correctionsBySignature.set( + JSON.stringify(normalizedCorrection), + normalizedCorrection + ) + }) + + return [...correctionsBySignature.entries()] + .sort(([firstSignature], [secondSignature]) => ( + firstSignature.localeCompare(secondSignature) + )) + .map(([, correction]) => correction) +} + +/** + * Returns the stable identity of the keyword a correction is intended to change. + * + * Generated scheme CSVs are shared by all native formats, so this identity selects the + * corrections relevant to one format without including the replacement value in the match. + * + * @param {Object} correction Correction descriptor. + * @returns {string} Stable correction-selection signature. + */ +const getCorrectionSelectionSignature = (correction) => JSON.stringify(sortObjectKeys({ + scheme: correction.scheme, + action: correction.action, + oldKeywordObject: normalizeCorrectionKeywordObject(correction.oldKeywordObject) +})) + +/** + * Generates corrections from published/draft CSV changes and compares them with checked fixtures. + * + * @param {Object} options Generation scenario options. + * @param {string} options.name Native format fixture name. + * @param {string} options.scenarioName Correction scenario name. + * @returns {Promise} + */ +const verifyGeneratedCorrections = async ({ + name, + scenarioName +}) => { + const expectedCorrections = readJsonFixture( + `${scenarioName}/${name}.corrections.json` + ).corrections + const schemes = [...new Set(expectedCorrections.map(({ scheme }) => scheme))] + const keywordChanges = new Map(schemes.map((scheme) => [ + scheme, + compareKeywordCsvContent({ + oldCsvContent: readGeneratedCsv({ + scheme, + version: 'published' + }), + newCsvContent: readGeneratedCsv({ + scheme, + version: 'draft' + }), + scheme + }) + ])) + const events = createKeywordEvents(keywordChanges) + const eventsByUuid = Object.fromEntries(events.map((event) => [event.UUID, event])) + + vi.mocked(getHistoricalConceptByKeyword).mockImplementation(async ({ keywordValue }) => { + const event = events.find(({ OldKeywordObject }) => OldKeywordObject === keywordValue) + + return event + ? { + uuid: event.UUID, + keywordObject: event.OldKeywordObject, + longName: event.OldKeywordObject.LongName + } + : undefined + }) + + vi.mocked(getPublishedConceptByUuid).mockImplementation(async ({ uuid }) => { + const event = eventsByUuid[uuid] + + return event?.NewKeywordObject + ? { + uuid, + keywordObject: event.NewKeywordObject, + longName: event.NewKeywordObject.LongName + } + : undefined + }) + + const generatedCorrections = await Promise.all(events.map(async (event) => ({ + ...await resolveOldKeywordConceptUuid({ + scheme: event.Scheme, + keywordValue: event.OldKeywordObject, + keywordEvent: { + eventType: event.EventType, + scheme: event.Scheme, + uuid: event.UUID + } + }), + scheme: event.Scheme + }))) + const expectedCorrectionSignatures = new Set( + expectedCorrections.map(getCorrectionSelectionSignature) + ) + const selectedGeneratedCorrections = generatedCorrections.filter((correction) => ( + expectedCorrectionSignatures.has(getCorrectionSelectionSignature(correction)) + )) + + expect(normalizeLogicalCorrections(selectedGeneratedCorrections)).toEqual( + normalizeLogicalCorrections(expectedCorrections) + ) +} + /** * Loads native metadata in the type expected by its correction delegate. * @@ -174,7 +387,63 @@ const verifyCorrectionScenario = async ({ } } +afterEach(() => { + vi.clearAllMocks() +}) + +describe('when publishing short-name keyword changes', () => { + test('preserves short names and auxiliary CSV fields in keyword events', () => { + const expectedEvents = readJsonFixture('generated/short-name-events.json') + const expectedUuids = new Set(expectedEvents.map(({ UUID }) => UUID)) + const schemes = [...new Set(expectedEvents.map(({ Scheme }) => Scheme))] + const keywordChanges = new Map(schemes.map((scheme) => [ + scheme, + compareKeywordCsvContent({ + oldCsvContent: readGeneratedCsv({ + scheme, + version: 'published' + }), + newCsvContent: readGeneratedCsv({ + scheme, + version: 'draft' + }), + scheme + }) + ])) + const actualEvents = createKeywordEvents(keywordChanges) + .filter(({ UUID }) => expectedUuids.has(UUID)) + .map(({ + EventType, + Scheme, + UUID, + OldKeywordObject, + NewKeywordObject + }) => ({ + EventType, + Scheme, + UUID, + OldKeywordObject, + NewKeywordObject + })) + .sort((first, second) => first.Scheme.localeCompare(second.Scheme)) + + expect(actualEvents).toEqual(expectedEvents) + }) +}) + describe('when correcting UMM-C metadata', () => { + test('matches corrections generated from published and draft CSV', async () => { + await verifyGeneratedCorrections({ + name: 'ummc', + scenarioName: 'updates' + }) + + await verifyGeneratedCorrections({ + name: 'ummc', + scenarioName: 'deletions' + }) + }) + test('matches the complete expected record for updates', async () => { await verifyCorrectionScenario({ action: 'replace', @@ -199,6 +468,18 @@ describe('when correcting UMM-C metadata', () => { }) describe('when correcting DIF10 metadata', () => { + test('matches corrections generated from published and draft CSV', async () => { + await verifyGeneratedCorrections({ + name: 'dif10', + scenarioName: 'updates' + }) + + await verifyGeneratedCorrections({ + name: 'dif10', + scenarioName: 'deletions' + }) + }) + test('matches the complete expected record for updates', async () => { await verifyCorrectionScenario({ action: 'replace', @@ -223,6 +504,18 @@ describe('when correcting DIF10 metadata', () => { }) describe('when correcting ECHO10 metadata', () => { + test('matches corrections generated from published and draft CSV', async () => { + await verifyGeneratedCorrections({ + name: 'echo10', + scenarioName: 'updates' + }) + + await verifyGeneratedCorrections({ + name: 'echo10', + scenarioName: 'deletions' + }) + }) + test('matches the complete expected record for updates', async () => { await verifyCorrectionScenario({ action: 'replace', @@ -247,6 +540,18 @@ describe('when correcting ECHO10 metadata', () => { }) describe('when correcting ISO19115 metadata', () => { + test('matches corrections generated from published and draft CSV', async () => { + await verifyGeneratedCorrections({ + name: 'iso19115', + scenarioName: 'updates' + }) + + await verifyGeneratedCorrections({ + name: 'iso19115', + scenarioName: 'deletions' + }) + }) + test('matches the complete expected record for updates', async () => { await verifyCorrectionScenario({ action: 'replace', @@ -271,6 +576,18 @@ describe('when correcting ISO19115 metadata', () => { }) describe('when correcting ISO-SMAP metadata', () => { + test('matches corrections generated from published and draft CSV', async () => { + await verifyGeneratedCorrections({ + name: 'isosmap', + scenarioName: 'updates' + }) + + await verifyGeneratedCorrections({ + name: 'isosmap', + scenarioName: 'deletions' + }) + }) + test('matches the complete expected record for updates', async () => { await verifyCorrectionScenario({ action: 'replace', diff --git a/serverless/src/shared/__tests__/applyUmmcMetadataCorrections.test.js b/serverless/src/shared/__tests__/applyUmmcMetadataCorrections.test.js index df6bd606..03809a1f 100644 --- a/serverless/src/shared/__tests__/applyUmmcMetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyUmmcMetadataCorrections.test.js @@ -403,11 +403,11 @@ describe('when correcting a UMM-C record', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Type: 'Space Stations/Crewed Spacecraft', + Category: 'Space Stations/Crewed Spacecraft', ShortName: 'ISS' }, newKeywordObject: { - Type: 'Space Stations/Crewed Spacecraft', + Category: 'Space Stations/Crewed Spacecraft', ShortName: 'ISS-UPDATED' }, newLongName: 'International Space Station Updated' @@ -427,20 +427,20 @@ describe('when correcting a UMM-C record', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'GLOBAL LAND', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'GLOBAL LAND', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'GEOGRAPHIC REGION', - Type: 'GLOBAL', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'GEOGRAPHIC REGION', + LocationType: 'GLOBAL', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } }, { @@ -490,20 +490,20 @@ describe('when correcting a UMM-C record', () => { scheme: 'isotopiccategory', action: 'replace', oldKeywordObject: { - Value: 'GEOSCIENTIFIC INFORMATION' + ISOTopicCategory: 'GEOSCIENTIFIC INFORMATION' }, newKeywordObject: { - Value: 'OCEANS' + ISOTopicCategory: 'OCEANS' } }, { scheme: 'productlevelid', action: 'replace', oldKeywordObject: { - Value: '2' + ProductLevelId: '2' }, newKeywordObject: { - Value: '3' + ProductLevelId: '3' } } ] @@ -1580,20 +1580,20 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'SOUTH AMERICA', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'SOUTH AMERICA', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1617,20 +1617,20 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'CANADA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'CANADA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1650,20 +1650,20 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: 'CALIFORNIA', - Subregion3: 'LOS ANGELES', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: 'CALIFORNIA', + LocationSubregion3: 'LOS ANGELES', + LocationSubregion4: '' } } ] @@ -1685,20 +1685,20 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'CONTINENT', - Type: 'NORTH AMERICA', - Subregion1: 'UNITED STATES OF AMERICA', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'NORTH AMERICA', + LocationSubregion1: 'UNITED STATES OF AMERICA', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'CONTINENT', - Type: 'EUROPE', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'CONTINENT', + LocationType: 'EUROPE', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1719,12 +1719,12 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'OCEAN', - Type: 'PACIFIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'PACIFIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1755,12 +1755,12 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'delete', oldKeywordObject: { - Category: 'OCEAN', - Type: 'ATLANTIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'ATLANTIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1784,20 +1784,20 @@ describe('when applying location UMM-C corrections', () => { scheme: 'locations', action: 'replace', oldKeywordObject: { - Category: 'OCEAN', - Type: 'ATLANTIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'ATLANTIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' }, newKeywordObject: { - Category: 'OCEAN', - Type: 'PACIFIC OCEAN', - Subregion1: '', - Subregion2: '', - Subregion3: '', - DetailedLocation: '' + LocationCategory: 'OCEAN', + LocationType: 'PACIFIC OCEAN', + LocationSubregion1: '', + LocationSubregion2: '', + LocationSubregion3: '', + LocationSubregion4: '' } } ] @@ -1845,11 +1845,11 @@ describe('when applying platform UMM-C corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Type: 'Earth Observation Satellites', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newKeywordObject: { - Type: 'Earth Observation Satellites', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-4' }, newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -1873,11 +1873,11 @@ describe('when applying platform UMM-C corrections', () => { scheme: 'platforms', action: 'replace', oldKeywordObject: { - Type: 'Aircraft', + Category: 'Aircraft', ShortName: 'SPOT-5' }, newKeywordObject: { - Type: 'Earth Observation Satellites', + Category: 'Earth Observation Satellites', ShortName: 'SPOT-7-New' }, newLongName: 'Systeme Observation de la Terre-7 New' diff --git a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js index 2b6021f1..530e666a 100644 --- a/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js +++ b/serverless/src/shared/__tests__/persistMetadataCorrectionAuditLog.test.js @@ -196,15 +196,15 @@ describe('persistMetadataCorrectionAuditLog', () => { scheme: 'platforms', keywordConceptUuid: 'uuid-optional', oldKeywordObject: { - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Platforms', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', ShortName: 'OLD PLATFORM' }, newKeywordObject: { - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Platforms', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', ShortName: 'NEW PLATFORM' } } diff --git a/serverless/src/shared/__tests__/resolveOldKeywordConceptUuid.test.js b/serverless/src/shared/__tests__/resolveOldKeywordConceptUuid.test.js index 3a399416..e312afab 100644 --- a/serverless/src/shared/__tests__/resolveOldKeywordConceptUuid.test.js +++ b/serverless/src/shared/__tests__/resolveOldKeywordConceptUuid.test.js @@ -163,14 +163,14 @@ describe('resolveOldKeywordConceptUuid', () => { vi.mocked(getHistoricalConceptByKeyword).mockResolvedValue({ uuid: 'resolved-scalar', keywordObject: { - Value: '' + ISOTopicCategory: '' } }) vi.mocked(getPublishedConceptByUuid).mockResolvedValue({ uuid: 'resolved-scalar', keywordObject: { - Value: 'BOUNDARIES' + ISOTopicCategory: 'BOUNDARIES' } }) @@ -180,10 +180,10 @@ describe('resolveOldKeywordConceptUuid', () => { })).resolves.toEqual({ keywordConceptUuid: 'resolved-scalar', oldKeywordObject: { - Value: '' + ISOTopicCategory: '' }, newKeywordObject: { - Value: 'BOUNDARIES' + ISOTopicCategory: 'BOUNDARIES' }, newKeywordPath: 'BOUNDARIES', action: 'replace' diff --git a/serverless/src/shared/dif10DomEditor.js b/serverless/src/shared/dif10DomEditor.js index 53394163..73c04021 100644 --- a/serverless/src/shared/dif10DomEditor.js +++ b/serverless/src/shared/dif10DomEditor.js @@ -1,4 +1,4 @@ -import { FULL_PATH_VALUE_FIELDS } from './redis-path-store/helpers/constants' +import { CSV_FIELDS, DIF10_FIELDS } from './redis-path-store/helpers/constants' import XmlMetadataPathEditor, { sequentialValueReplace } from './XmlMetadataPathEditor' // Wrap a block-style scheme config in the shared editor contract used by the DIF10 delegate. @@ -27,17 +27,8 @@ export const DIF10_SCHEME_EDITORS = { sciencekeywords: blockScheme({ nodeXPath: '//DIF/Science_Keywords', find: { - fieldPaths: [ - // XML fields to read from - 'Category', - 'Topic', - 'Term', - 'Variable_Level_1', - 'Variable_Level_2', - 'Variable_Level_3', - 'Detailed_Variable' - ], - valueKeys: FULL_PATH_VALUE_FIELDS.sciencekeywords + fieldPaths: DIF10_FIELDS.sciencekeywords, + valueKeys: CSV_FIELDS.sciencekeywords }, // Example correction input: // { @@ -61,127 +52,76 @@ export const DIF10_SCHEME_EDITORS = { // - Topic <- 'OCEANS' // - Term <- 'MARINE SEDIMENTS' replace: sequentialValueReplace( - [ - // XML fields to write to - 'Category', - 'Topic', - 'Term', - 'Variable_Level_1', - 'Variable_Level_2', - 'Variable_Level_3', - 'Detailed_Variable' - ], - FULL_PATH_VALUE_FIELDS.sciencekeywords + DIF10_FIELDS.sciencekeywords, + CSV_FIELDS.sciencekeywords ) }), locations: blockScheme({ nodeXPath: '//DIF/Location', find: { - fieldPaths: [ - // XML fields to read from - 'Location_Category', - 'Location_Type', - 'Location_Subregion1', - 'Location_Subregion2', - 'Location_Subregion3', - 'Detailed_Location' - ], - valueKeys: FULL_PATH_VALUE_FIELDS.locations + fieldPaths: DIF10_FIELDS.locations, + valueKeys: CSV_FIELDS.locations }, replace: sequentialValueReplace( - [ - // XML fields to write to - 'Location_Category', - 'Location_Type', - 'Location_Subregion1', - 'Location_Subregion2', - 'Location_Subregion3', - 'Detailed_Location' - ], - FULL_PATH_VALUE_FIELDS.locations + DIF10_FIELDS.locations, + CSV_FIELDS.locations ) }), chronounits: blockScheme({ nodeXPath: '//DIF/Temporal_Coverage/Paleo_DateTime/Chronostratigraphic_Unit', find: { - fieldPaths: [ - // XML fields to read from - 'Eon', - 'Era', - 'Period', - 'Epoch', - 'Stage', - 'Detailed_Classification' - ], - valueKeys: FULL_PATH_VALUE_FIELDS.chronounits + fieldPaths: DIF10_FIELDS.chronounits, + valueKeys: CSV_FIELDS.chronounits }, replace: sequentialValueReplace( - [ - // XML fields to write to - 'Eon', - 'Era', - 'Period', - 'Epoch', - 'Stage', - 'Detailed_Classification' - ], - FULL_PATH_VALUE_FIELDS.chronounits + DIF10_FIELDS.chronounits, + CSV_FIELDS.chronounits ) }), platforms: blockScheme({ - // Platform corrections are normalized into an object that can carry: - // - Class: the GCMD platform class, for example "Space-based Platforms" - // - Type: DIF10 - // - ShortName: DIF10 - // - // DIF10 Platform only persists `Type` and `ShortName`, so those are the - // only object fields written back into the XML. + // The platform CSV Category maps to the native DIF10 field. nodeXPath: '//DIF/Platform', find: { - fieldPaths: [ - // XML fields to read from - 'Short_Name' - ], - valueKeys: [ - // Keys from oldKeywordObject to compare against - 'ShortName' - ] + fieldPaths: [DIF10_FIELDS.shortName], + valueKeys: [CSV_FIELDS.shortName] }, replace: [ { // XML field to write to - fieldPath: 'Type', + fieldPath: DIF10_FIELDS.platformType, source: { // Key from newKeywordObject to read from type: 'value', - key: 'Type' + key: CSV_FIELDS.category } }, { // XML field to write to - fieldPath: 'Short_Name', + fieldPath: DIF10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'Long_Name', + fieldPath: DIF10_FIELDS.longName, source: { // Example correction input: // { // scheme: 'platforms', // action: 'replace', // oldKeywordObject: { - // Class: 'Space-based Platforms', - // Type: 'Earth Observation Satellites', + // Basis: 'Space-based Platforms', + // Category: 'Earth Observation Satellites', + // SubCategory: '', // ShortName: 'SPOT-4' // }, // newKeywordObject: { - // Class: 'Space-based Platforms', - // Type: 'Earth Observation Satellites', + // Basis: 'Space-based Platforms', + // Category: 'Earth Observation Satellites', + // SubCategory: '', // ShortName: 'SPOT-4-UPDATED' // }, // newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -199,28 +139,25 @@ export const DIF10_SCHEME_EDITORS = { instruments: blockScheme({ nodeXPath: '//DIF/Platform/Instrument', find: { - fieldPaths: [ - // XML fields to read from - 'Short_Name' - ], + fieldPaths: [DIF10_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'Short_Name', + fieldPath: DIF10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'Long_Name', + fieldPath: DIF10_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -232,28 +169,25 @@ export const DIF10_SCHEME_EDITORS = { projects: blockScheme({ nodeXPath: '//DIF/Project', find: { - fieldPaths: [ - // XML fields to read from - 'Short_Name' - ], + fieldPaths: [DIF10_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'Short_Name', + fieldPath: DIF10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'Long_Name', + fieldPath: DIF10_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -265,28 +199,25 @@ export const DIF10_SCHEME_EDITORS = { providers: blockScheme({ nodeXPath: '//DIF/Organization', find: { - fieldPaths: [ - // XML fields to read from - 'Organization_Name/Short_Name' - ], + fieldPaths: [DIF10_FIELDS.providerShortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'Organization_Name/Short_Name', + fieldPath: DIF10_FIELDS.providerShortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'Organization_Name/Long_Name', + fieldPath: DIF10_FIELDS.providerLongName, source: { // Correction param to read from type: 'param', @@ -298,40 +229,32 @@ export const DIF10_SCHEME_EDITORS = { rucontenttype: blockScheme({ nodeXPath: '//DIF/Related_URL/URL_Content_Type', find: { - fieldPaths: [ - // XML fields to read from - 'Type', - 'Subtype' - ], + fieldPaths: DIF10_FIELDS.rucontenttype, valueKeys: [ // Keys from oldKeywordObject to compare against - 'Type', - 'Subtype' + CSV_FIELDS.type, + CSV_FIELDS.subtype ] }, - replace: sequentialValueReplace([ - // XML fields to write to and keys from newKeywordObject to read from - 'Type', - 'Subtype' - ]), + replace: sequentialValueReplace( + DIF10_FIELDS.rucontenttype, + CSV_FIELDS.rucontenttype.slice(1) + ), removeNodeIfEmptyAfterReplace: true }), idnnode: blockScheme({ nodeXPath: '//DIF/IDN_Node', find: { - fieldPaths: [ - // XML fields to read from - 'Short_Name' - ], + fieldPaths: [DIF10_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'Short_Name', + fieldPath: DIF10_FIELDS.shortName, source: { // Example correction input: // { @@ -350,12 +273,12 @@ export const DIF10_SCHEME_EDITORS = { // correction object carries the replacement in `newKeywordObject.ShortName`. // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'Long_Name', + fieldPath: DIF10_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -381,10 +304,10 @@ export const DIF10_SCHEME_EDITORS = { // scheme: 'productlevelid', // action: 'replace', // oldKeywordObject: { - // Value: 'NA' + // ProductLevelId: 'NA' // }, // newKeywordObject: { - // Value: '1A' + // ProductLevelId: '1A' // } // } // diff --git a/serverless/src/shared/echo10DomEditor.js b/serverless/src/shared/echo10DomEditor.js index b0621dfe..d8d586f8 100644 --- a/serverless/src/shared/echo10DomEditor.js +++ b/serverless/src/shared/echo10DomEditor.js @@ -1,3 +1,4 @@ +import { CSV_FIELDS, ECHO10_FIELDS } from './redis-path-store/helpers/constants' import XmlMetadataPathEditor, { sequentialValueReplace } from './XmlMetadataPathEditor' // Wrap a block-style scheme config in the shared editor contract used by the ECHO10 delegate. @@ -25,26 +26,8 @@ export const ECHO10_SCHEME_EDITORS = { nodeXPath: '//Collection/ScienceKeywords/ScienceKeyword', removeEmptyParent: true, find: { - fieldPaths: [ - // XML fields to read from - 'CategoryKeyword', - 'TopicKeyword', - 'TermKeyword', - 'VariableLevel1Keyword/Value', - 'VariableLevel1Keyword/VariableLevel2Keyword/Value', - 'VariableLevel1Keyword/VariableLevel2Keyword/VariableLevel3Keyword', - 'DetailedVariableKeyword' - ], - valueKeys: [ - // Keys from oldKeywordObject to compare against - 'Category', - 'Topic', - 'Term', - 'VariableLevel1', - 'VariableLevel2', - 'VariableLevel3', - 'DetailedVariable' - ] + fieldPaths: ECHO10_FIELDS.sciencekeywords, + valueKeys: CSV_FIELDS.sciencekeywords }, // Example correction input: // { @@ -68,82 +51,54 @@ export const ECHO10_SCHEME_EDITORS = { // - TopicKeyword <- 'OCEANS' // - TermKeyword <- 'MARINE SEDIMENTS' replace: sequentialValueReplace( - [ - // XML fields to write to - 'CategoryKeyword', - 'TopicKeyword', - 'TermKeyword', - 'VariableLevel1Keyword/Value', - 'VariableLevel1Keyword/VariableLevel2Keyword/Value', - 'VariableLevel1Keyword/VariableLevel2Keyword/VariableLevel3Keyword', - 'DetailedVariableKeyword' - ], - [ - // Keys from newKeywordObject to read from - 'Category', - 'Topic', - 'Term', - 'VariableLevel1', - 'VariableLevel2', - 'VariableLevel3', - 'DetailedVariable' - ] + ECHO10_FIELDS.sciencekeywords, + CSV_FIELDS.sciencekeywords ) }), platforms: blockScheme({ - // Platform corrections are normalized into an object that can carry: - // - Class: the GCMD platform class, for example "Space-based Platforms" - // - Type: ECHO10 - // - ShortName: ECHO10 - // - // ECHO10 Platform only persists `Type` and `ShortName`, so those are the - // only object fields written back into the XML. + // The platform CSV Category maps to the native ECHO10 field. nodeXPath: '//Collection/Platforms/Platform', find: { - fieldPaths: [ - // XML fields to read from - 'ShortName' - ], - valueKeys: [ - // Keys from oldKeywordObject to compare against - 'ShortName' - ] + fieldPaths: [ECHO10_FIELDS.shortName], + valueKeys: [CSV_FIELDS.shortName] }, replace: [ { // XML field to write to - fieldPath: 'Type', + fieldPath: ECHO10_FIELDS.platformType, source: { // Key from newKeywordObject to read from type: 'value', - key: 'Type' + key: CSV_FIELDS.category } }, { // XML field to write to - fieldPath: 'ShortName', + fieldPath: ECHO10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'LongName', + fieldPath: ECHO10_FIELDS.longName, source: { // Example correction input: // { // scheme: 'platforms', // action: 'replace', // oldKeywordObject: { - // Class: 'Space-based Platforms', - // Type: 'Earth Observation Satellites', + // Basis: 'Space-based Platforms', + // Category: 'Earth Observation Satellites', + // SubCategory: '', // ShortName: 'SPOT-4' // }, // newKeywordObject: { - // Class: 'Space-based Platforms', - // Type: 'Earth Observation Satellites', + // Basis: 'Space-based Platforms', + // Category: 'Earth Observation Satellites', + // SubCategory: '', // ShortName: 'SPOT-4-UPDATED' // }, // newLongName: 'Systeme Observation de la Terre-4 Updated' @@ -161,28 +116,25 @@ export const ECHO10_SCHEME_EDITORS = { instruments: blockScheme({ nodeXPath: '//Collection/Platforms/Platform/Instruments/Instrument', find: { - fieldPaths: [ - // XML fields to read from - 'ShortName' - ], + fieldPaths: [ECHO10_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'ShortName', + fieldPath: ECHO10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'LongName', + fieldPath: ECHO10_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -194,28 +146,25 @@ export const ECHO10_SCHEME_EDITORS = { projects: blockScheme({ nodeXPath: '//Collection/Campaigns/Campaign', find: { - fieldPaths: [ - // XML fields to read from - 'ShortName' - ], + fieldPaths: [ECHO10_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // XML field to write to - fieldPath: 'ShortName', + fieldPath: ECHO10_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: 'LongName', + fieldPath: ECHO10_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -228,35 +177,31 @@ export const ECHO10_SCHEME_EDITORS = { nodeXPath: '//Collection/Contacts/Contact', removeEmptyParent: true, find: { - fieldPaths: [ - // XML fields to read from - 'Role', - 'OrganizationName' - ], + fieldPaths: ECHO10_FIELDS.providers, valueKeys: [ // Keys from oldKeywordObject to compare against - 'BucketLevel0', - 'ShortName' + CSV_FIELDS.providerRole, + CSV_FIELDS.shortName ] }, delete: [ { - fieldPath: '//Collection/ProcessingCenter', - matchOldValueKey: 'ShortName', + fieldPath: ECHO10_FIELDS.processingCenter, + matchOldValueKey: CSV_FIELDS.shortName, condition: ({ correction, editor: currentEditor }) => ( - correction.oldKeywordObject?.BucketLevel0 === 'PROCESSOR' - && currentEditor.resolveAbsoluteFieldElement('//Collection/ProcessingCenter', { - expectedText: correction.oldKeywordObject?.ShortName + correction.oldKeywordObject?.[CSV_FIELDS.providerRole] === 'PROCESSOR' + && currentEditor.resolveAbsoluteFieldElement(ECHO10_FIELDS.processingCenter, { + expectedText: correction.oldKeywordObject?.[CSV_FIELDS.shortName] }) !== null ) }, { - fieldPath: '//Collection/ArchiveCenter', - matchOldValueKey: 'ShortName', + fieldPath: ECHO10_FIELDS.archiveCenter, + matchOldValueKey: CSV_FIELDS.shortName, condition: ({ correction, editor: currentEditor }) => ( - correction.oldKeywordObject?.BucketLevel0 === 'ARCHIVER' - && currentEditor.resolveAbsoluteFieldElement('//Collection/ArchiveCenter', { - expectedText: correction.oldKeywordObject?.ShortName + correction.oldKeywordObject?.[CSV_FIELDS.providerRole] === 'ARCHIVER' + && currentEditor.resolveAbsoluteFieldElement(ECHO10_FIELDS.archiveCenter, { + expectedText: correction.oldKeywordObject?.[CSV_FIELDS.shortName] }) !== null ) } @@ -264,45 +209,45 @@ export const ECHO10_SCHEME_EDITORS = { replace: [ { // XML field to write to - fieldPath: 'OrganizationName', + fieldPath: ECHO10_FIELDS.providerOrganizationName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: '//Collection/ProcessingCenter', - matchOldValueKey: 'ShortName', + fieldPath: ECHO10_FIELDS.processingCenter, + matchOldValueKey: CSV_FIELDS.shortName, // Condition to satisfy before replacement can occur condition: ({ correction, editor: currentEditor }) => ( - correction.oldKeywordObject?.BucketLevel0 === 'PROCESSOR' - && currentEditor.resolveAbsoluteFieldElement('//Collection/ProcessingCenter', { - expectedText: correction.oldKeywordObject?.ShortName + correction.oldKeywordObject?.[CSV_FIELDS.providerRole] === 'PROCESSOR' + && currentEditor.resolveAbsoluteFieldElement(ECHO10_FIELDS.processingCenter, { + expectedText: correction.oldKeywordObject?.[CSV_FIELDS.shortName] }) !== null ), source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // XML field to write to - fieldPath: '//Collection/ArchiveCenter', - matchOldValueKey: 'ShortName', + fieldPath: ECHO10_FIELDS.archiveCenter, + matchOldValueKey: CSV_FIELDS.shortName, // Condition to satisfy before replacement can occur condition: ({ correction, editor: currentEditor }) => ( - correction.oldKeywordObject?.BucketLevel0 === 'ARCHIVER' - && currentEditor.resolveAbsoluteFieldElement('//Collection/ArchiveCenter', { - expectedText: correction.oldKeywordObject?.ShortName + correction.oldKeywordObject?.[CSV_FIELDS.providerRole] === 'ARCHIVER' + && currentEditor.resolveAbsoluteFieldElement(ECHO10_FIELDS.archiveCenter, { + expectedText: correction.oldKeywordObject?.[CSV_FIELDS.shortName] }) !== null ), source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } } ] @@ -310,29 +255,26 @@ export const ECHO10_SCHEME_EDITORS = { rucontenttype: blockScheme({ nodeXPath: '//Collection/OnlineResources/OnlineResource', find: { - fieldPaths: [ - // XML fields to read from - 'Type' - ], + fieldPaths: ECHO10_FIELDS.rucontenttype, valueKeys: ['CombinedType'], getExpectedValueObject: ({ correction }) => ({ CombinedType: [ - correction.oldKeywordObject?.URLContentType, - correction.oldKeywordObject?.Type, - correction.oldKeywordObject?.Subtype + correction.oldKeywordObject?.[CSV_FIELDS.urlContentType], + correction.oldKeywordObject?.[CSV_FIELDS.type], + correction.oldKeywordObject?.[CSV_FIELDS.subtype] ].filter(Boolean).join(' : ') }) }, replace: [ { // XML fields to write to and keys from newKeywordObject to read from - fieldPath: 'Type', + fieldPath: ECHO10_FIELDS.relatedUrlType, source: { type: 'computed', getValue: ({ correction }) => [ - correction.newKeywordObject?.URLContentType, - correction.newKeywordObject?.Type, - correction.newKeywordObject?.Subtype + correction.newKeywordObject?.[CSV_FIELDS.urlContentType], + correction.newKeywordObject?.[CSV_FIELDS.type], + correction.newKeywordObject?.[CSV_FIELDS.subtype] ].filter(Boolean).join(' : ') } } @@ -348,10 +290,10 @@ export const ECHO10_SCHEME_EDITORS = { // scheme: 'productlevelid', // action: 'replace', // oldKeywordObject: { - // Value: 'NA' + // ProductLevelId: 'NA' // }, // newKeywordObject: { - // Value: '1A' + // ProductLevelId: '1A' // } // } // diff --git a/serverless/src/shared/isCsvLongNameFlag.js b/serverless/src/shared/isCsvLongNameFlag.js index 11503b49..4c46dc52 100644 --- a/serverless/src/shared/isCsvLongNameFlag.js +++ b/serverless/src/shared/isCsvLongNameFlag.js @@ -21,7 +21,17 @@ */ export const isCsvLongNameFlag = (scheme) => { // Check if the scheme is in the list of schemes that require long names - if (['platforms', 'instruments', 'projects', 'providers', 'idnnode', 'dataformat'].includes(scheme)) { + if ( + [ + 'platforms', + 'instruments', + 'projects', + 'providers', + 'idnnode', + 'dataformat', + 'granuledataformat' + ].includes(scheme) + ) { return true } diff --git a/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js b/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js index 3158f223..304e7510 100644 --- a/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/createCsvForScheme.test.js @@ -83,7 +83,7 @@ describe('createCsvForScheme', () => { scheme: 'instruments', path: ['Instruments'], isLeaf: false - })).toEqual(['Instruments', '', '', '']) + })).toEqual(['Instruments', '', '', '', '']) expect(formatKeywordCsvPath({ scheme: 'projects', diff --git a/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByKeyword.test.js b/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByKeyword.test.js index 22aac26f..42753e5b 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByKeyword.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByKeyword.test.js @@ -102,7 +102,7 @@ describe('getHistoricalConceptByKeyword', () => { statusCode: 200, body: JSON.stringify({ uuid: 'platform-uuid', - fullPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > AQUA', + fullPath: 'Space-based Platforms > Earth Observation Satellites > > AQUA', longName: 'Aqua satellite' }) }) @@ -116,12 +116,12 @@ describe('getHistoricalConceptByKeyword', () => { cachedJsonResponseReader })).resolves.toMatchObject({ uuid: 'platform-uuid', - fullPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > AQUA', + fullPath: 'Space-based Platforms > Earth Observation Satellites > > AQUA', longName: 'Aqua satellite', keywordObject: { - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', + SubCategory: '', ShortName: 'AQUA', LongName: 'Aqua satellite' } diff --git a/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByShortName.test.js b/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByShortName.test.js index ef621dfa..76c8aab9 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByShortName.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getHistoricalConceptByShortName.test.js @@ -11,7 +11,7 @@ describe('getHistoricalConceptByShortName', () => { statusCode: 200, body: JSON.stringify({ uuid: 'uuid-123', - fullPath: 'AIR-BASED PLATFORMS > PROPELLER > > MODIS' + fullPath: 'Earth Remote Sensing Instruments > > > > MODIS' }) }) @@ -22,11 +22,12 @@ describe('getHistoricalConceptByShortName', () => { cachedJsonResponseReader })).resolves.toMatchObject({ uuid: 'uuid-123', - fullPath: 'AIR-BASED PLATFORMS > PROPELLER > > MODIS', + fullPath: 'Earth Remote Sensing Instruments > > > > MODIS', keywordObject: { - Category: 'AIR-BASED PLATFORMS', - Class: 'PROPELLER', - Subclass: '', + Category: 'Earth Remote Sensing Instruments', + Class: '', + Type: '', + Subtype: '', ShortName: 'MODIS' } }) @@ -43,7 +44,7 @@ describe('getHistoricalConceptByShortName', () => { statusCode: 200, body: JSON.stringify({ uuid: 'uuid-123', - fullPath: 'AIR-BASED PLATFORMS > PROPELLER > > AC-690A', + fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > > AC-690A', longName: 'Aerocommander aircraft' }) }) @@ -55,12 +56,13 @@ describe('getHistoricalConceptByShortName', () => { cachedJsonResponseReader })).resolves.toMatchObject({ uuid: 'uuid-123', - fullPath: 'AIR-BASED PLATFORMS > PROPELLER > > AC-690A', + fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > > AC-690A', longName: 'Aerocommander aircraft', keywordObject: { - Category: 'AIR-BASED PLATFORMS', - Class: 'PROPELLER', - Subclass: '', + Category: 'EARTH REMOTE SENSING INSTRUMENTS', + Class: 'ACTIVE REMOTE SENSING', + Type: 'ALTIMETERS', + Subtype: '', ShortName: 'AC-690A', LongName: 'Aerocommander aircraft' } diff --git a/serverless/src/shared/redis-path-store/__tests__/getKeywordPathFromKeywordObject.test.js b/serverless/src/shared/redis-path-store/__tests__/getKeywordPathFromKeywordObject.test.js index 82e4f1de..36a2507d 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getKeywordPathFromKeywordObject.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getKeywordPathFromKeywordObject.test.js @@ -1,24 +1,17 @@ import { getKeywordPathFromKeywordObject } from '../getKeywordPathFromKeywordObject' describe('getKeywordPathFromKeywordObject', () => { - test('returns canonical scalar keyword paths only when the object has a usable value', () => { + test('returns undefined for unknown schemes and invalid keyword objects', () => { expect(getKeywordPathFromKeywordObject({ scheme: 'customscalar', keywordObject: { Value: 'P1D' } - })).toBe('P1D') - - expect(getKeywordPathFromKeywordObject({ - scheme: 'customscalar', - keywordObject: null })).toBeUndefined() expect(getKeywordPathFromKeywordObject({ scheme: 'customscalar', - keywordObject: { - Value: '' - } + keywordObject: null })).toBeUndefined() }) @@ -26,12 +19,12 @@ describe('getKeywordPathFromKeywordObject', () => { expect(getKeywordPathFromKeywordObject({ scheme: 'platforms', keywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', + SubCategory: '', ShortName: 'Aqua' } - })).toBe('Space-based Platforms > Earth Observation Satellites > Aqua') + })).toBe('Space-based Platforms > Earth Observation Satellites > > Aqua') expect(getKeywordPathFromKeywordObject({ scheme: 'sciencekeywords', @@ -67,9 +60,9 @@ describe('getKeywordPathFromKeywordObject', () => { expect(getKeywordPathFromKeywordObject({ scheme: 'platforms', keywordObject: { + Basis: '', Category: '', - Class: '', - Type: '', + SubCategory: '', ShortName: '' } })).toBeUndefined() diff --git a/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js b/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js index 2f257115..ff5ef80e 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getPublishKeywordEvents.test.js @@ -116,54 +116,15 @@ describe('getPublishKeywordEvents', () => { }) }) - test('maps Short_Name to ShortName and Long_Name to LongName in platform update events', () => { - const createPlatformsCsv = ( - shortName, - longName = 'Greenhouse Gases Observing Satellite' - ) => [ + test('publishes an update when only an auxiliary long name changes', () => { + const createPlatformsCsv = (longName) => [ '"Keyword Version: test"', - '"Category","Class","Type","Short_Name","Long_Name","UUID"', - `"Platforms","Space-based Platforms","Earth Observation Satellites","${shortName}","${longName}","uuid-1"` + '"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"', + `"Space-based Platforms","Earth Observation Satellites","","GOSAT","${longName}","uuid-1"` ].join('\n') - const comparison = compareKeywordCsvContent({ - oldCsvContent: createPlatformsCsv('GOSAT'), - newCsvContent: createPlatformsCsv('GOSAT - Test1'), - scheme: 'platforms' - }) - - expect(comparison.changedKeywords.get('uuid-1')).toMatchObject({ - oldPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', - newPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT - Test1' - }) - - const [keywordEvent] = createKeywordEvents(new Map([ - ['platforms', comparison] - ])) - - expect(keywordEvent).toMatchObject({ - EventType: 'UPDATED', - UUID: 'uuid-1' - }) - - expect(keywordEvent.OldKeywordObject).toEqual({ - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', - ShortName: 'GOSAT', - LongName: 'Greenhouse Gases Observing Satellite' - }) - - expect(keywordEvent.NewKeywordObject).toEqual({ - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', - ShortName: 'GOSAT - Test1', - LongName: 'Greenhouse Gases Observing Satellite' - }) - const longNameComparison = compareKeywordCsvContent({ - oldCsvContent: createPlatformsCsv('GOSAT'), - newCsvContent: createPlatformsCsv('GOSAT', 'Updated GOSAT Satellite'), + oldCsvContent: createPlatformsCsv('Greenhouse Gases Observing Satellite'), + newCsvContent: createPlatformsCsv('Updated GOSAT Satellite'), scheme: 'platforms' }) const [longNameEvent] = createKeywordEvents(new Map([ @@ -171,8 +132,8 @@ describe('getPublishKeywordEvents', () => { ])) expect(longNameComparison.changedKeywords.get('uuid-1')).toMatchObject({ - oldPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', - newPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT' + oldPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT', + newPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT' }) expect(longNameEvent.NewKeywordObject.LongName).toBe('Updated GOSAT Satellite') diff --git a/serverless/src/shared/redis-path-store/__tests__/getPublishedConceptByKeyword.test.js b/serverless/src/shared/redis-path-store/__tests__/getPublishedConceptByKeyword.test.js index 21436bba..01579f00 100644 --- a/serverless/src/shared/redis-path-store/__tests__/getPublishedConceptByKeyword.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/getPublishedConceptByKeyword.test.js @@ -52,7 +52,7 @@ describe('getPublishedConceptByKeyword', () => { statusCode: 200, body: JSON.stringify({ uuid: 'instrument-uuid', - fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > MODIS', + fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > > MODIS', longName: 'Moderate Resolution Imaging Spectroradiometer' }) }) @@ -66,12 +66,13 @@ describe('getPublishedConceptByKeyword', () => { cachedJsonResponseReader })).resolves.toMatchObject({ uuid: 'instrument-uuid', - fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > MODIS', + fullPath: 'EARTH REMOTE SENSING INSTRUMENTS > ACTIVE REMOTE SENSING > ALTIMETERS > > MODIS', longName: 'Moderate Resolution Imaging Spectroradiometer', keywordObject: { Category: 'EARTH REMOTE SENSING INSTRUMENTS', Class: 'ACTIVE REMOTE SENSING', - Subclass: 'ALTIMETERS', + Type: 'ALTIMETERS', + Subtype: '', ShortName: 'MODIS', LongName: 'Moderate Resolution Imaging Spectroradiometer' } @@ -116,7 +117,7 @@ describe('getPublishedConceptByKeyword', () => { BucketLevel3: '', ShortName: 'NZ/NZAI/ANZ', LongName: 'National Archive', - DataCenterUrl: 'https://example.com/provider' + DataCenterURL: 'https://example.com/provider' } }) @@ -177,7 +178,7 @@ describe('getPublishedConceptByKeyword', () => { })).resolves.toMatchObject({ uuid: 'project-uuid', keywordObject: { - Category: 'A - C', + Bucket: 'A - C', ShortName: 'ALIENS' } }) diff --git a/serverless/src/shared/redis-path-store/__tests__/rebuildHistoricalConceptCache.test.js b/serverless/src/shared/redis-path-store/__tests__/rebuildHistoricalConceptCache.test.js index e2b292a1..c8a55705 100644 --- a/serverless/src/shared/redis-path-store/__tests__/rebuildHistoricalConceptCache.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/rebuildHistoricalConceptCache.test.js @@ -111,8 +111,8 @@ describe('rebuildHistoricalConceptCache', () => { }) .mockResolvedValueOnce({ Body: Readable.from([Buffer.from(`"Version" -"Category","Class","Type","Short_Name","Long_Name","UUID" -"Platforms","Space-based Platforms","Earth Observation Satellites","AQUA","Aqua satellite","uuid-2"`)]) +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites","","AQUA","Aqua satellite","uuid-2"`)]) }) const result = await rebuildHistoricalConceptCache(createContext({ @@ -206,8 +206,8 @@ describe('rebuildHistoricalConceptCache', () => { .mockResolvedValueOnce({ Contents: [{ Key: '1.0/platforms.csv' }] }) .mockResolvedValueOnce({ Body: Readable.from([Buffer.from(`"Version" -"Category","Class","Type","Short_Name","Long_Name","UUID" -"Platforms","Space-based Platforms","Earth Observation Satellites","AQUA","Aqua satellite","uuid-1"`)]) +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites","","AQUA","Aqua satellite","uuid-1"`)]) }) await expect(rebuildHistoricalConceptCache(createContext({ @@ -336,8 +336,8 @@ describe('rebuildHistoricalConceptCache', () => { .mockResolvedValueOnce({ Contents: [{ Key: '2.0/platforms.csv' }] }) .mockResolvedValueOnce({ Body: Readable.from([Buffer.from(`"Version" -"Category","Class","Type","Short_Name","Long_Name","UUID" -"Platforms","Space-based Platforms","Earth Observation Satellites","AQUA","Aqua satellite","uuid-2"`)]) +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites","","AQUA","Aqua satellite","uuid-2"`)]) }) const redisClient = { diff --git a/serverless/src/shared/redis-path-store/__tests__/writePublishedConceptCaches.test.js b/serverless/src/shared/redis-path-store/__tests__/writePublishedConceptCaches.test.js index 00d1ebe8..e1fb60cd 100644 --- a/serverless/src/shared/redis-path-store/__tests__/writePublishedConceptCaches.test.js +++ b/serverless/src/shared/redis-path-store/__tests__/writePublishedConceptCaches.test.js @@ -216,8 +216,8 @@ describe('writePublishedConceptCaches', () => { const context = createContext({ redisClient }) vi.mocked(downloadConcepts).mockResolvedValue(`"Version" -"Category","Class","Type","Short_Name","Long_Name","UUID" -"Platforms","Space-based Platforms","Earth Observation Satellites","AQUA","Aqua satellite","uuid-2"`) +"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID" +"Space-based Platforms","Earth Observation Satellites","","AQUA","Aqua satellite","uuid-2"`) vi.mocked(getConceptSchemeDetails).mockResolvedValue(['platforms']) diff --git a/serverless/src/shared/redis-path-store/getHistoricalConceptByKeyword.js b/serverless/src/shared/redis-path-store/getHistoricalConceptByKeyword.js index f5c28a03..955e3d5d 100644 --- a/serverless/src/shared/redis-path-store/getHistoricalConceptByKeyword.js +++ b/serverless/src/shared/redis-path-store/getHistoricalConceptByKeyword.js @@ -39,8 +39,9 @@ const defaultContext = { * const concept = await getHistoricalConceptByKeyword({ * scheme: 'platforms', * keywordObject: { - * Class: 'Space-based Platforms', - * Type: 'Earth Observation Satellites', + * Basis: 'Space-based Platforms', + * Category: 'Earth Observation Satellites', + * SubCategory: '', * ShortName: 'Aqua Legacy' * } * }) diff --git a/serverless/src/shared/redis-path-store/getHistoricalConceptByShortName.js b/serverless/src/shared/redis-path-store/getHistoricalConceptByShortName.js index a1f2b69d..732dc669 100644 --- a/serverless/src/shared/redis-path-store/getHistoricalConceptByShortName.js +++ b/serverless/src/shared/redis-path-store/getHistoricalConceptByShortName.js @@ -35,9 +35,9 @@ const defaultContext = { * // shortName: 'Aqua Legacy', * // longName: 'Aqua Legacy', * // keywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Earth Observation Satellites', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'Aqua Legacy' * // } * // } diff --git a/serverless/src/shared/redis-path-store/getKeywordPathFromKeywordObject.js b/serverless/src/shared/redis-path-store/getKeywordPathFromKeywordObject.js index d9ba3ca6..ac06b6ed 100644 --- a/serverless/src/shared/redis-path-store/getKeywordPathFromKeywordObject.js +++ b/serverless/src/shared/redis-path-store/getKeywordPathFromKeywordObject.js @@ -1,7 +1,5 @@ -import { buildFullPathLookupValue } from './helpers/buildFullPathLookupValue' import { buildKeywordPathFromObject } from './helpers/buildKeywordPathFromObject' -import { isLookupFullPathScheme } from './helpers/isLookupFullPathScheme' -import { isLookupShortNameScheme } from './helpers/isLookupShortNameScheme' +import { CSV_FIELDS } from './helpers/constants' import { normalizeKeywordScheme } from './helpers/normalizeKeywordScheme' import { splitKeywordPath } from './helpers/splitKeywordPath' import { trimKeywordPathSegment } from './helpers/trimKeywordPathSegment' @@ -9,8 +7,8 @@ import { trimKeywordPathSegment } from './helpers/trimKeywordPathSegment' /** * Converts a normalized keyword object into the human-readable path string used by KMS lookups. * - * Full-path schemes return `Category > Topic > ...` style paths, short-name schemes return their - * hierarchical short-name path, and scalar schemes return the trimmed `Value` field. + * Configured schemes use their normalized CSV fields in column order. Unknown schemes do not + * produce a keyword path. * * @param {object} params - Keyword path inputs. * @param {string} params.scheme - KMS keyword scheme. @@ -42,26 +40,17 @@ export const getKeywordPathFromKeywordObject = ({ return undefined } - if (isLookupFullPathScheme(normalizedScheme)) { - return buildFullPathLookupValue({ - scheme: normalizedScheme, - keywordValue: keywordObject - }) - } - - if (isLookupShortNameScheme(normalizedScheme)) { - const keywordPath = buildKeywordPathFromObject({ - scheme: normalizedScheme, - keywordObject - }) - - return splitKeywordPath(keywordPath) - .some((segment) => trimKeywordPathSegment(segment).length > 0) - ? keywordPath - : undefined + if (!Array.isArray(CSV_FIELDS[normalizedScheme])) { + return undefined } - const scalarValue = trimKeywordPathSegment(keywordObject.Value) + const keywordPath = buildKeywordPathFromObject({ + scheme: normalizedScheme, + keywordObject + }) - return scalarValue.length > 0 ? scalarValue : undefined + return splitKeywordPath(keywordPath) + .some((segment) => trimKeywordPathSegment(segment).length > 0) + ? keywordPath + : undefined } diff --git a/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js b/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js index 036d22ee..329d319b 100644 --- a/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js +++ b/serverless/src/shared/redis-path-store/getPublishKeywordEvents.js @@ -11,7 +11,7 @@ export { parseCsvContent } /** * Compares two keyword CSV payloads and groups differences into added, removed, and changed maps. * Keyword objects contain the scheme's named hierarchy fields. They also include `LongName` for - * schemes that support it and `DataCenterUrl` for providers when those values are present. + * schemes that support it and `DataCenterURL` for providers when those values are present. * * @param {object} params - Keyword CSV comparison inputs. * @param {string} params.oldCsvContent - Baseline CSV content, typically `published`. @@ -41,10 +41,10 @@ export { parseCsvContent } * @example * // Request * const comparison = compareKeywordCsvContent({ - * oldCsvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' - * + '"Platforms","Space-based Platforms","Satellite","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', - * newCsvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' - * + '"Platforms","Space-based Platforms","Satellite","GOSAT - Test1","Greenhouse Gases Observing Satellite","uuid-1"', + * oldCsvContent: '"Version"\n"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"\n' + * + '"Space-based Platforms","Earth Observation Satellites","","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', + * newCsvContent: '"Version"\n"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"\n' + * + '"Space-based Platforms","Earth Observation Satellites","","GOSAT - Test1","Greenhouse Gases Observing Satellite","uuid-1"', * scheme: 'platforms' * }) * @@ -54,19 +54,19 @@ export { parseCsvContent } * // removedKeywords: Map(0) {}, * // changedKeywords: Map([ * // ['uuid-1', { - * // oldPath: 'Platforms > Space-based Platforms > Satellite > GOSAT', - * // newPath: 'Platforms > Space-based Platforms > Satellite > GOSAT - Test1', + * // oldPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT', + * // newPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT - Test1', * // oldKeywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT', * // LongName: 'Greenhouse Gases Observing Satellite' * // }, * // newKeywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT - Test1', * // LongName: 'Greenhouse Gases Observing Satellite' * // } @@ -388,16 +388,16 @@ const getKeywordChangesForSchemeWithRetry = async ({ * // Scheme: 'platforms', * // UUID: 'uuid-1', * // OldKeywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT', * // LongName: 'Greenhouse Gases Observing Satellite' * // }, * // NewKeywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT - Test1', * // LongName: 'Greenhouse Gases Observing Satellite' * // }, diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordObjectFromPath.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordObjectFromPath.test.js index f54a2570..432252ca 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordObjectFromPath.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordObjectFromPath.test.js @@ -30,16 +30,31 @@ describe('buildKeywordObjectFromPath', () => { VariableLevel3: '', DetailedVariable: '' }) + + expect(buildKeywordPathObjectFromPath({ + scheme: 'unsupported', + keywordPath: 'VALUE' + })).toEqual({}) }) test('reconstructs platforms short-name paths into keyword objects', () => { expect(buildKeywordObjectFromPath({ scheme: 'platforms', - keywordPath: 'Space-based Platforms > Earth Observation Satellites > Aqua' + keywordPath: 'Other > > > Auxiliary Data' })).toEqual({ + Basis: 'Other', Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + SubCategory: '', + ShortName: 'Auxiliary Data' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'platforms', + keywordPath: 'Space-based Platforms > Earth Observation Satellites > Aqua' + })).toEqual({ + Basis: '', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', ShortName: 'Aqua' }) @@ -47,11 +62,53 @@ describe('buildKeywordObjectFromPath', () => { scheme: 'platforms', keywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > Aqua' })).toEqual({ - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Platforms', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', ShortName: 'Aqua' }) + + expect(buildKeywordObjectFromPath({ + scheme: 'platforms', + keywordPath: ' > Space-based Platforms > Earth Observation Satellites > Amazonia-1' + })).toEqual({ + Basis: '', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', + ShortName: 'Amazonia-1' + }) + }) + + test('reconstructs short-name paths with intentional blank slots from CSV-formatted inputs', () => { + expect(buildKeywordObjectFromPath({ + scheme: 'instruments', + keywordPath: 'Earth Remote Sensing Instruments > > > > MODIS' + })).toEqual({ + Category: 'Earth Remote Sensing Instruments', + Class: '', + Type: '', + Subtype: '', + ShortName: 'MODIS' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'projects', + keywordPath: ' > EOSDIS' + })).toEqual({ + Bucket: '', + ShortName: 'EOSDIS' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'providers', + keywordPath: 'NASA > > EOSDIS > GHRC > GHRC_DAAC' + })).toEqual({ + BucketLevel0: 'NASA', + BucketLevel1: '', + BucketLevel2: 'EOSDIS', + BucketLevel3: 'GHRC', + ShortName: 'GHRC_DAAC' + }) }) test('reconstructs provider paths into bucketed short-name keyword objects', () => { @@ -72,9 +129,9 @@ describe('buildKeywordObjectFromPath', () => { scheme: 'platforms', keywordPath: 'Aqua' })).toEqual({ + Basis: '', Category: '', - Class: '', - Type: '', + SubCategory: '', ShortName: 'Aqua' }) @@ -82,30 +139,20 @@ describe('buildKeywordObjectFromPath', () => { scheme: 'platforms', keywordPath: 'Platforms > Aqua' })).toEqual({ - Category: 'Platforms', - Class: '', - Type: '', + Basis: '', + Category: '', + SubCategory: 'Platforms', ShortName: 'Aqua' }) expect(buildKeywordObjectFromPath({ scheme: 'platforms', - keywordPath: 'Sensors > Aqua' + keywordPath: 'Air-based Platforms > A340-600' })).toEqual({ + Basis: '', Category: '', - Class: 'Sensors', - Type: '', - ShortName: 'Aqua' - }) - - expect(buildKeywordObjectFromPath({ - scheme: 'instruments', - keywordPath: 'Sensors > MODIS' - })).toEqual({ - Category: 'Sensors', - Class: '', - Subclass: '', - ShortName: 'MODIS' + SubCategory: 'Air-based Platforms', + ShortName: 'A340-600' }) expect(buildKeywordObjectFromPath({ @@ -114,7 +161,8 @@ describe('buildKeywordObjectFromPath', () => { })).toEqual({ Category: '', Class: '', - Subclass: '', + Type: '', + Subtype: '', ShortName: 'MODIS' }) @@ -122,13 +170,13 @@ describe('buildKeywordObjectFromPath', () => { scheme: 'projects', keywordPath: 'EOSDIS' })).toEqual({ - Category: '', + Bucket: '', ShortName: 'EOSDIS' }) expect(buildKeywordObjectFromPath({ scheme: 'providers', - keywordPath: 'NASA > GHRC_DAAC' + keywordPath: 'NASA > > > > GHRC_DAAC' })).toEqual({ BucketLevel0: 'NASA', BucketLevel1: '', @@ -152,9 +200,9 @@ describe('buildKeywordObjectFromPath', () => { test('reconstructs idnnode paths into joined short-name values', () => { expect(buildKeywordObjectFromPath({ scheme: 'idnnode', - keywordPath: 'NASA > Earthdata' + keywordPath: 'AMD/AU' })).toEqual({ - ShortName: 'NASA > Earthdata' + ShortName: 'AMD/AU' }) }) @@ -186,12 +234,12 @@ describe('buildKeywordObjectFromPath', () => { }) }) - test('returns scalar Value objects for non-slotted schemes and empty objects for blank input', () => { + test('returns CSV-shaped scalar objects and empty objects for blank input', () => { expect(buildKeywordObjectFromPath({ scheme: 'temporalresolutionrange', - keywordPath: 'P1D' + keywordPath: '< 1 second' })).toEqual({ - Value: 'P1D' + TemporalResolutionRange: '< 1 second' }) expect(buildKeywordObjectFromPath({ @@ -202,7 +250,12 @@ describe('buildKeywordObjectFromPath', () => { expect(buildKeywordPathObjectFromPath({ scheme: 'platforms', keywordPath: 'Aqua' - })).toEqual({}) + })).toEqual({ + Basis: '', + Category: '', + SubCategory: '', + ShortName: 'Aqua' + }) }) test('returns empty short-name values when single-field paths contain only blanks', () => { @@ -222,4 +275,85 @@ describe('buildKeywordObjectFromPath', () => { Value: 'NASA > AQUA' }) }) + + test('supports all known full-path lookup schemes', () => { + expect(buildKeywordObjectFromPath({ + scheme: 'locations', + keywordPath: 'CONTINENT > AFRICA > CENTRAL AFRICA > CAMEROON' + })).toEqual({ + LocationCategory: 'CONTINENT', + LocationType: 'AFRICA', + LocationSubregion1: 'CENTRAL AFRICA', + LocationSubregion2: 'CAMEROON', + LocationSubregion3: '', + LocationSubregion4: '' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'chronounits', + keywordPath: 'ARCHAEAN > MESOARCHEAN' + })).toEqual({ + Eon: 'ARCHAEAN', + Era: 'MESOARCHEAN', + Period: '', + Epoch: '', + Age: '', + SubAge: '' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'rucontenttype', + keywordPath: 'DistributionURL > USE SERVICE API > THREDDS DATA' + })).toEqual({ + URLContentType: 'DistributionURL', + Type: 'USE SERVICE API', + Subtype: 'THREDDS DATA' + }) + }) + + test('supports remaining short-name schemes, including single-value and joined-value variants', () => { + expect(buildKeywordObjectFromPath({ + scheme: 'dataformat', + keywordPath: 'NetCDF' + })).toEqual({ + ShortName: 'NetCDF' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'granuledataformat', + keywordPath: 'HDF5' + })).toEqual({ + ShortName: 'HDF5' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'idnnode', + keywordPath: 'ACADIS' + })).toEqual({ + ShortName: 'ACADIS' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'idnnode', + keywordPath: ' ACADIS ' + })).toEqual({ + ShortName: 'ACADIS' + }) + }) + + test('returns exact CSV fields for single-column schemes', () => { + expect(buildKeywordObjectFromPath({ + scheme: 'isotopiccategory', + keywordPath: 'OCEANS' + })).toEqual({ + ISOTopicCategory: 'OCEANS' + }) + + expect(buildKeywordObjectFromPath({ + scheme: 'verticalresolutionrange', + keywordPath: '1 meter - < 10 meters' + })).toEqual({ + VerticalResolutionRange: '1 meter - < 10 meters' + }) + }) }) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathFromObject.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathFromObject.test.js index 2b8a1a44..03ce4571 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathFromObject.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathFromObject.test.js @@ -16,9 +16,9 @@ describe('buildKeywordPathFromObject', () => { expect(buildKeywordPathFromObject({ scheme: 'platforms', keywordObject: { - Category: '', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: '', + Category: 'Space-based Platforms', + SubCategory: 'Earth Observation Satellites', ShortName: 'Aqua' } })).toEqual('Space-based Platforms > Earth Observation Satellites > Aqua') @@ -26,17 +26,40 @@ describe('buildKeywordPathFromObject', () => { expect(buildKeywordPathFromObject({ scheme: 'platforms', keywordObject: { + Basis: '', Category: '', - Class: '', - Type: '', + SubCategory: '', ShortName: '' } })).toEqual(' > > > ') }) + test('rebuilds short-name paths that include intentional blank slots', () => { + expect(buildKeywordPathFromObject({ + scheme: 'platforms', + keywordObject: { + Basis: 'Other', + Category: '', + SubCategory: '', + ShortName: 'Auxiliary Data' + } + })).toEqual('Other > > > Auxiliary Data') + + expect(buildKeywordPathFromObject({ + scheme: 'instruments', + keywordObject: { + Category: 'Earth Remote Sensing Instruments', + Class: '', + Type: '', + Subtype: '', + ShortName: 'MODIS' + } + })).toEqual('Earth Remote Sensing Instruments > > > > MODIS') + }) + test('flattens scalar objects for non-slotted schemes', () => { expect(buildKeywordPathFromObject({ - scheme: 'temporalresolutionrange', + scheme: 'unsupported', keywordObject: { Value: 'P1D' } diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromSegments.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromSegments.test.js index 268066d2..6b5fcd5e 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromSegments.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromSegments.test.js @@ -16,11 +16,16 @@ describe('buildKeywordPathObjectFromSegments', () => { }) }) - test('returns an empty object for schemes without slotted fields', () => { + test('maps short-name scheme segments to their CSV fields', () => { expect(buildKeywordPathObjectFromSegments({ scheme: 'platforms', - segments: ['Aqua'] - })).toEqual({}) + segments: ['Space-based Platforms', 'Earth Observation Satellites', '', 'Aqua'] + })).toEqual({ + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', + SubCategory: '', + ShortName: 'Aqua' + }) }) test('defaults missing segments to an empty slotted keyword object', () => { diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromValue.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromValue.test.js index f4b8fe55..ee9da5d3 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromValue.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/buildKeywordPathObjectFromValue.test.js @@ -20,7 +20,7 @@ describe('buildKeywordPathObjectFromValue', () => { }) }) - test('flattens raw array values for slotted schemes and ignores unsupported schemes', () => { + test('flattens raw values into the configured CSV fields', () => { expect(buildKeywordPathObjectFromValue({ scheme: 'sciencekeywords', keywordValue: ['EARTH SCIENCE', 'OCEANS'] @@ -37,6 +37,13 @@ describe('buildKeywordPathObjectFromValue', () => { expect(buildKeywordPathObjectFromValue({ scheme: 'temporalresolutionrange', keywordValue: 'P1D' + })).toEqual({ + TemporalResolutionRange: 'P1D' + }) + + expect(buildKeywordPathObjectFromValue({ + scheme: 'unsupported', + keywordValue: 'P1D' })).toEqual({}) }) }) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js index 8b92f920..e0ba30f5 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js @@ -7,12 +7,11 @@ import { } from '../keywordCsv' describe('when preparing CSV rows by header name', () => { - test('places GOSAT platform values under reordered headers', () => { + test('places a platform leaf hierarchy under reordered headers', () => { expect(prepareCsvRows({ - csvHeaders: ['Long_Name', 'UUID', 'Type', 'Short_Name', 'Category', 'Class'], + csvHeaders: ['Long_Name', 'UUID', 'Sub_Category', 'Short_Name', 'Category', 'Basis'], csvRows: [{ path: [ - 'Platforms', 'Space-based Platforms', 'Earth Observation Satellites', 'GOSAT' @@ -26,9 +25,9 @@ describe('when preparing CSV rows by header name', () => { })).toEqual([[ 'Greenhouse Gases Observing Satellite', 'uuid-gosat', - 'Earth Observation Satellites', + '', 'GOSAT', - 'Platforms', + 'Earth Observation Satellites', 'Space-based Platforms' ]]) }) @@ -50,14 +49,28 @@ describe('when parsing CSV rows by header name', () => { shortName: 'GOSAT', longName: 'Greenhouse Gases Observing Satellite', providerUrl: '', - keywordPath: ' > > > GOSAT' + keywordPath: ' > > > GOSAT', + keywordObject: { + Basis: '', + Category: '', + SubCategory: '', + ShortName: 'GOSAT', + LongName: 'Greenhouse Gases Observing Satellite' + } }, { uuid: '', shortName: 'AQUA', longName: '', providerUrl: '', - keywordPath: ' > > > AQUA' + keywordPath: ' > > > AQUA', + keywordObject: { + Basis: '', + Category: '', + SubCategory: '', + ShortName: 'AQUA', + LongName: '' + } } ]) }) @@ -75,10 +88,40 @@ describe('when parsing CSV rows by header name', () => { shortName: '"GOSAT "Test""', longName: '', providerUrl: '', - keywordPath: ' > > > "GOSAT "Test""' + keywordPath: ' > > > "GOSAT "Test""', + keywordObject: { + Basis: '', + Category: '', + SubCategory: '', + ShortName: '"GOSAT "Test""', + LongName: '' + } } ]) }) + + test('uses exact platform CSV columns regardless of header order', () => { + expect(parseCsv([ + '"Keyword Version: test"', + '"Short_Name","Sub_Category","UUID","Basis","Category"', + '"A1","Auxiliary","uuid-a","Other","Aqua"' + ].join('\n'), { + scheme: 'platforms' + })).toEqual([{ + uuid: 'uuid-a', + shortName: 'A1', + longName: '', + providerUrl: '', + keywordPath: 'Other > Aqua > Auxiliary > A1', + keywordObject: { + Basis: 'Other', + Category: 'Aqua', + SubCategory: 'Auxiliary', + ShortName: 'A1', + LongName: '' + } + }]) + }) }) describe('when parsing keyword CSV content for publication', () => { @@ -119,7 +162,8 @@ describe('when parsing keyword CSV content for publication', () => { BucketLevel2: 'EOSDIS', BucketLevel3: 'GHRC', ShortName: 'GHRC_DAAC', - DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' + LongName: '', + DataCenterURL: 'https://ghrc.nsstc.nasa.gov' } }] ])) @@ -184,8 +228,8 @@ describe('when parsing short-name keyword CSV content', () => { test('builds a GOSAT platform record with its long name', () => { const csvContent = [ '"Keyword Version: test"', - '"Long_Name","UUID","Type","Short_Name","Category","Class"', - '"Greenhouse Gases Observing Satellite","uuid-gosat","Earth Observation Satellites","GOSAT","Platforms","Space-based Platforms"' + '"Long_Name","UUID","Sub_Category","Short_Name","Category","Basis"', + '"Greenhouse Gases Observing Satellite","uuid-gosat","","GOSAT","Earth Observation Satellites","Space-based Platforms"' ].join('\n') expect(parseShortNameCsvRecords({ @@ -194,13 +238,13 @@ describe('when parsing short-name keyword CSV content', () => { })).toEqual(new Map([ ['GOSAT', { uuid: 'uuid-gosat', - fullPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT', + fullPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT', longName: 'Greenhouse Gases Observing Satellite', providerUrl: '', keywordObject: { - Category: 'Platforms', - Class: 'Space-based Platforms', - Type: 'Earth Observation Satellites', + Basis: 'Space-based Platforms', + Category: 'Earth Observation Satellites', + SubCategory: '', ShortName: 'GOSAT', LongName: 'Greenhouse Gases Observing Satellite' } @@ -208,6 +252,78 @@ describe('when parsing short-name keyword CSV content', () => { ])) }) + test('parses other short-name types with blank slots without shifting hierarchy', () => { + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Category","Class","Type","Subtype","Short_Name","Long_Name","UUID"', + '"Earth Remote Sensing Instruments","","","","MODIS","Moderate Resolution Imaging Spectroradiometer","uuid-instrument-modis"' + ].join('\n'), + scheme: 'instruments' + })).toEqual(new Map([ + ['MODIS', { + uuid: 'uuid-instrument-modis', + fullPath: 'Earth Remote Sensing Instruments > > > > MODIS', + longName: 'Moderate Resolution Imaging Spectroradiometer', + providerUrl: '', + keywordObject: { + Category: 'Earth Remote Sensing Instruments', + Class: '', + Type: '', + Subtype: '', + ShortName: 'MODIS', + LongName: 'Moderate Resolution Imaging Spectroradiometer' + } + }] + ])) + + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Bucket","Short_Name","Long_Name","UUID"', + '"","EOSDIS","EOSDIS","uuid-project-eosdis"' + ].join('\n'), + scheme: 'projects' + })).toEqual(new Map([ + ['EOSDIS', { + uuid: 'uuid-project-eosdis', + fullPath: ' > EOSDIS', + longName: 'EOSDIS', + providerUrl: '', + keywordObject: { + Bucket: '', + ShortName: 'EOSDIS', + LongName: 'EOSDIS' + } + }] + ])) + + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Bucket_Level_0","Bucket_Level_1","Bucket_Level_2","Bucket_Level_3","Short_Name","Long_Name","UUID"', + '"NASA","","EOSDIS","GHRC","GHRC_DAAC","NASA GHRC_DAAC","uuid-provider-middle-blank"' + ].join('\n'), + scheme: 'providers' + })).toEqual(new Map([ + ['GHRC_DAAC', { + uuid: 'uuid-provider-middle-blank', + fullPath: 'NASA > > EOSDIS > GHRC > GHRC_DAAC', + longName: 'NASA GHRC_DAAC', + providerUrl: '', + keywordObject: { + BucketLevel0: 'NASA', + BucketLevel1: '', + BucketLevel2: 'EOSDIS', + BucketLevel3: 'GHRC', + ShortName: 'GHRC_DAAC', + LongName: 'NASA GHRC_DAAC', + DataCenterURL: '' + } + }] + ])) + }) + test('builds a GHRC_DAAC provider record and skips the row without a short name', () => { const csvContent = [ '"Keyword Version: test"', @@ -231,7 +347,70 @@ describe('when parsing short-name keyword CSV content', () => { BucketLevel2: 'EOSDIS', BucketLevel3: 'GHRC', ShortName: 'GHRC_DAAC', - DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' + LongName: '', + DataCenterURL: 'https://ghrc.nsstc.nasa.gov' + } + }] + ])) + }) + + test('builds one-slot short-name scheme records without hierarchy slots', () => { + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Short_Name","Long_Name","UUID"', + '"NetCDF","Network Common Data Form","uuid-netcdf"' + ].join('\n'), + scheme: 'dataformat' + })).toEqual(new Map([ + ['NetCDF', { + uuid: 'uuid-netcdf', + fullPath: 'NetCDF', + longName: 'Network Common Data Form', + providerUrl: '', + keywordObject: { + ShortName: 'NetCDF', + LongName: 'Network Common Data Form' + } + }] + ])) + + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Short_Name","Long_Name","UUID"', + '"HDF5","Hierarchical Data Format version 5","uuid-hdf5"' + ].join('\n'), + scheme: 'granuledataformat' + })).toEqual(new Map([ + ['HDF5', { + uuid: 'uuid-hdf5', + fullPath: 'HDF5', + longName: 'Hierarchical Data Format version 5', + providerUrl: '', + keywordObject: { + ShortName: 'HDF5', + LongName: 'Hierarchical Data Format version 5' + } + }] + ])) + + expect(parseShortNameCsvRecords({ + csvContent: [ + '"Keyword Version: test"', + '"Short_Name","Long_Name","UUID"', + '"ACADIS","Australia Data and...","uuid-acadis"' + ].join('\n'), + scheme: 'idnnode' + })).toEqual(new Map([ + ['ACADIS', { + uuid: 'uuid-acadis', + fullPath: 'ACADIS', + longName: 'Australia Data and...', + providerUrl: '', + keywordObject: { + ShortName: 'ACADIS', + LongName: 'Australia Data and...' } }] ])) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/parseCachedConceptResponse.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/parseCachedConceptResponse.test.js index 9752988d..2bd9f652 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/parseCachedConceptResponse.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/parseCachedConceptResponse.test.js @@ -27,7 +27,7 @@ describe('parseCachedConceptResponse', () => { BucketLevel3: 'GHRC', ShortName: 'GHRC_DAAC', LongName: 'Global Hydrology Resource Center', - DataCenterUrl: 'https://ghrc.nsstc.nasa.gov' + DataCenterURL: 'https://ghrc.nsstc.nasa.gov' } }) }) diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/resolveLookupKeywordObject.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/resolveLookupKeywordObject.test.js index 05a5c09b..75617de4 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/resolveLookupKeywordObject.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/resolveLookupKeywordObject.test.js @@ -70,12 +70,21 @@ describe('resolveLookupKeywordObject', () => { expect(buildShortNameLookupValue(42)).toBe('42') }) - test('buildKeywordLookupObject falls back to scalar full-path values for non-slotted full-path schemes', () => { + test('buildKeywordLookupObject uses the configured CSV field for scalar full-path schemes', () => { expect(buildKeywordLookupObject({ scheme: 'temporalresolutionrange', keywordValue: 'P1D' })).toEqual({ - Value: 'P1D' + TemporalResolutionRange: 'P1D' + }) + + expect(buildKeywordLookupObject({ + scheme: 'productlevelid', + keywordValue: { + ProductLevelId: '2' + } + })).toEqual({ + ProductLevelId: '2' }) }) diff --git a/serverless/src/shared/redis-path-store/helpers/buildFullPathLookupValue.js b/serverless/src/shared/redis-path-store/helpers/buildFullPathLookupValue.js index 1696fa0e..8b4a4ba1 100644 --- a/serverless/src/shared/redis-path-store/helpers/buildFullPathLookupValue.js +++ b/serverless/src/shared/redis-path-store/helpers/buildFullPathLookupValue.js @@ -1,6 +1,6 @@ import { buildKeywordPathFromObject } from './buildKeywordPathFromObject' import { buildKeywordPathObjectFromValue } from './buildKeywordPathObjectFromValue' -import { FULL_PATH_VALUE_FIELDS } from './constants' +import { CSV_FIELDS } from './constants' import { flattenKeywordPathValue } from './flattenKeywordPathValue' import { hasKeywordObjectValue } from './hasKeywordObjectValue' import { joinKeywordPath } from './joinKeywordPath' @@ -20,7 +20,7 @@ export const buildKeywordPathFromValue = ({ keywordValue }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const slotFields = FULL_PATH_VALUE_FIELDS[normalizedScheme] + const slotFields = CSV_FIELDS[normalizedScheme] if (Array.isArray(slotFields)) { return buildKeywordPathFromObject({ @@ -49,7 +49,7 @@ export const buildFullPathLookupValue = ({ keywordValue }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const slotFields = FULL_PATH_VALUE_FIELDS[normalizedScheme] + const slotFields = CSV_FIELDS[normalizedScheme] if (Array.isArray(slotFields)) { const keywordObject = buildKeywordPathObjectFromValue({ diff --git a/serverless/src/shared/redis-path-store/helpers/buildKeywordObjectFromPath.js b/serverless/src/shared/redis-path-store/helpers/buildKeywordObjectFromPath.js index 1fa373cd..43be00ba 100644 --- a/serverless/src/shared/redis-path-store/helpers/buildKeywordObjectFromPath.js +++ b/serverless/src/shared/redis-path-store/helpers/buildKeywordObjectFromPath.js @@ -1,27 +1,9 @@ -import { buildKeywordPathObjectFromSegments } from './buildKeywordPathObjectFromSegments' -import { FULL_PATH_VALUE_FIELDS, SHORT_NAME_OBJECT_FIELDS } from './constants' +import { CSV_FIELDS } from './constants' import { isLookupShortNameScheme } from './isLookupShortNameScheme' -import { joinKeywordPath } from './joinKeywordPath' import { normalizeKeywordScheme } from './normalizeKeywordScheme' import { splitKeywordPath } from './splitKeywordPath' import { trimKeywordPathSegment } from './trimKeywordPathSegment' -/** - * Builds a slotted keyword-path object from a canonical keyword path string. - * - * @param {Object} params The keyword-path parsing input. - * @param {string} params.scheme Keyword scheme name. - * @param {string} params.keywordPath Canonical ` > `-delimited keyword path. - * @returns {Object} Slotted keyword-path object for schemes that use fixed path fields. - */ -export const buildKeywordPathObjectFromPath = ({ - scheme, - keywordPath -}) => buildKeywordPathObjectFromSegments({ - scheme, - segments: splitKeywordPath(keywordPath) -}) - const stripLeadingSchemeLabel = ({ normalizedScheme, pathSegments @@ -41,68 +23,41 @@ const stripLeadingSchemeLabel = ({ return pathSegments } -const buildShortNameKeywordObjectFromSegments = ({ +/** + * Builds a slotted keyword-path object from a canonical keyword path string. + * + * @param {Object} params The keyword-path parsing input. + * @param {string} params.scheme Keyword scheme name. + * @param {string} params.keywordPath Canonical ` > `-delimited keyword path. + * @returns {Object} Slotted keyword-path object for schemes that use fixed path fields. + */ +export const buildKeywordPathObjectFromPath = ({ scheme, - segments + keywordPath }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const fieldNames = SHORT_NAME_OBJECT_FIELDS[normalizedScheme] - const normalizedSegments = segments.map( - (segment) => trimKeywordPathSegment(segment) - ) - - if (fieldNames.length === 1) { - const singleValue = normalizedScheme === 'idnnode' - ? joinKeywordPath(normalizedSegments) - : [...normalizedSegments].reverse().find((segment) => segment.length > 0) || '' - - return { - [fieldNames[0]]: singleValue - } - } - - const lastNonEmptyIndex = normalizedSegments.reduce((lastIndex, segment, index) => ( - segment.length > 0 ? index : lastIndex - ), -1) - const shortName = lastNonEmptyIndex >= 0 ? normalizedSegments[lastNonEmptyIndex] : '' - const hierarchySegments = lastNonEmptyIndex >= 0 - ? normalizedSegments.slice(0, lastNonEmptyIndex) - : normalizedSegments.slice() - - if (normalizedScheme === 'platforms') { - const hasCategoryPrefix = normalizeKeywordScheme(hierarchySegments[0]) === 'platforms' + const fields = CSV_FIELDS[normalizedScheme] - return { - Category: hasCategoryPrefix ? hierarchySegments[0] : '', - Class: hasCategoryPrefix ? hierarchySegments[1] || '' : hierarchySegments[0] || '', - Type: hasCategoryPrefix ? hierarchySegments[2] || '' : hierarchySegments[1] || '', - ShortName: shortName - } + if (!Array.isArray(fields)) { + return {} } - if (normalizedScheme === 'instruments') { - return { - Category: hierarchySegments[0] || '', - Class: hierarchySegments[1] || '', - Subclass: hierarchySegments[2] || '', - ShortName: shortName - } - } + let segments = stripLeadingSchemeLabel({ + normalizedScheme, + pathSegments: splitKeywordPath(keywordPath) + }) - if (normalizedScheme === 'projects') { - return { - Category: hierarchySegments[0] || '', - ShortName: shortName - } + if (isLookupShortNameScheme(normalizedScheme) && segments.length < fields.length) { + segments = [ + ...Array(fields.length - segments.length).fill(''), + ...segments + ] } - return { - BucketLevel0: hierarchySegments[0] || '', - BucketLevel1: hierarchySegments[1] || '', - BucketLevel2: hierarchySegments[2] || '', - BucketLevel3: hierarchySegments[3] || '', - ShortName: shortName - } + return Object.fromEntries(fields.map((field, index) => [ + field, + trimKeywordPathSegment(segments[index]) + ])) } /** @@ -124,22 +79,10 @@ export const buildKeywordObjectFromPath = ({ return {} } - const slotFields = FULL_PATH_VALUE_FIELDS[normalizedScheme] - - if (Array.isArray(slotFields)) { + if (Array.isArray(CSV_FIELDS[normalizedScheme])) { return buildKeywordPathObjectFromPath({ scheme: normalizedScheme, - keywordPath: joinKeywordPath(stripLeadingSchemeLabel({ - normalizedScheme, - pathSegments: splitKeywordPath(normalizedKeywordPath) - })) - }) - } - - if (isLookupShortNameScheme(normalizedScheme)) { - return buildShortNameKeywordObjectFromSegments({ - scheme: normalizedScheme, - segments: splitKeywordPath(normalizedKeywordPath) + keywordPath: normalizedKeywordPath }) } diff --git a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathFromObject.js b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathFromObject.js index ff6eca82..03d6742e 100644 --- a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathFromObject.js +++ b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathFromObject.js @@ -1,4 +1,4 @@ -import { FULL_PATH_VALUE_FIELDS, SHORT_NAME_OBJECT_FIELDS } from './constants' +import { CSV_FIELDS } from './constants' import { flattenKeywordPathValue } from './flattenKeywordPathValue' import { joinKeywordPath } from './joinKeywordPath' import { normalizeKeywordScheme } from './normalizeKeywordScheme' @@ -17,27 +17,16 @@ export const buildKeywordPathFromObject = ({ keywordObject }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const slotFields = FULL_PATH_VALUE_FIELDS[normalizedScheme] - const shortNameObjectFields = SHORT_NAME_OBJECT_FIELDS[normalizedScheme] - - if (!Array.isArray(slotFields)) { - if (Array.isArray(shortNameObjectFields)) { - const keywordPathSegments = shortNameObjectFields.map( - (fieldName) => keywordObject?.[fieldName] - ) - const firstNonEmptyIndex = keywordPathSegments.findIndex( - (segment) => trimKeywordPathSegment(segment).length > 0 - ) - - return joinKeywordPath( - firstNonEmptyIndex >= 0 - ? keywordPathSegments.slice(firstNonEmptyIndex) - : keywordPathSegments - ) - } + const fields = CSV_FIELDS[normalizedScheme] + if (!Array.isArray(fields)) { return joinKeywordPath(flattenKeywordPathValue(keywordObject)) } - return joinKeywordPath(slotFields.map((fieldName) => keywordObject?.[fieldName])) + const segments = fields.map((fieldName) => keywordObject?.[fieldName]) + const firstNonEmptyIndex = segments.findIndex( + (segment) => trimKeywordPathSegment(segment).length > 0 + ) + + return joinKeywordPath(firstNonEmptyIndex >= 0 ? segments.slice(firstNonEmptyIndex) : segments) } diff --git a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromSegments.js b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromSegments.js index a391722c..c092df1f 100644 --- a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromSegments.js +++ b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromSegments.js @@ -1,4 +1,4 @@ -import { FULL_PATH_VALUE_FIELDS } from './constants' +import { CSV_FIELDS } from './constants' import { normalizeKeywordScheme } from './normalizeKeywordScheme' import { trimKeywordPathSegment } from './trimKeywordPathSegment' @@ -14,7 +14,7 @@ export const buildKeywordPathObjectFromSegments = ({ scheme, segments = [] }) => { - const slotFields = FULL_PATH_VALUE_FIELDS[normalizeKeywordScheme(scheme)] + const slotFields = CSV_FIELDS[normalizeKeywordScheme(scheme)] if (!Array.isArray(slotFields)) { return {} diff --git a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromValue.js b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromValue.js index 2039555c..030ddce8 100644 --- a/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromValue.js +++ b/serverless/src/shared/redis-path-store/helpers/buildKeywordPathObjectFromValue.js @@ -1,5 +1,5 @@ import { buildKeywordPathObjectFromSegments } from './buildKeywordPathObjectFromSegments' -import { FULL_PATH_VALUE_FIELDS } from './constants' +import { CSV_FIELDS } from './constants' import { flattenKeywordPathValue } from './flattenKeywordPathValue' import { normalizeKeywordScheme } from './normalizeKeywordScheme' @@ -17,7 +17,7 @@ export const buildKeywordPathObjectFromValue = ({ keywordValue }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const slotFields = FULL_PATH_VALUE_FIELDS[normalizedScheme] + const slotFields = CSV_FIELDS[normalizedScheme] if (!Array.isArray(slotFields)) { return {} diff --git a/serverless/src/shared/redis-path-store/helpers/constants.js b/serverless/src/shared/redis-path-store/helpers/constants.js index a55bb2f5..6bb4f1df 100644 --- a/serverless/src/shared/redis-path-store/helpers/constants.js +++ b/serverless/src/shared/redis-path-store/helpers/constants.js @@ -13,11 +13,20 @@ export const KEYWORD_PATH_SEPARATOR = ' > ' export const KEYWORD_DIFF_SKIP_HEADER_ROWS = 2 /** - * Full-path schemes mapped to their canonical slot order. + * Exact CSV column names and canonical hierarchy order used by each keyword scheme. + * UUID, Long_Name, and Data_Center_URL are auxiliary fields and are intentionally omitted. * - * @type {Readonly>} + * @type {Readonly>} */ -export const FULL_PATH_VALUE_FIELDS = Object.freeze({ +export const CSV_FIELDS = Object.freeze({ + category: 'Category', + shortName: 'ShortName', + providerRole: 'BucketLevel0', + urlContentType: 'URLContentType', + type: 'Type', + subtype: 'Subtype', + isoTopicCategory: 'ISOTopicCategory', + productLevelId: 'ProductLevelId', sciencekeywords: [ 'Category', 'Topic', @@ -28,12 +37,12 @@ export const FULL_PATH_VALUE_FIELDS = Object.freeze({ 'DetailedVariable' ], locations: [ - 'Category', - 'Type', - 'Subregion1', - 'Subregion2', - 'Subregion3', - 'DetailedLocation' + 'LocationCategory', + 'LocationType', + 'LocationSubregion1', + 'LocationSubregion2', + 'LocationSubregion3', + 'LocationSubregion4' ], chronounits: [ 'Eon', @@ -47,29 +56,22 @@ export const FULL_PATH_VALUE_FIELDS = Object.freeze({ 'URLContentType', 'Type', 'Subtype' - ] -}) - -/** - * Short-name schemes mapped to the fields used to rebuild canonical keyword paths. - * - * @type {Readonly>} - */ -export const SHORT_NAME_OBJECT_FIELDS = Object.freeze({ + ], platforms: [ + 'Basis', 'Category', - 'Class', - 'Type', + 'SubCategory', 'ShortName' ], instruments: [ 'Category', 'Class', - 'Subclass', + 'Type', + 'Subtype', 'ShortName' ], projects: [ - 'Category', + 'Bucket', 'ShortName' ], providers: [ @@ -79,26 +81,9 @@ export const SHORT_NAME_OBJECT_FIELDS = Object.freeze({ 'BucketLevel3', 'ShortName' ], - idnnode: [ - 'ShortName' - ], - dataformat: [ - 'ShortName' - ], - granuledataformat: [ - 'ShortName' - ] -}) - -/** - * CSV hierarchy fields in canonical keyword-path order. - * UUID, Long_Name, and Data_Center_URL are auxiliary fields and are intentionally omitted. - * - * @type {Readonly>} - */ -export const CSV_PATH_FIELDS = Object.freeze({ - ...FULL_PATH_VALUE_FIELDS, - ...SHORT_NAME_OBJECT_FIELDS, + idnnode: ['ShortName'], + dataformat: ['ShortName'], + granuledataformat: ['ShortName'], discipline: ['DisciplineName', 'Subdiscipline'], isotopiccategory: ['ISOTopicCategory'], temporalresolutionrange: ['TemporalResolutionRange'], @@ -108,6 +93,124 @@ export const CSV_PATH_FIELDS = Object.freeze({ measurementname: ['ContextMedium', 'Object', 'Quantity'] }) +/** + * Native UMM-C JSON fields used when reading and writing metadata. + * + * @type {Readonly>} + */ +export const UMMC_FIELDS = Object.freeze({ + sciencekeywords: [ + 'Category', + 'Topic', + 'Term', + 'VariableLevel1', + 'VariableLevel2', + 'VariableLevel3', + 'DetailedVariable' + ], + locations: [ + 'Category', + 'Type', + 'Subregion1', + 'Subregion2', + 'Subregion3', + 'DetailedLocation' + ], + chronounits: [ + 'Eon', + 'Era', + 'Period', + 'Epoch', + 'Stage', + 'DetailedClassification' + ], + platformType: 'Type', + shortName: 'ShortName', + longName: 'LongName', + rucontenttype: ['URLContentType', 'Type', 'Subtype'], + dataformat: 'Format', + processingLevelId: 'Id' +}) + +/** + * Native ECHO10 XML fields used when reading and writing metadata. + * + * @type {Readonly>} + */ +export const ECHO10_FIELDS = Object.freeze({ + sciencekeywords: [ + 'CategoryKeyword', + 'TopicKeyword', + 'TermKeyword', + 'VariableLevel1Keyword/Value', + 'VariableLevel1Keyword/VariableLevel2Keyword/Value', + 'VariableLevel1Keyword/VariableLevel2Keyword/VariableLevel3Keyword', + 'DetailedVariableKeyword' + ], + platformType: 'Type', + shortName: 'ShortName', + longName: 'LongName', + providers: ['Role', 'OrganizationName'], + providerOrganizationName: 'OrganizationName', + processingCenter: '//Collection/ProcessingCenter', + archiveCenter: '//Collection/ArchiveCenter', + rucontenttype: ['Type'], + relatedUrlType: 'Type' +}) + +/** + * Native DIF10 XML fields used when reading and writing metadata. + * + * @type {Readonly>} + */ +export const DIF10_FIELDS = Object.freeze({ + sciencekeywords: [ + 'Category', + 'Topic', + 'Term', + 'Variable_Level_1', + 'Variable_Level_2', + 'Variable_Level_3', + 'Detailed_Variable' + ], + locations: [ + 'Location_Category', + 'Location_Type', + 'Location_Subregion1', + 'Location_Subregion2', + 'Location_Subregion3', + 'Detailed_Location' + ], + chronounits: [ + 'Eon', + 'Era', + 'Period', + 'Epoch', + 'Stage', + 'Detailed_Classification' + ], + platformType: 'Type', + shortName: 'Short_Name', + longName: 'Long_Name', + providerShortName: 'Organization_Name/Short_Name', + providerLongName: 'Organization_Name/Long_Name', + rucontenttype: ['Type', 'Subtype'] +}) + +/** + * Native ISO 19115 XML fields used when reading and writing metadata. + * + * @type {Readonly>} + */ +export const ISO19115_FIELDS = Object.freeze({ + keywordValue: ['gmx:Anchor', 'gco:CharacterString'], + keywordLongName: 'LongName', + isoTopicCategory: 'gmd:MD_TopicCategoryCode', + isoTopicCategoryCodeListValue: 'gmd:MD_TopicCategoryCode/@codeListValue', + productLevelId: 'gmd:code/gco:CharacterString', + dataformat: 'gmd:MD_Format/gmd:name/gco:CharacterString' +}) + /** * Schemes that resolve lookups by canonical full path. * diff --git a/serverless/src/shared/redis-path-store/helpers/keywordCsv.js b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js index da843d54..570db374 100644 --- a/serverless/src/shared/redis-path-store/helpers/keywordCsv.js +++ b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js @@ -5,7 +5,7 @@ import { isCsvProviderUrlFlag } from '@/shared/isCsvProviderUrlFlag' import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' import { - CSV_PATH_FIELDS, + CSV_FIELDS, KEYWORD_DIFF_SKIP_HEADER_ROWS, KEYWORD_PATH_SEPARATOR } from './constants' @@ -55,7 +55,7 @@ export const formatKeywordCsvPath = ({ isLeaf }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const pathFieldsCount = CSV_PATH_FIELDS[normalizedScheme]?.length ?? path.length + const pathFieldsCount = CSV_FIELDS[normalizedScheme]?.length ?? path.length while (pathFieldsCount > path.length) { if (isLookupShortNameScheme(normalizedScheme) && isLeaf) { @@ -97,6 +97,32 @@ export const getCsvRowValues = ({ uuid ] +/** + * Maps a hierarchy row to its canonical named CSV path fields. + * + * @param {object} params CSV path-record input. + * @param {string[]} params.configuredFields Canonical fields for the scheme. + * @param {object} params.row Hierarchy row to map. + * @param {string} params.scheme Keyword scheme. + * @returns {Record} Values keyed by normalized CSV header. + */ +const buildCsvPathRecord = ({ + configuredFields, + row, + scheme +}) => { + const path = formatKeywordCsvPath({ + scheme, + path: [...row.path], + isLeaf: row.isLeaf + }) + + return Object.fromEntries(configuredFields.map((field, index) => [ + normalizeCsvHeader(field), + path[index] || '' + ])) +} + /** * Sorts hierarchy rows and places each value under its named CSV header. * @@ -125,7 +151,7 @@ export const prepareCsvRows = ({ csvRows, scheme }) => { - const configuredFields = CSV_PATH_FIELDS[normalizeKeywordScheme(scheme)] + const configuredFields = CSV_FIELDS[normalizeKeywordScheme(scheme)] const sortedRows = [...csvRows].sort((first, second) => { const firstValues = getCsvRowValues({ ...first, @@ -152,17 +178,12 @@ export const prepareCsvRows = ({ })) } - const pathHeaders = configuredFields.map(normalizeCsvHeader) - return sortedRows.map((row) => { - const path = formatKeywordCsvPath({ - scheme, - path: [...row.path], - isLeaf: row.isLeaf + const record = buildCsvPathRecord({ + configuredFields, + row, + scheme }) - const record = Object.fromEntries( - pathHeaders.map((header, index) => [header, path[index] || '']) - ) record.uuid = row.uuid @@ -181,13 +202,13 @@ export const prepareCsvRows = ({ * @param {string} options.scheme Keyword scheme. * @param {number} [options.headerRow=2] One-based line containing the CSV headers. * @param {string} [options.pathSeparator=KEYWORD_PATH_SEPARATOR] Keyword-path separator. - * @returns {{uuid: string, shortName: string, longName: string, providerUrl: string, keywordPath: string}[]} - * Parsed keyword records. + * @returns {{uuid: string, shortName: string, longName: string, providerUrl: string, + * keywordPath: string, keywordObject: Record|undefined}[]} Parsed keyword records. * * @example * parseCsv( - * '"Version"\n"Long_Name","UUID","Type","Short_Name","Category","Class"\n' - * + '"Greenhouse Gases Observing Satellite","uuid-1","Earth Observation Satellites","GOSAT","Platforms","Space-based Platforms"', + * '"Version"\n"Long_Name","UUID","Category","Short_Name","Basis","Sub_Category"\n' + * + '"Greenhouse Gases Observing Satellite","uuid-1","Earth Observation Satellites","GOSAT","Space-based Platforms",""', * { scheme: 'platforms' } * ) * // Returns: [{ @@ -195,7 +216,7 @@ export const prepareCsvRows = ({ * // shortName: 'GOSAT', * // longName: 'Greenhouse Gases Observing Satellite', * // providerUrl: '', - * // keywordPath: 'Platforms > Space-based Platforms > Earth Observation Satellites > GOSAT' + * // keywordPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT' * // }] */ export const parseCsv = ( @@ -207,7 +228,7 @@ export const parseCsv = ( } = {} ) => { const normalizedScheme = normalizeKeywordScheme(scheme) - const configuredFields = CSV_PATH_FIELDS[normalizedScheme] + const configuredFields = CSV_FIELDS[normalizedScheme] let pathHeaders = [] const rows = parse(csvContent || '', { columns: (headers) => { @@ -225,15 +246,33 @@ export const parseCsv = ( relax_column_count: true }) - return rows.map((row) => ({ - uuid: trimKeywordPathSegment(row.uuid), - shortName: trimKeywordPathSegment(row.shortname), - longName: trimKeywordPathSegment(row.longname), - providerUrl: trimKeywordPathSegment(row.datacenterurl), - keywordPath: pathHeaders + return rows.map((row) => { + const keywordPath = pathHeaders .map((header) => trimKeywordPathSegment(row[header])) .join(pathSeparator) - })) + const keywordObject = configuredFields + ? Object.fromEntries([ + ...configuredFields, + ...(isCsvLongNameFlag(normalizedScheme) ? ['LongName'] : []), + ...(isCsvProviderUrlFlag(normalizedScheme) ? ['DataCenterURL'] : []) + ].map((field) => [ + field, + trimKeywordPathSegment(row[normalizeCsvHeader(field)]) + ])) + : buildKeywordObjectFromPath({ + scheme: normalizedScheme, + keywordPath + }) + + return { + uuid: trimKeywordPathSegment(row.uuid), + shortName: trimKeywordPathSegment(row.shortname), + longName: trimKeywordPathSegment(row.longname), + providerUrl: trimKeywordPathSegment(row.datacenterurl), + keywordPath, + keywordObject + } + }) } /** @@ -248,16 +287,16 @@ export const parseCsv = ( * * @example * parseKeywordCsvContent( - * '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' - * + '"Platforms","Space-based Platforms","Satellite","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', + * '"Version"\n"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"\n' + * + '"Space-based Platforms","Earth Observation Satellites","","GOSAT","Greenhouse Gases Observing Satellite","uuid-1"', * { scheme: 'platforms' } * ) * // Returns: Map([['uuid-1', { - * // path: 'Platforms > Space-based Platforms > Satellite > GOSAT', + * // path: 'Space-based Platforms > Earth Observation Satellites > > GOSAT', * // keywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT', * // LongName: 'Greenhouse Gases Observing Satellite' * // } @@ -278,33 +317,15 @@ export const parseKeywordCsvContent = ( pathSeparator }).filter(({ uuid }) => uuid) - return new Map( - records - .map(({ - keywordPath, - longName, - providerUrl, - uuid - }) => { - const keywordObject = buildKeywordObjectFromPath({ - scheme: normalizedScheme, - keywordPath - }) - - if (isCsvLongNameFlag(normalizedScheme) && longName) { - keywordObject.LongName = longName - } - - if (isCsvProviderUrlFlag(normalizedScheme) && providerUrl) { - keywordObject.DataCenterUrl = providerUrl - } - - return [uuid, { - path: keywordPath, - keywordObject: hasKeywordObjectValue(keywordObject) ? keywordObject : undefined - }] - }) - ) + return new Map(records + .map(({ + keywordPath, + keywordObject, + uuid + }) => [uuid, { + path: keywordPath, + keywordObject: hasKeywordObjectValue(keywordObject) ? keywordObject : undefined + }])) } /** @@ -344,19 +365,19 @@ export const parseFullPathCsvRecords = ({ * * @example * parseShortNameCsvRecords({ - * csvContent: '"Version"\n"Category","Class","Type","Short_Name","Long_Name","UUID"\n' - * + '"Platforms","Space-based Platforms","Satellite","GOSAT","GOSAT Satellite","uuid-1"', + * csvContent: '"Version"\n"Basis","Category","Sub_Category","Short_Name","Long_Name","UUID"\n' + * + '"Space-based Platforms","Earth Observation Satellites","","GOSAT","GOSAT Satellite","uuid-1"', * scheme: 'platforms' * }) * // Returns: Map([['GOSAT', { * // uuid: 'uuid-1', - * // fullPath: 'Platforms > Space-based Platforms > Satellite > GOSAT', + * // fullPath: 'Space-based Platforms > Earth Observation Satellites > > GOSAT', * // longName: 'GOSAT Satellite', * // providerUrl: '', * // keywordObject: { - * // Category: 'Platforms', - * // Class: 'Space-based Platforms', - * // Type: 'Satellite', + * // Basis: 'Space-based Platforms', + * // Category: 'Earth Observation Satellites', + * // SubCategory: '', * // ShortName: 'GOSAT', * // LongName: 'GOSAT Satellite' * // } @@ -368,36 +389,23 @@ export const parseShortNameCsvRecords = ({ }) => { const normalizedScheme = normalizeKeywordScheme(scheme) - return new Map(parseCsv(csvContent, { + const records = parseCsv(csvContent, { scheme: normalizedScheme - }) - .filter(({ shortName }) => shortName) + }).filter(({ shortName }) => shortName) + + return new Map(records .map(({ keywordPath, longName, providerUrl, shortName, - uuid - }) => { - const keywordObject = buildKeywordObjectFromPath({ - scheme: normalizedScheme, - keywordPath - }) - - if (longName) { - keywordObject.LongName = longName - } - - if (providerUrl) { - keywordObject.DataCenterUrl = providerUrl - } - - return [shortName, { - uuid, - fullPath: keywordPath, - longName, - providerUrl, - keywordObject - }] - })) + uuid, + keywordObject + }) => [shortName, { + uuid, + fullPath: keywordPath, + longName, + providerUrl, + keywordObject + }])) } diff --git a/serverless/src/shared/redis-path-store/helpers/parseCachedConceptResponse.js b/serverless/src/shared/redis-path-store/helpers/parseCachedConceptResponse.js index b30a48eb..55788972 100644 --- a/serverless/src/shared/redis-path-store/helpers/parseCachedConceptResponse.js +++ b/serverless/src/shared/redis-path-store/helpers/parseCachedConceptResponse.js @@ -10,8 +10,8 @@ const attachSupplementalKeywordFields = ({ nextKeywordObject.LongName = concept.longName } - if (concept.providerUrl && !nextKeywordObject.DataCenterUrl) { - nextKeywordObject.DataCenterUrl = concept.providerUrl + if (concept.providerUrl && !nextKeywordObject.DataCenterURL) { + nextKeywordObject.DataCenterURL = concept.providerUrl } return nextKeywordObject diff --git a/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js b/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js index 78c05c50..22132043 100644 --- a/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js +++ b/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js @@ -1,4 +1,5 @@ import { buildFullPathLookupValue } from './buildFullPathLookupValue' +import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' import { buildKeywordPathObjectFromValue } from './buildKeywordPathObjectFromValue' import { flattenKeywordPathValue } from './flattenKeywordPathValue' import { hasKeywordObjectValue } from './hasKeywordObjectValue' @@ -65,7 +66,12 @@ export const buildKeywordLookupObject = ({ keywordValue }) - return fullPathLookupValue ? { Value: fullPathLookupValue } : {} + return fullPathLookupValue + ? buildKeywordObjectFromPath({ + scheme: normalizedScheme, + keywordPath: fullPathLookupValue + }) + : {} } if (isLookupShortNameScheme(normalizedScheme)) { diff --git a/serverless/src/shared/ummcDomEditor.js b/serverless/src/shared/ummcDomEditor.js index 11764513..e5342783 100644 --- a/serverless/src/shared/ummcDomEditor.js +++ b/serverless/src/shared/ummcDomEditor.js @@ -1,5 +1,5 @@ import JsonMetadataPathEditor, { sequentialValueReplace } from './JsonMetadataPathEditor' -import { FULL_PATH_VALUE_FIELDS } from './redis-path-store/helpers/constants' +import { CSV_FIELDS, UMMC_FIELDS } from './redis-path-store/helpers/constants' /** * A unified scheme creator that selects the appropriate @@ -85,10 +85,13 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'ScienceKeywords') }, find: { - fieldPaths: FULL_PATH_VALUE_FIELDS.sciencekeywords, - valueKeys: FULL_PATH_VALUE_FIELDS.sciencekeywords + fieldPaths: UMMC_FIELDS.sciencekeywords, + valueKeys: CSV_FIELDS.sciencekeywords }, - replace: sequentialValueReplace(FULL_PATH_VALUE_FIELDS.sciencekeywords) + replace: sequentialValueReplace( + UMMC_FIELDS.sciencekeywords, + CSV_FIELDS.sciencekeywords + ) }), locations: unifiedBlockScheme({ nodePath: '//LocationKeywords', @@ -96,10 +99,13 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'LocationKeywords') }, find: { - fieldPaths: FULL_PATH_VALUE_FIELDS.locations, - valueKeys: FULL_PATH_VALUE_FIELDS.locations + fieldPaths: UMMC_FIELDS.locations, + valueKeys: CSV_FIELDS.locations }, - replace: sequentialValueReplace(FULL_PATH_VALUE_FIELDS.locations) + replace: sequentialValueReplace( + UMMC_FIELDS.locations, + CSV_FIELDS.locations + ) }), chronounits: unifiedBlockScheme({ // 1. Path to the parent container @@ -108,13 +114,13 @@ export const UMMC_SCHEME_EDITORS = { childKey: 'ChronostratigraphicUnits', // 3. Match using all relevant hierarchy fields find: { - fieldPaths: ['Eon', 'Era', 'Period', 'Epoch', 'Stage', 'DetailedClassification'], - valueKeys: FULL_PATH_VALUE_FIELDS.chronounits + fieldPaths: UMMC_FIELDS.chronounits, + valueKeys: CSV_FIELDS.chronounits }, // 4. Map the replacement values replace: sequentialValueReplace( - ['Eon', 'Era', 'Period', 'Epoch', 'Stage', 'DetailedClassification'], - FULL_PATH_VALUE_FIELDS.chronounits + UMMC_FIELDS.chronounits, + CSV_FIELDS.chronounits ), afterDelete: (editor) => { cleanupArray(editor.document, 'PaleoTemporalCoverages', 'ChronostratigraphicUnits') @@ -126,37 +132,34 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'Platforms') }, find: { - fieldPaths: [ - // JSON fields to read from - 'ShortName' - ], + fieldPaths: [UMMC_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // JSON field to write to - fieldPath: 'Type', + fieldPath: UMMC_FIELDS.platformType, source: { // Key from newKeywordObject to read from type: 'value', - key: 'Type' + key: CSV_FIELDS.category } }, { // JSON field to write to - fieldPath: 'ShortName', + fieldPath: UMMC_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // JSON field to write to - fieldPath: 'LongName', + fieldPath: UMMC_FIELDS.longName, source: { type: 'param', key: 'newLongName' @@ -168,19 +171,19 @@ export const UMMC_SCHEME_EDITORS = { containerPath: '//Platforms', childKey: 'Instruments', find: { - fieldPaths: ['ShortName'], - valueKeys: ['ShortName'] + fieldPaths: [UMMC_FIELDS.shortName], + valueKeys: [CSV_FIELDS.shortName] }, replace: [ { - fieldPath: 'ShortName', + fieldPath: UMMC_FIELDS.shortName, source: { type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { - fieldPath: 'LongName', + fieldPath: UMMC_FIELDS.longName, source: { type: 'param', key: 'newLongName' @@ -194,28 +197,25 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'Projects') }, find: { - fieldPaths: [ - // JSON fields to read from - 'ShortName' - ], + fieldPaths: [UMMC_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // JSON field to write to - fieldPath: 'ShortName', + fieldPath: UMMC_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // JSON field to write to - fieldPath: 'LongName', + fieldPath: UMMC_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -230,28 +230,25 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'DataCenters') }, find: { - fieldPaths: [ - // JSON fields to read from - 'ShortName' - ], + fieldPaths: [UMMC_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // JSON field to write to - fieldPath: 'ShortName', + fieldPath: UMMC_FIELDS.shortName, source: { // Key from newKeywordObject to read from type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // JSON field to write to - fieldPath: 'LongName', + fieldPath: UMMC_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -266,10 +263,13 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'RelatedUrls') }, find: { - fieldPaths: FULL_PATH_VALUE_FIELDS.rucontenttype, - valueKeys: FULL_PATH_VALUE_FIELDS.rucontenttype + fieldPaths: UMMC_FIELDS.rucontenttype, + valueKeys: CSV_FIELDS.rucontenttype }, - replace: sequentialValueReplace(FULL_PATH_VALUE_FIELDS.rucontenttype) + replace: sequentialValueReplace( + UMMC_FIELDS.rucontenttype, + CSV_FIELDS.rucontenttype + ) }), idnnode: unifiedBlockScheme({ nodePath: '//DirectoryNames', @@ -277,27 +277,24 @@ export const UMMC_SCHEME_EDITORS = { cleanupArray(editor.document, 'DirectoryNames') }, find: { - fieldPaths: [ - // JSON fields to read from - 'ShortName' - ], + fieldPaths: [UMMC_FIELDS.shortName], valueKeys: [ // Keys from oldKeywordObject to compare against - 'ShortName' + CSV_FIELDS.shortName ] }, replace: [ { // JSON field to write to - fieldPath: 'ShortName', + fieldPath: UMMC_FIELDS.shortName, source: { type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }, { // JSON field to write to - fieldPath: 'LongName', + fieldPath: UMMC_FIELDS.longName, source: { // Correction param to read from type: 'param', @@ -314,7 +311,7 @@ export const UMMC_SCHEME_EDITORS = { }), productlevelid: scalarScheme({ nodePath: '//ProcessingLevel/Id', - fieldName: 'Id', + fieldName: UMMC_FIELDS.processingLevelId, afterDelete: (editor) => { // When ProcessingLevel.Id is deleted, remove the entire ProcessingLevel object // This matches the expected behavior where the Id is the primary field @@ -329,14 +326,14 @@ export const UMMC_SCHEME_EDITORS = { containerPath: '//ArchiveAndDistributionInformation', childKey: 'FileArchiveInformation', find: { - fieldPaths: ['Format'], - valueKeys: ['ShortName'] + fieldPaths: [UMMC_FIELDS.dataformat], + valueKeys: [CSV_FIELDS.shortName] }, replace: [{ - fieldPath: 'Format', + fieldPath: UMMC_FIELDS.dataformat, source: { type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }] }, @@ -344,14 +341,14 @@ export const UMMC_SCHEME_EDITORS = { containerPath: '//ArchiveAndDistributionInformation', childKey: 'FileDistributionInformation', find: { - fieldPaths: ['Format'], - valueKeys: ['ShortName'] + fieldPaths: [UMMC_FIELDS.dataformat], + valueKeys: [CSV_FIELDS.shortName] }, replace: [{ - fieldPath: 'Format', + fieldPath: UMMC_FIELDS.dataformat, source: { type: 'value', - key: 'ShortName' + key: CSV_FIELDS.shortName } }] } From 42db0bcb882598a2e7d036b2cfa029d8fea67ea2 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Thu, 20 Aug 2026 20:33:05 -0400 Subject: [PATCH 3/5] KMS-705: Updated test coverage. --- .../__tests__/Iso19115DomEditor.test.js | 25 +++++++++++++++++++ .../helpers/__tests__/keywordCsv.test.js | 20 +++++++++++++++ .../helpers/resolveLookupKeywordObject.js | 14 +---------- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/serverless/src/shared/__tests__/Iso19115DomEditor.test.js b/serverless/src/shared/__tests__/Iso19115DomEditor.test.js index 25b3a053..eef2b554 100644 --- a/serverless/src/shared/__tests__/Iso19115DomEditor.test.js +++ b/serverless/src/shared/__tests__/Iso19115DomEditor.test.js @@ -36,6 +36,31 @@ describe('ISO_19115_SCHEME_EDITORS', () => { ) }) + test('should return empty fields when an ISO keyword block has no value node', () => { + ISO_19115_SCHEME_EDITORS.sciencekeywords(mockEditor, { + action: 'replace', + oldKeywordObject: {} + }) + + const config = mockEditor.updateBlockNode.mock.calls[0][1] + const keywordObject = config.find.getNodeValueObject({ + node: {}, + editor: { + selectNodes: vi.fn().mockReturnValue([]) + } + }) + + expect(keywordObject).toEqual({ + Category: '', + Topic: '', + Term: '', + VariableLevel1: '', + VariableLevel2: '', + VariableLevel3: '', + DetailedVariable: '' + }) + }) + test('should trigger updateLeafNode for leaf types', () => { const correction = { action: 'replace', diff --git a/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js index e0ba30f5..99d260ff 100644 --- a/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js +++ b/serverless/src/shared/redis-path-store/helpers/__tests__/keywordCsv.test.js @@ -1,4 +1,5 @@ import { + getCsvRowValues, parseCsv, parseFullPathCsvRecords, parseKeywordCsvContent, @@ -7,6 +8,25 @@ import { } from '../keywordCsv' describe('when preparing CSV rows by header name', () => { + test('includes provider long names and URLs in CSV row values', () => { + expect(getCsvRowValues({ + path: ['NASA', 'GSFC', 'EOSDIS', 'GHRC', 'GHRC_DAAC'], + scheme: 'providers', + longName: 'Global Hydrometeorology Resource Center DAAC', + dataCenterUrl: 'https://ghrc.example.test', + uuid: 'uuid-ghrc' + })).toEqual([ + 'NASA', + 'GSFC', + 'EOSDIS', + 'GHRC', + 'GHRC_DAAC', + 'Global Hydrometeorology Resource Center DAAC', + 'https://ghrc.example.test', + 'uuid-ghrc' + ]) + }) + test('places a platform leaf hierarchy under reordered headers', () => { expect(prepareCsvRows({ csvHeaders: ['Long_Name', 'UUID', 'Sub_Category', 'Short_Name', 'Category', 'Basis'], diff --git a/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js b/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js index 22132043..0b29eeb1 100644 --- a/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js +++ b/serverless/src/shared/redis-path-store/helpers/resolveLookupKeywordObject.js @@ -1,5 +1,3 @@ -import { buildFullPathLookupValue } from './buildFullPathLookupValue' -import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' import { buildKeywordPathObjectFromValue } from './buildKeywordPathObjectFromValue' import { flattenKeywordPathValue } from './flattenKeywordPathValue' import { hasKeywordObjectValue } from './hasKeywordObjectValue' @@ -61,17 +59,7 @@ export const buildKeywordLookupObject = ({ return keywordObject } - const fullPathLookupValue = buildFullPathLookupValue({ - scheme: normalizedScheme, - keywordValue - }) - - return fullPathLookupValue - ? buildKeywordObjectFromPath({ - scheme: normalizedScheme, - keywordPath: fullPathLookupValue - }) - : {} + return {} } if (isLookupShortNameScheme(normalizedScheme)) { From 4adaabb37bfe4ec2e43826273a793eea9d017b02 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Fri, 21 Aug 2026 13:37:49 -0400 Subject: [PATCH 4/5] KMS-705: Ignore DataFormat corrections when only LongName changes --- .../src/shared/XmlMetadataPathEditor.js | 4 ++++ .../applyEcho10MetadataCorrections.test.js | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/serverless/src/shared/XmlMetadataPathEditor.js b/serverless/src/shared/XmlMetadataPathEditor.js index 49614955..a4365228 100644 --- a/serverless/src/shared/XmlMetadataPathEditor.js +++ b/serverless/src/shared/XmlMetadataPathEditor.js @@ -891,6 +891,10 @@ export class XmlMetadataPathEditor { } if (targetNode) { + if (trimString(this.getElementText(targetNode)) === value) { + return false + } + this.setElementText(targetNode, value) } else { const root = this.document?.documentElement diff --git a/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js b/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js index fe87c325..fe785bbe 100644 --- a/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js +++ b/serverless/src/shared/__tests__/applyEcho10MetadataCorrections.test.js @@ -1546,6 +1546,28 @@ describe('when applying Dataformat ECHO10 corrections', () => { expect(result.correctedMetadata).toContain('HDF5') }) + test('should not apply a correction when only the unsupported long name changed', async () => { + const result = await applyEcho10MetadataCorrections({ + metadataPayload: 'NetCDF', + corrections: [{ + scheme: 'dataformat', + action: 'replace', + oldKeywordObject: { + ShortName: 'NetCDF', + LongName: 'Network Common Data Form' + }, + newKeywordObject: { + ShortName: 'NetCDF', + LongName: 'Network Common Data Form Updated' + } + }] + }) + + expect(result.correctionCount).toBe(0) + expect(result.correctionsApplied).toEqual([]) + expect(result.correctedMetadata).toContain('NetCDF') + }) + test('should return false and not modify the field if newKeywordObject is empty or invalid', async () => { const result = await applyEcho10MetadataCorrections({ metadataPayload: mockEcho10WithDataFormat, From 9a38be79d63ae863b9ea814ea543423859474575 Mon Sep 17 00:00:00 2001 From: "Christopher D. Gokey" Date: Tue, 1 Sep 2026 22:26:43 -0400 Subject: [PATCH 5/5] KMS-705: Moved AUXILIARY_CSV_HEADERS to constants --- .../src/shared/redis-path-store/helpers/constants.js | 11 +++++++++++ .../src/shared/redis-path-store/helpers/keywordCsv.js | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/serverless/src/shared/redis-path-store/helpers/constants.js b/serverless/src/shared/redis-path-store/helpers/constants.js index 6bb4f1df..76aaae47 100644 --- a/serverless/src/shared/redis-path-store/helpers/constants.js +++ b/serverless/src/shared/redis-path-store/helpers/constants.js @@ -12,6 +12,17 @@ export const KEYWORD_PATH_SEPARATOR = ' > ' */ export const KEYWORD_DIFF_SKIP_HEADER_ROWS = 2 +/** + * Normalized CSV column names excluded from keyword hierarchy paths. + * + * @type {ReadonlySet} + */ +export const AUXILIARY_CSV_HEADERS = new Set([ + 'uuid', + 'longname', + 'datacenterurl' +]) + /** * Exact CSV column names and canonical hierarchy order used by each keyword scheme. * UUID, Long_Name, and Data_Center_URL are auxiliary fields and are intentionally omitted. diff --git a/serverless/src/shared/redis-path-store/helpers/keywordCsv.js b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js index 570db374..fe444d71 100644 --- a/serverless/src/shared/redis-path-store/helpers/keywordCsv.js +++ b/serverless/src/shared/redis-path-store/helpers/keywordCsv.js @@ -5,6 +5,7 @@ import { isCsvProviderUrlFlag } from '@/shared/isCsvProviderUrlFlag' import { buildKeywordObjectFromPath } from './buildKeywordObjectFromPath' import { + AUXILIARY_CSV_HEADERS, CSV_FIELDS, KEYWORD_DIFF_SKIP_HEADER_ROWS, KEYWORD_PATH_SEPARATOR @@ -29,8 +30,6 @@ export const normalizeCsvHeader = (header) => String(header) .toLowerCase() .replace(/[^a-z0-9]/g, '') -const AUXILIARY_CSV_HEADERS = new Set(['uuid', 'longname', 'datacenterurl']) - /** * Pads a hierarchy path to the number of fields configured for its scheme. Leaf values for * short-name schemes stay in the final field.