Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import {
mapGranularityToExternalFormat,
mapGranularityToInternalFormat,
} from '@hyperdx/common-utils/dist/externalSource';
import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
import mongoose from 'mongoose';
import request, { SuperAgentTest } from 'supertest';
Expand All @@ -20,10 +24,6 @@ import {
} from '@/models/source';
import { ITeam } from '@/models/team';
import { IUser } from '@/models/user';
import {
mapGranularityToExternalFormat,
mapGranularityToInternalFormat,
} from '@/routers/external-api/v2/sources';

describe('External API v2 Sources', () => {
const server = getServer();
Expand Down
94 changes: 6 additions & 88 deletions packages/api/src/routers/external-api/v2/sources.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import {
SourceKind,
SourceSchema,
SourceSchemaNoId,
type TSource,
} from '@hyperdx/common-utils/dist/types';
mapGranularityToInternalFormat,
toExternalSource,
} from '@hyperdx/common-utils/dist/externalSource';
import { SourceKind, SourceSchemaNoId } from '@hyperdx/common-utils/dist/types';
import express from 'express';
import { z } from 'zod';

Expand All @@ -21,44 +20,6 @@ import { processRequestWithEnhancedErrors as validateRequest } from '@/utils/enh
import logger from '@/utils/logger';
import { objectIdSchema } from '@/utils/zod';

export function mapGranularityToExternalFormat(granularity: string): string {
const matches = granularity.match(/^(\d+) (second|minute|hour|day)$/);
if (matches == null) return granularity;

const [, amount, unit] = matches;
switch (unit) {
case 'second':
return `${amount}s`;
case 'minute':
return `${amount}m`;
case 'hour':
return `${amount}h`;
case 'day':
return `${amount}d`;
default:
return granularity;
}
}

export function mapGranularityToInternalFormat(granularity: string): string {
const matches = granularity.match(/^(\d+)(s|m|h|d)$/);
if (matches == null) return granularity;

const [, amount, unit] = matches;
switch (unit) {
case 's':
return `${amount} second`;
case 'm':
return `${amount} minute`;
case 'h':
return `${amount} hour`;
case 'd':
return `${amount} day`;
default:
return granularity;
}
}

// Maps external short-form granularities (e.g. "5m") in the request body to
// the internal SQL interval format (e.g. "5 minute") expected by SourceSchema.
// Runs before body validation so external clients can use the same format the
Expand Down Expand Up @@ -127,44 +88,6 @@ function rejectPromqlSourceWhenDisabled(
next();
}

function mapSourceToExternalSource(source: TSource): TSource {
if (!('materializedViews' in source)) return source;

const mapped = { ...source };

if (Array.isArray(source.materializedViews)) {
mapped.materializedViews = source.materializedViews.map(view => ({
...view,
minGranularity: mapGranularityToExternalFormat(view.minGranularity),
}));
}

if (
'metadataMaterializedViews' in source &&
source.metadataMaterializedViews
) {
mapped.metadataMaterializedViews = {
...source.metadataMaterializedViews,
granularity: mapGranularityToExternalFormat(
source.metadataMaterializedViews.granularity,
),
};
}

return mapped;
}

function applyLegacyDefaults(
parsed: Record<string, unknown>,
): Record<string, unknown> {
// Legacy Session sources were created before timestampValueExpression was
// required. The old code defaulted it to 'TimestampTime' at query time.
if (parsed.kind === SourceKind.Session && !parsed.timestampValueExpression) {
return { ...parsed, timestampValueExpression: 'TimestampTime' };
}
return parsed;
}

function formatExternalSource(source: SourceDocument) {
// Convert to JSON so that any ObjectIds are converted to strings
const json = JSON.stringify(
Expand All @@ -187,13 +110,8 @@ function formatExternalSource(source: SourceDocument) {
})(),
);

// Parse using the SourceSchema to strip out any fields not defined in the schema
const parseResult = SourceSchema.safeParse(
applyLegacyDefaults(JSON.parse(json)),
);
if (parseResult.success) {
return mapSourceToExternalSource(parseResult.data);
}
const parseResult = toExternalSource(JSON.parse(json));
if (parseResult.success) return parseResult.data;

// If parsing fails, log the error and return undefined
logger.error(
Expand Down
96 changes: 96 additions & 0 deletions packages/common-utils/src/externalSource.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import { SourceKind, SourceSchema, type TSource } from '@/types';

export function mapGranularityToExternalFormat(granularity: string): string {
const matches = granularity.match(/^(\d+) (second|minute|hour|day)$/);
if (matches == null) return granularity;

const [, amount, unit] = matches;
switch (unit) {
case 'second':
return `${amount}s`;
case 'minute':
return `${amount}m`;
case 'hour':
return `${amount}h`;
case 'day':
return `${amount}d`;
default:
return granularity;
}
}

export function mapGranularityToInternalFormat(granularity: string): string {
const matches = granularity.match(/^(\d+)(s|m|h|d)$/);
if (matches == null) return granularity;

const [, amount, unit] = matches;
switch (unit) {
case 's':
return `${amount} second`;
case 'm':
return `${amount} minute`;
case 'h':
return `${amount} hour`;
case 'd':
return `${amount} day`;
default:
return granularity;
}
}

function mapGranularitiesToExternalFormat(source: TSource): TSource {
if (!('materializedViews' in source)) return source;

const mapped = { ...source };

if (Array.isArray(source.materializedViews)) {
mapped.materializedViews = source.materializedViews.map(view => ({
...view,
minGranularity: mapGranularityToExternalFormat(view.minGranularity),
}));
}

if (
'metadataMaterializedViews' in source &&
source.metadataMaterializedViews
) {
mapped.metadataMaterializedViews = {
...source.metadataMaterializedViews,
granularity: mapGranularityToExternalFormat(
source.metadataMaterializedViews.granularity,
),
};
}

return mapped;
}

function applyLegacyDefaults(parsed: unknown): unknown {
// Legacy Session sources were created before timestampValueExpression was
// required. The old code defaulted it to 'TimestampTime' at query time.
if (
parsed != null &&
typeof parsed === 'object' &&
'kind' in parsed &&
parsed.kind === SourceKind.Session &&
!('timestampValueExpression' in parsed && parsed.timestampValueExpression)
) {
return { ...parsed, timestampValueExpression: 'TimestampTime' };
}
return parsed;
}

/**
* A source in the shape the external API (`/api/v2/sources`) returns: only
* SourceSchema fields, with short-form granularities ("5m"). `json` must be
* plain JSON (ids as strings).
*/
export function toExternalSource(json: unknown) {
const result = SourceSchema.safeParse(applyLegacyDefaults(json));
return result.success
? {
success: true as const,
data: mapGranularitiesToExternalFormat(result.data),
}
: { success: false as const, error: result.error };
}
Loading