feat: Support JSON Sessions (#1628)

Author: pulpdrew

.changeset/breezy-cougars-shave.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+feat: Support JSON Sessions

docker-compose.dev.yml

@@ -72,8 +72,6 @@ services:
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
-      # Add a custom config file
-      - ./docker/otel-collector/dev.json.config.yaml:/etc/otelcol-contrib/custom.config.yaml
     ports:
       - '14318:4318' # OTLP http receiver
     restart: always

docker/otel-collector/dev.json.config.yaml

@@ -17,6 +17,8 @@ import {
 import { useRRWebEventStream } from '@/sessions';
 import { useDebugMode } from '@/utils';
 
+import { FieldExpressionGenerator } from './hooks/useFieldExpressionGenerator';
+
 import styles from '../styles/SessionSubpanelV2.module.scss';
 
 function getPlayerCurrentTime(player: Replayer) {
@@ -27,7 +29,7 @@ const URLHoverCard = memo(({ url }: { url: string }) => {
   let parsedUrl: URL | undefined;
   try {
     parsedUrl = new URL(url);
-  } catch (e) {
+  } catch {
     // ignore
   }
 
@@ -38,7 +40,7 @@ const URLHoverCard = memo(({ url }: { url: string }) => {
     for (const [key, value] of _searchParams.entries()) {
       searchParams.push({ key, value });
     }
-  } catch (e) {
+  } catch {
     // ignore
   }
 
@@ -92,6 +94,7 @@ export default function DOMPlayer({
   setPlayerFullWidth,
   playerFullWidth,
   resizeKey,
+  getSessionSourceFieldExpression,
 }: {
   config: {
     dateRange: [Date, Date];
@@ -106,14 +109,11 @@ export default function DOMPlayer({
   playerSpeed: number;
   setPlayerStartTimestamp?: (ts: number) => void;
   setPlayerEndTimestamp?: (ts: number) => void;
-  // setPlayerSpeed: (playerSpeed: number) => void;
   skipInactive: boolean;
-  // setSkipInactive: (skipInactive: boolean) => void;
-  // highlightedResultId: string | undefined;
-  // onClick: (logId: string, sortKey: number) => void;
   resizeKey?: string;
   setPlayerFullWidth: (fullWidth: boolean) => void;
   playerFullWidth: boolean;
+  getSessionSourceFieldExpression: FieldExpressionGenerator;
 }) {
   const debug = useDebugMode();
   const wrapper = useRef<HTMLDivElement>(null);
@@ -141,7 +141,7 @@ export default function DOMPlayer({
       limit: 1000000, // large enough to get all events
       onEvent: (event: { b: string; ck: number; tcks: number; t: number }) => {
         try {
-          const { b: body, ck: chunk, tcks: totalChunks, t: type } = event;
+          const { b: body, ck: chunk, tcks: totalChunks } = event;
           currentRrwebEvent += body;
           if (!chunk || chunk === totalChunks) {
             const parsedEvent = JSON.parse(currentRrwebEvent);
@@ -202,11 +202,10 @@ export default function DOMPlayer({
           }
         }
       },
+      getSessionSourceFieldExpression,
     },
     {
-      enabled: dateRange != null,
-      // @ts-ignore
-      keepPreviousData: true, // TODO: support streaming
+      keepPreviousData: true,
       shouldAbortPendingRequest: true,
     },
   );
@@ -358,7 +357,7 @@ export default function DOMPlayer({
     setIsReplayerInitialized(true);
 
     if (debug) {
-      // @ts-ignore
+      // @ts-expect-error this is for debugging purposes only
       window.__hdx_replayer = replayer.current;
     }
 

packages/app/src/DOMPlayer.tsx

@@ -1,9 +1,11 @@
 import { useMemo } from 'react';
 import uniqBy from 'lodash/uniqBy';
 import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types';
+import { keepPreviousData } from '@tanstack/react-query';
 
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
 
+import { SessionRow, sessionRowSchema } from './utils/sessions';
 import type { PlaybarMarker } from './PlaybarSlider';
 import { PlaybarSlider } from './PlaybarSlider';
 import { getShortUrl } from './utils';
@@ -29,14 +31,19 @@ export default function Playbar({
   const maxSliderVal = Math.ceil(playbackRange[1].getTime() / 1000) * 1000;
   const minSliderVal = Math.floor(playbackRange[0].getTime() / 1000) * 1000;
 
-  const { data, isLoading, isError, error } = useQueriedChartConfig(
-    queriedConfig,
-    {
-      placeholderData: (prev: any) => prev,
-      queryKey: ['PlayBar', queriedConfig],
-    },
+  const { data } = useQueriedChartConfig(queriedConfig, {
+    placeholderData: keepPreviousData,
+    queryKey: ['PlayBar', queriedConfig],
+  });
+
+  const events: SessionRow[] = useMemo(
+    () =>
+      data?.data
+        ?.map(row => sessionRowSchema.safeParse(row))
+        .filter(parsed => parsed.success)
+        .map(parsed => parsed.data) ?? [],
+    [data?.data],
   );
-  const events: any[] = useMemo(() => data?.data ?? [], [data?.data]);
 
   const markers = useMemo<PlaybarMarker[]>(() => {
     return uniqBy(
@@ -59,14 +66,14 @@ export default function Playbar({
         )
         .map(event => {
           const spanName = event['span_name'];
-          const locationHref = event['location.href'];
+          const locationHref = event['location.href'] ?? '';
           const shortLocationHref = getShortUrl(locationHref);
 
-          const errorMessage = event['error.message'];
+          const errorMessage = event['error.message'] ?? '';
 
-          const url = event['http.url'];
-          const statusCode = event['http.status_code'];
-          const method = event['http.method'];
+          const url = event['http.url'] ?? '';
+          const statusCode = Number(event['http.status_code'] ?? '');
+          const method = event['http.method'] ?? '';
           const shortUrl = getShortUrl(url);
 
           const isNavigation =
@@ -94,7 +101,7 @@ export default function Playbar({
                   ? errorMessage
                   : spanName === 'intercom.onShow'
                     ? 'Intercom Chat Opened'
-                    : event.body,
+                    : (event.body ?? ''),
             isError,
             isSuccess,
           };

packages/app/src/Playbar.tsx

@@ -2,7 +2,7 @@ import * as React from 'react';
 import cx from 'classnames';
 import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types';
 import { ScrollArea, Skeleton, Stack } from '@mantine/core';
-import { useThrottledCallback, useThrottledValue } from '@mantine/hooks';
+import { useThrottledValue } from '@mantine/hooks';
 import {
   IconArrowsLeftRight,
   IconMapPin,
@@ -16,18 +16,18 @@ import { useVirtualizer } from '@tanstack/react-virtual';
 import useRowWhere, { RowWhereResult } from '@/hooks/useRowWhere';
 
 import { useQueriedChartConfig } from './hooks/useChartConfig';
+import { SessionRow, sessionRowSchema } from './utils/sessions';
 import { useFormatTime } from './useFormatTime';
 import { formatmmss, getShortUrl } from './utils';
 
 import styles from '../styles/SessionSubpanelV2.module.scss';
 
 type SessionEvent = {
   id: string;
-  row: Record<string, string>; // original row object
-  sortKey: string;
+  row: SessionRow; // original row object
   isError: boolean;
   isSuccess: boolean;
-  eventSource: 'navigation' | 'chat' | 'network' | 'custom';
+  eventSource: 'navigation' | 'chat' | 'network' | 'custom' | 'log';
   title: string;
   description: string;
   timestamp: Date;
@@ -112,47 +112,48 @@ export const SessionEventList = ({
   onTimeClick: (ts: number) => void;
   eventsFollowPlayerPosition: boolean;
 }) => {
-  const { data, isLoading, isError, error, isPlaceholderData, isSuccess } =
-    useQueriedChartConfig(queriedConfig, {
-      placeholderData: (prev: any) => prev,
-      queryKey: ['SessionEventList', queriedConfig],
-    });
+  const { data, isLoading } = useQueriedChartConfig(queriedConfig, {
+    placeholderData: (prev: any) => prev,
+    queryKey: ['SessionEventList', queriedConfig],
+  });
   const formatTime = useFormatTime();
 
-  const events = React.useMemo(() => data?.data ?? [], [data?.data]);
+  const events: SessionRow[] = React.useMemo(
+    () =>
+      data?.data
+        ?.map(row => sessionRowSchema.safeParse(row))
+        .filter(parsed => parsed.success)
+        .map(parsed => parsed.data) ?? [],
+    [data?.data],
+  );
 
   const getRowWhere = useRowWhere({ meta: data?.meta, aliasMap });
 
   const rows = React.useMemo(() => {
     return (
-      events.map((event, i) => {
+      events.map(event => {
         const { timestamp, durationInMs } = event;
 
         // TODO: we should just use timestamp and durationInMs instead of startOffset and endOffset
         const startOffset = new Date(timestamp).getTime();
         const endOffset = new Date(startOffset).getTime() + durationInMs;
 
-        const isHighlighted = false;
-
-        const url = event['http.url'];
-        const statusCode = event['http.status_code'];
-        const method = event['http.method'];
+        const url = event['http.url'] ?? '';
+        const statusCode = Number(event['http.status_code'] ?? '');
+        const method = event['http.method'] ?? '';
         const shortUrl = getShortUrl(url);
 
         const isNetworkRequest =
           method != '' && method != null && url != null && url != '';
 
         const errorMessage = event['error.message'];
 
-        const body = event['body'];
+        const body = event['body'] ?? '';
         const component = event['component'];
         const spanName = event['span_name'];
-        const locationHref = event['location.href'];
+        const locationHref = event['location.href'] ?? '';
         const otelLibraryName = event['otel.library.name'];
         const shortLocationHref = getShortUrl(locationHref);
-        const isException =
-          event['exception.group_id'] != '' &&
-          event['exception.group_id'] != null;
 
         const isCustomEvent = otelLibraryName === 'custom-action';
         const isNavigation =
@@ -167,7 +168,6 @@ export const SessionEventList = ({
 
         return {
           id: event.id,
-          sortKey: event.sort_key,
           row: event,
           isError,
           isSuccess,
@@ -182,29 +182,27 @@ export const SessionEventList = ({
                   : 'log',
           title: isNavigation
             ? `Navigated`
-            : isException
-              ? 'Exception'
-              : spanName === 'console.error'
-                ? 'console.error'
-                : spanName === 'console.log'
-                  ? 'console.log'
-                  : spanName === 'console.warn'
-                    ? 'console.warn'
-                    : url.length > 0
-                      ? `${statusCode} ${method}`
-                      : spanName === 'intercom.onShow'
-                        ? 'Intercom Chat Opened'
-                        : isCustomEvent
+            : spanName === 'console.error'
+              ? 'console.error'
+              : spanName === 'console.log'
+                ? 'console.log'
+                : spanName === 'console.warn'
+                  ? 'console.warn'
+                  : url.length > 0
+                    ? `${statusCode} ${method}`
+                    : spanName === 'intercom.onShow'
+                      ? 'Intercom Chat Opened'
+                      : isCustomEvent
+                        ? spanName
+                        : component === 'console'
                           ? spanName
-                          : component === 'console'
-                            ? spanName
-                            : 'console.error',
+                          : 'console.error',
           description: isNavigation
             ? shortLocationHref
             : errorMessage != null && errorMessage.length > 0
               ? errorMessage
               : component === 'console'
-                ? body
+                ? (body ?? '')
                 : url.length > 0
                   ? shortUrl
                   : '',
@@ -215,7 +213,7 @@ export const SessionEventList = ({
                 format: 'time',
               }),
           duration: endOffset - startOffset,
-        } as SessionEvent;
+        } satisfies SessionEvent;
       }) ?? []
     );
   }, [events, showRelativeTime, minTs, formatTime]);