Timeline resource streaming

e2e-tests/fixtures/Plan.ts

@@ -292,7 +292,7 @@
   }
 
   async fillActivityPresetName(presetName: string) {
     await this.panelActivityForm.getByRole('combobox', { name: 'None' }).click();
     await this.panelActivityForm.locator('.dropdown-header').waitFor({ state: 'attached' });
     await this.panelActivityForm.getByPlaceholder('Enter preset name').click();
     await this.panelActivityForm.getByPlaceholder('Enter preset name').fill(presetName);
@@ -337,6 +337,7 @@
   async goto(planId = this.plans.planId) {
     await this.page.goto(`/plans/${planId}`, { waitUntil: 'load' });
     await this.page.waitForURL(`/plans/${planId}`, { waitUntil: 'load' });
+    await this.planTitle.waitFor({ state: 'visible' });
     await this.waitForTimelineLoading();
   }
 

e2e-tests/tests/plan-external-source.test.ts

@@ -1,6 +1,7 @@
 import test, { expect } from '@playwright/test';
 import { ExternalSources } from '../fixtures/ExternalSources.js';
 import { PanelNames, Plan } from '../fixtures/Plan.js';
+import { anyCanvasHasContent } from '../utilities/canvas.js';
 import {
   cleanupApiResources,
   closeBrowserResources,
@@ -157,22 +158,7 @@
   });
 
   test('Zero-duration events are properly drawn in the timeline', async () => {
-    // Get the current timeline canvas' pixels - use a set to just determine that non-0 RGB values exist
-    const doPixelsExist: boolean = await setup.page.evaluate(() => {
-      const canvas = document.querySelector('canvas');
-      if (canvas !== null && canvas !== undefined) {
-        const context = canvas.getContext('2d');
-        if (context !== null && context !== undefined) {
-          const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
-          const pixelData = Array.from(imageData.data);
-          return pixelData.length > 0 ? true : false;
-          // Assert that the number of unique RGB pixel values for the canvas is more than 0 (i.e., not empty)
-        }
-      }
-      return false;
-    });
-
-    expect(doPixelsExist).toBeTruthy();
+    expect(await anyCanvasHasContent(setup.page, '[data-component-name="TimelinePanel"] canvas')).toBeTruthy();
   });
 
   test('Linked derivation groups should be expandable in panel', async () => {

e2e-tests/tests/simulation.test.ts

@@ -2,6 +2,7 @@ import test, { expect } from '@playwright/test';
 import { Status } from '../../src/enums/status.js';
 import { PanelNames } from '../fixtures/Plan.js';
 import { setupTest, teardownTest, type FullSetupResult } from '../utilities/api.js';
+import { anyCanvasHasContent } from '../utilities/canvas.js';
 
 let setup: FullSetupResult;
 
@@ -61,9 +62,31 @@ test.describe.serial('Simulation', async () => {
     await setup.plan.showPanel(PanelNames.SIMULATION, true);
   });
 
+  // Smoke test for the windowed-pull pipeline: indicator settles cleanly and
+  // canvases render non-transparent content (catches the "blank plot" bug an
+  // indicator-only check would miss).
+  test(`Streaming pipeline: indicator settles + canvases render across two re-sims`, async () => {
+    const timelineErrorIndicator = setup.plan.page.getByRole('status', { name: 'Timeline data error' });
+    const timelineLoadingIndicator = setup.plan.page.getByRole('status', { name: 'Timeline loading' });
+    const timelineCanvasContent = () => anyCanvasHasContent(setup.page, '[data-component-name="TimelinePanel"] canvas');
+
+    await setup.plan.reRunSimulation();
+    await expect(timelineErrorIndicator).not.toBeVisible();
+    await expect(timelineLoadingIndicator).not.toBeVisible();
+    await expect.poll(timelineCanvasContent, { timeout: 10000 }).toBe(true);
+
+    await setup.plan.reRunSimulation();
+    await expect(timelineErrorIndicator).not.toBeVisible();
+    await expect(timelineLoadingIndicator).not.toBeVisible();
+    await expect.poll(timelineCanvasContent, { timeout: 10000 }).toBe(true);
+  });
+
   test(`Plans with an invalid activity should fail simulation`, async () => {
+    const timelineLoadingIndicator = setup.plan.page.getByRole('status', { name: 'Timeline loading' });
     await setup.plan.addActivity('BakeBananaBread');
     await setup.plan.runSimulation(Status.Failed);
+    // Regression: indicator must settle for terminal-null sims too.
+    await expect(timelineLoadingIndicator).not.toBeVisible();
   });
 
   test(`Modified plans should indicate that simulation is out of date`, async () => {

e2e-tests/utilities/canvas.ts

@@ -0,0 +1,25 @@
+import type { Page } from '@playwright/test';
+
+/**
+ * True if any canvas matched by `selector` has a pixel with non-zero alpha.
+ * Stronger than checking the canvas exists — a transparent canvas would
+ * pass that. Pair with `expect.poll` for post-async-update checks.
+ */
+export function anyCanvasHasContent(page: Page, selector: string = 'canvas'): Promise<boolean> {
+  return page.evaluate(sel => {
+    const canvases = document.querySelectorAll<HTMLCanvasElement>(sel);
+    for (const canvas of Array.from(canvases)) {
+      const ctx = canvas.getContext('2d');
+      if (!ctx) {
+        continue;
+      }
+      const data = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
+      for (let i = 3; i < data.length; i += 4) {
+        if (data[i] > 0) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }, selector);
+}

src/components/ResourceList.svelte

@@ -4,12 +4,7 @@
   import CloseIcon from '@nasa-jpl/stellar/icons/close.svg?component';
   import UploadIcon from '@nasa-jpl/stellar/icons/upload.svg?component';
   import { plan } from '../stores/plan';
-  import {
-    allResourceTypes,
-    fetchingResourcesExternal,
-    resourceTypesLoading,
-    simulationDatasetId,
-  } from '../stores/simulation';
+  import { allResourceTypes, resourceTypesLoading, simulationDatasetId } from '../stores/simulation';
   import type { User } from '../types/app';
   import type { ResourceType } from '../types/simulation';
   import type { TimelineItemType } from '../types/timeline';
@@ -34,7 +29,7 @@
   let loading: boolean = false;
 
   $: resourceDataTypes = [...new Set($allResourceTypes.map(t => t.schema.type))];
-  $: loading = $fetchingResourcesExternal || $resourceTypesLoading;
+  $: loading = $resourceTypesLoading;
   $: if (user !== null && $plan !== null) {
     hasUploadPermission = featurePermissions.externalResources.canCreate(user, $plan);
   }

src/components/timeline/Row.svelte

@@ -4,24 +4,19 @@
   import type { ScaleTime } from 'd3-scale';
   import { select, type Selection } from 'd3-selection';
   import { zoom as d3Zoom, zoomIdentity, type D3ZoomEvent, type ZoomBehavior, type ZoomTransform } from 'd3-zoom';
-  import { createEventDispatcher } from 'svelte';
+  import { createEventDispatcher, onDestroy } from 'svelte';
   import FilterWithXIcon from '../../assets/filter-with-x.svg?component';
   import { ViewDefaultDiscreteOptions } from '../../constants/view';
-  import { Status } from '../../enums/status';
   import { activityArgumentDefaultsMap } from '../../stores/activities';
-  import { catchError, logMessage } from '../../stores/errors';
   import {
     derivationGroupVisibilityMap,
     externalSources,
     planDerivationGroupLinks,
   } from '../../stores/external-source';
   import { planModelActivityTypes } from '../../stores/plan';
-  import {
-    externalResources,
-    fetchingResourcesExternal,
-    resourceTypes,
-    resourceTypesLoading,
-  } from '../../stores/simulation';
+  import { createExternalResourceSubscription } from '../../stores/externalResource';
+  import { createProfileSubscription } from '../../stores/profile';
+  import { resourceTypes, resourceTypesLoading } from '../../stores/simulation';
   import { selectedRow, viewAddFilterToRow } from '../../stores/views';
   import type {
     ActivityDirective,
@@ -66,8 +61,6 @@
   import { getExternalEventRowId } from '../../utilities/externalEvents';
   import { classNames } from '../../utilities/generic';
   import { showConfirmActivityCreationModal } from '../../utilities/modal';
-  import { sampleProfiles } from '../../utilities/resources';
-  import { getSimulationStatus } from '../../utilities/simulation';
   import { pluralize } from '../../utilities/text';
   import { getDoyTime } from '../../utilities/time';
   import {
@@ -220,7 +213,7 @@
     }
   });
 
-  $: if (plan && simulationDataset !== null && layers && $externalResources && !$resourceTypesLoading) {
+  $: if (plan && simulationDataset !== null && layers && !$resourceTypesLoading) {
     const simulationDatasetId = simulationDataset.dataset_id;
     const resourceNamesSet = new Set<string>();
     layers.map(layer => {
@@ -232,120 +225,66 @@
     });
     const resourceNames = Array.from(resourceNamesSet);
 
-    // Cancel and delete unused and stale requests as well as any external resources that
-    // are not in the list of current external resources
+    // Drop entries no longer referenced by any layer or whose sim dataset
+    // changed. Both factories own their own registry cleanup on unsubscribe.
     Object.entries(resourceRequestMap).forEach(([key, value]) => {
-      if (
-        resourceNames.indexOf(key) < 0 ||
-        value.simulationDatasetId !== simulationDatasetId ||
-        (value.type === 'external' && !$resourceTypes.find(type => type.name === name))
-      ) {
-        value.controller?.abort();
+      if (resourceNames.indexOf(key) < 0 || value.simulationDatasetId !== simulationDatasetId) {
+        value.unsubscribe?.();
         delete resourceRequestMap[key];
         resourceRequestMap = { ...resourceRequestMap };
       }
     });
 
-    // Only update if simulation is complete
-    if (
-      getSimulationStatus(simulationDataset) === Status.Complete ||
-      getSimulationStatus(simulationDataset) === Status.Canceled
-    ) {
-      const startTimeYmd = simulationDataset?.simulation_start_time ?? plan.start_time;
-      resourceNames.forEach(async name => {
-        // Check if resource is external
-        const isExternal = !$resourceTypes.find(type => type.name === name);
-        if (isExternal) {
-          // Handle external datasets separately as they are globally loaded and subscribed to
-          let resource = null;
-          if (!$fetchingResourcesExternal) {
-            resource = $externalResources.find(resource => resource.name === name) || null;
-          }
-          let error = !resource && !$fetchingResourcesExternal ? 'External Profile not Found' : '';
-
-          resourceRequestMap = {
-            ...resourceRequestMap,
-            [name]: {
-              ...resourceRequestMap[name],
-              error,
-              loading: $fetchingResourcesExternal,
-              resource,
-              simulationDatasetId,
-              type: 'external',
-            },
-          };
-        } else {
-          // Skip matching resources requests that have already been added for this simulation
-          if (
-            resourceRequestMap[name] &&
-            simulationDatasetId === resourceRequestMap[name].simulationDatasetId &&
-            (resourceRequestMap[name].loading || resourceRequestMap[name].error || resourceRequestMap[name].resource)
-          ) {
-            return;
-          }
+    const startTimeYmd = simulationDataset?.simulation_start_time ?? plan.start_time;
+    resourceNames.forEach(name => {
+      if (
+        resourceRequestMap[name] &&
+        simulationDatasetId === resourceRequestMap[name].simulationDatasetId &&
+        resourceRequestMap[name].unsubscribe
+      ) {
+        return;
+      }
 
-          const controller = new AbortController();
-          resourceRequestMap = {
-            ...resourceRequestMap,
-            [name]: {
-              ...resourceRequestMap[name],
-              controller,
-              error: '',
-              loading: true,
-              resource: null,
-              simulationDatasetId,
-              type: 'internal',
+      const isExternal = !$resourceTypes.find(type => type.name === name);
+      const subscription = isExternal
+        ? createExternalResourceSubscription(simulationDatasetId, name, startTimeYmd, user)
+        : createProfileSubscription(simulationDatasetId, name, startTimeYmd, user);
+      const type: 'external' | 'internal' = isExternal ? 'external' : 'internal';
+      // Declared before .subscribe() so the closure in `unsubscribe` below
+      // doesn't lean on TDZ-via-const initialization order.
+      let storeUnsubscribe: (() => void) | null = null;
+      storeUnsubscribe = subscription.store.subscribe(({ error, loading, resource }) => {
+        resourceRequestMap = {
+          ...resourceRequestMap,
+          [name]: {
+            ...resourceRequestMap[name],
+            error,
+            loading,
+            resource,
+            simulationDatasetId,
+            type,
+            unsubscribe: () => {
+              storeUnsubscribe?.();
+              subscription.unsubscribe();
             },
-          };
-
-          let resource = null;
-          let error = '';
-          let aborted = false;
-          try {
-            const startTime = performance.now();
-            const response = await effects.getResource(simulationDatasetId, name, user, controller.signal);
-            const { profile } = response;
-            if (profile && profile.length === 1) {
-              resource = sampleProfiles([profile[0]], startTimeYmd)[0];
-              logMessage(
-                `Retrieved profile ${name} (${profile[0].profile_segments.length} segment${pluralize(profile[0].profile_segments.length)}) for simulation ${simulationDatasetId}.`,
-                '',
-                performance.now() - startTime,
-              );
-            } else {
-              throw new Error('Profile not Found');
-            }
-          } catch (e) {
-            const err = e as Error;
-            if (err.name === 'AbortError') {
-              aborted = true;
-            } else {
-              catchError(`Profile Download Failed for ${name}`, e as Error);
-              error = err.message;
-            }
-          } finally {
-            if (!aborted) {
-              resourceRequestMap = {
-                ...resourceRequestMap,
-                [name]: {
-                  ...resourceRequestMap[name],
-                  error,
-                  loading: false,
-                  resource,
-                },
-              };
-            }
-          }
-        }
+          },
+        };
       });
-    }
+    });
   } else if (simulationDataset === null) {
-    Object.entries(resourceRequestMap).forEach(([_key, value]) => {
-      value.controller?.abort();
+    Object.values(resourceRequestMap).forEach(value => {
+      value.unsubscribe?.();
     });
     resourceRequestMap = {};
   }
 
+  onDestroy(() => {
+    Object.values(resourceRequestMap).forEach(value => {
+      value.unsubscribe?.();
+    });
+    resourceRequestMap = {};
+  });
+
   $: onDragenter(dragenter);
   $: onDragleave(dragleave);
   $: onDragover(dragover);
@@ -384,21 +323,23 @@
   $: if (resourceRequestMap) {
     const newLoadedResources: Resource[] = [];
     const newLoadingErrors: string[] = [];
+    let anyLoading = false;
     Object.values(resourceRequestMap).forEach(resourceRequest => {
       if (resourceRequest.resource) {
         newLoadedResources.push(resourceRequest.resource);
       }
       if (resourceRequest.error) {
         newLoadingErrors.push(resourceRequest.error);
       }
+      if (resourceRequest.loading) {
+        anyLoading = true;
+      }
     });
     loadedResources = newLoadedResources;
     resourceLoadingErrors = newLoadingErrors;
-
-    // Consider row to be loading if the number of completed resource requests (loaded or error state)
-    // is not equal to the total number of resource requests
-    anyResourcesLoading =
-      loadedResources.length + resourceLoadingErrors.length !== Object.keys(resourceRequestMap).length;
+    // Use per-request loading flag, not loaded+errored vs total: a request
+    // with both data and an error would be double-counted and stick true.
+    anyResourcesLoading = anyLoading;
   }
 
   // Compute scale domains for axes since it is optionally defined in the view