Upgrade glassnode-api to 1.0.0; recover rate-limited startup metadata

package.json

@@ -66,10 +66,10 @@
     "@x402/fetch": "2.26.0",
     "canvas": "^3.2.3",
     "chafa-wasm": "^0.3.3",
-    "glassnode-api": "^0.29.3",
+    "glassnode-api": "^1.0.0",
     "ink": "^7.1.1",
     "ink-spinner": "^5.0.0",
-    "ink-uplot": "^0.2.15",
+    "ink-uplot": "^0.2.16",
     "path2d": "^0.3.4",
     "react": "^19.3.0",
     "uplot": "^1.6.32",

src/lib/api-client.ts

@@ -8,6 +8,10 @@ const { version } = require('../../package.json') as { version: string };
 export const USER_AGENT = `glassnode-terminal-${version}`;
 
 let clientInstance: GlassnodeAPI | null = null;
+// API-key mode only: see createStartupClient().
+let startupClientInstance: GlassnodeAPI | null = null;
+const STARTUP_MAX_RETRIES = 5;
+const STARTUP_MAX_RETRY_DELAY_MS = 60_000; // the API's rate-limit window
 let x402Enabled = false;
 let x402FetchStore: typeof globalThis.fetch | null = null;
 let x402BaseUrl = 'https://x402.glassnode.com';
@@ -67,6 +71,23 @@ const userAgentFetch: typeof globalThis.fetch = (input, init) => {
   return globalThis.fetch(req);
 };
 
+/**
+ * The API's 429s carry `x-rate-limit-reset` (seconds until the per-minute window resets)
+ * but no `Retry-After`, so glassnode-api falls back to short jittered backoff and can give
+ * up before the window resets. Expose the reset as `Retry-After`, which glassnode-api honours
+ * (capped at `maxRetryDelay`), so a retry waits for the new window.
+ */
+export function rateLimitAwareFetch(inner: typeof globalThis.fetch): typeof globalThis.fetch {
+  return async (input, init) => {
+    const res = await inner(input, init);
+    const reset = res.headers.get('x-rate-limit-reset');
+    if (res.status !== 429 || res.headers.has('retry-after') || !reset || !/^\d+$/.test(reset)) return res;
+    const headers = new Headers(res.headers);
+    headers.set('retry-after', reset);
+    return new Response(res.body, { status: res.status, statusText: res.statusText, headers });
+  };
+}
+
 /**
  * Initialize the shared GlassnodeAPI client. Must be awaited once at startup,
  * before any createClient() call. Credentials are chosen from the environment:
@@ -131,6 +152,13 @@ export async function initClient(): Promise<GlassnodeAPI> {
   }
 
   clientInstance = new GlassnodeAPI({ apiKey, fetch: userAgentFetch, logger });
+  startupClientInstance = new GlassnodeAPI({
+    apiKey,
+    fetch: rateLimitAwareFetch(userAgentFetch),
+    logger,
+    maxRetries: STARTUP_MAX_RETRIES,
+    maxRetryDelay: STARTUP_MAX_RETRY_DELAY_MS,
+  });
   log('info', 'API-key mode enabled');
   return clientInstance;
 }
@@ -145,3 +173,15 @@ export function createClient(): GlassnodeAPI {
   }
   return clientInstance;
 }
+
+/**
+ * Client for the startup metadata burst (~1,800 calls against a 1,200/minute limit). With
+ * glassnode-api's default 2 short retries ~16% of the metadata was rate-limited (429) and
+ * lost. This client retries up to 5 times and, via rateLimitAwareFetch, waits for the
+ * rate-limit window to reset. Interactive calls keep the default so a rate limit doesn't
+ * stall the UI. x402 mode has no separate client: retrying a paid call could pay twice,
+ * so it stays at 0 retries.
+ */
+export function createStartupClient(): GlassnodeAPI {
+  return startupClientInstance ?? createClient();
+}

src/lib/startup-data.ts

@@ -1,5 +1,5 @@
 import type { AssetMetadata, MetricMetadata } from 'glassnode-api';
-import { createClient, isX402Enabled, fetchX402AdvancedPaths } from './api-client.js';
+import { createClient, createStartupClient, isX402Enabled, fetchX402AdvancedPaths } from './api-client.js';
 import { readCache, writeCache } from './cache.js';
 import type { MetricListItem } from './types.js';
 import { X402_CATALOG, X402_ASSET_SETS, X402_ASSETS, type X402CatalogEntry } from './x402-catalog.js';
@@ -11,14 +11,16 @@ export interface StartupData {
   metricMetadataMap: Record<string, MetricMetadata>;
 }
 
-const BATCH_SIZE = 20;
+const BATCH_SIZE = 10;
 const BATCH_DELAY_MS = 100;
 
 async function fetchAllMetricMetadata(
   metricPaths: string[],
   onProgress?: (done: number, total: number) => void,
 ): Promise<Record<string, MetricMetadata>> {
-  const client = createClient();
+  // Batches of 10 plus the startup client's rate-limit-aware retries keep lost (429) calls
+  // near zero; measured: 2 retries/batch 20 lost ~296 of 1,817.
+  const client = createStartupClient();
   const result: Record<string, MetricMetadata> = {};
 
   for (let i = 0; i < metricPaths.length; i += BATCH_SIZE) {
@@ -46,15 +48,16 @@ async function fetchAllMetricMetadata(
 }
 
 /** Sort assets by descending market cap, then symbol. */
-function sortAssetsByMcap(
+export function sortAssetsByMcap(
   assetData: AssetMetadata[],
-  mcapData: Array<{ bulk: Array<{ a: string; v: number }> }>,
+  // v is null for an asset with no value at that time (glassnode-api >= 0.30).
+  mcapData: Array<{ bulk: Array<{ a: string; v: number | null }> }>,
 ): AssetMetadata[] {
   const mcapBySymbol = new Map<string, number>();
   if (mcapData.length > 0) {
     const latest = mcapData[mcapData.length - 1]!;
     for (const entry of latest.bulk) {
-      mcapBySymbol.set(entry.a.toUpperCase(), entry.v);
+      if (entry.v !== null) mcapBySymbol.set(entry.a.toUpperCase(), entry.v);
     }
   }
   return [...assetData].sort((a, b) => {
@@ -65,6 +68,15 @@ function sortAssetsByMcap(
   });
 }
 
+/**
+ * Metrics to list: everything except point-in-time (`_pit`) variants. Checks the path suffix
+ * as well as `is_pit`, so a variant whose metadata failed to load (e.g. rate-limited) is still
+ * hidden; every `is_pit` metric's path ends in `_pit` and no other does.
+ */
+export function listedMetrics(paths: string[], metadata: Record<string, MetricMetadata>): string[] {
+  return paths.filter((p) => !(metadata[p]?.is_pit || p.endsWith('_pit')));
+}
+
 /** Turn catalog entries into the metrics list + a metadata map buildMetricList understands. */
 function catalogToStartupParts(entries: readonly X402CatalogEntry[]): {
   metrics: string[];
@@ -93,7 +105,9 @@ export async function loadStartupData(
   onProgress?: (message: string) => void,
 ): Promise<StartupData> {
   // Cache is per-mode: x402 shows only the ~326 advanced metrics, API mode shows all.
-  const cacheKey = isX402Enabled() ? 'startup-data-x402' : 'startup-data';
+  // v2: glassnode-api 1.0 + rate-limit-aware metadata fetch — don't reuse a day-old cache
+  // that lost metadata to rate limits (missing names, leaked _pit metrics).
+  const cacheKey = isX402Enabled() ? 'startup-data-x402' : 'startup-data-v2';
 
   const cached = readCache<StartupData>(cacheKey);
   if (cached) {
@@ -131,8 +145,7 @@ async function loadStartupDataApi(
     onProgress?.(`Fetching metric metadata... ${done}/${total}`);
   });
 
-  const filteredMetrics = metricPaths.filter((p) => !metricMetadataMap[p]?.is_pit);
-  return { assets: sortedAssets, metrics: filteredMetrics, metricMetadataMap };
+  return { assets: sortedAssets, metrics: listedMetrics(metricPaths, metricMetadataMap), metricMetadataMap };
 }
 
 /**

test/lib/api-client.spec.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const constructed = vi.hoisted(() => [] as Array<Record<string, unknown>>);
+vi.mock('glassnode-api', () => ({
+  GlassnodeAPI: class {
+    constructor(public config: Record<string, unknown>) {
+      constructed.push(config);
+    }
+  },
+}));
+
+describe('api-client startup client', () => {
+  beforeEach(() => {
+    constructed.length = 0;
+    vi.resetModules();
+    delete process.env['X402_PRIVATE_KEY'];
+    process.env['GLASSNODE_API_KEY'] = 'test-key';
+    process.env['GLASSNODE_MODE'] = 'api';
+  });
+
+  it('uses a separate client with more retries for the startup metadata burst (API-key mode)', async () => {
+    const { initClient, createClient, createStartupClient } = await import('../../src/lib/api-client.js');
+    await initClient();
+    const interactive = createClient() as unknown as { config: Record<string, unknown> };
+    const startup = createStartupClient() as unknown as { config: Record<string, unknown> };
+    expect(startup).not.toBe(interactive);
+    // Interactive calls keep glassnode-api's default; the startup burst retries rate limits more.
+    expect(interactive.config['maxRetries']).toBeUndefined();
+    expect(startup.config['maxRetries']).toBe(5);
+    // Long enough to wait out the API's 60 s rate-limit window (via Retry-After).
+    expect(startup.config['maxRetryDelay']).toBe(60_000);
+    expect(startup.config['fetch']).not.toBe(interactive.config['fetch']);
+    expect(startup.config['apiKey']).toBe('test-key');
+  });
+});
+
+describe('rateLimitAwareFetch', () => {
+  const respond = (status: number, headers: Record<string, string>) =>
+    vi.fn(async () => new Response('{}', { status, headers }));
+
+  it("exposes the API's x-rate-limit-reset as Retry-After on a 429", async () => {
+    const { rateLimitAwareFetch } = await import('../../src/lib/api-client.js');
+    const res = await rateLimitAwareFetch(respond(429, { 'x-rate-limit-reset': '37' }))('https://x');
+    expect(res.status).toBe(429);
+    expect(res.headers.get('retry-after')).toBe('37');
+  });
+
+  it('leaves other responses and an existing Retry-After alone', async () => {
+    const { rateLimitAwareFetch } = await import('../../src/lib/api-client.js');
+    const ok = await rateLimitAwareFetch(respond(200, { 'x-rate-limit-reset': '37' }))('https://x');
+    expect(ok.headers.get('retry-after')).toBeNull();
+    const own = await rateLimitAwareFetch(respond(429, { 'x-rate-limit-reset': '37', 'retry-after': '5' }))('https://x');
+    expect(own.headers.get('retry-after')).toBe('5');
+  });
+});

test/lib/startup-data.spec.ts

@@ -0,0 +1,28 @@
+import { describe, it, expect, vi } from 'vitest';
+
+vi.mock('../../src/lib/api-client.js', () => ({ createClient: vi.fn(), createStartupClient: vi.fn(), isX402Enabled: () => false }));
+const { sortAssetsByMcap, listedMetrics } = await import('../../src/lib/startup-data.js');
+
+const asset = (symbol: string) => ({ id: symbol.toLowerCase(), symbol, name: symbol }) as never;
+
+describe('sortAssetsByMcap', () => {
+  it('orders by the latest market cap, treating a null value (no data) like a missing one', () => {
+    const sorted = sortAssetsByMcap(
+      [asset('ADA'), asset('BTC'), asset('XYZ'), asset('ETH')],
+      [
+        { bulk: [{ a: 'btc', v: 1 }, { a: 'eth', v: 2 }] },
+        // glassnode-api >= 0.30: an asset without a value comes back as v: null.
+        { bulk: [{ a: 'btc', v: 900 }, { a: 'eth', v: 400 }, { a: 'xyz', v: null }] },
+      ],
+    );
+    expect(sorted.map((a) => a.symbol)).toEqual(['BTC', 'ETH', 'ADA', 'XYZ']);
+  });
+});
+
+describe('listedMetrics', () => {
+  it('hides point-in-time variants, even when their metadata failed to load', () => {
+    const paths = ['/a/count', '/a/count_pit', '/b/sum_pit', '/c/ratio'];
+    const metadata = { '/a/count_pit': { is_pit: true }, '/a/count': { is_pit: false } } as never;
+    expect(listedMetrics(paths, metadata)).toEqual(['/a/count', '/c/ratio']);
+  });
+});