diff --git a/package.json b/package.json index eb7fd85..3dbd10f 100644 --- a/package.json +++ b/package.json @@ -66,10 +66,10 @@ "@x402/fetch": "2.26.0", "canvas": "^3.2.3", "chafa-wasm": "^0.3.3", - "glassnode-api": "^0.29.3", + "glassnode-api": "^1.0.0", "ink": "^7.1.1", "ink-spinner": "^5.0.0", - "ink-uplot": "^0.2.15", + "ink-uplot": "^0.2.16", "path2d": "^0.3.4", "react": "^19.3.0", "uplot": "^1.6.32", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 29af624..68890fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,8 +25,8 @@ importers: specifier: ^0.3.3 version: 0.3.3 glassnode-api: - specifier: ^0.29.3 - version: 0.29.3(@x402/evm@2.26.0(typescript@7.0.2))(@x402/fetch@2.26.0)(viem@2.56.8(typescript@7.0.2)(zod@4.6.5)) + specifier: ^1.0.0 + version: 1.0.0(@x402/evm@2.26.0(typescript@7.0.2))(@x402/fetch@2.26.0)(viem@2.56.8(typescript@7.0.2)(zod@4.6.5)) ink: specifier: ^7.1.1 version: 7.1.1(@types/react@19.3.0)(react@19.3.0) @@ -34,8 +34,8 @@ importers: specifier: ^5.0.0 version: 5.0.0(ink@7.1.1(@types/react@19.3.0)(react@19.3.0))(react@19.3.0) ink-uplot: - specifier: ^0.2.15 - version: 0.2.15(ink@7.1.1(@types/react@19.3.0)(react@19.3.0))(react@19.3.0)(uplot@1.6.32) + specifier: ^0.2.16 + version: 0.2.16(ink@7.1.1(@types/react@19.3.0)(react@19.3.0))(react@19.3.0)(uplot@1.6.32) path2d: specifier: ^0.3.4 version: 0.3.4 @@ -722,9 +722,9 @@ packages: github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - glassnode-api@0.29.3: - resolution: {integrity: sha512-oCLTVHddtKxc8Ow4muf2ELOk72ibF4QNLypxtpCmlWfOkAvyekd8jRI23N8dkWECofxhn+aj6ocCaMWHBkbfkQ==} - engines: {node: '>=18.0.0'} + glassnode-api@1.0.0: + resolution: {integrity: sha512-oZMeZaamV4T1n7hpL6KCyykwU6DZR0kJFtkaXEIbtFTLOK1PBZw6b7lYzgd2Rs0QTBgWYEm1L3CX4LPzW8MsUw==} + engines: {node: '>=22.0.0'} peerDependencies: '@x402/evm': ^2.18.0 '@x402/fetch': ^2.18.0 @@ -766,8 +766,8 @@ packages: '@types/react': optional: true - ink-uplot@0.2.15: - resolution: {integrity: sha512-k9rOhV/ryoAdyDZOeXHOCp62ONhdgD1gBMRan9ck/fctHQ9AYr6N8cHecVHdPxEyTQsiFUQj3l2HVxSWUFYE2g==} + ink-uplot@0.2.16: + resolution: {integrity: sha512-n5ct2/cQZBEOiru/lMpUvYYg7HtPEJenWjrC7Kn/z7IRAG+LF4oLt/LHBdnfHSuSDFbt+wjkz1nYcYggTxwmlg==} engines: {node: '>=18.0.0'} peerDependencies: ink: '>=4.0.0' @@ -1697,7 +1697,7 @@ snapshots: github-from-package@0.0.0: {} - glassnode-api@0.29.3(@x402/evm@2.26.0(typescript@7.0.2))(@x402/fetch@2.26.0)(viem@2.56.8(typescript@7.0.2)(zod@4.6.5)): + glassnode-api@1.0.0(@x402/evm@2.26.0(typescript@7.0.2))(@x402/fetch@2.26.0)(viem@2.56.8(typescript@7.0.2)(zod@4.6.5)): dependencies: zod: 4.6.5 optionalDependencies: @@ -1723,7 +1723,7 @@ snapshots: optionalDependencies: '@types/react': 19.3.0 - ink-uplot@0.2.15(ink@7.1.1(@types/react@19.3.0)(react@19.3.0))(react@19.3.0)(uplot@1.6.32): + ink-uplot@0.2.16(ink@7.1.1(@types/react@19.3.0)(react@19.3.0))(react@19.3.0)(uplot@1.6.32): dependencies: canvas: 3.2.3 chafa-wasm: 0.3.3 diff --git a/src/lib/api-client.ts b/src/lib/api-client.ts index cc86b24..8d4d58e 100644 --- a/src/lib/api-client.ts +++ b/src/lib/api-client.ts @@ -8,6 +8,10 @@ const { version } = require('../../package.json') as { version: string }; export const USER_AGENT = `glassnode-terminal-${version}`; let clientInstance: GlassnodeAPI | null = null; +// API-key mode only: see createStartupClient(). +let startupClientInstance: GlassnodeAPI | null = null; +const STARTUP_MAX_RETRIES = 5; +const STARTUP_MAX_RETRY_DELAY_MS = 60_000; // the API's rate-limit window let x402Enabled = false; let x402FetchStore: typeof globalThis.fetch | null = null; let x402BaseUrl = 'https://x402.glassnode.com'; @@ -67,6 +71,23 @@ const userAgentFetch: typeof globalThis.fetch = (input, init) => { return globalThis.fetch(req); }; +/** + * The API's 429s carry `x-rate-limit-reset` (seconds until the per-minute window resets) + * but no `Retry-After`, so glassnode-api falls back to short jittered backoff and can give + * up before the window resets. Expose the reset as `Retry-After`, which glassnode-api honours + * (capped at `maxRetryDelay`), so a retry waits for the new window. + */ +export function rateLimitAwareFetch(inner: typeof globalThis.fetch): typeof globalThis.fetch { + return async (input, init) => { + const res = await inner(input, init); + const reset = res.headers.get('x-rate-limit-reset'); + if (res.status !== 429 || res.headers.has('retry-after') || !reset || !/^\d+$/.test(reset)) return res; + const headers = new Headers(res.headers); + headers.set('retry-after', reset); + return new Response(res.body, { status: res.status, statusText: res.statusText, headers }); + }; +} + /** * Initialize the shared GlassnodeAPI client. Must be awaited once at startup, * before any createClient() call. Credentials are chosen from the environment: @@ -131,6 +152,13 @@ export async function initClient(): Promise { } clientInstance = new GlassnodeAPI({ apiKey, fetch: userAgentFetch, logger }); + startupClientInstance = new GlassnodeAPI({ + apiKey, + fetch: rateLimitAwareFetch(userAgentFetch), + logger, + maxRetries: STARTUP_MAX_RETRIES, + maxRetryDelay: STARTUP_MAX_RETRY_DELAY_MS, + }); log('info', 'API-key mode enabled'); return clientInstance; } @@ -145,3 +173,15 @@ export function createClient(): GlassnodeAPI { } return clientInstance; } + +/** + * Client for the startup metadata burst (~1,800 calls against a 1,200/minute limit). With + * glassnode-api's default 2 short retries ~16% of the metadata was rate-limited (429) and + * lost. This client retries up to 5 times and, via rateLimitAwareFetch, waits for the + * rate-limit window to reset. Interactive calls keep the default so a rate limit doesn't + * stall the UI. x402 mode has no separate client: retrying a paid call could pay twice, + * so it stays at 0 retries. + */ +export function createStartupClient(): GlassnodeAPI { + return startupClientInstance ?? createClient(); +} diff --git a/src/lib/startup-data.ts b/src/lib/startup-data.ts index e818d17..a7a6be7 100644 --- a/src/lib/startup-data.ts +++ b/src/lib/startup-data.ts @@ -1,5 +1,5 @@ import type { AssetMetadata, MetricMetadata } from 'glassnode-api'; -import { createClient, isX402Enabled, fetchX402AdvancedPaths } from './api-client.js'; +import { createClient, createStartupClient, isX402Enabled, fetchX402AdvancedPaths } from './api-client.js'; import { readCache, writeCache } from './cache.js'; import type { MetricListItem } from './types.js'; import { X402_CATALOG, X402_ASSET_SETS, X402_ASSETS, type X402CatalogEntry } from './x402-catalog.js'; @@ -11,14 +11,16 @@ export interface StartupData { metricMetadataMap: Record; } -const BATCH_SIZE = 20; +const BATCH_SIZE = 10; const BATCH_DELAY_MS = 100; async function fetchAllMetricMetadata( metricPaths: string[], onProgress?: (done: number, total: number) => void, ): Promise> { - const client = createClient(); + // Batches of 10 plus the startup client's rate-limit-aware retries keep lost (429) calls + // near zero; measured: 2 retries/batch 20 lost ~296 of 1,817. + const client = createStartupClient(); const result: Record = {}; for (let i = 0; i < metricPaths.length; i += BATCH_SIZE) { @@ -46,15 +48,16 @@ async function fetchAllMetricMetadata( } /** Sort assets by descending market cap, then symbol. */ -function sortAssetsByMcap( +export function sortAssetsByMcap( assetData: AssetMetadata[], - mcapData: Array<{ bulk: Array<{ a: string; v: number }> }>, + // v is null for an asset with no value at that time (glassnode-api >= 0.30). + mcapData: Array<{ bulk: Array<{ a: string; v: number | null }> }>, ): AssetMetadata[] { const mcapBySymbol = new Map(); if (mcapData.length > 0) { const latest = mcapData[mcapData.length - 1]!; for (const entry of latest.bulk) { - mcapBySymbol.set(entry.a.toUpperCase(), entry.v); + if (entry.v !== null) mcapBySymbol.set(entry.a.toUpperCase(), entry.v); } } return [...assetData].sort((a, b) => { @@ -65,6 +68,15 @@ function sortAssetsByMcap( }); } +/** + * Metrics to list: everything except point-in-time (`_pit`) variants. Checks the path suffix + * as well as `is_pit`, so a variant whose metadata failed to load (e.g. rate-limited) is still + * hidden; every `is_pit` metric's path ends in `_pit` and no other does. + */ +export function listedMetrics(paths: string[], metadata: Record): string[] { + return paths.filter((p) => !(metadata[p]?.is_pit || p.endsWith('_pit'))); +} + /** Turn catalog entries into the metrics list + a metadata map buildMetricList understands. */ function catalogToStartupParts(entries: readonly X402CatalogEntry[]): { metrics: string[]; @@ -93,7 +105,9 @@ export async function loadStartupData( onProgress?: (message: string) => void, ): Promise { // Cache is per-mode: x402 shows only the ~326 advanced metrics, API mode shows all. - const cacheKey = isX402Enabled() ? 'startup-data-x402' : 'startup-data'; + // v2: glassnode-api 1.0 + rate-limit-aware metadata fetch — don't reuse a day-old cache + // that lost metadata to rate limits (missing names, leaked _pit metrics). + const cacheKey = isX402Enabled() ? 'startup-data-x402' : 'startup-data-v2'; const cached = readCache(cacheKey); if (cached) { @@ -131,8 +145,7 @@ async function loadStartupDataApi( onProgress?.(`Fetching metric metadata... ${done}/${total}`); }); - const filteredMetrics = metricPaths.filter((p) => !metricMetadataMap[p]?.is_pit); - return { assets: sortedAssets, metrics: filteredMetrics, metricMetadataMap }; + return { assets: sortedAssets, metrics: listedMetrics(metricPaths, metricMetadataMap), metricMetadataMap }; } /** diff --git a/test/lib/api-client.spec.ts b/test/lib/api-client.spec.ts new file mode 100644 index 0000000..2fa9e67 --- /dev/null +++ b/test/lib/api-client.spec.ts @@ -0,0 +1,55 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const constructed = vi.hoisted(() => [] as Array>); +vi.mock('glassnode-api', () => ({ + GlassnodeAPI: class { + constructor(public config: Record) { + constructed.push(config); + } + }, +})); + +describe('api-client startup client', () => { + beforeEach(() => { + constructed.length = 0; + vi.resetModules(); + delete process.env['X402_PRIVATE_KEY']; + process.env['GLASSNODE_API_KEY'] = 'test-key'; + process.env['GLASSNODE_MODE'] = 'api'; + }); + + it('uses a separate client with more retries for the startup metadata burst (API-key mode)', async () => { + const { initClient, createClient, createStartupClient } = await import('../../src/lib/api-client.js'); + await initClient(); + const interactive = createClient() as unknown as { config: Record }; + const startup = createStartupClient() as unknown as { config: Record }; + expect(startup).not.toBe(interactive); + // Interactive calls keep glassnode-api's default; the startup burst retries rate limits more. + expect(interactive.config['maxRetries']).toBeUndefined(); + expect(startup.config['maxRetries']).toBe(5); + // Long enough to wait out the API's 60 s rate-limit window (via Retry-After). + expect(startup.config['maxRetryDelay']).toBe(60_000); + expect(startup.config['fetch']).not.toBe(interactive.config['fetch']); + expect(startup.config['apiKey']).toBe('test-key'); + }); +}); + +describe('rateLimitAwareFetch', () => { + const respond = (status: number, headers: Record) => + vi.fn(async () => new Response('{}', { status, headers })); + + it("exposes the API's x-rate-limit-reset as Retry-After on a 429", async () => { + const { rateLimitAwareFetch } = await import('../../src/lib/api-client.js'); + const res = await rateLimitAwareFetch(respond(429, { 'x-rate-limit-reset': '37' }))('https://x'); + expect(res.status).toBe(429); + expect(res.headers.get('retry-after')).toBe('37'); + }); + + it('leaves other responses and an existing Retry-After alone', async () => { + const { rateLimitAwareFetch } = await import('../../src/lib/api-client.js'); + const ok = await rateLimitAwareFetch(respond(200, { 'x-rate-limit-reset': '37' }))('https://x'); + expect(ok.headers.get('retry-after')).toBeNull(); + const own = await rateLimitAwareFetch(respond(429, { 'x-rate-limit-reset': '37', 'retry-after': '5' }))('https://x'); + expect(own.headers.get('retry-after')).toBe('5'); + }); +}); diff --git a/test/lib/startup-data.spec.ts b/test/lib/startup-data.spec.ts new file mode 100644 index 0000000..ed5a049 --- /dev/null +++ b/test/lib/startup-data.spec.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('../../src/lib/api-client.js', () => ({ createClient: vi.fn(), createStartupClient: vi.fn(), isX402Enabled: () => false })); +const { sortAssetsByMcap, listedMetrics } = await import('../../src/lib/startup-data.js'); + +const asset = (symbol: string) => ({ id: symbol.toLowerCase(), symbol, name: symbol }) as never; + +describe('sortAssetsByMcap', () => { + it('orders by the latest market cap, treating a null value (no data) like a missing one', () => { + const sorted = sortAssetsByMcap( + [asset('ADA'), asset('BTC'), asset('XYZ'), asset('ETH')], + [ + { bulk: [{ a: 'btc', v: 1 }, { a: 'eth', v: 2 }] }, + // glassnode-api >= 0.30: an asset without a value comes back as v: null. + { bulk: [{ a: 'btc', v: 900 }, { a: 'eth', v: 400 }, { a: 'xyz', v: null }] }, + ], + ); + expect(sorted.map((a) => a.symbol)).toEqual(['BTC', 'ETH', 'ADA', 'XYZ']); + }); +}); + +describe('listedMetrics', () => { + it('hides point-in-time variants, even when their metadata failed to load', () => { + const paths = ['/a/count', '/a/count_pit', '/b/sum_pit', '/c/ratio']; + const metadata = { '/a/count_pit': { is_pit: true }, '/a/count': { is_pit: false } } as never; + expect(listedMetrics(paths, metadata)).toEqual(['/a/count', '/c/ratio']); + }); +});