test(builder): cover Builder lifecycle regressions

packages/builder/test/unit/genesis.test.ts

@@ -0,0 +1,68 @@
+import {afterEach, beforeEach, describe, expect, it, vi} from "vitest";
+import {ErrorAborted} from "@lodestar/utils";
+import {waitForGenesis} from "../../src/genesis.js";
+import {getApiClientStub, mockApiErrorResponse, mockApiResponse} from "./utils/apiStub.js";
+import {getMockedLogger} from "./utils/logger.js";
+
+describe("Genesis", () => {
+  const logger = getMockedLogger();
+  const api = getApiClientStub();
+  const genesis = {
+    genesisTime: 1,
+    genesisValidatorsRoot: new Uint8Array(32),
+    genesisForkVersion: new Uint8Array(4),
+  };
+
+  let controller: AbortController;
+
+  beforeEach(() => {
+    controller = new AbortController();
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+    vi.useRealTimers();
+  });
+
+  it("waits for a not-found genesis response without logging a stack trace", async () => {
+    api.beacon.getGenesis
+      .mockResolvedValueOnce(await mockApiErrorResponse(404))
+      .mockResolvedValueOnce(mockApiResponse({data: genesis}));
+
+    const promise = waitForGenesis(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toEqual(genesis);
+    expect(api.beacon.getGenesis).toHaveBeenCalledTimes(2);
+    expect(logger.info).toHaveBeenCalledWith(
+      "Waiting for genesis",
+      expect.objectContaining({message: expect.any(String)})
+    );
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it("warns without a stack trace and retries an unexpected genesis failure", async () => {
+    const error = Error("genesis endpoint unavailable");
+    api.beacon.getGenesis.mockRejectedValueOnce(error).mockResolvedValueOnce(mockApiResponse({data: genesis}));
+
+    const promise = waitForGenesis(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toEqual(genesis);
+    expect(api.beacon.getGenesis).toHaveBeenCalledTimes(2);
+    expect(logger.warn).toHaveBeenCalledWith("Failed to fetch genesis", {message: error.message});
+  });
+
+  it("aborts while sleeping between genesis polls", async () => {
+    api.beacon.getGenesis.mockResolvedValue(await mockApiErrorResponse(404));
+
+    const promise = waitForGenesis(api, logger, controller.signal);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.abort();
+
+    await expect(promise).rejects.toThrow(ErrorAborted);
+    expect(api.beacon.getGenesis).toHaveBeenCalledOnce();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+});

packages/builder/test/unit/identity.test.ts

@@ -43,11 +43,34 @@ describe("Identity", () => {
     expect(res?.balance).toEqual(balance);
   });
 
-  it("fails to fetch the builder status", async () => {
+  it("returns an inert result and preserves API error detail when status lookup fails", async () => {
     api.beacon.getStateBuilders.mockResolvedValue(await mockApiErrorResponse(500));
     const res = await getBuilderStatus(api, logger, index);
     expect(res).toBeNull();
-    expect(logger.warn).toHaveBeenCalledOnce();
+    expect(logger.warn).toHaveBeenCalledWith(
+      "Couldn't fetch the builder",
+      {},
+      expect.objectContaining({status: 500, message: expect.stringMatching(/status 500/)})
+    );
+  });
+
+  it("distinguishes an empty successful status response from a beacon node failure", async () => {
+    api.beacon.getStateBuilders.mockResolvedValue(
+      mockApiResponse({data: [], meta: {executionOptimistic: true, finalized: false}})
+    );
+
+    await expect(getBuilderStatus(api, logger, index)).resolves.toBeNull();
+    expect(logger.warn).toHaveBeenCalledWith("Builder status not available in beacon node");
+    expect(logger.warn).not.toHaveBeenCalledWith("Couldn't fetch the builder", expect.anything(), expect.anything());
+  });
+
+  it("returns a non-active status without conflating it with a lookup failure", async () => {
+    api.beacon.getStateBuilders.mockResolvedValue(
+      mockGetStateBuildersResponse(index, {status: "pending", pubkey, balance, version})
+    );
+
+    await expect(getBuilderStatus(api, logger, index)).resolves.toEqual({status: "pending", balance});
+    expect(logger.warn).not.toHaveBeenCalled();
   });
 
   it("successfully resolves builder identity", async () => {
@@ -101,6 +124,21 @@ describe("Identity", () => {
     expect(api.beacon.getStateBuilders).toHaveBeenCalledTimes(2);
   });
 
+  it("aborts while waiting for the beacon node to return the builder", async () => {
+    vi.useFakeTimers();
+    api.beacon.getStateBuilders.mockResolvedValue(
+      mockApiResponse({data: [], meta: {executionOptimistic: true, finalized: false}})
+    );
+
+    const promise = resolveBuilderIdentity(api, logger, pubkeyString, abortController.signal, clock, config);
+    await vi.advanceTimersByTimeAsync(0);
+    abortController.abort();
+
+    await expect(promise).rejects.toThrow(ErrorAborted);
+    expect(api.beacon.getStateBuilders).toHaveBeenCalledOnce();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
   it("waits for a pending builder to become active", async () => {
     vi.useFakeTimers();
     api.beacon.getStateBuilders.mockResolvedValueOnce(

packages/builder/test/unit/readiness.test.ts

@@ -0,0 +1,135 @@
+import {afterEach, beforeEach, describe, expect, it, vi} from "vitest";
+import {routes} from "@lodestar/api";
+import {ErrorAborted} from "@lodestar/utils";
+import {logNodeVersion, waitForNodeReady} from "../../src/readiness.js";
+import {getApiClientStub, mockApiErrorResponse, mockApiResponse} from "./utils/apiStub.js";
+import {getMockedLogger} from "./utils/logger.js";
+
+describe("Readiness", () => {
+  const logger = getMockedLogger();
+  const api = getApiClientStub();
+
+  let controller: AbortController;
+
+  beforeEach(() => {
+    controller = new AbortController();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+    vi.useRealTimers();
+  });
+
+  function mockSyncingStatus(overrides: Partial<routes.node.SyncingStatus> = {}) {
+    return mockApiResponse<routes.node.SyncingStatus, void, routes.node.Endpoints["getSyncingStatus"]>({
+      data: {
+        headSlot: 1,
+        syncDistance: 0,
+        isSyncing: false,
+        isOptimistic: false,
+        elOffline: false,
+        ...overrides,
+      },
+    });
+  }
+
+  it("waits for a syncing beacon node to become ready", async () => {
+    vi.useFakeTimers();
+    api.node.getSyncingStatus
+      .mockResolvedValueOnce(mockSyncingStatus({headSlot: 0, syncDistance: 1, isSyncing: true}))
+      .mockResolvedValueOnce(mockSyncingStatus());
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toBeUndefined();
+    expect(api.node.getSyncingStatus).toHaveBeenCalledTimes(2);
+    expect(logger.info).toHaveBeenCalledWith(
+      "Beacon node is not ready yet",
+      expect.objectContaining({headSlot: 0, syncDistance: 1, elOffline: false})
+    );
+    expect(logger.info).toHaveBeenCalledWith("Beacon node is ready", {headSlot: 1});
+  });
+
+  it("waits for an offline execution client to become ready", async () => {
+    vi.useFakeTimers();
+    api.node.getSyncingStatus
+      .mockResolvedValueOnce(mockSyncingStatus({elOffline: true}))
+      .mockResolvedValueOnce(mockSyncingStatus());
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toBeUndefined();
+    expect(api.node.getSyncingStatus).toHaveBeenCalledTimes(2);
+    expect(logger.info).toHaveBeenCalledWith(
+      "Beacon node EL is offline, unable to submit bids",
+      expect.objectContaining({elOffline: true})
+    );
+  });
+
+  it("retries a non-ok sync response and then becomes ready", async () => {
+    vi.useFakeTimers();
+    api.node.getSyncingStatus
+      .mockResolvedValueOnce(await mockApiErrorResponse(500))
+      .mockResolvedValueOnce(mockSyncingStatus());
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toBeUndefined();
+    expect(api.node.getSyncingStatus).toHaveBeenCalledTimes(2);
+    expect(logger.warn).toHaveBeenCalledWith("Cannot get node sync status", expect.objectContaining({status: 500}));
+  });
+
+  it("retries an unreachable beacon node and then becomes ready", async () => {
+    vi.useFakeTimers();
+    const error = Error("connect ECONNREFUSED");
+    api.node.getSyncingStatus.mockRejectedValueOnce(error).mockResolvedValueOnce(mockSyncingStatus());
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toBeUndefined();
+    expect(api.node.getSyncingStatus).toHaveBeenCalledTimes(2);
+    expect(logger.error).toHaveBeenCalledWith("Cannot reach the beacon node", {}, error);
+  });
+
+  it("aborts while sleeping between readiness polls", async () => {
+    vi.useFakeTimers();
+    api.node.getSyncingStatus.mockResolvedValue(mockSyncingStatus({isSyncing: true}));
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersByTimeAsync(0);
+    controller.abort();
+
+    await expect(promise).rejects.toThrow(ErrorAborted);
+    expect(api.node.getSyncingStatus).toHaveBeenCalledOnce();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it("waits while the beacon node head is optimistic", async () => {
+    vi.useFakeTimers();
+    api.node.getSyncingStatus
+      .mockResolvedValueOnce(mockSyncingStatus({isOptimistic: true}))
+      .mockResolvedValueOnce(mockSyncingStatus());
+
+    const promise = waitForNodeReady(api, logger, controller.signal);
+    await vi.advanceTimersToNextTimerAsync();
+
+    await expect(promise).resolves.toBeUndefined();
+    expect(api.node.getSyncingStatus).toHaveBeenCalledTimes(2);
+    expect(logger.warn).toHaveBeenCalledWith(
+      "Beacon node head is optimistic, execution payloads are not yet verified - unable to submit bids",
+      expect.objectContaining({headSlot: 1, syncDistance: 0})
+    );
+  });
+
+  it("keeps node-version lookup failure non-fatal", async () => {
+    const error = Error("version endpoint unavailable");
+    api.node.getNodeVersionV2.mockRejectedValue(error);
+
+    await expect(logNodeVersion(api, logger)).resolves.toBeUndefined();
+    expect(logger.warn).toHaveBeenCalledWith("Failed to get node version", {}, error);
+  });
+});

packages/builder/test/unit/utils/apiStub.ts

@@ -6,10 +6,12 @@ export {type ApiClientStub, mockApiErrorResponse, mockApiResponse} from "@lodest
 export function getApiClientStub(): ApiClientStub {
   return {
     beacon: {
+      getGenesis: vi.fn(),
       getStateBuilders: vi.fn(),
     },
     node: {
       getSyncingStatus: vi.fn(),
+      getNodeVersionV2: vi.fn(),
     },
   } as unknown as ApiClientStub;
 }