Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions packages/beacon-node/src/chain/blocks/importBlock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,11 @@ export async function importBlock(
const currentEpoch = computeEpochAtSlot(currentSlot);
const blockEpoch = computeEpochAtSlot(blockSlot);
const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
const receiveDelaySec =
fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
const slotTimeSec = computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
const receiveDelaySec = fullyVerifiedBlock.seenTimestampSec - slotTimeSec;
// A repeat proposal ignored on gossip and imported later keeps its gossip arrival for PTC timeliness
const ptcReceiveDelaySec =
(fullyVerifiedBlock.firstSeenTimestampSec ?? fullyVerifiedBlock.seenTimestampSec) - slotTimeSec;
const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
const fork = this.config.getForkSeq(blockSlot);

Expand Down Expand Up @@ -149,7 +152,8 @@ export async function importBlock(
importDelaySec,
currentSlot,
executionStatus,
dataAvailabilityStatus
dataAvailabilityStatus,
ptcReceiveDelaySec
);

// This adds the state necessary to process the next block
Expand Down
1 change: 1 addition & 0 deletions packages/beacon-node/src/chain/blocks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ export async function processBlocks(
indexedAttestations: indexedAttestationsByBlock[i],
// TODO: Make this param mandatory and capture in gossip
seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
firstSeenTimestampSec: opts.firstSeenTimestampSec,
});
}

Expand Down
4 changes: 4 additions & 0 deletions packages/beacon-node/src/chain/blocks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ export type ImportBlockOpts = {
validBlobSidecars?: BlobSidecarValidation;
/** Seen timestamp seconds */
seenTimestampSec?: number;
/** Seconds the block was first seen on gossip, if it was ignored there as a repeat proposal. Only affects PTC timeliness */

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only affects PTC timeliness

so confusing that the ptc deadline was reused for this, also kinda arbitrary

firstSeenTimestampSec?: number;
};

/**
Expand All @@ -108,6 +110,8 @@ export type FullyVerifiedBlock = {
indexedAttestations: IndexedAttestation[];
/** Seen timestamp seconds */
seenTimestampSec: number;
/** Seconds the block was first seen on gossip, if it was ignored there as a repeat proposal. Only affects PTC timeliness */
firstSeenTimestampSec?: number;
/** If the execution payload couldn't be verified because of EL syncing status, used in optimistic sync */
executionStatus: BlockExecutionStatus | PayloadExecutionStatus;
};
Expand Down
32 changes: 32 additions & 0 deletions packages/beacon-node/src/chain/seenCache/seenBlockProposers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {MapDef} from "@lodestar/utils";

/** Two distinct block roots signed by the same proposer for the same slot are sufficient to establish an equivocation */
const MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL = 2;
/** Far above what a node can deserialize within a slot, so a flood of repeat proposals cannot crowd out a real sibling */
const MAX_FIRST_SEEN_BLOCK_ROOTS_PER_SLOT = 4096;

/**
* Keeps a cache to filter block proposals from the same validator in the same slot.
Expand All @@ -16,6 +18,9 @@ const MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL = 2;
* The signed block header of each observed root is kept so a proposer slashing can be produced from the two
* conflicting headers once an equivocation is established.
*
* The time a repeat proposal was first seen on gossip is kept even if it is not imported at that point, so a later
* import through sync records its PTC timeliness from arrival rather than from the download.
*
* The cache is pruned on finalization and bounds the number of roots stored per proposer and slot
*/
export class SeenBlockProposers {
Expand All @@ -26,6 +31,9 @@ export class SeenBlockProposers {
Slot,
MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>
>(() => new MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>(() => new Map()));
private readonly firstSeenTimestampSecBySlot = new MapDef<Slot, Map<RootHex, number>>(
() => new Map<RootHex, number>()
);
private finalizedSlot: Slot = 0;

isKnown(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
Expand Down Expand Up @@ -94,6 +102,25 @@ export class SeenBlockProposers {
}
}

/** Record when a block root was first seen on gossip, later sightings keep the first timestamp */
observeFirstSeen(blockSlot: Slot, blockRoot: RootHex, seenTimestampSec: number): void {
if (blockSlot < this.finalizedSlot) {
return;
}

const firstSeenTimestampSecByRoot = this.firstSeenTimestampSecBySlot.getOrDefault(blockSlot);
if (
firstSeenTimestampSecByRoot.size < MAX_FIRST_SEEN_BLOCK_ROOTS_PER_SLOT &&
!firstSeenTimestampSecByRoot.has(blockRoot)
) {
firstSeenTimestampSecByRoot.set(blockRoot, seenTimestampSec);
}
}

getFirstSeenTimestampSec(blockSlot: Slot, blockRoot: RootHex): number | undefined {
return this.firstSeenTimestampSecBySlot.get(blockSlot)?.get(blockRoot);
}

/** Mark a block as known from gossip or another block import path */
add(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): void {
if (blockSlot < this.finalizedSlot) {
Expand All @@ -115,6 +142,11 @@ export class SeenBlockProposers {
this.signedBlockHeadersBySlot.delete(slot);
}
}
for (const slot of this.firstSeenTimestampSecBySlot.keys()) {
if (slot < finalizedSlot) {
this.firstSeenTimestampSecBySlot.delete(slot);
Comment on lines +145 to +147

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Evict first-seen entries without waiting for finalization

If finality stalls, this cleanup never removes entries for later slots, so the new cache can grow by up to 4096 roots for every affected slot indefinitely. After a proposer has supplied two signature-verified siblings, additional repeat proposals skip signature verification, making it practical for an equivocating proposer and its peers to fill this allowance on each selected slot and steadily exhaust node memory during an extended non-finalizing period. Retain only the small recent-slot window in which PTC timeliness can still matter, or enforce a global bound.

Useful? React with 👍 / 👎.

}
}
}

seenAtEpoch(epoch: Epoch, index: ValidatorIndex): boolean {
Expand Down
17 changes: 15 additions & 2 deletions packages/beacon-node/src/network/processor/gossipHandlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,20 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand

// IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
// Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
// (provably invalid) and unexpected errors prune below.
// (provably invalid), unexpected errors and repeat proposals that are not imported prune.
if (e.action === GossipAction.IGNORE) {
if (e.type.code === BlockErrorCode.REPEAT_PROPOSAL) {
// Keep the arrival time so a sibling imported later through sync still gets its PTC timeliness from gossip
chain.seenBlockProposers.observeFirstSeen(slot, blockRootHex, seenTimestampSec);
// Only a signature-verified sibling is imported by the beacon_block handler, any other repeat proposal
// is dropped from the caches and re-downloaded by sync if it ever becomes relevant
if (!chain.seenBlockProposers.hasBlockRoot(slot, signedBlock.message.proposerIndex, blockRootHex)) {
chain.seenBlockInputCache.prune(blockRootHex);
if (isForkPostGloas(fork)) {
chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
}
}
}
throw e;
}

Expand Down Expand Up @@ -738,7 +750,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
if (
e instanceof BlockGossipError &&
e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
// this is make sure the block's proposer signature was verified, it should be true anyway
// Only a signature-verified sibling recorded in the seen cache is imported. The cache holds at most two
// roots per proposer and slot, which bounds full imports over gossip to one alternate
chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
) {
// blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE
Expand Down
1 change: 1 addition & 0 deletions packages/beacon-node/src/sync/unknownBlock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -929,6 +929,7 @@ export class BlockInputSync {
// see https://github.com/ChainSafe/lodestar/issues/5650
ignoreIfFinalized: true,
blsVerifyOnMainThread: true,
firstSeenTimestampSec: this.chain.seenBlockProposers.getFirstSeenTimestampSec(blockSlot, blockRootHex),
})
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,46 @@ describe("LodestarForkChoice", () => {
/**
* finalized - slot 8 (finalized 1) - slot 12 - slot 16 (finalized 2) - slot 20 - slot 24 (finalized 3) - slot 28 - slot 32 (finalized 4)
*/
it("onBlock - evaluates PTC timeliness with the first seen delay when provided", () => {
const {blockHeader} = computeAnchorCheckpoint(config, anchorState);
const finalizedRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader);
const block16 = generateSignedBlockAtSlot(16);
block16.message.parentRoot = finalizedRoot;
const state16 = runStateTransition(anchorStateView, block16);
block16.message.stateRoot = state16.hashTreeRoot();
const {block: block17, state: state17} = makeChild({block: block16, state: state16}, 17);

// Imported late in its slot but first seen on gossip well before the PTC deadline
const lateDelaySec = config.SECONDS_PER_SLOT - 1;
forkChoice.updateTime(16);
const summary16 = forkChoice.onBlock(
block16.message,
state16,
lateDelaySec,
lateDelaySec,
16,
executionStatus,
dataAvailabilityStatus,
1
);
expect(summary16.timeliness).toBe(false);
expect(summary16.ptcTimeliness).toBe(true);

// Without a first seen delay the import delay decides
forkChoice.updateTime(17);
const summary17 = forkChoice.onBlock(
block17.message,
state17,
lateDelaySec,
lateDelaySec,
17,
executionStatus,
dataAvailabilityStatus
);
expect(summary17.timeliness).toBe(false);
expect(summary17.ptcTimeliness).toBe(false);
});

it("prune - should prune old blocks", () => {
const {blockHeader} = computeAnchorCheckpoint(config, anchorState);
const finalizedRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,43 @@ describe("SeenBlockProposers", () => {
expect(cache.getEquivocationHeaders(slot, proposerIndex)).toBe(null);
});

it("keeps the first seen timestamp of a block root", () => {
const cache = new SeenBlockProposers();

cache.observeFirstSeen(slot, blockRoot, 10);
cache.observeFirstSeen(slot, blockRoot, 20);
cache.observeFirstSeen(slot, conflictingBlockRoot, 30);

expect(cache.getFirstSeenTimestampSec(slot, blockRoot)).toBe(10);
expect(cache.getFirstSeenTimestampSec(slot, conflictingBlockRoot)).toBe(30);
expect(cache.getFirstSeenTimestampSec(slot, additionalBlockRoot)).toBe(undefined);
expect(cache.getFirstSeenTimestampSec(slot + 1, blockRoot)).toBe(undefined);
});

it("bounds first seen block roots per slot", () => {
const cache = new SeenBlockProposers();

for (let i = 0; i < 4096; i++) {
cache.observeFirstSeen(slot, toRootHex(Buffer.from(i.toString(16).padStart(64, "0"), "hex")), i);
}
cache.observeFirstSeen(slot, blockRoot, 10);
cache.observeFirstSeen(slot + 1, blockRoot, 10);

expect(cache.getFirstSeenTimestampSec(slot, blockRoot)).toBe(undefined);
expect(cache.getFirstSeenTimestampSec(slot + 1, blockRoot)).toBe(10);
});

it("prunes first seen block roots and ignores finalized slots", () => {
const cache = new SeenBlockProposers();
cache.observeFirstSeen(slot, blockRoot, 10);

cache.prune(slot + 1);
cache.observeFirstSeen(slot, conflictingBlockRoot, 20);

expect(cache.getFirstSeenTimestampSec(slot, blockRoot)).toBe(undefined);
expect(cache.getFirstSeenTimestampSec(slot, conflictingBlockRoot)).toBe(undefined);
});

it("rejects updates for slots before the finalized slot", () => {
const cache = new SeenBlockProposers();
cache.prune(slot + 1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,12 @@ describe("getGossipHandlers", () => {
});

it("imports a signature-verified REPEAT_PROPOSAL (equivocating) block into fork choice but keeps IGNORE", async () => {
const {processBlock, threw} = await runBeaconBlockRepeatProposal(denebConfig, {recorded: true});
const {processBlock, threw, firstSeenTimestampSec, pruneBlockInput} = await runBeaconBlockRepeatProposal(
denebConfig,
{recorded: true}
);
expect(firstSeenTimestampSec).toBe(12);
expect(pruneBlockInput).not.toHaveBeenCalled();

// imported so LMD-GHOST can weigh it ...
expect(processBlock).toHaveBeenCalledOnce();
Expand All @@ -74,7 +79,13 @@ describe("getGossipHandlers", () => {
});

it("does not import a REPEAT_PROPOSAL block whose root was not recorded (unverified 3rd+ proposal)", async () => {
const {processBlock, threw} = await runBeaconBlockRepeatProposal(denebConfig, {recorded: false});
const {processBlock, threw, firstSeenTimestampSec, pruneBlockInput} = await runBeaconBlockRepeatProposal(
denebConfig,
{recorded: false}
);
// arrival time is kept so a later import through sync gets its PTC timeliness from gossip, the block itself is dropped
expect(firstSeenTimestampSec).toBe(12);
expect(pruneBlockInput).toHaveBeenCalledOnce();

expect(processBlock).not.toHaveBeenCalled();
expect(threw).toBe(true);
Expand Down Expand Up @@ -162,7 +173,12 @@ async function runBeaconBlockProcessingError(
async function runBeaconBlockRepeatProposal(
config: BeaconConfig,
{recorded}: {recorded: boolean}
): Promise<{processBlock: ReturnType<typeof vi.fn>; threw: boolean}> {
): Promise<{
processBlock: ReturnType<typeof vi.fn>;
threw: boolean;
firstSeenTimestampSec: number | undefined;
pruneBlockInput: ReturnType<typeof vi.fn>;
}> {
const logger = testLogger();
const peerIdStr = "16Uiu2HAmTestGossipPeer" as PeerIdStr;
const signedBlock = ssz.deneb.SignedBeaconBlock.defaultValue();
Expand Down Expand Up @@ -201,6 +217,7 @@ async function runBeaconBlockRepeatProposal(
}

const processBlock = vi.fn().mockResolvedValue(undefined);
const pruneBlockInput = vi.fn();
const chain = {
clock: new ClockStopped(1),
custodyConfig: {sampledColumns: [], custodyColumns: []} as unknown as CustodyConfig,
Expand All @@ -213,7 +230,7 @@ async function runBeaconBlockRepeatProposal(
seenBlockInputCache: {
getByBlock: vi.fn().mockReturnValue(blockInput),
get: vi.fn().mockReturnValue(blockInput),
prune: vi.fn(),
prune: pruneBlockInput,
} as unknown as SeenBlockInput,
seenPayloadEnvelopeInputCache: {
add: vi.fn(),
Expand Down Expand Up @@ -242,7 +259,7 @@ async function runBeaconBlockRepeatProposal(
await beaconBlockHandler({
gossipData: {serializedData: ssz.deneb.SignedBeaconBlock.serialize(signedBlock)},
peerIdStr,
seenTimestampSec: 0,
seenTimestampSec: 12,
topic: {
boundary: {fork: ForkName.deneb, epoch: 0},
type: GossipType.beacon_block,
Expand All @@ -257,7 +274,12 @@ async function runBeaconBlockRepeatProposal(
await new Promise((resolve) => setTimeout(resolve, 0));
await new Promise((resolve) => setTimeout(resolve, 0));

return {processBlock, threw};
return {
processBlock,
threw,
firstSeenTimestampSec: seenBlockProposers.getFirstSeenTimestampSec(signedBlock.message.slot, blockRootHex),
pruneBlockInput,
};
}

function getExecutionBlockError(
Expand Down
Loading
Loading