feat(#96): FRC-0089 EC finality calculator (observed-data finality)

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to Lantern.
 
 ## Unreleased (v1.9.0)
 
+### Added
+
+- **FRC-0089 EC finality calculator** ([#96](https://github.com/Reiers/lantern/issues/96)).
+  New `chain/ecfinality` package (ported from lotus, math verbatim so the
+  reference test vectors apply unchanged) computes an upper bound on reorg
+  probability from **observed** block counts per epoch: on a healthy chain
+  the 2^-30 guarantee is met around depth ~30 (~15 min) instead of the
+  static worst-case 900 epochs. On-demand + cached per head (an idle node
+  pays nothing). Windowed-store aware: a node with a shallow observed
+  window reports "not computable" instead of an over-confident number.
+  Surfaced on the dashboard dev page (new "EC finality" card) and via
+  `Daemon.ECFinality()` for embedded consumers. Semantics for consumers:
+  `finalized = max(ec-finalized, f3-finalized)` (matches lotus v2 API).
+  New dep: `github.com/rvagg/go-skellam-pmf` (tiny, pure Go, same as
+  lotus). TRUST-MODEL.md section 2.7 documents what it does and does NOT
+  guarantee.
+
 ### Security / head-path hardening
 
 - **Head-source corroboration** ([#80](https://github.com/Reiers/lantern/issues/80) part 2).

TRUST-MODEL.md

@@ -119,6 +119,21 @@ eclipse/fork-selection exposure on the un-finalized tip is documented in
 
 ---
 
+### 2.7 Observed-data EC finality (FRC-0089) — a computed bound, not a proof
+
+`chain/ecfinality` (#96) computes an upper bound on the probability that
+a tipset at depth D could be reorged, from the OBSERVED block counts per
+epoch in the local header store. Under healthy conditions the 2^-30
+bound is met around depth ~30 (~15 min) instead of the static worst-case
+900. Lantern uses it as an honesty instrument (dashboard, stats) and as
+an input for retention decisions - NOT as a substitute for F3: the
+calculation assumes the observed chain is the honest heavy chain, so its
+guarantees are only as good as the head path that observed it (see 3.3).
+Where F3 finality is available it always takes precedence; the useful
+semantics for consumers is `finalized = max(ec-finalized, f3-finalized)`.
+A node with a shallow observed window (< 30 epochs of history) reports
+"not computable" rather than an over-confident number.
+
 ## 3. What Lantern does NOT trust
 
 These are deliberately untrusted. A Lantern node treats responses from any
@@ -143,6 +158,18 @@ Inbound block + message gossip is decoded then validated by the consumer
 (`chain/header.ValidateBlock` for blocks; `crypto/sigs.Verify` for
 messages). Malformed traffic is dropped.
 
+Head adoption additionally applies (v1.9.0): heaviest-ParentWeight fork
+choice, the divergence gate (independent-source head monitor), and
+optional head-source corroboration (#80): a head advance requires
+forwarding by N distinct peers or one trusted floor peer. Peers earn
+score through first-delivery history on the blocks/msgs topics (#97) and
+lose it for invalid messages and IP-colocation; the trusted floor
+(bootstrap/beacon/direct peers) is connmgr-protected and cannot be
+evicted by a dial flood. Honest boundary: peer IDs are Sybil-cheap, so
+corroboration raises eclipse cost only in combination with scoring and
+the floor. It is hardening, not finality; finality comes from F3
+(section 2.2) and the observed-data EC finality bound (section 2.7).
+
 ### 3.4 Forest / Lotus nodes that are NOT the wired bridge
 
 A Forest or Lotus node that happens to be a libp2p peer of ours is in the

chain/ecfinality/cache.go

@@ -0,0 +1,154 @@
+package ecfinality
+
+import (
+	"errors"
+	"math"
+	"sync"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	ltypes "github.com/Reiers/lantern/chain/types"
+)
+
+// MinWindow is the minimum number of epochs of observed history required
+// before the calculator reports a threshold at all. Below this the L
+// distribution has too little data to be meaningful, so Status reports
+// depth -1 (not computable) rather than an over-confident number. 30
+// epochs (~15 min of history) is the depth at which a healthy chain
+// typically meets 2^-30 in the first place.
+const MinWindow = 30
+
+// HeaderSource is the minimal store surface the cache walks. Lantern's
+// chain/header/store.Store satisfies it.
+type HeaderSource interface {
+	Head() *ltypes.TipSet
+	GetTipSet(tsk ltypes.TipSetKey) (*ltypes.TipSet, error)
+}
+
+// Status is the resolved EC finality state for a given head.
+type Status struct {
+	// ThresholdDepth is the shallowest epoch depth at which the reorg
+	// probability drops below 2^-30. -1 when not met or not computable
+	// (degraded chain, or observed window < MinWindow).
+	ThresholdDepth int
+	// FinalizedEpoch is head.Height() - ThresholdDepth, or -1 when
+	// ThresholdDepth is -1.
+	FinalizedEpoch abi.ChainEpoch
+	// HeadEpoch is the head height the computation ran against.
+	HeadEpoch abi.ChainEpoch
+	// WindowEpochs is how many epochs of observed history the calculator
+	// actually had (honesty signal: a freshly-booted node has only its
+	// anchor depth until #91/#92 give it a deeper tail).
+	WindowEpochs int
+}
+
+// Cache computes the FRC-0089 threshold for the current head and caches
+// the result, recomputing only when the head changes. Computation is
+// on-demand (dashboard/stats pulls), NOT on every head change: the cost
+// is dominated by the Skellam PMF loops, so an idle node pays nothing.
+// Safe for concurrent use.
+type Cache struct {
+	src      HeaderSource
+	finality int // L-distribution lookback (900 mainnet/calibration)
+
+	mu     sync.Mutex
+	cached *Status
+	calls  uint64 // lifetime Status calls
+	comps  uint64 // lifetime recomputes (cache misses)
+}
+
+// NewCache builds a Cache over the given header source. finality is the
+// lookback depth for the L distribution (900 for both mainnet and
+// calibration).
+func NewCache(src HeaderSource, finality int) *Cache {
+	return &Cache{src: src, finality: finality}
+}
+
+// Status returns the EC finality state for the current head, cached per
+// head tipset key. Errors are not cached; a transient store failure
+// allows retry on the next call.
+func (c *Cache) Status() (*Status, error) {
+	if c == nil || c.src == nil {
+		return nil, errors.New("ecfinality: no header source")
+	}
+	head := c.src.Head()
+	if head == nil {
+		return nil, errors.New("ecfinality: no head")
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+
+	if c.cached != nil && c.cached.HeadEpoch == head.Height() {
+		return c.cached, nil
+	}
+	c.comps++
+
+	chain, err := c.walkChain(head)
+	if err != nil {
+		return nil, err
+	}
+
+	st := &Status{
+		ThresholdDepth: -1,
+		FinalizedEpoch: -1,
+		HeadEpoch:      head.Height(),
+		WindowEpochs:   len(chain),
+	}
+	if len(chain) >= MinWindow {
+		guarantee := math.Pow(2, float64(DefaultSafetyExponent))
+		st.ThresholdDepth = FindThresholdDepth(chain, c.finality, DefaultBlocksPerEpoch, DefaultByzantineFraction, guarantee)
+		if st.ThresholdDepth >= 0 {
+			st.FinalizedEpoch = head.Height() - abi.ChainEpoch(st.ThresholdDepth)
+		}
+	}
+
+	c.cached = st
+	return st, nil
+}
+
+// Stats returns (lifetime Status calls, lifetime recomputes).
+func (c *Cache) Stats() (calls, recomputes uint64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls, c.comps
+}
+
+// walkChain walks back from head collecting block counts per epoch, oldest
+// first. Null rounds appear as 0 entries so the array index maps directly
+// to epoch height differences.
+//
+// Lantern's header store is windowed (anchor-rooted, pruning per #92), so
+// the walk stops gracefully where history runs out: a missing parent ends
+// the walk instead of erroring, and Status reports the achieved window.
+// The target is finality+5 epochs (the reference lookback); nodes with a
+// deeper tail (#91) converge to the exact reference behavior.
+func (c *Cache) walkChain(head *ltypes.TipSet) ([]int, error) {
+	needed := c.finality + 5
+	chain := make([]int, 0, min(needed, 1024))
+	ts := head
+	for {
+		chain = append(chain, len(ts.Cids()))
+		if len(chain) >= needed {
+			break
+		}
+		if ts.Height() == 0 {
+			break // genesis
+		}
+		parent, err := c.src.GetTipSet(ts.Parents())
+		if err != nil || parent == nil {
+			break // window exhausted: use what we observed
+		}
+		// Insert 0 entries for null rounds between this tipset and parent.
+		for nulls := int(ts.Height()-parent.Height()) - 1; nulls > 0 && len(chain) < needed; nulls-- {
+			chain = append(chain, 0)
+		}
+		ts = parent
+	}
+	// Reverse to chronological order (oldest first).
+	for i, j := 0, len(chain)-1; i < j; i, j = i+1, j-1 {
+		chain[i], chain[j] = chain[j], chain[i]
+	}
+	return chain, nil
+}

chain/ecfinality/cache_test.go

@@ -0,0 +1,161 @@
+package ecfinality
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+	"github.com/stretchr/testify/require"
+
+	ltypes "github.com/Reiers/lantern/chain/types"
+)
+
+func tCID(t *testing.T, s string) cid.Cid {
+	t.Helper()
+	hash, err := mh.Sum([]byte(s), mh.SHA2_256, -1)
+	require.NoError(t, err)
+	return cid.NewCidV1(cid.DagCBOR, hash)
+}
+
+func tHeader(t *testing.T, h abi.ChainEpoch, parents []cid.Cid, tag string) *ltypes.BlockHeader {
+	t.Helper()
+	miner, err := address.NewIDAddress(1000)
+	require.NoError(t, err)
+	return &ltypes.BlockHeader{
+		Miner:                 miner,
+		Ticket:                &ltypes.Ticket{VRFProof: []byte("t-" + tag)},
+		ElectionProof:         &ltypes.ElectionProof{WinCount: 1, VRFProof: []byte("e-" + tag)},
+		Parents:               parents,
+		ParentWeight:          ltypes.NewInt(uint64(h)),
+		Height:                h,
+		ParentStateRoot:       tCID(t, "state-"+tag),
+		ParentMessageReceipts: tCID(t, "receipts-"+tag),
+		Messages:              tCID(t, "msgs-"+tag),
+		Timestamp:             1_700_000_000 + uint64(h)*30,
+		ParentBaseFee:         ltypes.NewInt(100),
+	}
+}
+
+// fakeSource is an in-memory HeaderSource over a linked chain of tipsets.
+type fakeSource struct {
+	head *ltypes.TipSet
+	byK  map[string]*ltypes.TipSet
+}
+
+func (f *fakeSource) Head() *ltypes.TipSet { return f.head }
+func (f *fakeSource) GetTipSet(tsk ltypes.TipSetKey) (*ltypes.TipSet, error) {
+	ts, ok := f.byK[tsk.String()]
+	if !ok {
+		return nil, fmt.Errorf("not found")
+	}
+	return ts, nil
+}
+
+// buildChain builds a linked single-block-per-epoch chain of n epochs
+// starting at startEpoch, optionally skipping heights in skip (null
+// rounds). Blocks-per-epoch is 1 for simplicity; blockCount overrides via
+// widthAt (height -> number of blocks in the tipset).
+func buildChain(t *testing.T, startEpoch abi.ChainEpoch, n int, widthAt func(abi.ChainEpoch) int, skip map[abi.ChainEpoch]bool) *fakeSource {
+	t.Helper()
+	src := &fakeSource{byK: make(map[string]*ltypes.TipSet)}
+	var prev []cid.Cid
+	for i := 0; i < n; i++ {
+		h := startEpoch + abi.ChainEpoch(i)
+		if skip != nil && skip[h] {
+			continue // null round: no tipset at this height
+		}
+		width := 1
+		if widthAt != nil {
+			width = widthAt(h)
+		}
+		var blks []*ltypes.BlockHeader
+		for w := 0; w < width; w++ {
+			blks = append(blks, tHeader(t, h, prev, fmt.Sprintf("h%d-w%d", h, w)))
+		}
+		ts, err := ltypes.NewTipSet(blks)
+		require.NoError(t, err)
+		src.byK[ts.Key().String()] = ts
+		src.head = ts
+		prev = ts.Cids()
+	}
+	return src
+}
+
+// TestCache_HealthyWindowFindsThreshold: a healthy 5-wide chain with 120
+// epochs of history reports a threshold around the ~30-epoch mark.
+func TestCache_HealthyWindowFindsThreshold(t *testing.T) {
+	src := buildChain(t, 100, 120, func(abi.ChainEpoch) int { return 5 }, nil)
+	c := NewCache(src, 900)
+
+	st, err := c.Status()
+	require.NoError(t, err)
+	require.Equal(t, 120, st.WindowEpochs, "walk must cover the full available window")
+	require.Greater(t, st.ThresholdDepth, 0, "healthy chain must meet the guarantee")
+	require.Less(t, st.ThresholdDepth, 35, "healthy chain finalizes shallow")
+	require.Equal(t, st.HeadEpoch-abi.ChainEpoch(st.ThresholdDepth), st.FinalizedEpoch)
+}
+
+// TestCache_ShallowWindowNotComputable: a node with only a few epochs of
+// observed history (fresh boot from anchor) must report -1, not an
+// over-confident threshold.
+func TestCache_ShallowWindowNotComputable(t *testing.T) {
+	src := buildChain(t, 100, MinWindow-5, func(abi.ChainEpoch) int { return 5 }, nil)
+	c := NewCache(src, 900)
+
+	st, err := c.Status()
+	require.NoError(t, err)
+	require.Equal(t, MinWindow-5, st.WindowEpochs)
+	require.Equal(t, -1, st.ThresholdDepth, "shallow window must not report a threshold")
+	require.Equal(t, abi.ChainEpoch(-1), st.FinalizedEpoch)
+}
+
+// TestCache_NullRoundsCounted: a gap in heights (null rounds) appears as
+// 0-entries in the walked window, so WindowEpochs counts epochs, not
+// tipsets.
+func TestCache_NullRoundsCounted(t *testing.T) {
+	skip := map[abi.ChainEpoch]bool{150: true, 151: true}
+	src := buildChain(t, 100, 100, func(abi.ChainEpoch) int { return 5 }, skip)
+	c := NewCache(src, 900)
+
+	st, err := c.Status()
+	require.NoError(t, err)
+	require.Equal(t, 100, st.WindowEpochs, "null rounds count as epochs (0 blocks)")
+	require.Greater(t, st.ThresholdDepth, 0)
+}
+
+// TestCache_RecomputesOnlyOnHeadChange: repeated Status calls on the same
+// head are cache hits; a head advance triggers exactly one recompute.
+func TestCache_RecomputesOnlyOnHeadChange(t *testing.T) {
+	src := buildChain(t, 100, 120, func(abi.ChainEpoch) int { return 5 }, nil)
+	c := NewCache(src, 900)
+
+	_, err := c.Status()
+	require.NoError(t, err)
+	_, err = c.Status()
+	require.NoError(t, err)
+	calls, comps := c.Stats()
+	require.Equal(t, uint64(2), calls)
+	require.Equal(t, uint64(1), comps, "same head must be a cache hit")
+
+	// Advance head by one epoch.
+	newHead := tHeader(t, src.head.Height()+1, src.head.Cids(), "advance")
+	ts, err := ltypes.NewTipSet([]*ltypes.BlockHeader{newHead})
+	require.NoError(t, err)
+	src.byK[ts.Key().String()] = ts
+	src.head = ts
+
+	_, err = c.Status()
+	require.NoError(t, err)
+	_, comps = c.Stats()
+	require.Equal(t, uint64(2), comps, "head advance must recompute once")
+}
+
+// TestCache_NoHead: an empty source errors cleanly.
+func TestCache_NoHead(t *testing.T) {
+	c := NewCache(&fakeSource{byK: map[string]*ltypes.TipSet{}}, 900)
+	_, err := c.Status()
+	require.Error(t, err)
+}