feat: backward and forward LET testing + runbook + tool

.dockerignore

@@ -0,0 +1,29 @@
+# Version control
+.git/
+.gitignore
+
+# Sensitive / secret files
+.env
+.env.*
+*.pem
+*.key
+*.p12
+*.pfx
+*.crt
+*.cer
+
+# Editor and OS artifacts
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
+
+# Build artifacts (not needed — built inside the container)
+/out/
+
+# Test fixtures and E2E env (may contain keys/configs)
+test/e2e/envs/
+
+# Local developer configs
+*.local

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach to aggkit in Docker (dlv)",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "host": "127.0.0.1",
+            "port": 40000,
+            "apiVersion": 2,
+            "showLog": true
+        }
+    ]
+}

Dockerfile.debug

@@ -0,0 +1,47 @@
+# ================================
+# STAGE 1: Build debug binary + dlv
+# ================================
+FROM golang:1.25.7-alpine AS builder
+
+RUN apk add --no-cache gcc musl-dev make sqlite-dev git
+
+WORKDIR /home/aigent/repos/aggkit
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+# Install delve
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+# Build debug binary (disable inlining and optimizations for proper debugging)
+RUN CGO_ENABLED=1 go build \
+    -gcflags="all=-N -l" \
+    -o /out/aggkit \
+    ./cmd
+
+# ================================
+# STAGE 2: Debug runtime image
+# ================================
+FROM alpine:3.22
+
+RUN apk add --no-cache sqlite-libs ca-certificates \
+    && addgroup -S appgroup \
+    && adduser -S appuser -G appgroup
+
+COPY --from=builder /go/bin/dlv /usr/local/bin/dlv
+COPY --from=builder /out/aggkit /usr/local/bin/aggkit
+
+# Run as non-root. Note: the container must be started with
+# --cap-add=SYS_PTRACE so that delve can trace the target process.
+USER appuser
+
+EXPOSE 5576/tcp
+EXPOSE 40000/tcp
+
+# Default: run aggkit under delve headless. App args are provided by
+# the compose file's `command:` block (appended after the `--` separator).
+CMD ["dlv", "exec", "/usr/local/bin/aggkit", \
+     "--headless", "--listen=:40000", "--api-version=2", \
+     "--accept-multiclient", "--log"]

Makefile

@@ -97,6 +97,10 @@ build-docker-ci: ## Builds a docker image with the aggkit binary for CI (include
 build-docker-nc: ## Builds a docker image with the aggkit binary - but without build cache
 	docker build --no-cache=true -t aggkit:local -f ./Dockerfile .
 
+.PHONY: build-docker-debug
+build-docker-debug: ## Builds a debug docker image (dlv headless on :40000, no optimizations)
+	docker build -t aggkit:local-debug -f ./Dockerfile.debug .
+
 .PHONY: test-unit
 test-unit: ## Runs the unit tests
 	trap '$(STOP)' EXIT; MallocNanoZone=0 go test -count=1 -short -race -p 1 -covermode=atomic -coverprofile=coverage.out -timeout 15m ./...

agglayer/types/types.go

@@ -713,7 +713,13 @@ func (b *BridgeExit) UnmarshalJSON(data []byte) error {
 	b.DestinationAddress = aux.DestinationAddress
 	var ok bool
 	if !strings.Contains(aux.Amount, nilStr) {
-		b.Amount, ok = new(big.Int).SetString(aux.Amount, base10)
+		base := base10
+		amountStr := aux.Amount
+		if strings.HasPrefix(amountStr, "0x") || strings.HasPrefix(amountStr, "0X") {
+			base = 16
+			amountStr = amountStr[2:]
+		}
+		b.Amount, ok = new(big.Int).SetString(amountStr, base)
 		if !ok {
 			return fmt.Errorf("failed to convert amount to big.Int: %s", aux.Amount)
 		}

agglayer/types/types_test.go

@@ -1897,3 +1897,50 @@ func createTreeDBForTest(t *testing.T) *sql.DB {
 	require.NoError(t, err)
 	return treeDB
 }
+
+// TestBridgeExit_UnmarshalJSON_HexAmount verifies that BridgeExit correctly deserializes
+// amounts with a "0x"-prefixed hex string (added in this branch).
+func TestBridgeExit_UnmarshalJSON_HexAmount(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		amountJSON     string
+		expectedAmount *big.Int
+	}{
+		{
+			name:           "decimal amount",
+			amountJSON:     `"1000"`,
+			expectedAmount: big.NewInt(1000),
+		},
+		{
+			name:           "0x-prefixed hex amount",
+			amountJSON:     `"0x3e8"`,
+			expectedAmount: big.NewInt(1000),
+		},
+		{
+			name:           "0X-prefixed hex amount (uppercase)",
+			amountJSON:     `"0X3E8"`,
+			expectedAmount: big.NewInt(1000),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			jsonData := fmt.Sprintf(`{
+				"leaf_type": "Transfer",
+				"token_info": {"origin_network": 1, "origin_token_address": "0x0000000000000000000000000000000000000001"},
+				"dest_network": 2,
+				"dest_address": "0x0000000000000000000000000000000000000002",
+				"amount": %s,
+				"metadata": null
+			}`, tc.amountJSON)
+
+			var be BridgeExit
+			require.NoError(t, json.Unmarshal([]byte(jsonData), &be))
+			require.Equal(t, tc.expectedAmount, be.Amount)
+		})
+	}
+}

aggsender/aggsender.go

@@ -133,6 +133,18 @@ func newAggsender(
 		return nil, err
 	}
 
+	aggchainFEPCaller, err := query.NewAggchainFEPQuerier(logger, cfg.Mode,
+		cfg.SovereignRollupAddr, l1Client)
+	if err != nil {
+		return nil, fmt.Errorf("error creating aggchain FEP caller: %w", err)
+	}
+
+	certQuerier := query.NewCertificateQuerier(
+		l2Syncer,
+		aggchainFEPCaller,
+		aggLayerClient,
+	)
+
 	flowManager, err := flows.NewBuilderFlow(
 		ctx,
 		cfg,
@@ -144,6 +156,7 @@ func newAggsender(
 		l2Syncer,
 		rollupDataQuerier,
 		committeeQuerier,
+		certQuerier,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error creating flow manager: %w", err)
@@ -161,18 +174,6 @@ func newAggsender(
 		compatibility.NewKeyValueToCompatibilityStorage[db.RuntimeData](storage, aggkitcommon.AGGSENDER),
 	)
 
-	aggchainFEPCaller, err := query.NewAggchainFEPQuerier(logger, cfg.Mode,
-		cfg.SovereignRollupAddr, l1Client)
-	if err != nil {
-		return nil, fmt.Errorf("error creating aggchain FEP caller: %w", err)
-	}
-
-	certQuerier := query.NewCertificateQuerier(
-		l2Syncer,
-		aggchainFEPCaller,
-		aggLayerClient,
-	)
-
 	verifierFlow, err := flows.NewLocalVerifier(
 		ctx,
 		cfg,
@@ -252,8 +253,13 @@ func (a *AggSender) GetRPCServices() []jRPC.Service {
 	logger := log.WithFields("module", "aggsender-rpc")
 	return []jRPC.Service{
 		{
-			Name:    "aggsender",
-			Service: aggsenderrpc.NewAggsenderRPC(logger, a.storage, a),
+			Name: "aggsender",
+			Service: aggsenderrpc.NewAggsenderRPC(
+				logger, a.storage, a,
+				a.cfg.EnableDebugSendCertificate,
+				a.cfg.DebugSendCertificateAuthAddress,
+				a.aggLayerClient,
+			),
 		},
 	}
 }
@@ -264,6 +270,11 @@ func (a *AggSender) Start(ctx context.Context) {
 	metrics.Register()
 	a.status.Start(time.Now().UTC())
 
+	if a.cfg.EnableDebugSendCertificate {
+		a.log.Warn("Debug send certificate endpoint enabled — aggsender certificate sending is DISABLED")
+		return
+	}
+
 	a.checkDBCompatibility(ctx)
 	a.certStatusChecker.CheckInitialStatus(ctx, a.cfg.DelayBetweenRetries.Duration, a.status)
 	if err := a.flow.CheckInitialStatus(ctx); err != nil {

aggsender/aggsender_test.go

@@ -74,7 +74,8 @@ func TestConfigString(t *testing.T) {
 		"RetriesToBuildAndSendCertificate: RetryPolicyConfig{Mode: , Config: RetryDelaysConfig{Delays: [], MaxRetries: NO RETRIES}}\n"+
 		"StorageRetainCertificatesPolicy: retain all certificates, keep history: false\n"+
 		"BlockFinalityForL1InfoTree: FinalizedBlock\n"+
-		"TriggerCertMode: Auto\nTriggerEpochBased: EpochNotificationPercentage: 50\n",
+		"TriggerCertMode: Auto\nTriggerEpochBased: EpochNotificationPercentage: 50\n"+
+		"EnableDebugSendCertificate: false\n",
 		config.AgglayerClient.String())
 
 	require.Equal(t, expected, config.String())
@@ -250,7 +251,7 @@ func TestSendCertificate_NoClaims(t *testing.T) {
 		localValidator:         mockLocalValidator,
 		flow: flows.NewPPBuilderFlow(logger,
 			flows.NewBaseFlow(logger, mockL2BridgeQuerier, mockStorage,
-				mockL1Querier, mockLERQuerier, flows.NewBaseFlowConfigDefault()),
+				mockL1Querier, mockLERQuerier, nil, flows.NewBaseFlowConfigDefault()),
 			mockStorage, mockL1Querier, mockL2BridgeQuerier, signer, true, 0),
 	}
 
@@ -862,7 +863,7 @@ func newAggsenderTestData(t *testing.T, creationFlags testDataFlags) *aggsenderT
 		},
 		flow: flows.NewPPBuilderFlow(logger,
 			flows.NewBaseFlow(logger, l2BridgeQuerier, storage,
-				l1InfoTreeQuerierMock, lerQuerier, flows.NewBaseFlowConfigDefault()),
+				l1InfoTreeQuerierMock, lerQuerier, nil, flows.NewBaseFlowConfigDefault()),
 			storage, l1InfoTreeQuerierMock, l2BridgeQuerier, signer, true, 0),
 	}
 	var flowMock *mocks.AggsenderBuilderFlow

aggsender/config/config.go

@@ -150,6 +150,13 @@ type Config struct {
 	TriggerEpochBased TriggerEpochBasedConfig `mapstructure:"TriggerEpochBased"`
 	// TriggerASAP is the configuration for the ASAP trigger mode (TriggerCertMode==ASAP)
 	TriggerASAP TriggerASAPConfig `mapstructure:"TriggerASAP"`
+	// EnableDebugSendCertificate enables the debug RPC endpoint for sending arbitrary certificates.
+	// When true, the aggsender's normal certificate-sending loop is disabled.
+	// Default false. NEVER enable in production.
+	EnableDebugSendCertificate bool `mapstructure:"EnableDebugSendCertificate"`
+	// DebugSendCertificateAuthAddress is the Ethereum address authorized to sign debug send requests.
+	// Only used when EnableDebugSendCertificate is true.
+	DebugSendCertificateAuthAddress ethCommon.Address `mapstructure:"DebugSendCertificateAuthAddress"`
 }
 
 func (c Config) CheckCertConfigBriefString() string {
@@ -174,7 +181,8 @@ func (c Config) String() string {
 		"StorageRetainCertificatesPolicy: " + c.StorageRetainCertificatesPolicy.String() + "\n" +
 		"BlockFinalityForL1InfoTree: " + c.BlockFinalityForL1InfoTree.String() + "\n" +
 		"TriggerCertMode: " + c.TriggerCertMode.String() + "\n" +
-		"TriggerEpochBased: " + c.TriggerEpochBased.String() + "\n"
+		"TriggerEpochBased: " + c.TriggerEpochBased.String() + "\n" +
+		"EnableDebugSendCertificate: " + fmt.Sprintf("%t", c.EnableDebugSendCertificate) + "\n"
 }
 
 // Validate checks if the configuration is valid
@@ -200,5 +208,8 @@ func (c Config) Validate() error {
 	if err := c.TriggerCertMode.Validate(); err != nil {
 		return fmt.Errorf("invalid TriggerCertMode config: %w", err)
 	}
+	if c.EnableDebugSendCertificate && c.DebugSendCertificateAuthAddress == (ethCommon.Address{}) {
+		return fmt.Errorf("DebugSendCertificateAuthAddress must be set when EnableDebugSendCertificate is enabled")
+	}
 	return nil
 }

aggsender/db/aggsender_db_storage.go

@@ -100,6 +100,8 @@ type AggSenderStorage interface {
 	SaveOrUpdateCertificate(ctx context.Context, certificate types.Certificate) error
 	// GetLastSettledCertificate returns the last settled certificate from the storage
 	GetLastSettledCertificate() (*types.CertificateHeader, error)
+	// GetCertificateBridgeExits returns the bridge exits for the signed certificate at the given height
+	GetCertificateBridgeExits(height uint64) ([]*agglayertypes.BridgeExit, error)
 }
 
 var _ AggSenderStorage = (*AggSenderSQLStorage)(nil)
@@ -286,6 +288,23 @@ func (a *AggSenderSQLStorage) GetLastSettledCertificate() (*types.CertificateHea
 	return &certificateHeader, nil
 }
 
+// GetCertificateBridgeExits returns the bridge exits for the signed certificate at the given height.
+// Returns nil if no certificate exists at that height or the certificate has no signed certificate data.
+func (a *AggSenderSQLStorage) GetCertificateBridgeExits(height uint64) ([]*agglayertypes.BridgeExit, error) {
+	cert, err := a.GetCertificateByHeight(height)
+	if err != nil {
+		return nil, err
+	}
+	if cert == nil || cert.SignedCertificate == nil {
+		return nil, nil
+	}
+	var agglayerCert agglayertypes.Certificate
+	if err := json.Unmarshal([]byte(*cert.SignedCertificate), &agglayerCert); err != nil {
+		return nil, fmt.Errorf("GetCertificateBridgeExits: failed to unmarshal certificate at height %d: %w", height, err)
+	}
+	return agglayerCert.BridgeExits, nil
+}
+
 // SaveOrUpdateCertificate saves the certificate in the storage
 // It will insert a new certificate or update the existing one if it has the same height and certificate ID
 func (a *AggSenderSQLStorage) SaveOrUpdateCertificate(ctx context.Context, certificate types.Certificate) error {