From ab9ff72d2a2fe7bf776441df884ee5cb4782693f Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:56:56 +0100 Subject: [PATCH 01/74] Add ics23 integration --- ibc/store/proofs_ics23.go | 143 ++++++++++++++++++++++ shared/core/types/error.go | 6 + shared/core/types/proto/commitments.proto | 11 ++ 3 files changed, 160 insertions(+) create mode 100644 ibc/store/proofs_ics23.go create mode 100644 shared/core/types/proto/commitments.proto diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go new file mode 100644 index 000000000..498ad4a79 --- /dev/null +++ b/ibc/store/proofs_ics23.go @@ -0,0 +1,143 @@ +package store + +import ( + "bytes" + "crypto/sha256" + + ics23 "github.com/cosmos/ics23/go" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/smt" +) + +type position int + +const ( + left position = iota // 0 + right // 1 +) + +var ( + // Custom SMT spec as the store does not hash values + smtSpec *ics23.ProofSpec = &ics23.ProofSpec{ + LeafSpec: &ics23.LeafOp{ + Hash: ics23.HashOp_SHA256, + PrehashKey: ics23.HashOp_SHA256, + PrehashValue: ics23.HashOp_NO_HASH, + Length: ics23.LengthOp_NO_PREFIX, + Prefix: []byte{0}, + }, + InnerSpec: &ics23.InnerSpec{ + ChildOrder: []int32{0, 1}, + ChildSize: 32, + MinPrefixLength: 1, + MaxPrefixLength: 1, + EmptyChild: make([]byte, 32), + Hash: ics23.HashOp_SHA256, + }, + MaxDepth: 256, + PrehashKeyBeforeComparison: true, + } + innerPrefix = []byte{1} + + // defaultValue is the default placeholder value in a SparseMerkleTree + defaultValue = make([]byte, 32) +) + +// VerifyMembership verifies the CommitmentProof provided, checking whether it produces the same +// root as the one given. If it does, the key-value pair is a member of the tree +func VerifyMembership(root *coreTypes.CommitmentRoot, proof *ics23.CommitmentProof, key, value []byte) bool { + // verify the proof + return ics23.VerifyMembership(smtSpec, root.Hash, proof, key, value) +} + +// VerifyNonMembership verifies the CommitmentProof provided, checking whether it produces the same +// root as the one given. If it does, the key-value pair is not a member of the tree as the proof's +// value is either the default nil value for the SMT or an unrelated value at the path +func VerifyNonMembership(root *coreTypes.CommitmentRoot, proof *ics23.CommitmentProof, key []byte) bool { + // Verify the proof of the non-membership data doesn't belong to the key + valid := ics23.VerifyMembership(smtSpec, root.Hash, proof, key, proof.GetExist().GetValue()) + // Verify the key was actually empty + if bytes.Equal(proof.GetExist().GetValue(), defaultValue) { + return valid + } + // Verify the key was present with unrelated data + return !valid +} + +// createMembershipProof generates a CommitmentProof object verifying the membership of a key-value pair +// in the SMT provided +func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentProof, error) { + proof, err := tree.Prove(key) + if err != nil { + return nil, coreTypes.ErrCreatingProof(err.Error()) + } + return &ics23.CommitmentProof{ + Proof: &ics23.CommitmentProof_Exist{ + Exist: convertSMPToExistenceProof(&proof, key, value), + }, + }, nil +} + +// createNonMembershipProof generates a CommitmentProof object verifying the membership of an unrealted key at the given key in the SMT provided +func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof, error) { + proof, err := tree.Prove(key) + if err != nil { + return nil, coreTypes.ErrCreatingProof(err.Error()) + } + + value := defaultValue + if proof.NonMembershipLeafData != nil { + value = proof.NonMembershipLeafData[33:] + } + + return &ics23.CommitmentProof{ + Proof: &ics23.CommitmentProof_Exist{ + Exist: convertSMPToExistenceProof(&proof, key, value), + }, + }, nil +} + +// convertSMPToExistenceProof converts a SparseMerkleProof to an ICS23 ExistenceProof used for +// both membership and non-membership proof verification +func convertSMPToExistenceProof(proof *smt.SparseMerkleProof, key, value []byte) *ics23.ExistenceProof { + path := sha256.Sum256(key) + steps := make([]*ics23.InnerOp, 0, len(proof.SideNodes)) + for i := 0; i < len(proof.SideNodes); i++ { + var prefix, suffix []byte + prefix = append(prefix, innerPrefix...) + if getPathBit(path[:], len(proof.SideNodes)-1-i) == left { + suffix = make([]byte, 0, len(proof.SideNodes[i])) + suffix = append(suffix, proof.SideNodes[i]...) + } else { + prefix = append(prefix, proof.SideNodes[i]...) + } + op := &ics23.InnerOp{ + Hash: ics23.HashOp_SHA256, + Prefix: prefix, + Suffix: suffix, + } + steps = append(steps, op) + } + leaf := &ics23.LeafOp{ + Hash: ics23.HashOp_SHA256, + PrehashKey: ics23.HashOp_SHA256, + PrehashValue: ics23.HashOp_NO_HASH, + Length: ics23.LengthOp_NO_PREFIX, + Prefix: []byte{0}, + } + return &ics23.ExistenceProof{ + Key: key, + Value: value, + Leaf: leaf, + Path: steps, + } +} + +// getPathBit determines whether the hash of a node at a certain depth in the tree is the +// left or the right child of its parent +func getPathBit(data []byte, position int) position { + if int(data[position/8])&(1<<(8-1-uint(position)%8)) > 0 { + return right + } + return left +} diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 4de0bc6ef..a7593489c 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -180,6 +180,7 @@ const ( CodeHostAlreadyExists Code = 138 CodeIBCInvalidID Code = 139 CodeIBCInvalidPath Code = 140 + CodeCreatingProofError Code = 141 ) const ( @@ -318,6 +319,7 @@ const ( HostAlreadyExistsError = "an ibc host already exists" IBCInvalidIDError = "invalid ibc identifier" IBCInvalidPathError = "invalid ibc path" + CreatingProofError = "an error occurred creating the CommitmentProof" ) func ErrUnknownParam(paramName string) Error { @@ -864,3 +866,7 @@ func ErrIBCInvalidID(identifier, msg string) Error { func ErrIBCInvalidPath(path string) Error { return NewError(CodeIBCInvalidPath, fmt.Sprintf("%s: %s", IBCInvalidPathError, path)) } + +func ErrCreatingProof(err error) Error { + return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", CreatingProofError, err.Error())) +} diff --git a/shared/core/types/proto/commitments.proto b/shared/core/types/proto/commitments.proto new file mode 100644 index 000000000..aee5008b3 --- /dev/null +++ b/shared/core/types/proto/commitments.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package core; + +option go_package = "github.com/pokt-network/pocket/shared/core/types"; + +// CommitmentRoot represents the merkle root hash of a CommitmentState +// This is used in the verification of CommitmentProof objects +message CommitmentRoot { + bytes hash = 1; +} From b47402d33ec2b63daca60aa85c85e21511bbbd41 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:23:36 +0100 Subject: [PATCH 02/74] Add SMT proof conversion to ics23 Existence and Exclusion proofs with verification --- go.mod | 5 + go.sum | 2 + ibc/host/path_test.go | 18 +-- ibc/store/proofs_ics23.go | 110 +++++++------ ibc/store/proofs_ics23_test.go | 178 ++++++++++++++++++++++ shared/core/types/proto/commitments.proto | 11 -- 6 files changed, 256 insertions(+), 68 deletions(-) create mode 100644 ibc/store/proofs_ics23_test.go delete mode 100644 shared/core/types/proto/commitments.proto diff --git a/go.mod b/go.mod index 61a9ad1c1..937561931 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,9 @@ module github.com/pokt-network/pocket go 1.18 +// TECHDEBT: remove once upstream PR is merged (see: https://github.com/cosmos/ics23/pull/153) +replace github.com/cosmos/ics23/go => ../ics23/go + // TECHDEBT: remove once upstream PR is merged (see: https://github.com/regen-network/gocuke/pull/12) replace github.com/regen-network/gocuke => github.com/pokt-network/gocuke v0.0.1 @@ -21,6 +24,7 @@ require ( require ( github.com/benbjohnson/clock v1.3.0 + github.com/cosmos/ics23/go v0.10.0 github.com/deepmap/oapi-codegen v1.12.4 github.com/dgraph-io/badger/v3 v3.2103.2 github.com/foxcpp/go-mockdns v1.0.0 @@ -87,6 +91,7 @@ require ( github.com/cockroachdb/apd/v3 v3.1.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cosmos/gogoproto v1.4.3 // indirect github.com/cucumber/common/messages/go/v19 v19.1.2 // indirect github.com/cucumber/gherkin/go/v26 v26.0.3 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect diff --git a/go.sum b/go.sum index 7d7377ae4..4e8bb7827 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cosmos/gogoproto v1.4.3 h1:RP3yyVREh9snv/lsOvmsAPQt8f44LgL281X0IOIhhcI= +github.com/cosmos/gogoproto v1.4.3/go.mod h1:0hLIG5TR7IvV1fme1HCFKjfzW9X2x0Mo+RooWXCnOWU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= diff --git a/ibc/host/path_test.go b/ibc/host/path_test.go index 1c1c8d367..041b2a239 100644 --- a/ibc/host/path_test.go +++ b/ibc/host/path_test.go @@ -65,14 +65,14 @@ func TestPaths_CommitmentPrefix(t *testing.T) { result string }{ { - name: "Successfully applies and removes prefix to produce the same path", + // Successfully applies and removes prefix to produce the same path path: "path", prefix: coreTypes.CommitmentPrefix([]byte("test")), expected: []byte("test/path"), result: "path", }, { - name: "Fails to produce input path when given a different prefix", + // Fails to produce input path when given a different prefix path: "path", prefix: coreTypes.CommitmentPrefix([]byte("test2")), expected: []byte("test/path"), @@ -81,14 +81,12 @@ func TestPaths_CommitmentPrefix(t *testing.T) { } for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - commitment := ApplyPrefix(prefix, tc.path) - require.NotNil(t, commitment) - require.Equal(t, []byte(commitment), tc.expected) + commitment := ApplyPrefix(prefix, tc.path) + require.NotNil(t, commitment) + require.Equal(t, []byte(commitment), tc.expected) - path := RemovePrefix(tc.prefix, commitment) - require.NotNil(t, path) - require.Equal(t, path, tc.result) - }) + path := RemovePrefix(tc.prefix, commitment) + require.NotNil(t, path) + require.Equal(t, path, tc.result) } } diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 498ad4a79..c1ab2f390 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -1,7 +1,6 @@ package store import ( - "bytes" "crypto/sha256" ics23 "github.com/cosmos/ics23/go" @@ -45,23 +44,17 @@ var ( // VerifyMembership verifies the CommitmentProof provided, checking whether it produces the same // root as the one given. If it does, the key-value pair is a member of the tree -func VerifyMembership(root *coreTypes.CommitmentRoot, proof *ics23.CommitmentProof, key, value []byte) bool { +func VerifyMembership(root ics23.CommitmentRoot, proof *ics23.CommitmentProof, key, value []byte) bool { // verify the proof - return ics23.VerifyMembership(smtSpec, root.Hash, proof, key, value) + return ics23.VerifyMembership(smtSpec, root, proof, key, value) } // VerifyNonMembership verifies the CommitmentProof provided, checking whether it produces the same // root as the one given. If it does, the key-value pair is not a member of the tree as the proof's // value is either the default nil value for the SMT or an unrelated value at the path -func VerifyNonMembership(root *coreTypes.CommitmentRoot, proof *ics23.CommitmentProof, key []byte) bool { - // Verify the proof of the non-membership data doesn't belong to the key - valid := ics23.VerifyMembership(smtSpec, root.Hash, proof, key, proof.GetExist().GetValue()) - // Verify the key was actually empty - if bytes.Equal(proof.GetExist().GetValue(), defaultValue) { - return valid - } - // Verify the key was present with unrelated data - return !valid +func VerifyNonMembership(root ics23.CommitmentRoot, proof *ics23.CommitmentProof, key []byte) bool { + // verify the proof + return ics23.VerifyNonMembership(smtSpec, root, proof, key) } // createMembershipProof generates a CommitmentProof object verifying the membership of a key-value pair @@ -69,47 +62,82 @@ func VerifyNonMembership(root *coreTypes.CommitmentRoot, proof *ics23.Commitment func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) if err != nil { - return nil, coreTypes.ErrCreatingProof(err.Error()) + return nil, coreTypes.ErrCreatingProof(err) } - return &ics23.CommitmentProof{ - Proof: &ics23.CommitmentProof_Exist{ - Exist: convertSMPToExistenceProof(&proof, key, value), - }, - }, nil + return convertSMPToExistenceProof(&proof, key, value), nil } // createNonMembershipProof generates a CommitmentProof object verifying the membership of an unrealted key at the given key in the SMT provided func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) if err != nil { - return nil, coreTypes.ErrCreatingProof(err.Error()) + return nil, coreTypes.ErrCreatingProof(err) } - value := defaultValue - if proof.NonMembershipLeafData != nil { - value = proof.NonMembershipLeafData[33:] - } + return convertSMPToExclusionProof(&proof, key), nil +} +// convertSMPToExistenceProof converts a SparseMerkleProof to an ics23 +// ExistenceProof to verify membership of an element +func convertSMPToExistenceProof(proof *smt.SparseMerkleProof, key, value []byte) *ics23.CommitmentProof { + path := sha256.Sum256(key) + steps := convertSideNodesToSteps(proof.SideNodes, path[:]) return &ics23.CommitmentProof{ Proof: &ics23.CommitmentProof_Exist{ - Exist: convertSMPToExistenceProof(&proof, key, value), + Exist: &ics23.ExistenceProof{ + Key: key, + Value: value, + Leaf: smtSpec.LeafSpec, + Path: steps, + }, }, - }, nil + } } -// convertSMPToExistenceProof converts a SparseMerkleProof to an ICS23 ExistenceProof used for -// both membership and non-membership proof verification -func convertSMPToExistenceProof(proof *smt.SparseMerkleProof, key, value []byte) *ics23.ExistenceProof { +// convertSMPToExistenceProof converts a SparseMerkleProof to an ics23 +// ExclusionProof to verify non-membership of an element +func convertSMPToExclusionProof(proof *smt.SparseMerkleProof, key []byte) *ics23.CommitmentProof { path := sha256.Sum256(key) - steps := make([]*ics23.InnerOp, 0, len(proof.SideNodes)) - for i := 0; i < len(proof.SideNodes); i++ { + steps := convertSideNodesToSteps(proof.SideNodes, path[:]) + leaf := &ics23.LeafOp{ + Hash: ics23.HashOp_SHA256, + // Do not re-hash already hashed fields from NonMembershipLeafData + PrehashKey: ics23.HashOp_NO_HASH, + PrehashValue: ics23.HashOp_NO_HASH, + Length: ics23.LengthOp_NO_PREFIX, + Prefix: []byte{0}, + } + actualPath := path[:] + actualValue := defaultValue + if proof.NonMembershipLeafData != nil { + actualPath = proof.NonMembershipLeafData[1:33] + actualValue = proof.NonMembershipLeafData[33:] + } + return &ics23.CommitmentProof{ + Proof: &ics23.CommitmentProof_Exclusion{ + Exclusion: &ics23.ExclusionProof{ + Key: key, + ActualPath: actualPath, + ActualValueHash: actualValue, + Leaf: leaf, + Path: steps, + }, + }, + } +} + +// convertSideNodesToSteps converts the SideNodes field in the SparseMerkleProof +// into a list of InnerOps for the ics23 CommitmentProof +func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { + steps := make([]*ics23.InnerOp, 0, len(sideNodes)) + for i := 0; i < len(sideNodes); i++ { var prefix, suffix []byte prefix = append(prefix, innerPrefix...) - if getPathBit(path[:], len(proof.SideNodes)-1-i) == left { - suffix = make([]byte, 0, len(proof.SideNodes[i])) - suffix = append(suffix, proof.SideNodes[i]...) + if getPathBit(path[:], len(sideNodes)-1-i) == left { + suffix = make([]byte, 0, len(sideNodes[i])) + suffix = append(suffix, sideNodes[i]...) } else { - prefix = append(prefix, proof.SideNodes[i]...) + prefix = append(prefix, sideNodes[i]...) } op := &ics23.InnerOp{ Hash: ics23.HashOp_SHA256, @@ -118,19 +146,7 @@ func convertSMPToExistenceProof(proof *smt.SparseMerkleProof, key, value []byte) } steps = append(steps, op) } - leaf := &ics23.LeafOp{ - Hash: ics23.HashOp_SHA256, - PrehashKey: ics23.HashOp_SHA256, - PrehashValue: ics23.HashOp_NO_HASH, - Length: ics23.LengthOp_NO_PREFIX, - Prefix: []byte{0}, - } - return &ics23.ExistenceProof{ - Key: key, - Value: value, - Leaf: leaf, - Path: steps, - } + return steps } // getPathBit determines whether the hash of a node at a certain depth in the tree is the diff --git a/ibc/store/proofs_ics23_test.go b/ibc/store/proofs_ics23_test.go new file mode 100644 index 000000000..97cd63011 --- /dev/null +++ b/ibc/store/proofs_ics23_test.go @@ -0,0 +1,178 @@ +package store + +import ( + "crypto/sha256" + "testing" + + ics23 "github.com/cosmos/ics23/go" + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) + +// Proof generation cannot fail but verification can +func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { + nodeStore := kvstore.NewMemKVStore() + tree := smt.NewSparseMerkleTree(nodeStore, sha256.New(), smt.WithValueHasher(nil)) + require.NotNil(t, tree) + + // Set a value in the store + err := tree.Update([]byte("foo"), []byte("bar")) + require.NoError(t, err) + err = tree.Update([]byte("bar"), []byte("foo")) + require.NoError(t, err) + err = tree.Update([]byte("testKey"), []byte("testValue")) + require.NoError(t, err) + err = tree.Update([]byte("testKey2"), []byte("testValue2")) + require.NoError(t, err) + + testCases := []struct { + key []byte + value []byte + nonmembership bool + fails bool + expected error + }{ + { + // Successfully generates a membership proof for a key stored + key: []byte("foo"), + value: []byte("bar"), + nonmembership: false, + fails: false, + expected: nil, + }, + { + // Successfully generates a non-membership proof for a key not stored + key: []byte("baz"), + value: []byte("testValue2"), // unrelated leaf data + nonmembership: true, + fails: false, + expected: nil, + }, + { + // Successfully generates a non-membership proof for an unset nil key + key: nil, + value: []byte("foo"), // unrelated leaf data + nonmembership: true, + fails: false, + expected: nil, + }, + } + + for _, tc := range testCases { + var proof *ics23.CommitmentProof + if tc.nonmembership { + proof, err = createNonMembershipProof(tree, tc.key) + } else { + proof, err = createMembershipProof(tree, tc.key, tc.value) + } + if tc.fails { + require.EqualError(t, err, tc.expected.Error()) + require.Nil(t, proof) + return + } + require.NoError(t, err) + require.NotNil(t, proof) + if tc.nonmembership { + require.Equal(t, tc.value, proof.GetExclusion().GetActualValueHash()) + require.NotNil(t, proof.GetExclusion().GetLeaf()) + require.NotNil(t, proof.GetExclusion().GetPath()) + } else { + require.Equal(t, tc.value, proof.GetExist().GetValue()) + require.NotNil(t, proof.GetExist().GetLeaf()) + require.NotNil(t, proof.GetExist().GetPath()) + } + } + + err = nodeStore.Stop() + require.NoError(t, err) +} + +func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { + nodeStore := kvstore.NewMemKVStore() + tree := smt.NewSparseMerkleTree(nodeStore, sha256.New(), smt.WithValueHasher(nil)) + require.NotNil(t, tree) + + // Set a value in the store + err := tree.Update([]byte("foo"), []byte("bar")) + require.NoError(t, err) + err = tree.Update([]byte("bar"), []byte("foo")) + require.NoError(t, err) + err = tree.Update([]byte("testKey"), []byte("testValue")) + require.NoError(t, err) + err = tree.Update([]byte("testKey2"), []byte("testValue2")) + require.NoError(t, err) + + root := tree.Root() + require.NotNil(t, root) + + testCases := []struct { + key []byte + value []byte + nonmembership bool + valid bool + }{ + { + // Successfully verifies a membership proof for a key-value stored pair + key: []byte("foo"), + value: []byte("bar"), + nonmembership: false, + valid: true, + }, + { + // Successfully verifies a non-membership proof for a key-value pair not stored + key: []byte("not stored"), + value: nil, + nonmembership: true, + valid: true, + }, + { + // Fails to verify a membership proof for a key-value pair not stored + key: []byte("baz"), + value: []byte("bar"), + nonmembership: false, + valid: false, + }, + { + // Fails to verify a non-membership proof for a key stored in the tree + key: []byte("foo"), + value: nil, + nonmembership: true, + valid: false, + }, + } + + proof := new(ics23.CommitmentProof) + for _, tc := range testCases { + var err error + if tc.nonmembership { + proof, err = createNonMembershipProof(tree, tc.key) + } else { + proof, err = createMembershipProof(tree, tc.key, tc.value) + } + require.NoError(t, err) + require.NotNil(t, proof) + + if tc.nonmembership { + require.NotNil(t, proof.GetExclusion()) + } else { + require.NotNil(t, proof.GetExist()) + } + + var valid bool + if tc.nonmembership { + valid = VerifyNonMembership(root, proof, tc.key) + } else { + valid = VerifyMembership(root, proof, tc.key, tc.value) + } + + if tc.valid { + require.True(t, valid) + } else { + require.False(t, valid) + } + } + + err = nodeStore.Stop() + require.NoError(t, err) +} diff --git a/shared/core/types/proto/commitments.proto b/shared/core/types/proto/commitments.proto deleted file mode 100644 index aee5008b3..000000000 --- a/shared/core/types/proto/commitments.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package core; - -option go_package = "github.com/pokt-network/pocket/shared/core/types"; - -// CommitmentRoot represents the merkle root hash of a CommitmentState -// This is used in the verification of CommitmentProof objects -message CommitmentRoot { - bytes hash = 1; -} From 20fa2fc2307ce4d489bef343003bf8c4e2d881cb Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:42:39 +0100 Subject: [PATCH 03/74] Add ICS23 docs --- ibc/docs/README.md | 9 +++++- ibc/docs/ics23.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 ibc/docs/ics23.md diff --git a/ibc/docs/README.md b/ibc/docs/README.md index 26de85af7..0ea5dd98d 100644 --- a/ibc/docs/README.md +++ b/ibc/docs/README.md @@ -9,6 +9,7 @@ - [Persistence](#persistence) - [Components](#components) - [ICS-24 Host Requirements](#ics-24-host-requirements) + - [ICS-23 Vector Commitments](#ics-23-vector-commitments) ## Definitions @@ -106,6 +107,12 @@ The [IBC specification][ibc-spec] details numerous Interchain Standards (ICSs) t See: [ICS-24](./ics24.md) for more details on the specifics of the ICS-24 implementation for Pocket. +### ICS-23 Vector Commitments + +[ICS-23][ics23] defines the `CommitmentProof` type that is used to prove the membership/non-membership of a key-value pair in the IBC stores. As this type is serialisable the relayers can relay these proofs to a counterparty chain, as described above, for their light client (of the source chain) to verify the proof and thus react accordingly. In order to implement ICS-23, the `cosmos/ics23` library was used, specifically its `CommitmentProof` type and its methods to verify the proofs in a way that does not require the tree itself. + +See: [ICS-23](./ics23.md) for more details on the specifics of the ICS-23 implementation for Pocket. + [ibc-spec]: https://github.com/cosmos/ibc [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md -[smt]: https://github.com/pokt-network/smt +[ics23]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-023-vector-commitments/README.md diff --git a/ibc/docs/ics23.md b/ibc/docs/ics23.md new file mode 100644 index 000000000..95cc2895e --- /dev/null +++ b/ibc/docs/ics23.md @@ -0,0 +1,78 @@ +# ICS-23 Vector Commitments + +- [Overview](#overview) +- [Implementation](#implementation) + - [Custom SMT `ProofSpec`](#custom-smt-proofspec) + - [Converting `SparseMerkleProof` to `CommitmentProof`](#converting-sparsemerkleproof-to-commitmentproof) + - [Proof Verification](#proof-verification) + +## Overview + +[ICS-23][ics23] defines the types and functions needed to verify membership of a key-value pair in a `CommitmentState`. As the Pocket IBC implementation uses the [SMT][smt] for its provable stores this is what is referred to as the `CommitmentState` object. Cosmos has a library `cosmos/ics23` which is already SDK agnostic and defines many of the types necessary for ICS-23. This library was able to be used _mostly_ out of the box, with some minor adjustments detailed below. + +## Implementation + +The benefit of using `cosmos/ics23` over implementing similar types ourselves is twofold: + +1. It is already SDK agnostic, so can be used by Pocket (a non-cosmos chain) without any issues. +2. The functions defined for proof verification are separate from the tree structures themselves + - This means we do not need to interact with an SMT instance in order to verify proofs. + +However, there were some changes made specifically for Pocket's implementation of ICS-23. + +### Custom SMT `ProofSpec` + +The `ProofSpec` type in `cosmos/ics23` is used to define the steps needed to verify a proof, what hashing functions should be used, any node prefixes, etc. This is then passed into the verification functions in order to verify a proof, instead of having to interact with the tree itself. This is useful as proofs must be verified on a light-client and as such being able to verify a proof without creating a tree is much more memory efficient. + +As the SMT used by Pocket Network only stores hashed values by default, the IBC stores use the `WithValueHasher(nil)` option which stores the raw bytes of the values in the tree. As such the following `ProofSpec` was created: + +```go +smtSpec *ics23.ProofSpec = &ics23.ProofSpec{ + LeafSpec: &ics23.LeafOp{ + Hash: ics23.HashOp_SHA256, + PrehashKey: ics23.HashOp_SHA256, + PrehashValue: ics23.HashOp_NO_HASH, + Length: ics23.LengthOp_NO_PREFIX, + Prefix: []byte{0}, + }, + InnerSpec: &ics23.InnerSpec{ + ChildOrder: []int32{0, 1}, + ChildSize: 32, + MinPrefixLength: 1, + MaxPrefixLength: 1, + EmptyChild: make([]byte, 32), + Hash: ics23.HashOp_SHA256, + }, + MaxDepth: 256, + PrehashKeyBeforeComparison: true, +} +``` + +The main change from the `cosmos/ics23` `SmtSpec` object is that the `PrehashValue` field is set to not hash values prior to hashing the key-value pair. + +### Converting `SparseMerkleProof` to `CommitmentProof` + +In order to convert the proofs generated by the SMT into a serialisable proof used by `cosmos/ics23`, the `SideNodes` field of the `SparseMerkleProof` must be converted into a list of `InnerOp` types which define the order of the hashes. The order of the hashes is important as depending on whether the next hash is the left or right neighbour of the current hash, they will be hashed in a different order, ultimately creating a different root hash. This conversion allows the verification to produce the same root hash as the SMT would have produced when verifying the proof. + +As `SparseMerkleProof` objects represent both inclusion and exclusion proofs as defined in the [JMT whitepaper][jmt] the conversion step will convert the SMT proof into either a `ExistenceProof` or `ExclusionProof` as defined in `cosmos/ics23`. + +### Proof Verification + +When verifying membership of an element the logic is as follows: + +1. Use the key-value pair to generate a leaf hash +2. Hash the leaf with the `SideNodes` found in the `path` field of the `ExistenceProof` to generate the root hash +3. Compare the root hash with the one provided + +For non-membership the logic is as follows: + +1. If the `ActualValueHash` field in the `ExclusionProof` is the SMT's placeholder value, then use the placeholder value as the leaf node hash and follow steps 3-4 +2. If the `ActualValueHash` field is not the placeholder value, then use the `ActualPath` and `ActualValueHash` fields to generate the leaf node hash - do not hash these values before hashing the node as they are populated from the SMT proof's `NonMembershipLeafData` field and thus are already hashed +3. Hash the leaf node hash with the `SideNodes` found in the `Path` field of the `ExclusionProof` to generate the root hash +4. Compare the root hash with the one provided + +The full implementation of this logic can be found [here](../store/proofs_ics23.go) as well as in the `cosmos/ics23` [library](https://github.com/h5law/ics23/blob/56d948cafb83ded78dc4b9de3c8b04582734851a/go/proof.go#L171). + +[ics23]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-023-vector-commitments/README.md +[smt]: https://github.com/pokt-network/smt +[jmt]: https://developers.diem.com/papers/jellyfish-merkle-tree/2021-01-14.pdf From 52e62e4ad68914c4b7833d677731032d2e3b754a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:18:23 +0100 Subject: [PATCH 04/74] Fix errors --- go.mod | 2 +- go.sum | 2 ++ ibc/store/proofs_ics23.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 937561931..28281e5ed 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/pokt-network/pocket go 1.18 // TECHDEBT: remove once upstream PR is merged (see: https://github.com/cosmos/ics23/pull/153) -replace github.com/cosmos/ics23/go => ../ics23/go +replace github.com/cosmos/ics23/go => github.com/h5law/ics23/go v0.0.0-20230619152251-56d948cafb83 // TECHDEBT: remove once upstream PR is merged (see: https://github.com/regen-network/gocuke/pull/12) replace github.com/regen-network/gocuke => github.com/pokt-network/gocuke v0.0.1 diff --git a/go.sum b/go.sum index 4e8bb7827..271c59bc7 100644 --- a/go.sum +++ b/go.sum @@ -358,6 +358,8 @@ github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQ github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/h5law/ics23/go v0.0.0-20230619152251-56d948cafb83 h1:uG97IfYQttG5iVt/jHK2wnGZgKUxHUjnzAlWY6EDso8= +github.com/h5law/ics23/go v0.0.0-20230619152251-56d948cafb83/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index c1ab2f390..1ae2675e5 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -133,7 +133,7 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { for i := 0; i < len(sideNodes); i++ { var prefix, suffix []byte prefix = append(prefix, innerPrefix...) - if getPathBit(path[:], len(sideNodes)-1-i) == left { + if getPathBit(path, len(sideNodes)-1-i) == left { suffix = make([]byte, 0, len(sideNodes[i])) suffix = append(suffix, sideNodes[i]...) } else { From 06566581aef0ab80b2f4ff87845e75936e914633 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:20:53 +0100 Subject: [PATCH 05/74] Update next error comment --- shared/core/types/error.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index a7593489c..870b6b733 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 141 +// NextCode: 142 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials From d631bb3c9676f165ef2d99cca8a1fc0d66478e0a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:50:50 +0100 Subject: [PATCH 06/74] Address comments in docs --- ibc/docs/README.md | 2 +- ibc/docs/ics23.md | 47 ++++++++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/ibc/docs/README.md b/ibc/docs/README.md index 0ea5dd98d..44c1ac1d6 100644 --- a/ibc/docs/README.md +++ b/ibc/docs/README.md @@ -109,7 +109,7 @@ See: [ICS-24](./ics24.md) for more details on the specifics of the ICS-24 implem ### ICS-23 Vector Commitments -[ICS-23][ics23] defines the `CommitmentProof` type that is used to prove the membership/non-membership of a key-value pair in the IBC stores. As this type is serialisable the relayers can relay these proofs to a counterparty chain, as described above, for their light client (of the source chain) to verify the proof and thus react accordingly. In order to implement ICS-23, the `cosmos/ics23` library was used, specifically its `CommitmentProof` type and its methods to verify the proofs in a way that does not require the tree itself. +[ICS-23][ics23] defines the `CommitmentProof` type that is used to prove the membership/non-membership of a key-value pair in the IBC stores. As this type is serialisable the relayers can relay these proofs to a counterparty chain, as described above, for their ibc specific light client (of the source chain) to verify the proof and thus react accordingly. In order to implement ICS-23, the `cosmos/ics23` library was used, specifically its `CommitmentProof` type and its methods to verify the proofs in a way that does not require the tree itself. See: [ICS-23](./ics23.md) for more details on the specifics of the ICS-23 implementation for Pocket. diff --git a/ibc/docs/ics23.md b/ibc/docs/ics23.md index 95cc2895e..6f6839132 100644 --- a/ibc/docs/ics23.md +++ b/ibc/docs/ics23.md @@ -2,29 +2,37 @@ - [Overview](#overview) - [Implementation](#implementation) - - [Custom SMT `ProofSpec`](#custom-smt-proofspec) - - [Converting `SparseMerkleProof` to `CommitmentProof`](#converting-sparsemerkleproof-to-commitmentproof) - - [Proof Verification](#proof-verification) + - [Custom SMT `ProofSpec`](#custom-smt-proofspec) + - [Converting `SparseMerkleProof` to `CommitmentProof`](#converting-sparsemerkleproof-to-commitmentproof) + - [Proof Verification](#proof-verification) ## Overview -[ICS-23][ics23] defines the types and functions needed to verify membership of a key-value pair in a `CommitmentState`. As the Pocket IBC implementation uses the [SMT][smt] for its provable stores this is what is referred to as the `CommitmentState` object. Cosmos has a library `cosmos/ics23` which is already SDK agnostic and defines many of the types necessary for ICS-23. This library was able to be used _mostly_ out of the box, with some minor adjustments detailed below. +[ICS-23][ics23] defines the types and functions needed to verify membership of a key-value pair in a `CommitmentState`. As the Pocket IBC implementation uses the [SMT][smt] for its provable stores, this is referred to as the `CommitmentState` object. Cosmos has a library `cosmos/ics23` which is already SDK agnostic and defines many of the types necessary for ICS-23. This library was able to be used _mostly_ out of the box, with some minor adjustments detailed below. ## Implementation The benefit of using `cosmos/ics23` over implementing similar types ourselves is twofold: -1. It is already SDK agnostic, so can be used by Pocket (a non-cosmos chain) without any issues. -2. The functions defined for proof verification are separate from the tree structures themselves - - This means we do not need to interact with an SMT instance in order to verify proofs. +1. It is already SDK agnostic, so can be used by Pocket (a non-cosmos chain) without any issues or major changes. +2. The functions defined for proof verification are decoupled from the underlying tree structure, meaning proof verification is tree agnostic. However, there were some changes made specifically for Pocket's implementation of ICS-23. +See: [`cosmos/ics23` #152](https://github.com/cosmos/ics23/issues/152) and [`cosmos/ics23` #153](https://github.com/cosmos/ics23/pull/153ß) for the details of the changes made to allow for `ExclusionProof` verification. + ### Custom SMT `ProofSpec` -The `ProofSpec` type in `cosmos/ics23` is used to define the steps needed to verify a proof, what hashing functions should be used, any node prefixes, etc. This is then passed into the verification functions in order to verify a proof, instead of having to interact with the tree itself. This is useful as proofs must be verified on a light-client and as such being able to verify a proof without creating a tree is much more memory efficient. +The `ProofSpec` type in `cosmos/ics23` is used to define: + +1. The steps needed to verify a proof +2. The hash functions used +3. Node prefixes +4. Etc... + +The `ProofSpec` is then passed into the verification functions in order to verify a proof instead of having to interact with the tree itself. This is useful as proofs must be verified via an (IBC) light client, and as such being able to verify a proof without reconstructing a tree is much more memory efficient. -As the SMT used by Pocket Network only stores hashed values by default, the IBC stores use the `WithValueHasher(nil)` option which stores the raw bytes of the values in the tree. As such the following `ProofSpec` was created: +As the SMT used by Pocket Network only stores hashed values by default, the IBC store uses the `WithValueHasher(nil)` option which stores the source value (as raw bytes) in the tree. The following `ProofSpec` was created to support this: ```go smtSpec *ics23.ProofSpec = &ics23.ProofSpec{ @@ -33,14 +41,14 @@ smtSpec *ics23.ProofSpec = &ics23.ProofSpec{ PrehashKey: ics23.HashOp_SHA256, PrehashValue: ics23.HashOp_NO_HASH, Length: ics23.LengthOp_NO_PREFIX, - Prefix: []byte{0}, + Prefix: []byte{0}, }, InnerSpec: &ics23.InnerSpec{ ChildOrder: []int32{0, 1}, ChildSize: 32, MinPrefixLength: 1, MaxPrefixLength: 1, - EmptyChild: make([]byte, 32), + EmptyChild: make([]byte, 32), Hash: ics23.HashOp_SHA256, }, MaxDepth: 256, @@ -48,28 +56,31 @@ smtSpec *ics23.ProofSpec = &ics23.ProofSpec{ } ``` -The main change from the `cosmos/ics23` `SmtSpec` object is that the `PrehashValue` field is set to not hash values prior to hashing the key-value pair. +The main difference from the `cosmos/ics23` `SmtSpec` object is that the `PrehashValue` field is set to not hash values before hashing the key-value pair. ### Converting `SparseMerkleProof` to `CommitmentProof` In order to convert the proofs generated by the SMT into a serialisable proof used by `cosmos/ics23`, the `SideNodes` field of the `SparseMerkleProof` must be converted into a list of `InnerOp` types which define the order of the hashes. The order of the hashes is important as depending on whether the next hash is the left or right neighbour of the current hash, they will be hashed in a different order, ultimately creating a different root hash. This conversion allows the verification to produce the same root hash as the SMT would have produced when verifying the proof. -As `SparseMerkleProof` objects represent both inclusion and exclusion proofs as defined in the [JMT whitepaper][jmt] the conversion step will convert the SMT proof into either a `ExistenceProof` or `ExclusionProof` as defined in `cosmos/ics23`. +As `SparseMerkleProof` objects represent both inclusion and exclusion proofs as defined in the [JMT whitepaper][jmt]. The conversion step will convert the SMT proof into either an `ExistenceProof` or `ExclusionProof` as defined in `cosmos/ics23`. ### Proof Verification -When verifying membership of an element the logic is as follows: +Membership proofs are verified as follows: 1. Use the key-value pair to generate a leaf hash 2. Hash the leaf with the `SideNodes` found in the `path` field of the `ExistenceProof` to generate the root hash -3. Compare the root hash with the one provided +3. Compare the root hash with the one provided and expect them to be identical -For non-membership the logic is as follows: +Non-membership proofs are verified as follows: -1. If the `ActualValueHash` field in the `ExclusionProof` is the SMT's placeholder value, then use the placeholder value as the leaf node hash and follow steps 3-4 -2. If the `ActualValueHash` field is not the placeholder value, then use the `ActualPath` and `ActualValueHash` fields to generate the leaf node hash - do not hash these values before hashing the node as they are populated from the SMT proof's `NonMembershipLeafData` field and thus are already hashed +1. If the `ActualValueHash` field in the `ExclusionProof` is the SMT's placeholder value, then use the placeholder value as the leaf node hash and skip to step 3 below +2. If the `ActualValueHash` field is not the placeholder value, then use the `ActualPath` and `ActualValueHash` fields to generate the leaf node hash. + - **IMPORTANT**: DO NOT hash these values before hashing the node as they are populated from the SMT proof's `NonMembershipLeafData` field and thus are already hashed 3. Hash the leaf node hash with the `SideNodes` found in the `Path` field of the `ExclusionProof` to generate the root hash 4. Compare the root hash with the one provided + - As the non-membership proof uses the `ActualValueHash` field to generate the leaf node hash, the non-membership proof is actually proving membership of either a placeholder key or an unrelated key in the tree. + - This means that if the root hash computed and the one provided are equal then the key we were looking for was not in the tree. If they are not equal the proof is invalid and the key is in the tree. The full implementation of this logic can be found [here](../store/proofs_ics23.go) as well as in the `cosmos/ics23` [library](https://github.com/h5law/ics23/blob/56d948cafb83ded78dc4b9de3c8b04582734851a/go/proof.go#L171). From 1beb4bdb83a33cc1b3e7bf9a22617ee36b0f5fcd Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:54:18 +0100 Subject: [PATCH 07/74] Add names back to tests --- ibc/host/path_test.go | 18 ++--- ibc/store/proofs_ics23_test.go | 118 +++++++++++++++++---------------- 2 files changed, 72 insertions(+), 64 deletions(-) diff --git a/ibc/host/path_test.go b/ibc/host/path_test.go index 041b2a239..1c1c8d367 100644 --- a/ibc/host/path_test.go +++ b/ibc/host/path_test.go @@ -65,14 +65,14 @@ func TestPaths_CommitmentPrefix(t *testing.T) { result string }{ { - // Successfully applies and removes prefix to produce the same path + name: "Successfully applies and removes prefix to produce the same path", path: "path", prefix: coreTypes.CommitmentPrefix([]byte("test")), expected: []byte("test/path"), result: "path", }, { - // Fails to produce input path when given a different prefix + name: "Fails to produce input path when given a different prefix", path: "path", prefix: coreTypes.CommitmentPrefix([]byte("test2")), expected: []byte("test/path"), @@ -81,12 +81,14 @@ func TestPaths_CommitmentPrefix(t *testing.T) { } for _, tc := range testCases { - commitment := ApplyPrefix(prefix, tc.path) - require.NotNil(t, commitment) - require.Equal(t, []byte(commitment), tc.expected) + t.Run(tc.name, func(t *testing.T) { + commitment := ApplyPrefix(prefix, tc.path) + require.NotNil(t, commitment) + require.Equal(t, []byte(commitment), tc.expected) - path := RemovePrefix(tc.prefix, commitment) - require.NotNil(t, path) - require.Equal(t, path, tc.result) + path := RemovePrefix(tc.prefix, commitment) + require.NotNil(t, path) + require.Equal(t, path, tc.result) + }) } } diff --git a/ibc/store/proofs_ics23_test.go b/ibc/store/proofs_ics23_test.go index 97cd63011..cff486b9d 100644 --- a/ibc/store/proofs_ics23_test.go +++ b/ibc/store/proofs_ics23_test.go @@ -27,6 +27,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { require.NoError(t, err) testCases := []struct { + name string key []byte value []byte nonmembership bool @@ -34,7 +35,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { expected error }{ { - // Successfully generates a membership proof for a key stored + name: "Successfully generates a membership proof for a key stored", key: []byte("foo"), value: []byte("bar"), nonmembership: false, @@ -42,7 +43,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { expected: nil, }, { - // Successfully generates a non-membership proof for a key not stored + name: "Successfully generates a non-membership proof for a key not stored", key: []byte("baz"), value: []byte("testValue2"), // unrelated leaf data nonmembership: true, @@ -50,7 +51,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { expected: nil, }, { - // Successfully generates a non-membership proof for an unset nil key + name: "Successfully generates a non-membership proof for an unset nil key", key: nil, value: []byte("foo"), // unrelated leaf data nonmembership: true, @@ -60,28 +61,30 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { } for _, tc := range testCases { - var proof *ics23.CommitmentProof - if tc.nonmembership { - proof, err = createNonMembershipProof(tree, tc.key) - } else { - proof, err = createMembershipProof(tree, tc.key, tc.value) - } - if tc.fails { - require.EqualError(t, err, tc.expected.Error()) - require.Nil(t, proof) - return - } - require.NoError(t, err) - require.NotNil(t, proof) - if tc.nonmembership { - require.Equal(t, tc.value, proof.GetExclusion().GetActualValueHash()) - require.NotNil(t, proof.GetExclusion().GetLeaf()) - require.NotNil(t, proof.GetExclusion().GetPath()) - } else { - require.Equal(t, tc.value, proof.GetExist().GetValue()) - require.NotNil(t, proof.GetExist().GetLeaf()) - require.NotNil(t, proof.GetExist().GetPath()) - } + t.Run(tc.name, func(t *testing.T) { + var proof *ics23.CommitmentProof + if tc.nonmembership { + proof, err = createNonMembershipProof(tree, tc.key) + } else { + proof, err = createMembershipProof(tree, tc.key, tc.value) + } + if tc.fails { + require.EqualError(t, err, tc.expected.Error()) + require.Nil(t, proof) + return + } + require.NoError(t, err) + require.NotNil(t, proof) + if tc.nonmembership { + require.Equal(t, tc.value, proof.GetExclusion().GetActualValueHash()) + require.NotNil(t, proof.GetExclusion().GetLeaf()) + require.NotNil(t, proof.GetExclusion().GetPath()) + } else { + require.Equal(t, tc.value, proof.GetExist().GetValue()) + require.NotNil(t, proof.GetExist().GetLeaf()) + require.NotNil(t, proof.GetExist().GetPath()) + } + }) } err = nodeStore.Stop() @@ -107,34 +110,35 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { require.NotNil(t, root) testCases := []struct { + name string key []byte value []byte nonmembership bool valid bool }{ { - // Successfully verifies a membership proof for a key-value stored pair + name: "Successfully verifies a membership proof for a key-value stored pair", key: []byte("foo"), value: []byte("bar"), nonmembership: false, valid: true, }, { - // Successfully verifies a non-membership proof for a key-value pair not stored + name: "Successfully verifies a non-membership proof for a key-value pair not stored", key: []byte("not stored"), value: nil, nonmembership: true, valid: true, }, { - // Fails to verify a membership proof for a key-value pair not stored + name: "Fails to verify a membership proof for a key-value pair not stored", key: []byte("baz"), value: []byte("bar"), nonmembership: false, valid: false, }, { - // Fails to verify a non-membership proof for a key stored in the tree + name: "Fails to verify a non-membership proof for a key stored in the tree", key: []byte("foo"), value: nil, nonmembership: true, @@ -144,33 +148,35 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { proof := new(ics23.CommitmentProof) for _, tc := range testCases { - var err error - if tc.nonmembership { - proof, err = createNonMembershipProof(tree, tc.key) - } else { - proof, err = createMembershipProof(tree, tc.key, tc.value) - } - require.NoError(t, err) - require.NotNil(t, proof) - - if tc.nonmembership { - require.NotNil(t, proof.GetExclusion()) - } else { - require.NotNil(t, proof.GetExist()) - } - - var valid bool - if tc.nonmembership { - valid = VerifyNonMembership(root, proof, tc.key) - } else { - valid = VerifyMembership(root, proof, tc.key, tc.value) - } - - if tc.valid { - require.True(t, valid) - } else { - require.False(t, valid) - } + t.Run(tc.name, func(t *testing.T) { + var err error + if tc.nonmembership { + proof, err = createNonMembershipProof(tree, tc.key) + } else { + proof, err = createMembershipProof(tree, tc.key, tc.value) + } + require.NoError(t, err) + require.NotNil(t, proof) + + if tc.nonmembership { + require.NotNil(t, proof.GetExclusion()) + } else { + require.NotNil(t, proof.GetExist()) + } + + var valid bool + if tc.nonmembership { + valid = VerifyNonMembership(root, proof, tc.key) + } else { + valid = VerifyMembership(root, proof, tc.key, tc.value) + } + + if tc.valid { + require.True(t, valid) + } else { + require.False(t, valid) + } + }) } err = nodeStore.Stop() From 9a7a578edbcf70899d7a88941f58b612178e7e21 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:24:23 +0100 Subject: [PATCH 08/74] Address comments --- ibc/store/proofs_ics23.go | 38 ++++++++--- ibc/store/proofs_ics23_test.go | 114 ++++++++++++++++----------------- 2 files changed, 83 insertions(+), 69 deletions(-) diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 1ae2675e5..1d209f3a6 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -8,11 +8,14 @@ import ( "github.com/pokt-network/smt" ) +// position refers to whether the node is either the left or right child of its parent +// Ref: https://github.com/pokt-network/smt/blob/main/types.go type position int const ( - left position = iota // 0 - right // 1 + left position = iota // 0 + right // 1 + hashSize = 32 ) var ( @@ -27,10 +30,10 @@ var ( }, InnerSpec: &ics23.InnerSpec{ ChildOrder: []int32{0, 1}, - ChildSize: 32, + ChildSize: hashSize, MinPrefixLength: 1, MaxPrefixLength: 1, - EmptyChild: make([]byte, 32), + EmptyChild: make([]byte, hashSize), Hash: ics23.HashOp_SHA256, }, MaxDepth: 256, @@ -39,7 +42,7 @@ var ( innerPrefix = []byte{1} // defaultValue is the default placeholder value in a SparseMerkleTree - defaultValue = make([]byte, 32) + defaultValue = make([]byte, hashSize) ) // VerifyMembership verifies the CommitmentProof provided, checking whether it produces the same @@ -94,7 +97,7 @@ func convertSMPToExistenceProof(proof *smt.SparseMerkleProof, key, value []byte) } } -// convertSMPToExistenceProof converts a SparseMerkleProof to an ics23 +// convertSMPToExclusionProof converts a SparseMerkleProof to an ics23 // ExclusionProof to verify non-membership of an element func convertSMPToExclusionProof(proof *smt.SparseMerkleProof, key []byte) *ics23.CommitmentProof { path := sha256.Sum256(key) @@ -110,8 +113,8 @@ func convertSMPToExclusionProof(proof *smt.SparseMerkleProof, key []byte) *ics23 actualPath := path[:] actualValue := defaultValue if proof.NonMembershipLeafData != nil { - actualPath = proof.NonMembershipLeafData[1:33] - actualValue = proof.NonMembershipLeafData[33:] + actualPath = proof.NonMembershipLeafData[1 : 1+hashSize] // len(prefix): len(prefix) + hashSize + actualValue = proof.NonMembershipLeafData[1+hashSize:] } return &ics23.CommitmentProof{ Proof: &ics23.CommitmentProof_Exclusion{ @@ -134,9 +137,11 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { var prefix, suffix []byte prefix = append(prefix, innerPrefix...) if getPathBit(path, len(sideNodes)-1-i) == left { + // path is on the left so sidenode must be on the right suffix = make([]byte, 0, len(sideNodes[i])) suffix = append(suffix, sideNodes[i]...) } else { + // path is on the right so sidenode must be on the left prefix = append(prefix, sideNodes[i]...) } op := &ics23.InnerOp{ @@ -149,9 +154,22 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { return steps } -// getPathBit determines whether the hash of a node at a certain depth in the tree is the -// left or the right child of its parent +// getPathBit takes the hash of a key (the path) and a position (depth) and returns whether at +// that position in the tree the path goes left or right. This is used to determine the order +// of child nodes and the order in which they are hashed when verifying proofs. +// Ref: https://github.com/pokt-network/smt/blob/main/utils.go func getPathBit(data []byte, position int) position { + // get the byte at the position and then left shift one by the offset of the position + // from the leftmost bit in the byte. Check if the bitwise and is the same + // Path: []byte{ {0 1 0 1 1 0 1 0}, {0 1 1 0 1 1 0 1}, {1 0 0 1 0 0 1 0} } (length = 24 bits / 3 bytes) + // Position: 13 - 13/8=1 + // Path[1] = {0 1 1 0 1 1 0 1} + // uint(13)%8 = 5, 8-1-5=2 + // 00000001 << 2 = 00000100 + // {0 1 1 0 1 1 0 1} + // & {0 0 0 0 0 1 0 0} + // = {0 0 0 0 0 1 0 0} + // > 0 so Path is on the right at position 13 if int(data[position/8])&(1<<(8-1-uint(position)%8)) > 0 { return right } diff --git a/ibc/store/proofs_ics23_test.go b/ibc/store/proofs_ics23_test.go index cff486b9d..0695ac681 100644 --- a/ibc/store/proofs_ics23_test.go +++ b/ibc/store/proofs_ics23_test.go @@ -27,43 +27,43 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { require.NoError(t, err) testCases := []struct { - name string - key []byte - value []byte - nonmembership bool - fails bool - expected error + name string + key []byte + value []byte + isNonMembership bool + fails bool + expected error }{ { - name: "Successfully generates a membership proof for a key stored", - key: []byte("foo"), - value: []byte("bar"), - nonmembership: false, - fails: false, - expected: nil, + name: "Successfully generates a membership proof for a key stored", + key: []byte("foo"), + value: []byte("bar"), + isNonMembership: false, + fails: false, + expected: nil, }, { - name: "Successfully generates a non-membership proof for a key not stored", - key: []byte("baz"), - value: []byte("testValue2"), // unrelated leaf data - nonmembership: true, - fails: false, - expected: nil, + name: "Successfully generates a non-membership proof for a key not stored", + key: []byte("baz"), + value: []byte("testValue2"), // unrelated leaf data + isNonMembership: true, + fails: false, + expected: nil, }, { - name: "Successfully generates a non-membership proof for an unset nil key", - key: nil, - value: []byte("foo"), // unrelated leaf data - nonmembership: true, - fails: false, - expected: nil, + name: "Successfully generates a non-membership proof for an unset nil key", + key: nil, + value: []byte("foo"), // unrelated leaf data + isNonMembership: true, + fails: false, + expected: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var proof *ics23.CommitmentProof - if tc.nonmembership { + if tc.isNonMembership { proof, err = createNonMembershipProof(tree, tc.key) } else { proof, err = createMembershipProof(tree, tc.key, tc.value) @@ -75,7 +75,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { } require.NoError(t, err) require.NotNil(t, proof) - if tc.nonmembership { + if tc.isNonMembership { require.Equal(t, tc.value, proof.GetExclusion().GetActualValueHash()) require.NotNil(t, proof.GetExclusion().GetLeaf()) require.NotNil(t, proof.GetExclusion().GetPath()) @@ -110,39 +110,39 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { require.NotNil(t, root) testCases := []struct { - name string - key []byte - value []byte - nonmembership bool - valid bool + name string + key []byte + value []byte + isNonMembership bool + valid bool }{ { - name: "Successfully verifies a membership proof for a key-value stored pair", - key: []byte("foo"), - value: []byte("bar"), - nonmembership: false, - valid: true, + name: "Successfully verifies a membership proof for a key-value stored pair", + key: []byte("foo"), + value: []byte("bar"), + isNonMembership: false, + valid: true, }, { - name: "Successfully verifies a non-membership proof for a key-value pair not stored", - key: []byte("not stored"), - value: nil, - nonmembership: true, - valid: true, + name: "Successfully verifies a non-membership proof for a key-value pair not stored", + key: []byte("not stored"), + value: nil, + isNonMembership: true, + valid: true, }, { - name: "Fails to verify a membership proof for a key-value pair not stored", - key: []byte("baz"), - value: []byte("bar"), - nonmembership: false, - valid: false, + name: "Fails to verify a membership proof for a key-value pair not stored", + key: []byte("baz"), + value: []byte("bar"), + isNonMembership: false, + valid: false, }, { - name: "Fails to verify a non-membership proof for a key stored in the tree", - key: []byte("foo"), - value: nil, - nonmembership: true, - valid: false, + name: "Fails to verify a non-membership proof for a key stored in the tree", + key: []byte("foo"), + value: nil, + isNonMembership: true, + valid: false, }, } @@ -150,7 +150,7 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var err error - if tc.nonmembership { + if tc.isNonMembership { proof, err = createNonMembershipProof(tree, tc.key) } else { proof, err = createMembershipProof(tree, tc.key, tc.value) @@ -158,24 +158,20 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { require.NoError(t, err) require.NotNil(t, proof) - if tc.nonmembership { + if tc.isNonMembership { require.NotNil(t, proof.GetExclusion()) } else { require.NotNil(t, proof.GetExist()) } var valid bool - if tc.nonmembership { + if tc.isNonMembership { valid = VerifyNonMembership(root, proof, tc.key) } else { valid = VerifyMembership(root, proof, tc.key, tc.value) } - if tc.valid { - require.True(t, valid) - } else { - require.False(t, valid) - } + require.Equal(t, tc.valid, valid) }) } From 327ed964fe66b2d7f016d4203e4bb6329ba721df Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:11:34 +0100 Subject: [PATCH 09/74] Address comments --- ibc/docs/README.md | 1 + ibc/docs/ics23.md | 35 ++++++++++++++++++++++++++++++---- ibc/store/proofs_ics23.go | 2 ++ ibc/store/proofs_ics23_test.go | 4 ++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/ibc/docs/README.md b/ibc/docs/README.md index 44c1ac1d6..01a04c4ed 100644 --- a/ibc/docs/README.md +++ b/ibc/docs/README.md @@ -116,3 +116,4 @@ See: [ICS-23](./ics23.md) for more details on the specifics of the ICS-23 implem [ibc-spec]: https://github.com/cosmos/ibc [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics23]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-023-vector-commitments/README.md +[smt]: https://github.com/pokt-network/smt diff --git a/ibc/docs/ics23.md b/ibc/docs/ics23.md index 6f6839132..b6c5ff943 100644 --- a/ibc/docs/ics23.md +++ b/ibc/docs/ics23.md @@ -74,13 +74,40 @@ Membership proofs are verified as follows: Non-membership proofs are verified as follows: -1. If the `ActualValueHash` field in the `ExclusionProof` is the SMT's placeholder value, then use the placeholder value as the leaf node hash and skip to step 3 below -2. If the `ActualValueHash` field is not the placeholder value, then use the `ActualPath` and `ActualValueHash` fields to generate the leaf node hash. +1. If the `ActualValueHash` field in the `ExclusionProof` is the SMT's placeholder value (`[32]byte`, i.e. the key is not set in the tree), then use the placeholder value as the leaf node hash and skip to step 3 below +2. If the `ActualValueHash` field is not the placeholder value, then use the `ActualPath` and `ActualValueHash` fields (provided via `NonMembershipLeafData`) to generate the leaf node hash. - **IMPORTANT**: DO NOT hash these values before hashing the node as they are populated from the SMT proof's `NonMembershipLeafData` field and thus are already hashed 3. Hash the leaf node hash with the `SideNodes` found in the `Path` field of the `ExclusionProof` to generate the root hash 4. Compare the root hash with the one provided - - As the non-membership proof uses the `ActualValueHash` field to generate the leaf node hash, the non-membership proof is actually proving membership of either a placeholder key or an unrelated key in the tree. - - This means that if the root hash computed and the one provided are equal then the key we were looking for was not in the tree. If they are not equal the proof is invalid and the key is in the tree. + - if `computedRootHash == providedRootHash` + - `key` not in tree -> `Proof` is valid -> exclusion QED + - if `computedRootHash != providedRootHash` + - `key` is in tree -> `Proof` is invalid -> exclusion QED + +```mermaid +flowchart TD + I["Proof,Key"] + NMD{"proof.NonMembershipLeafData == nil ?"} + KP1["actualPath = sha256(key) \n actualValue = placeholder\ncurrentHash = [32]byte"] + KP2["actualPath = ProvidedKeyHash \n actualValue = ProvidedValueHash\ncurrentHash = sha256([]byte{0}+actualPath+actualValueHash)"] + C["nextHash = sha256(currentHash+sideNodeHash)"] + Compare{"ComputedRootHash == ProvidedRootHash ?"} + EV["Exclusion Prove VALID"] + EI["Exclusion Prove INVALID"] + + I --> NMD + NMD -- Yes --> KP1 + NMD -- No --> KP2 + + KP1 -- CurrentHash --> C + KP2 -- CurrentHash --> C + + C -- while NextSideNode != nil --> C + + C -- ComputedRootHash --> Compare + Compare -- Yes --> EV + Compare -- No --> EI +``` The full implementation of this logic can be found [here](../store/proofs_ics23.go) as well as in the `cosmos/ics23` [library](https://github.com/h5law/ics23/blob/56d948cafb83ded78dc4b9de3c8b04582734851a/go/proof.go#L171). diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 1d209f3a6..a7930dd1c 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -9,6 +9,7 @@ import ( ) // position refers to whether the node is either the left or right child of its parent +// for the binary SMT // Ref: https://github.com/pokt-network/smt/blob/main/types.go type position int @@ -154,6 +155,7 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { return steps } +// TECHDEBT(smt #14): Use exposed method from SMT library // getPathBit takes the hash of a key (the path) and a position (depth) and returns whether at // that position in the tree the path goes left or right. This is used to determine the order // of child nodes and the order in which they are hashed when verifying proofs. diff --git a/ibc/store/proofs_ics23_test.go b/ibc/store/proofs_ics23_test.go index 0695ac681..3dff554e4 100644 --- a/ibc/store/proofs_ics23_test.go +++ b/ibc/store/proofs_ics23_test.go @@ -16,7 +16,7 @@ func TestICS23Proofs_GenerateCommitmentProofs(t *testing.T) { tree := smt.NewSparseMerkleTree(nodeStore, sha256.New(), smt.WithValueHasher(nil)) require.NotNil(t, tree) - // Set a value in the store + // Prepare a tree with a predetermined set of key-value pairs err := tree.Update([]byte("foo"), []byte("bar")) require.NoError(t, err) err = tree.Update([]byte("bar"), []byte("foo")) @@ -96,7 +96,7 @@ func TestICS23Proofs_VerifyCommitmentProofs(t *testing.T) { tree := smt.NewSparseMerkleTree(nodeStore, sha256.New(), smt.WithValueHasher(nil)) require.NotNil(t, tree) - // Set a value in the store + // Prepare a tree with a predetermined set of key-value pairs err := tree.Update([]byte("foo"), []byte("bar")) require.NoError(t, err) err = tree.Update([]byte("bar"), []byte("foo")) From bc5a878ab7f98f3c9004e963dbb2096975269375 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:31:09 +0100 Subject: [PATCH 10/74] add isLeft helper and use smt.GetPathBit() --- ibc/store/proofs_ics23.go | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index a7930dd1c..f07c9d545 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -137,7 +137,7 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { for i := 0; i < len(sideNodes); i++ { var prefix, suffix []byte prefix = append(prefix, innerPrefix...) - if getPathBit(path, len(sideNodes)-1-i) == left { + if isLeft(path, len(sideNodes)-1-i) { // path is on the left so sidenode must be on the right suffix = make([]byte, 0, len(sideNodes[i])) suffix = append(suffix, sideNodes[i]...) @@ -155,25 +155,7 @@ func convertSideNodesToSteps(sideNodes [][]byte, path []byte) []*ics23.InnerOp { return steps } -// TECHDEBT(smt #14): Use exposed method from SMT library -// getPathBit takes the hash of a key (the path) and a position (depth) and returns whether at -// that position in the tree the path goes left or right. This is used to determine the order -// of child nodes and the order in which they are hashed when verifying proofs. -// Ref: https://github.com/pokt-network/smt/blob/main/utils.go -func getPathBit(data []byte, position int) position { - // get the byte at the position and then left shift one by the offset of the position - // from the leftmost bit in the byte. Check if the bitwise and is the same - // Path: []byte{ {0 1 0 1 1 0 1 0}, {0 1 1 0 1 1 0 1}, {1 0 0 1 0 0 1 0} } (length = 24 bits / 3 bytes) - // Position: 13 - 13/8=1 - // Path[1] = {0 1 1 0 1 1 0 1} - // uint(13)%8 = 5, 8-1-5=2 - // 00000001 << 2 = 00000100 - // {0 1 1 0 1 1 0 1} - // & {0 0 0 0 0 1 0 0} - // = {0 0 0 0 0 1 0 0} - // > 0 so Path is on the right at position 13 - if int(data[position/8])&(1<<(8-1-uint(position)%8)) > 0 { - return right - } - return left +// isLeft returns true is the i-th bit of path is a left child in the SMT +func isLeft(path []byte, i int) bool { + return smt.GetPathBit(path, i) == left } From 2f67dcd61c61fb92e1beceabd7f1aeb8febc98c8 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 21:40:00 +0100 Subject: [PATCH 11/74] go.mod --- go.mod | 2 +- go.sum | 7 +++++-- ibc/store/proofs_ics23.go | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 28281e5ed..8a4e44748 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiformats/go-multiaddr v0.8.0 - github.com/pokt-network/smt v0.5.0 + github.com/pokt-network/smt v0.6.0 github.com/quasilyte/go-ruleguard/dsl v0.3.21 github.com/regen-network/gocuke v0.6.2 github.com/rs/zerolog v1.27.0 diff --git a/go.sum b/go.sum index 271c59bc7..eb0a2dba4 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-ecvrf v0.0.1 h1:wv45+kZ0mG4G9oSTMjAlbgKqa4tPbNr4WLoCWqz5/bo= github.com/ProtonMail/go-ecvrf v0.0.1/go.mod h1:fhZbiRYn62/JGnBG2NGwCx0oT+gr/+I5R/hwiyAFpAU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -150,6 +152,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s= @@ -702,8 +705,8 @@ github.com/pokt-network/go-mockdns v0.0.1 h1:1Kb/kIFH6bNtY9F1bFhJyMRMCc7WyiqfGg0 github.com/pokt-network/go-mockdns v0.0.1/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/pokt-network/gocuke v0.0.1 h1:qJ/Ryf+hi5L6T9lsOZDNbiAclHkLlDio5/eVKQEYhgE= github.com/pokt-network/gocuke v0.0.1/go.mod h1:BowLKW4++696gTTU33teodtIhjjyaphEbhQT9D5Refw= -github.com/pokt-network/smt v0.5.0 h1:rNTW3FB6i0pNMnafDqsBySgs0zpbjs0spP3p7ltVjAE= -github.com/pokt-network/smt v0.5.0/go.mod h1:CWgC9UzDxXJNkL7TEADnJXutZVMYzK/+dmBb37RWkeQ= +github.com/pokt-network/smt v0.6.0 h1:W4NS6L0N5N0sDRq+2MskbWtCTnBiwzsgIJHxlHbwbgk= +github.com/pokt-network/smt v0.6.0/go.mod h1:CWgC9UzDxXJNkL7TEADnJXutZVMYzK/+dmBb37RWkeQ= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index f07c9d545..f07367224 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -68,7 +68,7 @@ func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentP if err != nil { return nil, coreTypes.ErrCreatingProof(err) } - return convertSMPToExistenceProof(&proof, key, value), nil + return convertSMPToExistenceProof(proof, key, value), nil } // createNonMembershipProof generates a CommitmentProof object verifying the membership of an unrealted key at the given key in the SMT provided @@ -78,7 +78,7 @@ func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof return nil, coreTypes.ErrCreatingProof(err) } - return convertSMPToExclusionProof(&proof, key), nil + return convertSMPToExclusionProof(proof, key), nil } // convertSMPToExistenceProof converts a SparseMerkleProof to an ics23 From 17e24199b5421e305c8cbaf4a83fa5b7d13bafea Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 21:46:20 +0100 Subject: [PATCH 12/74] Fix SMT repo --- go.mod | 2 +- go.sum | 7 ++----- ibc/store/proofs_ics23.go | 6 ++---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 8a4e44748..1be7fb2dd 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiformats/go-multiaddr v0.8.0 - github.com/pokt-network/smt v0.6.0 + github.com/pokt-network/smt v0.6.1 github.com/quasilyte/go-ruleguard/dsl v0.3.21 github.com/regen-network/gocuke v0.6.2 github.com/rs/zerolog v1.27.0 diff --git a/go.sum b/go.sum index eb0a2dba4..925ffa331 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-ecvrf v0.0.1 h1:wv45+kZ0mG4G9oSTMjAlbgKqa4tPbNr4WLoCWqz5/bo= github.com/ProtonMail/go-ecvrf v0.0.1/go.mod h1:fhZbiRYn62/JGnBG2NGwCx0oT+gr/+I5R/hwiyAFpAU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -152,7 +150,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s= @@ -705,8 +702,8 @@ github.com/pokt-network/go-mockdns v0.0.1 h1:1Kb/kIFH6bNtY9F1bFhJyMRMCc7WyiqfGg0 github.com/pokt-network/go-mockdns v0.0.1/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/pokt-network/gocuke v0.0.1 h1:qJ/Ryf+hi5L6T9lsOZDNbiAclHkLlDio5/eVKQEYhgE= github.com/pokt-network/gocuke v0.0.1/go.mod h1:BowLKW4++696gTTU33teodtIhjjyaphEbhQT9D5Refw= -github.com/pokt-network/smt v0.6.0 h1:W4NS6L0N5N0sDRq+2MskbWtCTnBiwzsgIJHxlHbwbgk= -github.com/pokt-network/smt v0.6.0/go.mod h1:CWgC9UzDxXJNkL7TEADnJXutZVMYzK/+dmBb37RWkeQ= +github.com/pokt-network/smt v0.6.1 h1:u5yTGNNND6edXv3vMQrAcjku1Ig4osehdu+EMYSXHUU= +github.com/pokt-network/smt v0.6.1/go.mod h1:CWgC9UzDxXJNkL7TEADnJXutZVMYzK/+dmBb37RWkeQ= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index f07367224..4f0d43fba 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -11,11 +11,9 @@ import ( // position refers to whether the node is either the left or right child of its parent // for the binary SMT // Ref: https://github.com/pokt-network/smt/blob/main/types.go -type position int - const ( - left position = iota // 0 - right // 1 + left int = iota // 0 + right // 1 hashSize = 32 ) From cf92d93fa247d095f34f7913448a20d48d63bdff Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:38:32 +0100 Subject: [PATCH 13/74] Add IBC proto types to protogen --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 96407cbf6..4174d4ecf 100644 --- a/Makefile +++ b/Makefile @@ -299,6 +299,9 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # P2P $(PROTOC_SHARED) -I=./p2p/raintree/types/proto --go_out=./p2p/types ./p2p/raintree/types/proto/*.proto + # IBC + $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto + # echo "View generated proto files by running: make protogen_show" # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following From a194cef1a132d46b391d9af9c3be3c2d86b2d004 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:38:57 +0100 Subject: [PATCH 14/74] Add provable stores and HandleMessage --- ibc/module.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/ibc/module.go b/ibc/module.go index c4f4c9c56..e60dfd6f0 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -1,11 +1,18 @@ package ibc import ( + "fmt" + "sync" + + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" + "google.golang.org/protobuf/types/known/anypb" ) var _ modules.IBCModule = &ibcModule{} @@ -13,7 +20,9 @@ var _ modules.IBCModule = &ibcModule{} type ibcModule struct { base_modules.IntegratableModule - cfg *configs.IBCConfig + cfg *configs.IBCConfig + m sync.Mutex + logger *modules.Logger // Only a single host is allowed at a time @@ -75,6 +84,59 @@ func (m *ibcModule) GetModuleName() string { return modules.IBCModuleName } +// HandleMessage accepts a generic IBC message and routes it to the specific handler +func (m *ibcModule) HandleMessage(message *anypb.Any) error { + m.m.Lock() + defer m.m.Unlock() + + switch message.MessageName() { + + case messaging.IbcMessageContentType: + msg, err := codec.GetCodec().FromAny(message) + if err != nil { + return err + } + ibcMessage, ok := msg.(*ibcTypes.IbcMessage) + if !ok { + return fmt.Errorf("failed to cast message to IBCMessage") + } + return m.handleIBCMessage(ibcMessage) + + default: + return coreTypes.ErrUnknownIBCMessageType(string(message.MessageName())) + } +} + +// handleIBCMessage unpacks the IBC message to its type and calls the appropriate handler +func (m *ibcModule) handleIBCMessage(message *ibcTypes.IbcMessage) error { + switch msg := message.Event.(type) { + case *ibcTypes.IbcMessage_Update: + return m.handleUpdateMessage(msg.Update) + case *ibcTypes.IbcMessage_Prune: + return m.handlePruneMessage(msg.Prune) + default: + return coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", msg)) + } +} + +// handleUpdateMessage adds the updated store entry to the IBC store change mempool +func (m *ibcModule) handleUpdateMessage(message *ibcTypes.UpdateIbcStore) error { + if m.host == nil { + return coreTypes.ErrHostDoesNotExist() + } + // TODO: implement this + return nil +} + +// handlePruneMessage adds a removal entry to the IBC store change mempool +func (m *ibcModule) handlePruneMessage(message *ibcTypes.PruneIbcStore) error { + if m.host == nil { + return coreTypes.ErrHostDoesNotExist() + } + // TODO: implement this + return nil +} + // newHost returns a new IBC host instance if one is not already created func (m *ibcModule) newHost() error { if m.host != nil { From c7206fcf2110b0b1b9da4e200f17c61027bc2893 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:39:14 +0100 Subject: [PATCH 15/74] Export treestore trees --- persistence/trees/trees.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index de51f0184..b2ef8fd76 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -23,9 +23,14 @@ import ( "github.com/pokt-network/smt" ) -// smtTreeHasher sets the hasher used by the tree SMT trees -// as a package level variable for visibility and internal use. -var smtTreeHasher hash.Hash = sha256.New() +var ( + // smtTreeHasher sets the hasher used by the tree SMT trees + // as a package level variable for visibility and internal use. + smtTreeHasher hash.Hash = sha256.New() + // smtValueHasher sets the hasher used by the tree SMT trees + // to be nil, which means that values are not prehashed + smtValueHasher smt.Option = smt.WithValueHasher(nil) +) const ( RootTreeName = "root" @@ -38,6 +43,7 @@ const ( TransactionsTreeName = "transactions" ParamsTreeName = "params" FlagsTreeName = "flags" + IbcTreeName = "ibc" ) var actorTypeToMerkleTreeName = map[coreTypes.ActorType]string{ @@ -60,7 +66,7 @@ var stateTreeNames = []string{ // Account Trees AccountTreeName, PoolTreeName, // Data Trees - TransactionsTreeName, ParamsTreeName, FlagsTreeName, + TransactionsTreeName, ParamsTreeName, FlagsTreeName, IbcTreeName, } // stateTree is a wrapper around the SMT that contains an identifying @@ -121,7 +127,7 @@ func (t *treeStore) DebugClearAll() error { if err := nodeStore.ClearAll(); err != nil { return fmt.Errorf("failed to clear %s node store: %w", treeName, err) } - stateTree.tree = smt.NewSparseMerkleTree(nodeStore, smtTreeHasher) + stateTree.tree = smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher) } return nil } @@ -190,7 +196,9 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height if err := t.updateFlagsTree(flags); err != nil { return "", fmt.Errorf("failed to update flags tree - %w", err) } - // Default should not happen; panic and log the treeType - this is a strong code smell. + case IbcTreeName: + // TODO: Implement IBC tree + // Default default: t.logger.Panic().Msgf("unhandled merkle tree type: %s", treeName) } From 8c9f38a220309159a4afc14404722cd69571cfe7 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:40:10 +0100 Subject: [PATCH 16/74] Add IBC message handling --- shared/messaging/events.go | 3 +++ shared/node.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/shared/messaging/events.go b/shared/messaging/events.go index a726a55a0..68a5eddec 100644 --- a/shared/messaging/events.go +++ b/shared/messaging/events.go @@ -12,6 +12,9 @@ const ( // Utility TxGossipMessageContentType = "utility.TxGossipMessage" + + // IBC + IbcMessageContentType = "ibc.IbcMessage" ) // Helper logger for state sync tranition events diff --git a/shared/node.go b/shared/node.go index ccdb806f7..c644cc793 100644 --- a/shared/node.go +++ b/shared/node.go @@ -189,6 +189,8 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) // TODO: Remove this lib once we move to Go 1.2 return multierr.Combine(err_consensus, err_p2p) + case messaging.IbcMessageContentType: + return node.GetBus().GetIBCModule().HandleMessage(message.Content) default: logger.Global.Warn().Msgf("Unsupported message content type: %s", contentType) } From 9b3d286877610d1d1ef9a78d18d8b714d6bb2c9c Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:40:28 +0100 Subject: [PATCH 17/74] Add ibc message types --- ibc/messages.go | 29 +++++++++++++++++++++++++++++ ibc/types/proto/messages.proto | 26 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 ibc/messages.go create mode 100644 ibc/types/proto/messages.proto diff --git a/ibc/messages.go b/ibc/messages.go new file mode 100644 index 000000000..020aa46ff --- /dev/null +++ b/ibc/messages.go @@ -0,0 +1,29 @@ +package ibc + +import ( + "github.com/pokt-network/pocket/ibc/types" + coreTypes "github.com/pokt-network/pocket/shared/core/types" +) + +func CreateUpdateStoreMessage(prefix coreTypes.CommitmentPrefix, key, value []byte) *types.IbcMessage { + return &types.IbcMessage{ + Event: &types.IbcMessage_Update{ + Update: &types.UpdateIbcStore{ + Prefix: prefix, + Key: key, + Value: value, + }, + }, + } +} + +func CreatePruneStoreMessage(prefix coreTypes.CommitmentPrefix, key []byte) *types.IbcMessage { + return &types.IbcMessage{ + Event: &types.IbcMessage_Prune{ + Prune: &types.PruneIbcStore{ + Prefix: prefix, + Key: key, + }, + }, + } +} diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto new file mode 100644 index 000000000..a436515b5 --- /dev/null +++ b/ibc/types/proto/messages.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package types; + +option go_package = "github.com/pokt-network/pocket/ibc/types"; + +// update_ibc_store defines a message representing the addition of a key/value pair to the IBC store +message update_ibc_store { + bytes prefix = 1; + bytes key = 2; + bytes value = 3; +} + +// prune_ibc_store defines a message representing the removal of a key from the IBC store +message prune_ibc_store { + bytes prefix = 1; + bytes key = 2; +} + +// ibc_message defines the different types of IBC message that can be sent across the network +message ibc_message { + oneof event { + update_ibc_store update = 1; + prune_ibc_store prune = 2; + } +} From 797ac03910da47bffee5dd50afaded709e4be330 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:40:56 +0100 Subject: [PATCH 18/74] Add provable stores and HandleMessage --- shared/modules/ibc_module.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index d006ee194..c8593e225 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -1,5 +1,10 @@ package modules +import ( + ics23 "github.com/cosmos/ics23/go" + "google.golang.org/protobuf/types/known/anypb" +) + //go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler const IBCModuleName = "ibc" @@ -9,6 +14,9 @@ type IBCModule interface { // GetHost returns the IBC host of the modules GetHost() IBCHost + + // HandleMessage handles the given IBC message + HandleMessage(*anypb.Any) error } // IBCHost is the interface used by the host machine (a Pocket node) to interact with the IBC module @@ -196,3 +204,13 @@ type IBCHandler interface { ) (Packet, error) **/ } + +type ProvableStore interface { + Get(key []byte) ([]byte, error) + Set(key, value []byte) error + Delete(key []byte) error + Root() []byte + GetCommitmentPrefix() coreTypes.CommitmentPrefix + CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) + CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) +} From b1f9a8875927dc74ab7e3e1732df585b5d97a18a Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:41:10 +0100 Subject: [PATCH 19/74] Add more IBC errors --- shared/core/types/error.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 870b6b733..1f707e605 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 142 +// NextCode: 144 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -178,9 +178,11 @@ const ( CodeUnknownMessageType Code = 131 CodeProposalBlockNotSet Code = 133 CodeHostAlreadyExists Code = 138 - CodeIBCInvalidID Code = 139 - CodeIBCInvalidPath Code = 140 - CodeCreatingProofError Code = 141 + CodeHostDoesNotExist Code = 139 + CodeIBCInvalidID Code = 140 + CodeIBCInvalidPath Code = 141 + CodeCreatingProofError Code = 142 + CodeUnknownIBCMessageTypeError Code = 143 ) const ( @@ -317,9 +319,11 @@ const ( UnknownActorTypeError = "the actor type is not recognized" UnknownMessageTypeError = "the message being by the utility message is not recognized" HostAlreadyExistsError = "an ibc host already exists" + HostDoesNotExistError = "an ibc host does not exist" IBCInvalidIDError = "invalid ibc identifier" IBCInvalidPathError = "invalid ibc path" CreatingProofError = "an error occurred creating the CommitmentProof" + UnknownIBCMessageTypeError = "the ibc message type is not recognized" ) func ErrUnknownParam(paramName string) Error { @@ -859,6 +863,10 @@ func ErrHostAlreadyExists() Error { return NewError(CodeHostAlreadyExists, HostAlreadyExistsError) } +func ErrHostDoesNotExist() Error { + return NewError(CodeHostDoesNotExist, HostDoesNotExistError) +} + func ErrIBCInvalidID(identifier, msg string) Error { return NewError(CodeIBCInvalidID, fmt.Sprintf("%s: %s (%s)", IBCInvalidIDError, identifier, msg)) } @@ -870,3 +878,7 @@ func ErrIBCInvalidPath(path string) Error { func ErrCreatingProof(err error) Error { return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", CreatingProofError, err.Error())) } + +func ErrUnknownIBCMessageType(messageType string) Error { + return NewError(CodeUnknownIBCMessageTypeError, fmt.Sprintf("%s: %s", UnknownIBCMessageTypeError, messageType)) +} From 1dca7d80d1c325b1e1edde3c3d71d045761cb944 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 11:16:55 +0100 Subject: [PATCH 20/74] Add GetProvableStore --- ibc/host.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ibc/host.go b/ibc/host.go index 603580cac..35b7d64b1 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -3,6 +3,7 @@ package ibc import ( "time" + coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" ) @@ -16,3 +17,8 @@ type host struct { func (h *host) GetTimestamp() uint64 { return uint64(time.Now().Unix()) } + +// GetProvableStore returns +func (h *host) GetProvableStore(prefix coreTypes.CommitmentPrefix) (modules.ProvableStore, error) { + return nil, nil +} From ff2d440c94206606a47045a59559a015ae1d48e9 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 11:17:19 +0100 Subject: [PATCH 21/74] Implement provableStore --- ibc/store/provable_store.go | 74 ++++++++++++++++++++++++++++++++++++ shared/modules/ibc_module.go | 1 + 2 files changed, 75 insertions(+) create mode 100644 ibc/store/provable_store.go diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go new file mode 100644 index 000000000..421ebb32d --- /dev/null +++ b/ibc/store/provable_store.go @@ -0,0 +1,74 @@ +package store + +import ( + "bytes" + + ics23 "github.com/cosmos/ics23/go" + "github.com/pokt-network/pocket/ibc/host" + "github.com/pokt-network/pocket/persistence/kvstore" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/smt" +) + +var _ modules.ProvableStore = &provableStore{} + +// provableStore implements the ProvableStore interface and wraps an SMT +type provableStore struct { + prefix coreTypes.CommitmentPrefix + tree *smt.SMT + nodeStore kvstore.KVStore +} + +// GetCommitmentPrefix returns the commitment prefix of the provable store +func (p *provableStore) GetCommitmentPrefix() coreTypes.CommitmentPrefix { + return p.prefix +} + +// Get returns the value in the tree of the key prefixed with the CommitmentPrefix +func (p *provableStore) Get(key []byte) ([]byte, error) { + prefixed := applyPrefix(p.prefix, key) + return p.tree.Get(prefixed) +} + +// Set sets the value in the tree of the key prefixed with the CommitmentPrefix +func (p *provableStore) Set(key, value []byte) error { + prefixed := applyPrefix(p.prefix, key) + return p.tree.Update(prefixed, value) +} + +// Delete deletes the value in the tree of the key prefixed with the CommitmentPrefix +func (p *provableStore) Delete(key []byte) error { + prefixed := applyPrefix(p.prefix, key) + return p.tree.Delete(prefixed) +} + +// CreateMembershipProof creates a membership proof for the key-value pair with the key +// prefixed with the CommitmentPrefix +func (p *provableStore) CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) { + prefixed := applyPrefix(p.prefix, key) + return createMembershipProof(p.tree, prefixed, value) +} + +// CreateNonMembershipProof creates a non-membership proof for the key prefixed with the CommitmentPrefix +func (p *provableStore) CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) { + prefixed := applyPrefix(p.prefix, key) + return createNonMembershipProof(p.tree, prefixed) +} + +// Root returns the root of the entire tree +// NOTE: Root does not work on a per-prefix basis but returns the root of the entire tree +func (p *provableStore) Root() []byte { + return p.tree.Root() +} + +// applyPrefix will apply the CommitmentPrefix to the key provided if not already applied +func applyPrefix(prefix coreTypes.CommitmentPrefix, key []byte) coreTypes.CommitmentPath { + slashed := make([]byte, 0, len(key)+1) + slashed = append(slashed, key...) + slashed = append(slashed, []byte("/")...) + if bytes.Equal(prefix[:len(slashed)], slashed) { + return key + } + return host.ApplyPrefix(prefix, string(key)) +} diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index c8593e225..52427a2e1 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -2,6 +2,7 @@ package modules import ( ics23 "github.com/cosmos/ics23/go" + coreTypes "github.com/pokt-network/pocket/shared/core/types" "google.golang.org/protobuf/types/known/anypb" ) From 83fb7a1118064613ee6f47108f5603ff7cfa52fd Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:40:37 +0100 Subject: [PATCH 22/74] Add private key to ibc config --- build/config/config.validator1.json | 3 ++- build/config/config.validator2.json | 3 ++- build/config/config.validator3.json | 3 ++- build/config/config.validator4.json | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build/config/config.validator1.json b/build/config/config.validator1.json index 702ad4232..746e74881 100644 --- a/build/config/config.validator1.json +++ b/build/config/config.validator1.json @@ -59,6 +59,7 @@ "chains": ["0001"] }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" } } diff --git a/build/config/config.validator2.json b/build/config/config.validator2.json index 3e2ef6f6e..052d8eeb7 100644 --- a/build/config/config.validator2.json +++ b/build/config/config.validator2.json @@ -52,6 +52,7 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" } } diff --git a/build/config/config.validator3.json b/build/config/config.validator3.json index 5cd45e544..834a72ff2 100644 --- a/build/config/config.validator3.json +++ b/build/config/config.validator3.json @@ -52,6 +52,7 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" } } diff --git a/build/config/config.validator4.json b/build/config/config.validator4.json index 2eb7747d7..df37cb69a 100644 --- a/build/config/config.validator4.json +++ b/build/config/config.validator4.json @@ -52,6 +52,7 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" } } From aa56b42e423cfdcf77345f3ed2b0e4b7046707d9 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:41:11 +0100 Subject: [PATCH 23/74] Allow conversion of IbcMessage to Transaction --- ibc/messages.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ibc/messages.go b/ibc/messages.go index 020aa46ff..7a97aae4d 100644 --- a/ibc/messages.go +++ b/ibc/messages.go @@ -1,8 +1,13 @@ package ibc import ( + "fmt" + "github.com/pokt-network/pocket/ibc/types" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" + "google.golang.org/protobuf/types/known/anypb" ) func CreateUpdateStoreMessage(prefix coreTypes.CommitmentPrefix, key, value []byte) *types.IbcMessage { @@ -27,3 +32,23 @@ func CreatePruneStoreMessage(prefix coreTypes.CommitmentPrefix, key []byte) *typ }, } } + +func ConvertIBCMessageToTx(ibcMessage *types.IbcMessage) (*coreTypes.Transaction, error) { + var anyMsg *anypb.Any + var err error + switch event := ibcMessage.Event.(type) { + case *types.IbcMessage_Update: + anyMsg, err = codec.GetCodec().ToAny(event.Update) + case *types.IbcMessage_Prune: + anyMsg, err = codec.GetCodec().ToAny(event.Prune) + default: + return nil, coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", event)) + } + if err != nil { + return nil, err + } + return &coreTypes.Transaction{ + Msg: anyMsg, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, nil +} From 454db6eb3980b06cc9b9d275f95753d4e54289cc Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:41:57 +0100 Subject: [PATCH 24/74] HandleEvent adds ibc message to TxMempool --- ibc/module.go | 88 ++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/ibc/module.go b/ibc/module.go index e60dfd6f0..6390479a5 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -1,6 +1,7 @@ package ibc import ( + "encoding/hex" "fmt" "sync" @@ -9,6 +10,7 @@ import ( "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" @@ -20,9 +22,9 @@ var _ modules.IBCModule = &ibcModule{} type ibcModule struct { base_modules.IntegratableModule - cfg *configs.IBCConfig - m sync.Mutex + m sync.Mutex + cfg *configs.IBCConfig logger *modules.Logger // Only a single host is allowed at a time @@ -89,51 +91,59 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { m.m.Lock() defer m.m.Unlock() - switch message.MessageName() { - - case messaging.IbcMessageContentType: - msg, err := codec.GetCodec().FromAny(message) - if err != nil { - return err - } - ibcMessage, ok := msg.(*ibcTypes.IbcMessage) - if !ok { - return fmt.Errorf("failed to cast message to IBCMessage") - } - return m.handleIBCMessage(ibcMessage) - - default: + // Check the message is actually a valid IBC message + if message.MessageName() != messaging.IbcMessageContentType { return coreTypes.ErrUnknownIBCMessageType(string(message.MessageName())) } -} + msg, err := codec.GetCodec().FromAny(message) + if err != nil { + return err + } + ibcMessage, ok := msg.(*ibcTypes.IbcMessage) + if !ok { + return fmt.Errorf("failed to cast message to IBCMessage") + } + if err := ibcMessage.ValidateBasic(); err != nil { + return err + } -// handleIBCMessage unpacks the IBC message to its type and calls the appropriate handler -func (m *ibcModule) handleIBCMessage(message *ibcTypes.IbcMessage) error { - switch msg := message.Event.(type) { - case *ibcTypes.IbcMessage_Update: - return m.handleUpdateMessage(msg.Update) - case *ibcTypes.IbcMessage_Prune: - return m.handlePruneMessage(msg.Prune) - default: - return coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", msg)) + // Convert IBC message to a utility Transaction + tx, err := ConvertIBCMessageToTx(ibcMessage) + if err != nil { + return err } -} -// handleUpdateMessage adds the updated store entry to the IBC store change mempool -func (m *ibcModule) handleUpdateMessage(message *ibcTypes.UpdateIbcStore) error { - if m.host == nil { - return coreTypes.ErrHostDoesNotExist() + // Sign the transaction + pkBz, err := hex.DecodeString(m.cfg.PrivateKey) + if err != nil { + return err + } + pk, err := crypto.NewPrivateKeyFromBytes(pkBz) + if err != nil { + return err + } + signableBz, err := tx.SignableBytes() + if err != nil { + return err + } + signature, err := pk.Sign(signableBz) + if err != nil { + return err + } + tx.Signature = &coreTypes.Signature{ + Signature: signature, + PublicKey: pk.PublicKey().Bytes(), } - // TODO: implement this - return nil -} -// handlePruneMessage adds a removal entry to the IBC store change mempool -func (m *ibcModule) handlePruneMessage(message *ibcTypes.PruneIbcStore) error { - if m.host == nil { - return coreTypes.ErrHostDoesNotExist() + // Marshall the Transaction and send it to the utility module + txBz, err := codec.GetCodec().Marshal(tx) + if err != nil { + return err + } + if err := m.GetBus().GetUtilityModule().HandleTransaction(txBz); err != nil { + return err } - // TODO: implement this + m.logger.Info().Str("message_type", "IbcMessage").Msg("Successfully added a new message to the mempool!") return nil } From 0d867f364197df4fd151fe4f7a170782b719815e Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:42:20 +0100 Subject: [PATCH 25/74] Add signer to messages --- ibc/types/proto/messages.proto | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto index a436515b5..e2a62b0f8 100644 --- a/ibc/types/proto/messages.proto +++ b/ibc/types/proto/messages.proto @@ -6,15 +6,17 @@ option go_package = "github.com/pokt-network/pocket/ibc/types"; // update_ibc_store defines a message representing the addition of a key/value pair to the IBC store message update_ibc_store { - bytes prefix = 1; - bytes key = 2; - bytes value = 3; + bytes signer = 1; + bytes prefix = 2; + bytes key = 3; + bytes value = 4; } // prune_ibc_store defines a message representing the removal of a key from the IBC store message prune_ibc_store { - bytes prefix = 1; - bytes key = 2; + bytes signer = 1; + bytes prefix = 2; + bytes key = 3; } // ibc_message defines the different types of IBC message that can be sent across the network From a3178e4b77657d6bb0e802e28b2c305ed951e6af Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:42:50 +0100 Subject: [PATCH 26/74] Implement utility Message interface for IbcMessage types --- ibc/types/messages.go | 91 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 ibc/types/messages.go diff --git a/ibc/types/messages.go b/ibc/types/messages.go new file mode 100644 index 000000000..8a38613b6 --- /dev/null +++ b/ibc/types/messages.go @@ -0,0 +1,91 @@ +package types + +import ( + "fmt" + "log" + + "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + utilityTypes "github.com/pokt-network/pocket/utility/types" +) + +// Implement the Message interface +var ( + _ utilityTypes.Message = &UpdateIbcStore{} + _ utilityTypes.Message = &PruneIbcStore{} +) + +func (m *IbcMessage) ValidateBasic() coreTypes.Error { + switch msg := m.Event.(type) { + case *IbcMessage_Update: + return msg.Update.ValidateBasic() + case *IbcMessage_Prune: + return msg.Prune.ValidateBasic() + default: + return coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", msg)) + } +} + +func (m *UpdateIbcStore) ValidateBasic() coreTypes.Error { + if m.Key == nil { + return coreTypes.ErrNilField("key") + } + if m.Prefix == nil { + return coreTypes.ErrNilField("prefix") + } + if m.Value == nil { + return coreTypes.ErrNilField("value") + } + return nil +} + +func (m *PruneIbcStore) ValidateBasic() coreTypes.Error { + if m.Key == nil { + return coreTypes.ErrNilField("key") + } + if m.Prefix == nil { + return coreTypes.ErrNilField("prefix") + } + return nil +} + +func (m *UpdateIbcStore) SetSigner(signer []byte) { m.Signer = signer } +func (m *PruneIbcStore) SetSigner(signer []byte) { m.Signer = signer } + +func (m *UpdateIbcStore) GetMessageName() string { + return string(m.ProtoReflect().Descriptor().Name()) +} + +func (m *PruneIbcStore) GetMessageName() string { + return string(m.ProtoReflect().Descriptor().Name()) +} + +func (m *UpdateIbcStore) GetMessageRecipient() string { return "" } +func (m *PruneIbcStore) GetMessageRecipient() string { return "" } + +func (m *UpdateIbcStore) GetSigner() []byte { return m.Signer } +func (m *PruneIbcStore) GetSigner() []byte { return m.Signer } + +func (m *UpdateIbcStore) GetActorType() coreTypes.ActorType { + return coreTypes.ActorType_ACTOR_TYPE_VAL +} + +func (m *PruneIbcStore) GetActorType() coreTypes.ActorType { + return coreTypes.ActorType_ACTOR_TYPE_VAL +} + +func (m *UpdateIbcStore) GetCanonicalBytes() []byte { + bz, err := codec.GetCodec().Marshal(m) + if err != nil { + log.Fatalf("must marshal %v", err) + } + return bz // DISCUSS(#142): should we also sort the JSON like in V0? +} + +func (m *PruneIbcStore) GetCanonicalBytes() []byte { + bz, err := codec.GetCodec().Marshal(m) + if err != nil { + log.Fatalf("must marshal %v", err) + } + return bz // DISCUSS(#142): should we also sort the JSON like in V0? +} From 3570e52a3dcae367a17f9f6c7c1a5c2329350d02 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:43:15 +0100 Subject: [PATCH 27/74] Update IBC config to have a private key --- runtime/configs/config.go | 1 + runtime/configs/proto/ibc_config.proto | 1 + 2 files changed, 2 insertions(+) diff --git a/runtime/configs/config.go b/runtime/configs/config.go index 3e387a8c8..ee2f6f382 100644 --- a/runtime/configs/config.go +++ b/runtime/configs/config.go @@ -181,6 +181,7 @@ func WithPK(pk string) func(*Config) { cfg.PrivateKey = pk cfg.Consensus.PrivateKey = pk cfg.P2P.PrivateKey = pk + cfg.IBC.PrivateKey = pk } } diff --git a/runtime/configs/proto/ibc_config.proto b/runtime/configs/proto/ibc_config.proto index 389e19528..b83d4f406 100644 --- a/runtime/configs/proto/ibc_config.proto +++ b/runtime/configs/proto/ibc_config.proto @@ -12,4 +12,5 @@ message IBCConfig { // 2. The node is a servicer and thus when IBC enabled is they are enabled // to relay IBC packets using an IBC relayer binary bool enabled = 1; + string private_key = 2; // hex encoded } From 22970b12d73af0b5c541a16d018c47808fde0483 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:43:29 +0100 Subject: [PATCH 28/74] Add IBC nil field errors --- shared/core/types/error.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 1f707e605..fbd9323c3 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 144 +// NextCode: 145 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -183,6 +183,7 @@ const ( CodeIBCInvalidPath Code = 141 CodeCreatingProofError Code = 142 CodeUnknownIBCMessageTypeError Code = 143 + CodeNilFieldError Code = 144 ) const ( @@ -324,6 +325,7 @@ const ( IBCInvalidPathError = "invalid ibc path" CreatingProofError = "an error occurred creating the CommitmentProof" UnknownIBCMessageTypeError = "the ibc message type is not recognized" + NilFieldError = "field cannot be nil" ) func ErrUnknownParam(paramName string) Error { @@ -882,3 +884,7 @@ func ErrCreatingProof(err error) Error { func ErrUnknownIBCMessageType(messageType string) Error { return NewError(CodeUnknownIBCMessageTypeError, fmt.Sprintf("%s: %s", UnknownIBCMessageTypeError, messageType)) } + +func ErrNilField(field string) Error { + return NewError(CodeNilFieldError, fmt.Sprintf("%s: %s", NilFieldError, field)) +} From f72f51391ac1e86bbafe71e841b307fec5d14880 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:43:46 +0100 Subject: [PATCH 29/74] Add IbcMessage Tx handling logic --- utility/unit_of_work/gov.go | 8 ++++---- utility/unit_of_work/transaction.go | 2 +- utility/unit_of_work/tx_message_handler.go | 15 +++++++++++++++ utility/unit_of_work/tx_message_signers.go | 13 +++++++++++++ utility/utility_message_handler.go | 1 - 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/utility/unit_of_work/gov.go b/utility/unit_of_work/gov.go index 2e18914ff..e1b893ac1 100644 --- a/utility/unit_of_work/gov.go +++ b/utility/unit_of_work/gov.go @@ -4,6 +4,7 @@ import ( "math/big" "strings" + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/persistence" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -24,9 +25,7 @@ func init() { } } -var ( - govParamTypes map[string]int -) +var govParamTypes map[string]int const ( BIGINT int = iota @@ -151,7 +150,8 @@ func (u *baseUtilityUnitOfWork) getParamOwner(paramName string) ([]byte, coreTyp func (u *baseUtilityUnitOfWork) getFee(msg typesUtil.Message, actorType coreTypes.ActorType) (amount *big.Int, err coreTypes.Error) { switch x := msg.(type) { - case *typesUtil.MessageSend: + // DISCUSS: What fee should be deducted for IBC store update messages + case *typesUtil.MessageSend, *ibcTypes.UpdateIbcStore, *ibcTypes.PruneIbcStore: return getGovParam[*big.Int](u, typesUtil.MessageSendFee) case *typesUtil.MessageStake: switch actorType { diff --git a/utility/unit_of_work/transaction.go b/utility/unit_of_work/transaction.go index 3e91c0c14..6e2d04c87 100644 --- a/utility/unit_of_work/transaction.go +++ b/utility/unit_of_work/transaction.go @@ -33,7 +33,7 @@ func (u *baseUtilityUnitOfWork) basicValidateTransaction(tx *coreTypes.Transacti // Get the address of the transaction signer pubKey, er := crypto.NewPublicKeyFromBytes(tx.Signature.PublicKey) - if err != nil { + if er != nil { return nil, coreTypes.ErrNewPublicKeyFromBytes(er) } address := pubKey.Address() diff --git a/utility/unit_of_work/tx_message_handler.go b/utility/unit_of_work/tx_message_handler.go index 9b3080af9..42de09a07 100644 --- a/utility/unit_of_work/tx_message_handler.go +++ b/utility/unit_of_work/tx_message_handler.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "math/big" + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" @@ -26,6 +27,10 @@ func (u *baseUtilityUnitOfWork) handleMessage(msg typesUtil.Message) (err coreTy return u.handleUnpauseMessage(x) case *typesUtil.MessageChangeParameter: return u.handleMessageChangeParameter(x) + case *ibcTypes.UpdateIbcStore: + return u.handleUpdateIbcStore(x) + case *ibcTypes.PruneIbcStore: + return u.handlePruneIbcStore(x) default: return coreTypes.ErrUnknownMessage(x) } @@ -220,6 +225,16 @@ func (u *baseUtilityUnitOfWork) handleMessageChangeParameter(message *typesUtil. return u.updateParam(message.ParameterKey, v) } +// TODO_IN_THIS_COMMIT: implement +func (u *baseUtilityUnitOfWork) handleUpdateIbcStore(message *ibcTypes.UpdateIbcStore) coreTypes.Error { + return nil +} + +// TODO_IN_THIS_COMMIT: implement +func (u *baseUtilityUnitOfWork) handlePruneIbcStore(message *ibcTypes.PruneIbcStore) coreTypes.Error { + return nil +} + func (u *baseUtilityUnitOfWork) checkBelowMaxChains(actorType coreTypes.ActorType, chains []string) coreTypes.Error { // validators don't have chains field if actorType == coreTypes.ActorType_ACTOR_TYPE_VAL { diff --git a/utility/unit_of_work/tx_message_signers.go b/utility/unit_of_work/tx_message_signers.go index 1da3b9acc..8e84bc133 100644 --- a/utility/unit_of_work/tx_message_signers.go +++ b/utility/unit_of_work/tx_message_signers.go @@ -1,6 +1,7 @@ package unit_of_work import ( + ibcTypes "github.com/pokt-network/pocket/ibc/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" typesUtil "github.com/pokt-network/pocket/utility/types" @@ -24,6 +25,10 @@ func (u *baseUtilityUnitOfWork) getSignerCandidates(msg typesUtil.Message) ([][] return u.getMessageUnpauseSignerCandidates(x) case *typesUtil.MessageChangeParameter: return u.getMessageChangeParameterSignerCandidates(x) + case *ibcTypes.UpdateIbcStore: + return u.getUpdateIbcStoreSingerCandidates(x) + case *ibcTypes.PruneIbcStore: + return u.getPruneIbcStoreSingerCandidates(x) default: return nil, coreTypes.ErrUnknownMessage(x) } @@ -72,3 +77,11 @@ func (u *baseUtilityUnitOfWork) getMessageUnpauseSignerCandidates(msg *typesUtil func (u *baseUtilityUnitOfWork) getMessageSendSignerCandidates(msg *typesUtil.MessageSend) ([][]byte, coreTypes.Error) { return [][]byte{msg.FromAddress}, nil } + +func (u *baseUtilityUnitOfWork) getUpdateIbcStoreSingerCandidates(msg *ibcTypes.UpdateIbcStore) ([][]byte, coreTypes.Error) { + return [][]byte{msg.Signer}, nil +} + +func (u *baseUtilityUnitOfWork) getPruneIbcStoreSingerCandidates(msg *ibcTypes.PruneIbcStore) ([][]byte, coreTypes.Error) { + return [][]byte{msg.Signer}, nil +} diff --git a/utility/utility_message_handler.go b/utility/utility_message_handler.go index 4cb6ee2a7..46079d1c5 100644 --- a/utility/utility_message_handler.go +++ b/utility/utility_message_handler.go @@ -44,7 +44,6 @@ func (u *utilityModule) HandleUtilityMessage(message *anypb.Any) error { } u.logger.Info().Str("message_type", "TxGossipMessage").Msg("Successfully added a new message to the mempool!") - default: return coreTypes.ErrUnknownMessageType(message.MessageName()) } From c7176e8d7760f2d0c7de18388d404e4d9e4f48fd Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:30:50 +0100 Subject: [PATCH 30/74] Add techdebt comment --- utility/unit_of_work/gov.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utility/unit_of_work/gov.go b/utility/unit_of_work/gov.go index e1b893ac1..19461e07b 100644 --- a/utility/unit_of_work/gov.go +++ b/utility/unit_of_work/gov.go @@ -150,7 +150,7 @@ func (u *baseUtilityUnitOfWork) getParamOwner(paramName string) ([]byte, coreTyp func (u *baseUtilityUnitOfWork) getFee(msg typesUtil.Message, actorType coreTypes.ActorType) (amount *big.Int, err coreTypes.Error) { switch x := msg.(type) { - // DISCUSS: What fee should be deducted for IBC store update messages + // TECHDEBT(M6): Decide on IBC store change fees and move into a governance parameter case *typesUtil.MessageSend, *ibcTypes.UpdateIbcStore, *ibcTypes.PruneIbcStore: return getGovParam[*big.Int](u, typesUtil.MessageSendFee) case *typesUtil.MessageStake: From 5c2bbc58e337080a3c91ca7ae88bd3d5ab9ef633 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:31:05 +0100 Subject: [PATCH 31/74] Remove duplicate method --- ibc/types/messages.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ibc/types/messages.go b/ibc/types/messages.go index 8a38613b6..35bd990a1 100644 --- a/ibc/types/messages.go +++ b/ibc/types/messages.go @@ -63,9 +63,6 @@ func (m *PruneIbcStore) GetMessageName() string { func (m *UpdateIbcStore) GetMessageRecipient() string { return "" } func (m *PruneIbcStore) GetMessageRecipient() string { return "" } -func (m *UpdateIbcStore) GetSigner() []byte { return m.Signer } -func (m *PruneIbcStore) GetSigner() []byte { return m.Signer } - func (m *UpdateIbcStore) GetActorType() coreTypes.ActorType { return coreTypes.ActorType_ACTOR_TYPE_VAL } From c05ab8eeb0f07457a43271527a05aede063c7508 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:39:02 +0100 Subject: [PATCH 32/74] Add IBC store change related postgres DB sql code --- persistence/db.go | 11 +++++++++++ persistence/ibc.go | 14 ++++++++++++++ persistence/sql/sql.go | 30 ++++++++++++++++++++++++++++++ persistence/types/ibc.go | 26 ++++++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 persistence/ibc.go create mode 100644 persistence/types/ibc.go diff --git a/persistence/db.go b/persistence/db.go index 43bf14e91..8b420c0ff 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -134,6 +134,10 @@ func initializeAllTables(ctx context.Context, db *pgxpool.Conn) error { } } + if err := initialiseIbcTables(ctx, db); err != nil { + return err + } + return nil } @@ -184,3 +188,10 @@ func initializeBlockTables(ctx context.Context, db *pgxpool.Conn) error { } return nil } + +func initialiseIbcTables(ctx context.Context, db *pgxpool.Conn) error { + if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IbcStoreTableName, types.IbcStoreTableSchema)); err != nil { + return err + } + return nil +} diff --git a/persistence/ibc.go b/persistence/ibc.go new file mode 100644 index 000000000..7e7bdb71f --- /dev/null +++ b/persistence/ibc.go @@ -0,0 +1,14 @@ +package persistence + +import ( + pTypes "github.com/pokt-network/pocket/persistence/types" +) + +// SetIBCStoreEntry sets the key value pair in the IBC store postgres table +func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { + ctx, tx := p.getCtxAndTx() + if _, err := tx.Exec(ctx, pTypes.InsertIbcStoreEntryQuery(p.Height, key, value)); err != nil { + return err + } + return nil +} diff --git a/persistence/sql/sql.go b/persistence/sql/sql.go index 48bbec2d3..274e20979 100644 --- a/persistence/sql/sql.go +++ b/persistence/sql/sql.go @@ -165,6 +165,36 @@ func GetParams(pgtx pgx.Tx, height uint64) ([]*coreTypes.Param, error) { return paramSlice, nil } +// GetIBCStoreUpdates returns the set of key-value pairs updated at the current height for the IBC store +func GetIBCStoreUpdates(pgtx pgx.Tx, height uint64) (keys [][]byte, values [][]byte, err error) { + fields := "key,value" + query := fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY key ASC", fields, ptypes.IbcStoreTableName, height) + rows, err := pgtx.Query(context.TODO(), query) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + var hexKey, hexValue string + for rows.Next() { + if err := rows.Scan(&hexKey, &hexValue); err != nil { + return nil, nil, err + } + key, err := hex.DecodeString(hexKey) + if err != nil { + return nil, nil, err + } + value, err := hex.DecodeString(hexValue) + if err != nil { + return nil, nil, err + } + keys = append(keys, key) + values = append(values, value) + } + + return keys, values, nil +} + func getActor(tx pgx.Tx, actorSchema ptypes.ProtocolActorSchema, address []byte, height int64) (actor *coreTypes.Actor, err error) { ctx := context.TODO() actor, height, err = getActorFromRow(actorSchema.GetActorType(), tx.QueryRow(ctx, actorSchema.GetQuery(hex.EncodeToString(address), height))) diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go new file mode 100644 index 000000000..c94b542d6 --- /dev/null +++ b/persistence/types/ibc.go @@ -0,0 +1,26 @@ +package types + +import ( + "encoding/hex" + "fmt" +) + +const ( + IbcStoreTableName = "ibc_messages" + IbcStoreTableSchema = `( + height BIGINT NOT NULL, + key TEXT NOT NULL, + value TEXT, + PRIMARY KEY (height, key) + )` +) + +func InsertIbcStoreEntryQuery(height int64, key, value []byte) string { + return fmt.Sprintf( + `INSERT INTO %s(height, key, value) VALUES(%d, '%s', '%s')`, + IbcStoreTableName, + height, + hex.EncodeToString(key), + hex.EncodeToString(value), + ) +} From 373209190b5b39fa4652853fd824a20e071a3b21 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:39:21 +0100 Subject: [PATCH 33/74] Add update IBC state tree logic from postgres DB changes --- persistence/trees/trees.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index b2ef8fd76..bb5c2b07f 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -197,7 +197,13 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height return "", fmt.Errorf("failed to update flags tree - %w", err) } case IbcTreeName: - // TODO: Implement IBC tree + keys, values, err := sql.GetIBCStoreUpdates(pgtx, height) + if err != nil { + return "", fmt.Errorf("failed to get IBC store updates: %w", err) + } + if err := t.updateIbcTree(keys, values); err != nil { + return "", fmt.Errorf("failed to update IBC tree: %w", err) + } // Default default: t.logger.Panic().Msgf("unhandled merkle tree type: %s", treeName) @@ -349,3 +355,23 @@ func (t *treeStore) updateFlagsTree(flags []*coreTypes.Flag) error { return nil } + +func (t *treeStore) updateIbcTree(keys [][]byte, values [][]byte) error { + if len(keys) != len(values) { + return fmt.Errorf("keys and values must be the same length") + } + for i := 0; i < len(keys); i++ { + key := keys[i] + value := values[i] + if value == nil { + if err := t.merkleTrees[IbcTreeName].tree.Delete(key); err != nil { + return err + } + } else { + if err := t.merkleTrees[IbcTreeName].tree.Update(key, value); err != nil { + return err + } + } + } + return nil +} From 5ffa511cf5790b470f425ed25d8cd80b9708e9bc Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:40:32 +0100 Subject: [PATCH 34/74] Remove prefix field from ibc messages --- ibc/types/messages.go | 6 ------ ibc/types/proto/messages.proto | 10 +++++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ibc/types/messages.go b/ibc/types/messages.go index 35bd990a1..e9163b2a1 100644 --- a/ibc/types/messages.go +++ b/ibc/types/messages.go @@ -30,9 +30,6 @@ func (m *UpdateIbcStore) ValidateBasic() coreTypes.Error { if m.Key == nil { return coreTypes.ErrNilField("key") } - if m.Prefix == nil { - return coreTypes.ErrNilField("prefix") - } if m.Value == nil { return coreTypes.ErrNilField("value") } @@ -43,9 +40,6 @@ func (m *PruneIbcStore) ValidateBasic() coreTypes.Error { if m.Key == nil { return coreTypes.ErrNilField("key") } - if m.Prefix == nil { - return coreTypes.ErrNilField("prefix") - } return nil } diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto index e2a62b0f8..39b6c839c 100644 --- a/ibc/types/proto/messages.proto +++ b/ibc/types/proto/messages.proto @@ -5,18 +5,18 @@ package types; option go_package = "github.com/pokt-network/pocket/ibc/types"; // update_ibc_store defines a message representing the addition of a key/value pair to the IBC store +// the key field should be the full key - prefixed with the stores CommitmentPrefix message update_ibc_store { bytes signer = 1; - bytes prefix = 2; - bytes key = 3; - bytes value = 4; + bytes key = 2; + bytes value = 3; } // prune_ibc_store defines a message representing the removal of a key from the IBC store +// the key field should be the full key - prefixed with the stores CommitmentPrefix message prune_ibc_store { bytes signer = 1; - bytes prefix = 2; - bytes key = 3; + bytes key = 2; } // ibc_message defines the different types of IBC message that can be sent across the network From eee1a18707452610397d70f0fee48c404a77c55e Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:40:52 +0100 Subject: [PATCH 35/74] Add IBC postgres db update error --- shared/core/types/error.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index fbd9323c3..c317ce50a 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 145 +// NextCode: 146 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -184,6 +184,7 @@ const ( CodeCreatingProofError Code = 142 CodeUnknownIBCMessageTypeError Code = 143 CodeNilFieldError Code = 144 + CodeUpdatingIBCStoreDBError Code = 145 ) const ( @@ -326,6 +327,7 @@ const ( CreatingProofError = "an error occurred creating the CommitmentProof" UnknownIBCMessageTypeError = "the ibc message type is not recognized" NilFieldError = "field cannot be nil" + UpdatingIBCStoreDBError = "an error occurred updating the ibc store postgres database" ) func ErrUnknownParam(paramName string) Error { @@ -888,3 +890,7 @@ func ErrUnknownIBCMessageType(messageType string) Error { func ErrNilField(field string) Error { return NewError(CodeNilFieldError, fmt.Sprintf("%s: %s", NilFieldError, field)) } + +func ErrUpdatingIBCStoreDB(err error) Error { + return NewError(CodeUpdatingIBCStoreDBError, fmt.Sprintf("%s: %s", UpdatingIBCStoreDBError, err.Error())) +} From cdec8cae1614e86a2c0e60bffa002be82390bda5 Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:41:24 +0100 Subject: [PATCH 36/74] Add SetIBCStoreEntry method to PostgresContext --- shared/modules/persistence_module.go | 3 +++ utility/unit_of_work/tx_message_handler.go | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 3160ef391..d6c0d8a6b 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -133,6 +133,9 @@ type PersistenceWriteContext interface { // Flag Operations InitFlags() error SetFlag(paramName string, value any, enabled bool) error + + // IBC Operations + SetIBCStoreEntry(key, value []byte) error } type PersistenceReadContext interface { diff --git a/utility/unit_of_work/tx_message_handler.go b/utility/unit_of_work/tx_message_handler.go index 42de09a07..31e6fe46d 100644 --- a/utility/unit_of_work/tx_message_handler.go +++ b/utility/unit_of_work/tx_message_handler.go @@ -225,13 +225,17 @@ func (u *baseUtilityUnitOfWork) handleMessageChangeParameter(message *typesUtil. return u.updateParam(message.ParameterKey, v) } -// TODO_IN_THIS_COMMIT: implement func (u *baseUtilityUnitOfWork) handleUpdateIbcStore(message *ibcTypes.UpdateIbcStore) coreTypes.Error { + if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, message.Value); err != nil { + return coreTypes.ErrUpdatingIBCStoreDB(err) + } return nil } -// TODO_IN_THIS_COMMIT: implement func (u *baseUtilityUnitOfWork) handlePruneIbcStore(message *ibcTypes.PruneIbcStore) coreTypes.Error { + if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, nil); err != nil { + return coreTypes.ErrUpdatingIBCStoreDB(err) + } return nil } From ac619b48434955ae5d86ad0f7a7467ab07a6ff8f Mon Sep 17 00:00:00 2001 From: Harry Law <53987565+h5law@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:05:38 +0100 Subject: [PATCH 37/74] Address linter errors --- ibc/store/provable_store.go | 9 +++++---- persistence/sql/sql.go | 2 +- persistence/trees/trees.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index 421ebb32d..789685c21 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -5,7 +5,6 @@ import ( ics23 "github.com/cosmos/ics23/go" "github.com/pokt-network/pocket/ibc/host" - "github.com/pokt-network/pocket/persistence/kvstore" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/smt" @@ -14,10 +13,12 @@ import ( var _ modules.ProvableStore = &provableStore{} // provableStore implements the ProvableStore interface and wraps an SMT +// it operates in memory and thus cannot make any changes to the underlying +// database. All changes must be propagated through the `IbcMessage` type +// and added to the mempool for inclusion in the next block type provableStore struct { - prefix coreTypes.CommitmentPrefix - tree *smt.SMT - nodeStore kvstore.KVStore + prefix coreTypes.CommitmentPrefix + tree *smt.SMT } // GetCommitmentPrefix returns the commitment prefix of the provable store diff --git a/persistence/sql/sql.go b/persistence/sql/sql.go index 274e20979..85b692c9e 100644 --- a/persistence/sql/sql.go +++ b/persistence/sql/sql.go @@ -166,7 +166,7 @@ func GetParams(pgtx pgx.Tx, height uint64) ([]*coreTypes.Param, error) { } // GetIBCStoreUpdates returns the set of key-value pairs updated at the current height for the IBC store -func GetIBCStoreUpdates(pgtx pgx.Tx, height uint64) (keys [][]byte, values [][]byte, err error) { +func GetIBCStoreUpdates(pgtx pgx.Tx, height uint64) (keys, values [][]byte, err error) { fields := "key,value" query := fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY key ASC", fields, ptypes.IbcStoreTableName, height) rows, err := pgtx.Query(context.TODO(), query) diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index bb5c2b07f..b06591969 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -356,7 +356,7 @@ func (t *treeStore) updateFlagsTree(flags []*coreTypes.Flag) error { return nil } -func (t *treeStore) updateIbcTree(keys [][]byte, values [][]byte) error { +func (t *treeStore) updateIbcTree(keys, values [][]byte) error { if len(keys) != len(values) { return fmt.Errorf("keys and values must be the same length") } From ff447f7bd9bed655f88bae7b0b22ef0afacab8fa Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:12:23 +0100 Subject: [PATCH 38/74] Add HandleMessage unit tests --- ibc/handle_message_test.go | 278 +++++++++++++++++++++++++++++++++++++ ibc/main_test.go | 144 +++++++++++++++++++ ibc/module.go | 4 - 3 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 ibc/handle_message_test.go create mode 100644 ibc/main_test.go diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go new file mode 100644 index 000000000..2ad4a7956 --- /dev/null +++ b/ibc/handle_message_test.go @@ -0,0 +1,278 @@ +package ibc + +import ( + "fmt" + "strconv" + "strings" + "testing" + + ibcTypes "github.com/pokt-network/pocket/ibc/types" + "github.com/pokt-network/pocket/persistence/indexer" + "github.com/pokt-network/pocket/shared/codec" + "github.com/pokt-network/pocket/shared/core/types" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestHandleMessage_ErrorAlreadyInMempool(t *testing.T) { + // Prepare test data + _, tx := prepareUpdateMessage(t, []byte("key"), []byte("value")) + txProtoBytes, err := proto.Marshal(tx) + require.NoError(t, err) + + // Prepare the environment + _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) + + // Manually add the tx to the mempool + err = utilityMod.GetMempool().AddTx(txProtoBytes) + require.NoError(t, err) + + // Error on having a duplciate transaction + err = utilityMod.HandleTransaction(txProtoBytes) + require.Error(t, err) + require.EqualError(t, err, coreTypes.ErrDuplicateTransaction().Error()) +} + +func TestHandleMessage_ErrorAlreadyCommitted(t *testing.T) { + // Prepare the environment + _, utilityMod, persistenceMod, _ := prepareEnvironment(t, 0, 0, 0, 0) + idxTx := prepareIndexedMessage(t, persistenceMod.GetTxIndexer()) + + // Error on having an indexed transaction + err := utilityMod.HandleTransaction(idxTx.Tx) + require.Error(t, err) + require.EqualError(t, err, coreTypes.ErrTransactionAlreadyCommitted().Error()) +} + +func TestHandleMessage_BasicValidation_Message(t *testing.T) { + updateMsg, _ := prepareUpdateMessage(t, []byte("key"), []byte("value")) + require.NoError(t, updateMsg.ValidateBasic()) + pruneMsg, _ := preparePruneMessage(t, []byte("key")) + require.NoError(t, pruneMsg.ValidateBasic()) + + testCases := []struct { + name string + msg *ibcTypes.IbcMessage + expected error + }{ + { + name: "Invalid Update Message: Empty Key", + msg: CreateUpdateStoreMessage(nil, []byte("value")), + expected: coreTypes.ErrNilField("key"), + }, + { + name: "Invalid Update Message: Empty Value", + msg: CreateUpdateStoreMessage([]byte("key"), nil), + expected: coreTypes.ErrNilField("value"), + }, + { + name: "Invalid Prune Message: Empty Key", + msg: CreatePruneStoreMessage(nil), + expected: coreTypes.ErrNilField("key"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.msg.ValidateBasic() + require.EqualError(t, err, tc.expected.Error()) + }) + } +} + +func TestHandleMessage_BasicValidation_Transaction(t *testing.T) { + privKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + + pubKey := privKey.PublicKey() + + message, validTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) + anyMessage, err := codec.GetCodec().ToAny(message) + require.NoError(t, err) + err = validTx.Sign(privKey) + require.NoError(t, err) + + testCases := []struct { + name string + txProto *coreTypes.Transaction + expectedErr error + }{ + { + name: "Invalid transaction: Missing Nonce", + txProto: &types.Transaction{}, + expectedErr: types.ErrEmptyNonce(), + }, + { + name: "Invalid transaction: Missing Signature Structure", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + }, + expectedErr: types.ErrEmptySignatureStructure(), + }, + { + name: "Invalid transaction: Missing Signature", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &types.Signature{ + PublicKey: nil, + Signature: nil, + }, + }, + expectedErr: types.ErrEmptySignature(), + }, + { + name: "Invalid transaction: Missing Public Key", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &types.Signature{ + PublicKey: nil, + Signature: []byte("bytes in place for signature but not actually valid"), + }, + }, + expectedErr: types.ErrEmptyPublicKey(), + }, + { + name: "Invalid transaction: Invalid Public Key", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &types.Signature{ + PublicKey: []byte("invalid pub key"), + Signature: []byte("bytes in place for signature but not actually valid"), + }, + }, + expectedErr: types.ErrNewPublicKeyFromBytes(fmt.Errorf("the public key length is not valid, expected length 32, actual length: 15")), + }, + { + name: "Invalid transaction: Invalid Message", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &types.Signature{ + PublicKey: pubKey.Bytes(), + Signature: []byte("bytes in place for signature but not actually valid"), + }, + Msg: nil, + }, + expectedErr: types.ErrDecodeMessage(fmt.Errorf("proto: invalid empty type URL")), + }, + { + name: "Invalid transaction: Invalid Signature", + txProto: &types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &types.Signature{ + PublicKey: pubKey.Bytes(), + Signature: []byte("invalid signature"), + }, + Msg: anyMessage, + }, + expectedErr: types.ErrSignatureVerificationFailed(), + }, + { + name: "Valid well-formatted transaction with valid signature", + txProto: validTx, + expectedErr: nil, + }, + } + + // Prepare the environment + _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + txProtoBytes, err := proto.Marshal(tc.txProto) + require.NoError(t, err) + + err = utilityMod.HandleTransaction(txProtoBytes) + if tc.expectedErr != nil { + expected := tc.expectedErr.Error() + if strings.Contains(err.Error(), "\u00a0") { + expected = strings.Replace(expected, "proto: ", "proto:\u00a0", 1) + } + require.EqualError(t, err, expected) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestHandleMessage_GetIndexedMessage(t *testing.T) { + // Prepare the environment + _, utilityMod, persistenceMod, _ := prepareEnvironment(t, 1, 0, 0, 0) + idxTx := prepareIndexedMessage(t, persistenceMod.GetTxIndexer()) + + tests := []struct { + name string + txProtoBytes []byte + txExists bool + expectErr error + }{ + {"returns indexed transaction when it exists", idxTx.Tx, true, nil}, + {"returns error when transaction doesn't exist", []byte("Does not exist"), false, types.ErrTransactionNotCommitted()}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + idTx, err := utilityMod.GetIndexedTransaction(test.txProtoBytes) + if test.expectErr != nil { + require.EqualError(t, err, test.expectErr.Error()) + require.Nil(t, idTx) + } else { + require.NoError(t, err) + require.NotNil(t, idTx) + } + }) + } +} + +func TestHandleMessage_AddToMempool(t *testing.T) { + // prepare the environment + _, _, _, ibcMod := prepareEnvironment(t, 1, 0, 0, 0) + require.Len(t, ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll(), 0) + msg, _ := prepareUpdateMessage(t, []byte("key"), []byte("value")) + anyMsg, err := codec.GetCodec().ToAny(msg) + require.NoError(t, err) + require.NoError(t, ibcMod.HandleMessage(anyMsg)) + require.Len(t, ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll(), 1) +} + +func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IbcMessage, *coreTypes.Transaction) { + t.Helper() + msg := CreateUpdateStoreMessage(key, value) + tx, err := ConvertIBCMessageToTx(msg) + require.NoError(t, err) + return msg, tx +} + +func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IbcMessage, *coreTypes.Transaction) { + t.Helper() + msg := CreatePruneStoreMessage(key) + tx, err := ConvertIBCMessageToTx(msg) + require.NoError(t, err) + return msg, tx +} + +func prepareIndexedMessage(t *testing.T, txIndexer indexer.TxIndexer) *coreTypes.IndexedTransaction { + t.Helper() + _, tx := preparePruneMessage(t, []byte{}) + txProtoBytes, err := proto.Marshal(tx) + require.NoError(t, err) + + // Test data - Prepare IndexedTransaction + idxTx := &coreTypes.IndexedTransaction{ + Tx: txProtoBytes, + Height: 0, + Index: 0, + ResultCode: 0, + Error: "h5law", + SignerAddr: "h5law", + RecipientAddr: "h5law", + MessageType: "h5law", + } + + // Index a test transaction + err = txIndexer.Index(idxTx) + require.NoError(t, err) + + return idxTx +} diff --git a/ibc/main_test.go b/ibc/main_test.go new file mode 100644 index 000000000..dfb2bf1a5 --- /dev/null +++ b/ibc/main_test.go @@ -0,0 +1,144 @@ +package ibc + +import ( + "log" + "os" + "testing" + + "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/runtime/test_artifacts/keygen" + "github.com/pokt-network/pocket/shared/messaging" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/utility" + "github.com/stretchr/testify/require" +) + +var dbURL string + +// NB: `TestMain` serves all tests in the immediate `ibc` package and not its children +func TestMain(m *testing.M) { + pool, resource, url := test_artifacts.SetupPostgresDocker() + dbURL = url + + exitCode := m.Run() + test_artifacts.CleanupPostgresDocker(m, pool, resource) + os.Exit(exitCode) +} + +func newTestUtilityModule(bus modules.Bus) modules.UtilityModule { + utilityMod, err := utility.Create(bus) + if err != nil { + log.Fatalf("Error creating utility module: %s", err) + } + return utilityMod.(modules.UtilityModule) +} + +func newTestPersistenceModule(bus modules.Bus) modules.PersistenceModule { + persistenceMod, err := persistence.Create(bus) + if err != nil { + log.Fatalf("Error creating persistence module: %s", err) + } + return persistenceMod.(modules.PersistenceModule) +} + +func newTestIbcModule(bus modules.Bus) modules.IBCModule { + ibcMod, err := Create(bus) + if err != nil { + log.Fatalf("Error creating ibc module: %s", err) + } + return ibcMod.(modules.IBCModule) +} + +// Prepares a runtime environment for testing along with a genesis state, a persistence module and a utility module +func prepareEnvironment( + t *testing.T, + numValidators, // nolint:unparam // we are not currently modifying parameter but want to keep it modifiable in the future + numServicers, + numApplications, + numFisherman int, + genesisOpts ...test_artifacts.GenesisOption, +) (*runtime.Manager, modules.UtilityModule, modules.PersistenceModule, modules.IBCModule) { + teardownDeterministicKeygen := keygen.GetInstance().SetSeed(42) + + runtimeCfg := newTestRuntimeConfig(numValidators, numServicers, numApplications, numFisherman, genesisOpts...) + bus, err := runtime.CreateBus(runtimeCfg) + require.NoError(t, err) + + testPersistenceMod := newTestPersistenceModule(bus) + err = testPersistenceMod.Start() + require.NoError(t, err) + + testUtilityMod := newTestUtilityModule(bus) + err = testUtilityMod.Start() + require.NoError(t, err) + + testIbcMod := newTestIbcModule(bus) + err = testIbcMod.Start() + require.NoError(t, err) + + // Reset database to genesis before every test + err = testPersistenceMod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + }) + require.NoError(t, err) + + t.Cleanup(func() { + teardownDeterministicKeygen() + err := testPersistenceMod.Stop() + require.NoError(t, err) + err = testUtilityMod.Stop() + require.NoError(t, err) + err = testIbcMod.Stop() + require.NoError(t, err) + }) + + return runtimeCfg, testUtilityMod, testPersistenceMod, testIbcMod +} + +func newTestRuntimeConfig( + numValidators, + numServicers, + numApplications, + numFisherman int, + genesisOpts ...test_artifacts.GenesisOption, +) *runtime.Manager { + cfg, err := configs.CreateTempConfig(&configs.Config{ + Utility: &configs.UtilityConfig{ + MaxMempoolTransactionBytes: 1000000, + MaxMempoolTransactions: 1000, + }, + Persistence: &configs.PersistenceConfig{ + PostgresUrl: dbURL, + NodeSchema: "test_schema", + BlockStorePath: ":memory:", + TxIndexerPath: ":memory:", + TreesStoreDir: ":memory:", + MaxConnsCount: 50, + MinConnsCount: 1, + MaxConnLifetime: "5m", + MaxConnIdleTime: "1m", + HealthCheckPeriod: "30s", + }, + Validator: &configs.ValidatorConfig{Enabled: true}, + IBC: &configs.IBCConfig{ + Enabled: true, + PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + }, + }) + if err != nil { + log.Fatalf("Error creating config: %s", err) + } + genesisState, _ := test_artifacts.NewGenesisState( + numValidators, + numServicers, + numApplications, + numFisherman, + genesisOpts..., + ) + runtimeCfg := runtime.NewManager(cfg, genesisState) + return runtimeCfg +} diff --git a/ibc/module.go b/ibc/module.go index 6390479a5..9a10cd163 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -11,7 +11,6 @@ import ( "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" "google.golang.org/protobuf/types/known/anypb" @@ -92,9 +91,6 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { defer m.m.Unlock() // Check the message is actually a valid IBC message - if message.MessageName() != messaging.IbcMessageContentType { - return coreTypes.ErrUnknownIBCMessageType(string(message.MessageName())) - } msg, err := codec.GetCodec().FromAny(message) if err != nil { return err From 6e4a510f0fc08b4f2382e533dfd11e374c98a23e Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:18:29 +0100 Subject: [PATCH 39/74] Simplify tests as covered in utility --- ibc/handle_message_test.go | 115 +++---------------------------------- 1 file changed, 8 insertions(+), 107 deletions(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index 2ad4a7956..4a89367d5 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -1,9 +1,6 @@ package ibc import ( - "fmt" - "strconv" - "strings" "testing" ibcTypes "github.com/pokt-network/pocket/ibc/types" @@ -83,118 +80,22 @@ func TestHandleMessage_BasicValidation_Message(t *testing.T) { } func TestHandleMessage_BasicValidation_Transaction(t *testing.T) { + // Prepare the environment + _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) + privKey, err := crypto.GeneratePrivateKey() require.NoError(t, err) - pubKey := privKey.PublicKey() - - message, validTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) - anyMessage, err := codec.GetCodec().ToAny(message) + _, validTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) require.NoError(t, err) err = validTx.Sign(privKey) require.NoError(t, err) - testCases := []struct { - name string - txProto *coreTypes.Transaction - expectedErr error - }{ - { - name: "Invalid transaction: Missing Nonce", - txProto: &types.Transaction{}, - expectedErr: types.ErrEmptyNonce(), - }, - { - name: "Invalid transaction: Missing Signature Structure", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - }, - expectedErr: types.ErrEmptySignatureStructure(), - }, - { - name: "Invalid transaction: Missing Signature", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ - PublicKey: nil, - Signature: nil, - }, - }, - expectedErr: types.ErrEmptySignature(), - }, - { - name: "Invalid transaction: Missing Public Key", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ - PublicKey: nil, - Signature: []byte("bytes in place for signature but not actually valid"), - }, - }, - expectedErr: types.ErrEmptyPublicKey(), - }, - { - name: "Invalid transaction: Invalid Public Key", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ - PublicKey: []byte("invalid pub key"), - Signature: []byte("bytes in place for signature but not actually valid"), - }, - }, - expectedErr: types.ErrNewPublicKeyFromBytes(fmt.Errorf("the public key length is not valid, expected length 32, actual length: 15")), - }, - { - name: "Invalid transaction: Invalid Message", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ - PublicKey: pubKey.Bytes(), - Signature: []byte("bytes in place for signature but not actually valid"), - }, - Msg: nil, - }, - expectedErr: types.ErrDecodeMessage(fmt.Errorf("proto: invalid empty type URL")), - }, - { - name: "Invalid transaction: Invalid Signature", - txProto: &types.Transaction{ - Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ - PublicKey: pubKey.Bytes(), - Signature: []byte("invalid signature"), - }, - Msg: anyMessage, - }, - expectedErr: types.ErrSignatureVerificationFailed(), - }, - { - name: "Valid well-formatted transaction with valid signature", - txProto: validTx, - expectedErr: nil, - }, - } - - // Prepare the environment - _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - txProtoBytes, err := proto.Marshal(tc.txProto) - require.NoError(t, err) + txProtoBytes, err := proto.Marshal(validTx) + require.NoError(t, err) - err = utilityMod.HandleTransaction(txProtoBytes) - if tc.expectedErr != nil { - expected := tc.expectedErr.Error() - if strings.Contains(err.Error(), "\u00a0") { - expected = strings.Replace(expected, "proto: ", "proto:\u00a0", 1) - } - require.EqualError(t, err, expected) - } else { - require.NoError(t, err) - } - }) - } + err = utilityMod.HandleTransaction(txProtoBytes) + require.NoError(t, err) } func TestHandleMessage_GetIndexedMessage(t *testing.T) { From 895dfd0ee7b7f5cb743ab5fd3fb0b4200aed84bd Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:24:35 +0100 Subject: [PATCH 40/74] Update validation testcases --- ibc/handle_message_test.go | 40 ++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index 4a89367d5..44076f7d3 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -10,13 +10,12 @@ import ( coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" ) func TestHandleMessage_ErrorAlreadyInMempool(t *testing.T) { // Prepare test data _, tx := prepareUpdateMessage(t, []byte("key"), []byte("value")) - txProtoBytes, err := proto.Marshal(tx) + txProtoBytes, err := codec.GetCodec().Marshal(tx) require.NoError(t, err) // Prepare the environment @@ -91,11 +90,44 @@ func TestHandleMessage_BasicValidation_Transaction(t *testing.T) { err = validTx.Sign(privKey) require.NoError(t, err) - txProtoBytes, err := proto.Marshal(validTx) + txProtoBytes, err := codec.GetCodec().Marshal(validTx) require.NoError(t, err) err = utilityMod.HandleTransaction(txProtoBytes) require.NoError(t, err) + + testCases := []struct { + name string + msg *ibcTypes.IbcMessage + expected error + }{ + { + name: "Invalid Update Message: Empty Key", + msg: CreateUpdateStoreMessage(nil, []byte("value")), + expected: coreTypes.ErrEmptySignatureStructure(), + }, + { + name: "Invalid Update Message: Empty Value", + msg: CreateUpdateStoreMessage([]byte("key"), nil), + expected: coreTypes.ErrEmptySignatureStructure(), + }, + { + name: "Invalid Prune Message: Empty Key", + msg: CreatePruneStoreMessage(nil), + expected: coreTypes.ErrEmptySignatureStructure(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tx, err := ConvertIBCMessageToTx(tc.msg) + require.NoError(t, err) + txProtoBytes, err := codec.GetCodec().Marshal(tx) + require.NoError(t, err) + err = utilityMod.HandleTransaction(txProtoBytes) + require.EqualError(t, err, tc.expected.Error()) + }) + } } func TestHandleMessage_GetIndexedMessage(t *testing.T) { @@ -156,7 +188,7 @@ func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IbcMessage, *coreT func prepareIndexedMessage(t *testing.T, txIndexer indexer.TxIndexer) *coreTypes.IndexedTransaction { t.Helper() _, tx := preparePruneMessage(t, []byte{}) - txProtoBytes, err := proto.Marshal(tx) + txProtoBytes, err := codec.GetCodec().Marshal(tx) require.NoError(t, err) // Test data - Prepare IndexedTransaction From ac2a97449fb0148e188e4e73faf7e9b967162db9 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:26:04 +0100 Subject: [PATCH 41/74] Fix importing twice --- ibc/handle_message_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index 44076f7d3..a6a2b22f4 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -6,7 +6,6 @@ import ( ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/persistence/indexer" "github.com/pokt-network/pocket/shared/codec" - "github.com/pokt-network/pocket/shared/core/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" "github.com/stretchr/testify/require" @@ -142,7 +141,7 @@ func TestHandleMessage_GetIndexedMessage(t *testing.T) { expectErr error }{ {"returns indexed transaction when it exists", idxTx.Tx, true, nil}, - {"returns error when transaction doesn't exist", []byte("Does not exist"), false, types.ErrTransactionNotCommitted()}, + {"returns error when transaction doesn't exist", []byte("Does not exist"), false, coreTypes.ErrTransactionNotCommitted()}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From dbf7f2bd8209f00fec8a43fe5dfee3a3ff7179ce Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:35:23 +0100 Subject: [PATCH 42/74] Add nolint comments --- ibc/main_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibc/main_test.go b/ibc/main_test.go index dfb2bf1a5..a2f9abe7c 100644 --- a/ibc/main_test.go +++ b/ibc/main_test.go @@ -53,6 +53,8 @@ func newTestIbcModule(bus modules.Bus) modules.IBCModule { } // Prepares a runtime environment for testing along with a genesis state, a persistence module and a utility module +// +//nolint:unparam // Test suite is not fully built out yet func prepareEnvironment( t *testing.T, numValidators, // nolint:unparam // we are not currently modifying parameter but want to keep it modifiable in the future @@ -99,6 +101,7 @@ func prepareEnvironment( return runtimeCfg, testUtilityMod, testPersistenceMod, testIbcMod } +//nolint:unparam // Test suite is not fully built out yet func newTestRuntimeConfig( numValidators, numServicers, From 0c05028ee5a3c7b2c9349724a2ef3f00b0abe67f Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 20:27:51 +0100 Subject: [PATCH 43/74] Check msg equality in test mempool test --- ibc/handle_message_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index a6a2b22f4..11aca854d 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -165,7 +165,23 @@ func TestHandleMessage_AddToMempool(t *testing.T) { anyMsg, err := codec.GetCodec().ToAny(msg) require.NoError(t, err) require.NoError(t, ibcMod.HandleMessage(anyMsg)) - require.Len(t, ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll(), 1) + mempoolTxs := ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll() + + require.Len(t, mempoolTxs, 1) + mTx := mempoolTxs[0] + require.NotNil(t, mTx) + txProto := &coreTypes.Transaction{} + err = codec.GetCodec().Unmarshal(mTx, txProto) + require.NoError(t, err) + ibcMsg := &ibcTypes.UpdateIbcStore{} + require.NoError(t, txProto.GetMsg().UnmarshalTo(ibcMsg)) + + // Compare messages are equal + origBz, err := codec.GetCodec().Marshal(msg.GetUpdate()) + require.NoError(t, err) + ibcMsgBz, err := codec.GetCodec().Marshal(ibcMsg) + require.NoError(t, err) + require.Equal(t, origBz, ibcMsgBz) } func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IbcMessage, *coreTypes.Transaction) { From 2cf0ff29a2668b1bf3a50cf48caccbf46a626c89 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 22 Jun 2023 21:50:32 +0100 Subject: [PATCH 44/74] Add new issue comments --- ibc/host.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ibc/host.go b/ibc/host.go index 35b7d64b1..c04e9b68a 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -1,3 +1,8 @@ +// TECHDEBT(#854): The host should be a submodule with access to the bus, this allows for the +// host to access the persistence module and thus the tree store in order to create local +// copies of the IBC state tree with the GetProvableStore() function. Bus access also will +// allow for the host to send any local changes to these stores as IbcMessage types through +// the P2P module's Broadcast() function to allow them to be included in the next block. package ibc import ( @@ -18,7 +23,10 @@ func (h *host) GetTimestamp() uint64 { return uint64(time.Now().Unix()) } -// GetProvableStore returns +// GetProvableStore returns a copy of the IBC state tree where all operations happen +// locally (in memory) and are not persisted to the database. All changes are instead +// broadcasted to the network for inclusion in the next block. +// TODO(#854): Implement this func (h *host) GetProvableStore(prefix coreTypes.CommitmentPrefix) (modules.ProvableStore, error) { return nil, nil } From 7cbbe6e7d41d6ec5f84992b2f4176983e5264c10 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 23 Jun 2023 15:08:02 +0100 Subject: [PATCH 45/74] Add mockgen flag for ProvableStore --- shared/modules/ibc_module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index 52427a2e1..1ebc5b601 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) -//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler +//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler,ProvableStore const IBCModuleName = "ibc" From a1edf4bf00f85e18cb13cbeb1aaf4177423be2c2 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 23 Jun 2023 15:22:02 +0100 Subject: [PATCH 46/74] Address comments --- ibc/docs/ics24.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++ ibc/host.go | 10 +++--- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 8834c4b88..cc3226a5a 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -4,6 +4,12 @@ - [Implementation](#implementation) - [Paths and Identifiers](#paths-and-identifiers) - [Timestamps](#timestamps) +- [IBC State](#ibc-state) + - [IBC State Tree](#ibc-state-tree) + - [IBC Messages](#ibc-messages) + - [IBC Message Handling](#ibc-message-handling) + - [Mempool](#mempool) + - [State Transition](#state-transition) ## Overview @@ -11,12 +17,35 @@ As token transfers as defined in [ICS-20][ics20] work on a lock and mint pattern, any tokens sent from **chain A** to **chain B** will have a denomination unique to the connection/channel/port combination that the packet was sent over. This means that if a host where to shutdown a connection or channel without warning any tokens yet to be returned to the host chain would be lost. For this reason, only validator nodes are able to become hosts, as they provide the most reliability out of the different node types. +## Host Configuration + +Only validators can be configured to be IBC hosts. If the IBC module, during its creation, detects the node is a validator (and the IBC `enabled` field in the config is `true`) it will automatically create a host. + +```json +"ibc": { + "enabled": bool, + "private_key": string +} +``` + +The `PrivateKey` field of the configuration is used to sign IBC store related messages and state transitions for inclusion in the block. + ## Implementation **Note**: The ICS-24 implementation is still a work in progress and is not yet fully implemented. ICS-24 has numerous sub components that must be implemented in order for the host to be fully functional. These range from type definitions for identifiers, paths and stores as well as the methods to interact with them. Alongside these ICS-24 also defines the Event Logging system which is used to store the packet data and timeouts for the relayers to read, as only the `CommitmentProof` objects are committed to the chain state. In addition to these numerous other features are part of ICS-24 that are closely linked to other ICS components such as consensus state introspection and client state validation. +### Persistence + +The IBC stores must be included in the networks consensus state as one of the many state trees. This is to ensure the IBC light clients verifying Pocket network's state can verify the inclusion or exclusion of IBC related information from the block headers. + +As the IBC host will make changes to the IBC store locally, in response to functions being called by relayers, they require these changes to be propagated throughout the network (i.e. the mempool) and included in all other node's IBC stores so that during block production these changes are reflected in the state transition. This is done by utilizing the existing transaction workflow, adding the IBC store change messages to the mempool and then handling them as a new message type in block production/application logic. + +TODO_IN_THIS_COMMIT: Mermaid Diagram Here + +See: [IBC State](#ibc-state) below for more details on the IBC state transition process. + ### Paths and Identifiers Paths are defined as bytestrings that are used to access the elements in the different stores. They are built with the function `ApplyPrefix()` which takes a store key as a prefix and a path string and will return the key to access an element in the specific store. The logic for paths can be found in [host/keys.go](../host/keys.go) and [host/prefix.go](../host/prefix.go) @@ -27,6 +56,59 @@ Identifiers are bytestrings constrained to specific characters and lengths depen The `GetTimestamp()` function returns the current unix timestamp of the host machine and is used to calculate timeout periods for packets +## IBC State + +As mentioned [above](#persistence) the IBC store **must** be included in the consensus state of the network. As such the IBC store as defined in [ICS-24][ics24] has been implemented as a single IBC state tree. + +### IBC State Tree + +The IBC state tree is a non-value hashing `SMT` backed by a persistent `KVStore`, this is due to its need for data retrieval as well as proof generation/verification. The root hash of the IBC state tree is included in the `rootTree` which computes the networks state hash for any given block. This allows verifiers to not only verify the inclusion/exclusion of any element in the IBC state tree itself but also that the IBC state tree was used to compute the network's state hash, by utilising the `CommitmentProof` object defined in [ICS-23][ics23]. + +### IBC Messages + +As the hosts make changes to this IBC store locally (creating light clients, opening connections, and sending packets for example); these changes must be propagated throughout the network to ensure the IBC state tree is consistent across all nodes. This is achieved by the new `IbcMessage` type defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: + +- `UpdateIbcStore`: Updating the store with a key-value pair; adding a new or updating an existing element +- `PruneIbcStore`: Pruning the store via its key; removal of an existing element + +_Note: In both types described above the `key` field **must** already be prefixed with the `CommitmentPrefix` and should be a valid path in the store._ + +When changes are made locally they are not committed to the IBC store itself but are instead used to create an `IbcMessage` which is broadcasted to the network. This is akin to a simple send transaction that has been propagated throughout the mempool but has not been committed to the on-chain state. + +### IBC Message Handling + +Upon a node receiving an `IbcMessage` from the event bus it will use the `HandleMessage()` method of the `IBCModule` to add this message to the transactions mempool via the following steps: + +1. Wrap the `IbcMessage` within a `Transaction` +2. Sign the `Transaction` using the `IBCModule`'s private key +3. Broadcast the `Transaction` throughout the mempool + +TODO_IN_THIS_COMMIT: Mermaid Diagram Here + +See: [ibc/module.go](../module.go) for the specific implementation details. + +### Mempool + +With the `IbcMessage` now propagated through the network's mempool, when it is reaped (by the block proposer) the message's validity will be handled by first determining the type of the `IbcMessage`: + +- `UpdateIbcStore`: The `key` and `value` fields of the `IbcMessage` are inserted into the `ibc_messages` Postgres table along with the current height +- `PruneIbcStore`: The `key` with a `nil` value is passed into the `ibc_messages` Postgres table along with the current height + +_Note: Prior to insertion the `key` and `value` fields of the messages are hexadecimally encoded into strings._ + +### State Transition + +When the new state hash is computed, the different state trees read the updates from their respective Postgres tables and update the trees accordingly. For each entry in the `ibc_message` table depending on the entries `value` field the tree will perform one of two operations: + +- `value == nil` + - This is a `PruneIbcStore` message and thus the tree will delete the entry with the given `key` + - `ibcTree.Delete(key)` +- `value != nil` + - This is an `UpdateIbcStore` message and thus the tree will update the entry with the given `key` to have the given `value` + - `ibcTree.Update(key, value)` + +The new root hash of the IBC store tree is then included in the `rootTree` and the new state hash is computed, ensuring the IBC store's state is consistent across the network. + [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md [smt]: https://github.com/pokt-network/smt diff --git a/ibc/host.go b/ibc/host.go index c04e9b68a..442017b80 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -2,7 +2,7 @@ // host to access the persistence module and thus the tree store in order to create local // copies of the IBC state tree with the GetProvableStore() function. Bus access also will // allow for the host to send any local changes to these stores as IbcMessage types through -// the P2P module's Broadcast() function to allow them to be included in the next block. +// the P2P module's Broadcast() function to allow them to propagate through the network's mempool. package ibc import ( @@ -23,9 +23,11 @@ func (h *host) GetTimestamp() uint64 { return uint64(time.Now().Unix()) } -// GetProvableStore returns a copy of the IBC state tree where all operations happen -// locally (in memory) and are not persisted to the database. All changes are instead -// broadcasted to the network for inclusion in the next block. +// GetProvableStore returns a copy of the IBC state tree where all operations observed by +// this specific ibc light client were applied to its ephemeral (in memory) state and have not +// yet been included in the next block. The aggregation of all light client-provable stores +// propagated throughout the network are happen validated by the block proposer when reaping +// the mempool, and lead to a valid on-chain state transition when consensus is reached. // TODO(#854): Implement this func (h *host) GetProvableStore(prefix coreTypes.CommitmentPrefix) (modules.ProvableStore, error) { return nil, nil From 416682b4bbf56814ab2f17a0824a5f72f3951072 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 23 Jun 2023 16:38:02 +0100 Subject: [PATCH 47/74] Add diagrams --- ibc/docs/ics24.md | 67 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index cc3226a5a..148f68469 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -40,9 +40,45 @@ ICS-24 has numerous sub components that must be implemented in order for the hos The IBC stores must be included in the networks consensus state as one of the many state trees. This is to ensure the IBC light clients verifying Pocket network's state can verify the inclusion or exclusion of IBC related information from the block headers. -As the IBC host will make changes to the IBC store locally, in response to functions being called by relayers, they require these changes to be propagated throughout the network (i.e. the mempool) and included in all other node's IBC stores so that during block production these changes are reflected in the state transition. This is done by utilizing the existing transaction workflow, adding the IBC store change messages to the mempool and then handling them as a new message type in block production/application logic. +The following is a simplified sequence diagram of an IBC fungible token transfer. This requires **Chain A** to commit to its state the packet data related to the transfer, so that **Chain B** can verify the inclusion of this packet data with the light client of **Chain A** it runs. + +```mermaid +sequenceDiagram + actor UA as User A + box Blue Chain A + participant A1 as Validator A + participant A2 as IBC Host A + participant A3 as Light Client B + end + box Transparent Relayer + actor R1 as Relayer + end + box Red Chain B + participant B1 as Validator B + participant B2 as IBC Host B + participant B3 as Light Client A + end + actor UB as User B + R1->>R1: Watch(Chain A) + R1->>R1: Watch(Chain B) + UA->>A1: Send 10$POKT to User B + A1->>A1: Lock(10$POKT) + A1->>A2: Create(FungibleTokenPacketData) + A2->>A1: Commit(FungibleTokenPacketData) + A1->>A1: NewBlock() + R1->>R1: CheckNewBlockForIBCPackets() + R1->>A2: QueryAndProve(FungibleTokenPacketData) + A2->>R1: FungibleTokenPacketData + A2->>R1: Proof(FungibleTokenPacketData) + R1->>B2: Validate(FungibleTokenPacketData, Proof(FungibleTokenPacketData)) + B2->>B3: Verify(Proof(FungibleTokenPacketData)) + B3->>B2: FoundInState(FungibleTokenPacketData) + B2->>B1: Send 10$POKT to User B + B1->>B1: Mint(10$POKT) + B1->>UB: Receive 10$POKT from User A +``` -TODO_IN_THIS_COMMIT: Mermaid Diagram Here +As the IBC host will make changes to the IBC store locally, in response to functions being called by relayers, they require these changes to be propagated throughout the network (i.e. the mempool) and included in all other node's IBC stores so that during block production these changes are reflected in the state transition. This is done by utilizing the existing transaction workflow, adding the IBC store change messages to the mempool and then handling them as a new message type in block production/application logic. See: [IBC State](#ibc-state) below for more details on the IBC state transition process. @@ -83,7 +119,32 @@ Upon a node receiving an `IbcMessage` from the event bus it will use the `Handle 2. Sign the `Transaction` using the `IBCModule`'s private key 3. Broadcast the `Transaction` throughout the mempool -TODO_IN_THIS_COMMIT: Mermaid Diagram Here +```mermaid +graph LR + subgraph Bus + A[Events] + end + subgraph I[IBC Host] + I1["HandleMessage(Message)"] + end + subgraph Handler + H1["ConvertIbcMessageToTransaction(IbcMessage)"] + subgraph Transaction + T1["coreTypes.Transaction{Msg: IbcMessage}"] + end + H2["SignTransaction(Transaction)"] + end + subgraph Mempool + M1["ValidateTransaction(Transaction)"] + M2["AddToMempool(Transaction)"] + end + Bus--Message-->I + I--IbcMessage-->Handler + H1--IbcMessage-->Transaction + Transaction--Transaction-->H2 + Handler--Transaction-->Mempool + M1--Transaction-->M2 +``` See: [ibc/module.go](../module.go) for the specific implementation details. From 953a146386fe03e700730d6da86cc7ee54b347ef Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:28:32 +0100 Subject: [PATCH 48/74] Update state hashes --- persistence/test/state_test.go | 6 +++--- utility/session_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/persistence/test/state_test.go b/persistence/test/state_test.go index d7a7555cd..37aeacf01 100644 --- a/persistence/test/state_test.go +++ b/persistence/test/state_test.go @@ -42,9 +42,9 @@ func TestStateHash_DeterministicStateWhenUpdatingAppStake(t *testing.T) { // logic changes, these hashes will need to be updated based on the test output. // TODO: Add an explicit updateSnapshots flag to the test to make this more clear. stateHashes := []string{ - "fb1c1b2da242eb6148884e1f11c184c673963b6fcb59feea4ea51c9840e4b56c", - "9ae40f9fd0864c01760c19d6688d1c7bba5ad746fc3f7123cf071b142c2a302a", - "5c13743e9d29e83cbff1301e3750c7583e365b2c3fd81b643ed8db985f46268f", + "00788e9a5133c31c869eb58d383f2fc30aac19b585947a8c82fe5808504ff84f", + "693450dee542f1455fdbfdeab22eb91e1888af50285330d45607b58c22f985f7", + "ad13b1646d02978f893919f413c7497b6f4d332011c40b6453499316d6705680", } stakeAmount := initialStakeAmount diff --git a/utility/session_test.go b/utility/session_test.go index ff544e508..c05e5bbdf 100644 --- a/utility/session_test.go +++ b/utility/session_test.go @@ -25,7 +25,7 @@ func TestSession_GetSession_SingleFishermanSingleServicerBaseCase(t *testing.T) numFishermen := 1 numServicers := 1 // needs to be manually updated if business logic changes - expectedSessionId := "7a915e89e4805095150e6826dff161ab2612e766b9e6893e6fe747e20a3abfa8" + expectedSessionId := "3de7deb1c64b32cf313838f8124b2b123ae78cff02bf5eca1564d9f78a764608" runtimeCfg, utilityMod, _ := prepareEnvironment(t, 5, numServicers, 1, numFishermen) From c39ea1d090aa06143a4c4d739130d66768358486 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:38:43 +0100 Subject: [PATCH 49/74] fixup: runtime key addition --- runtime/manager_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/manager_test.go b/runtime/manager_test.go index 8c3ce360f..35fef8094 100644 --- a/runtime/manager_test.go +++ b/runtime/manager_test.go @@ -1823,7 +1823,10 @@ func TestNewManagerFromReaders(t *testing.T) { }, Validator: &configs.ValidatorConfig{Enabled: true}, Fisherman: defaultCfg.Fisherman, - IBC: &configs.IBCConfig{Enabled: true}, + IBC: &configs.IBCConfig{ + Enabled: true, + PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + }, }, genesisState: expectedGenesis, clock: clock.New(), From 89550be82c8bb7ec96d5ae8ff613078aca097d7b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 12:00:16 +0100 Subject: [PATCH 50/74] fixup: remove prefixes from IbcMessages --- ibc/messages.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ibc/messages.go b/ibc/messages.go index 7a97aae4d..c50912926 100644 --- a/ibc/messages.go +++ b/ibc/messages.go @@ -10,24 +10,22 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) -func CreateUpdateStoreMessage(prefix coreTypes.CommitmentPrefix, key, value []byte) *types.IbcMessage { +func CreateUpdateStoreMessage(key, value []byte) *types.IbcMessage { return &types.IbcMessage{ Event: &types.IbcMessage_Update{ Update: &types.UpdateIbcStore{ - Prefix: prefix, - Key: key, - Value: value, + Key: key, + Value: value, }, }, } } -func CreatePruneStoreMessage(prefix coreTypes.CommitmentPrefix, key []byte) *types.IbcMessage { +func CreatePruneStoreMessage(key []byte) *types.IbcMessage { return &types.IbcMessage{ Event: &types.IbcMessage_Prune{ Prune: &types.PruneIbcStore{ - Prefix: prefix, - Key: key, + Key: key, }, }, } From 47668029a02fd6aa79d086944314c54222e9938b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:26:29 +0100 Subject: [PATCH 51/74] Clear IBC table state between tests --- persistence/debug.go | 1 + persistence/test/benchmark_state_test.go | 2 +- persistence/types/ibc.go | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/persistence/debug.go b/persistence/debug.go index 81b075365..90baf0da8 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -14,6 +14,7 @@ var nonActorClearFunctions = []func() string{ types.ClearAllGovParamsQuery, types.ClearAllGovFlagsQuery, types.ClearAllBlocksQuery, + types.ClearAllIbcQuery, } func (m *persistenceModule) HandleDebugMessage(debugMessage *messaging.DebugMessage) error { diff --git a/persistence/test/benchmark_state_test.go b/persistence/test/benchmark_state_test.go index 9084faf8a..2e40e2fbb 100644 --- a/persistence/test/benchmark_state_test.go +++ b/persistence/test/benchmark_state_test.go @@ -118,7 +118,7 @@ MethodLoop: case reflect.Slice: switch arg.Elem().Kind() { case reflect.Uint8: - v = reflect.ValueOf([]uint8{0}) + v = reflect.ValueOf([]uint8{uint8(rand.Intn(2 ^ 8 - 1))}) case reflect.String: v = reflect.ValueOf([]string{"abc"}) default: diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go index c94b542d6..4d68688bd 100644 --- a/persistence/types/ibc.go +++ b/persistence/types/ibc.go @@ -24,3 +24,7 @@ func InsertIbcStoreEntryQuery(height int64, key, value []byte) string { hex.EncodeToString(value), ) } + +func ClearAllIbcQuery() string { + return fmt.Sprintf(`DELETE FROM %s`, IbcStoreTableName) +} From ed94086daa4c608409561b2bc64b520a561c9853 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:54:10 +0100 Subject: [PATCH 52/74] Improve unit test cases --- ibc/handle_message_test.go | 145 +++++++++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 23 deletions(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index 11aca854d..cc2f6a563 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -1,6 +1,8 @@ package ibc import ( + "errors" + "fmt" "testing" ibcTypes "github.com/pokt-network/pocket/ibc/types" @@ -42,16 +44,21 @@ func TestHandleMessage_ErrorAlreadyCommitted(t *testing.T) { } func TestHandleMessage_BasicValidation_Message(t *testing.T) { - updateMsg, _ := prepareUpdateMessage(t, []byte("key"), []byte("value")) - require.NoError(t, updateMsg.ValidateBasic()) - pruneMsg, _ := preparePruneMessage(t, []byte("key")) - require.NoError(t, pruneMsg.ValidateBasic()) - testCases := []struct { name string msg *ibcTypes.IbcMessage expected error }{ + { + name: "Valid Update Message", + msg: CreateUpdateStoreMessage([]byte("key"), []byte("value")), + expected: nil, + }, + { + name: "Valid Prune Message", + msg: CreatePruneStoreMessage([]byte("key")), + expected: nil, + }, { name: "Invalid Update Message: Empty Key", msg: CreateUpdateStoreMessage(nil, []byte("value")), @@ -72,7 +79,11 @@ func TestHandleMessage_BasicValidation_Message(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { err := tc.msg.ValidateBasic() - require.EqualError(t, err, tc.expected.Error()) + if tc.expected != nil { + require.EqualError(t, err, tc.expected.Error()) + } else { + require.NoError(t, err) + } }) } } @@ -83,48 +94,136 @@ func TestHandleMessage_BasicValidation_Transaction(t *testing.T) { privKey, err := crypto.GeneratePrivateKey() require.NoError(t, err) + falseKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) - _, validTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) + validUpdateMsg, validUpdateTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) require.NoError(t, err) - err = validTx.Sign(privKey) + err = validUpdateTx.Sign(privKey) require.NoError(t, err) - - txProtoBytes, err := codec.GetCodec().Marshal(validTx) + updateAny, err := codec.GetCodec().ToAny(validUpdateMsg.GetUpdate()) + require.NoError(t, err) + bz, err := validUpdateTx.SignableBytes() + require.NoError(t, err) + falseUpdateSig, err := falseKey.Sign(bz) require.NoError(t, err) - err = utilityMod.HandleTransaction(txProtoBytes) + validPruneMsg, validPruneTx := preparePruneMessage(t, []byte("key")) + require.NoError(t, err) + err = validPruneTx.Sign(privKey) + require.NoError(t, err) + pruneAny, err := codec.GetCodec().ToAny(validPruneMsg.GetPrune()) + require.NoError(t, err) + bz, err = validPruneTx.SignableBytes() + require.NoError(t, err) + falsePruneSig, err := falseKey.Sign(bz) require.NoError(t, err) testCases := []struct { name string - msg *ibcTypes.IbcMessage + tx *coreTypes.Transaction expected error }{ { - name: "Invalid Update Message: Empty Key", - msg: CreateUpdateStoreMessage(nil, []byte("value")), - expected: coreTypes.ErrEmptySignatureStructure(), + name: "Valid Update Transaction", + tx: validUpdateTx, + expected: nil, }, { - name: "Invalid Update Message: Empty Value", - msg: CreateUpdateStoreMessage([]byte("key"), nil), + name: "Valid Prune Transaction", + tx: validPruneTx, + expected: nil, + }, + { + name: "Invalid Update Transaction: Empty Nonce", + tx: &coreTypes.Transaction{ + Msg: updateAny, + }, + expected: coreTypes.ErrEmptyNonce(), + }, + { + name: "Invalid Prune Transaction: Empty Nonce", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + }, + expected: coreTypes.ErrEmptyNonce(), + }, + { + name: "Invalid Update Transaction: Empty Signature", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, expected: coreTypes.ErrEmptySignatureStructure(), }, { - name: "Invalid Prune Message: Empty Key", - msg: CreatePruneStoreMessage(nil), + name: "Invalid Prune Transaction: Empty Signature", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, expected: coreTypes.ErrEmptySignatureStructure(), }, + { + name: "Invalid Update Transaction: Bad Key", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: []byte("bad key"), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrNewPublicKeyFromBytes(errors.New("the public key length is not valid, expected length 32, actual length: 7")), + }, + { + name: "Invalid Prune Transaction: Bad Signature", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: []byte("bad key"), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrNewPublicKeyFromBytes(errors.New("the public key length is not valid, expected length 32, actual length: 7")), + }, + { + name: "Invalid Update Transaction: Bad Signature", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: privKey.PublicKey().Bytes(), + Signature: falseUpdateSig, + }, + }, + expected: coreTypes.ErrSignatureVerificationFailed(), + }, + { + name: "Invalid Prune Transaction: Bad Key", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: privKey.PublicKey().Bytes(), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrSignatureVerificationFailed(), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tx, err := ConvertIBCMessageToTx(tc.msg) - require.NoError(t, err) - txProtoBytes, err := codec.GetCodec().Marshal(tx) + txProtoBytes, err := codec.GetCodec().Marshal(tc.tx) require.NoError(t, err) err = utilityMod.HandleTransaction(txProtoBytes) - require.EqualError(t, err, tc.expected.Error()) + if tc.expected != nil { + require.EqualError(t, err, tc.expected.Error()) + } else { + require.NoError(t, err) + } }) } } From 32964cb3bcc0228f3e8740bc5f0ac85b02409fc4 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:41:30 +0100 Subject: [PATCH 53/74] Add no valuehashing to state trees --- persistence/trees/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persistence/trees/module.go b/persistence/trees/module.go index 9c0d5eff0..cb21c9913 100644 --- a/persistence/trees/module.go +++ b/persistence/trees/module.go @@ -70,7 +70,7 @@ func (t *treeStore) setupTrees() error { } t.merkleTrees[stateTreeNames[i]] = &stateTree{ name: stateTreeNames[i], - tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher), + tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher), nodeStore: nodeStore, } } @@ -90,7 +90,7 @@ func (t *treeStore) setupInMemory() error { nodeStore := kvstore.NewMemKVStore() // For testing, `smt.NewSimpleMap()` can be used as well t.merkleTrees[stateTreeNames[i]] = &stateTree{ name: stateTreeNames[i], - tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher), + tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher), nodeStore: nodeStore, } } From 1d84d08809e1729809c31d20f77e1b801bc153d1 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:41:59 +0100 Subject: [PATCH 54/74] Add ibc.feature text file to track upcoming tests to be added --- ibc/ibc.feature | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ibc/ibc.feature diff --git a/ibc/ibc.feature b/ibc/ibc.feature new file mode 100644 index 000000000..e8c297bb2 --- /dev/null +++ b/ibc/ibc.feature @@ -0,0 +1,12 @@ +Tests to add: + - Add integration tests to cover the following: + - Local IBC store change is propagated through the network + - Check via P2P magic + - Local IBC store change is included in PostgresDB + - Check via SQL query + - Add E2E tests to cover the following: + - Local IBC store change gets included in the next block + - Check via inclusion proof + - Local IBC store change that is not included in next block stays in mempool + - Check by querying mempool + - Check via exclusion proof From f7e7c0ce16e394c7fb7c6b09e1879e5e513a112a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:53:48 +0100 Subject: [PATCH 55/74] Update docs --- ibc/docs/ics24.md | 23 +++++++---------------- persistence/docs/PROTOCOL_STATE_HASH.md | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 148f68469..82291862b 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -1,7 +1,9 @@ # ICS-24 Host Requirements - [Overview](#overview) +- [Host Configuration](#host-configuration) - [Implementation](#implementation) + - [Persistence](#persistence) - [Paths and Identifiers](#paths-and-identifiers) - [Timestamps](#timestamps) - [IBC State](#ibc-state) @@ -45,7 +47,7 @@ The following is a simplified sequence diagram of an IBC fungible token transfer ```mermaid sequenceDiagram actor UA as User A - box Blue Chain A + box Transparent Chain A participant A1 as Validator A participant A2 as IBC Host A participant A3 as Light Client B @@ -53,7 +55,7 @@ sequenceDiagram box Transparent Relayer actor R1 as Relayer end - box Red Chain B + box Transparent Chain B participant B1 as Validator B participant B2 as IBC Host B participant B3 as Light Client A @@ -152,23 +154,12 @@ See: [ibc/module.go](../module.go) for the specific implementation details. With the `IbcMessage` now propagated through the network's mempool, when it is reaped (by the block proposer) the message's validity will be handled by first determining the type of the `IbcMessage`: -- `UpdateIbcStore`: The `key` and `value` fields of the `IbcMessage` are inserted into the `ibc_messages` Postgres table along with the current height -- `PruneIbcStore`: The `key` with a `nil` value is passed into the `ibc_messages` Postgres table along with the current height - -_Note: Prior to insertion the `key` and `value` fields of the messages are hexadecimally encoded into strings._ +- `UpdateIbcStore`: The `key` and `value` fields are tracked by persistence and used to update the `ibc` store state tree +- `PruneIbcStore`: The `key` field is tracked by persistence and marked for removal in the `ibc` store state tree ### State Transition -When the new state hash is computed, the different state trees read the updates from their respective Postgres tables and update the trees accordingly. For each entry in the `ibc_message` table depending on the entries `value` field the tree will perform one of two operations: - -- `value == nil` - - This is a `PruneIbcStore` message and thus the tree will delete the entry with the given `key` - - `ibcTree.Delete(key)` -- `value != nil` - - This is an `UpdateIbcStore` message and thus the tree will update the entry with the given `key` to have the given `value` - - `ibcTree.Update(key, value)` - -The new root hash of the IBC store tree is then included in the `rootTree` and the new state hash is computed, ensuring the IBC store's state is consistent across the network. +See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-state-tree) for more information on how the persistence module uses the data it has tracked from the `IbcMessage` objects, in order to update the actual state trees and in turn the root hash. [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md diff --git a/persistence/docs/PROTOCOL_STATE_HASH.md b/persistence/docs/PROTOCOL_STATE_HASH.md index 75ffd4886..1d2e28c8e 100644 --- a/persistence/docs/PROTOCOL_STATE_HASH.md +++ b/persistence/docs/PROTOCOL_STATE_HASH.md @@ -10,6 +10,7 @@ Alternative implementation of the persistence module are free to choose their ow - [Block Proto](#block-proto) - [Trees](#trees) - [Compute State Hash](#compute-state-hash) + - [IBC State Tree](#ibc-state-tree) - [Store Block (Commit)](#store-block-commit) - [Failed Commitments](#failed-commitments) @@ -96,6 +97,23 @@ sequenceDiagram deactivate P ``` +### IBC State Tree + +When the new state hash is computed, the different state trees read the updates from their respective Postgres tables and update the trees accordingly. + +`IbcMessage` objects are inserted into the `ibc_message` table in two ways., depending on the IBC messages' type: 1. `UpdateIbcStore`: the `key` and `value` fields are inserted with the height into the table 2. `PruneIbcStore`: the `key` with a `nil` value is inserted into the table + +For each entry in the `ibc_message` table depending on the entries `value` field the tree will perform one of two operations: + +- `value == nil` + - This is a `PruneIbcStore` message and thus the tree will delete the entry with the given `key` + - `ibcTree.Delete(key)` +- `value != nil` + - This is an `UpdateIbcStore` message and thus the tree will update the entry with the given `key` to have the given `value` + - `ibcTree.Update(key, value)` + +_Note: Prior to insertion the `key` and `value` fields of the messages are hexadecimally encoded into strings._ + ## Store Block (Commit) When the `Commit(proposer, quorumCert)` function is invoked, the current context is committed to disk. The `PersistenceContext` does the following: From be48d99da1db268bdf2283cdf4470b003c1084a3 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:48:46 +0100 Subject: [PATCH 56/74] Fix proto naming --- ibc/handle_message_test.go | 8 ++--- ibc/messages.go | 22 +++++++------- ibc/module.go | 4 +-- ibc/types/messages.go | 34 +++++++++++----------- ibc/types/proto/messages.proto | 16 +++++----- utility/unit_of_work/gov.go | 2 +- utility/unit_of_work/tx_message_handler.go | 12 ++++---- utility/unit_of_work/tx_message_signers.go | 12 ++++---- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index cc2f6a563..b547d2a87 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -46,7 +46,7 @@ func TestHandleMessage_ErrorAlreadyCommitted(t *testing.T) { func TestHandleMessage_BasicValidation_Message(t *testing.T) { testCases := []struct { name string - msg *ibcTypes.IbcMessage + msg *ibcTypes.IBCMessage expected error }{ { @@ -272,7 +272,7 @@ func TestHandleMessage_AddToMempool(t *testing.T) { txProto := &coreTypes.Transaction{} err = codec.GetCodec().Unmarshal(mTx, txProto) require.NoError(t, err) - ibcMsg := &ibcTypes.UpdateIbcStore{} + ibcMsg := &ibcTypes.UpdateIBCStore{} require.NoError(t, txProto.GetMsg().UnmarshalTo(ibcMsg)) // Compare messages are equal @@ -283,7 +283,7 @@ func TestHandleMessage_AddToMempool(t *testing.T) { require.Equal(t, origBz, ibcMsgBz) } -func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IbcMessage, *coreTypes.Transaction) { +func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { t.Helper() msg := CreateUpdateStoreMessage(key, value) tx, err := ConvertIBCMessageToTx(msg) @@ -291,7 +291,7 @@ func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IbcMessage return msg, tx } -func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IbcMessage, *coreTypes.Transaction) { +func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { t.Helper() msg := CreatePruneStoreMessage(key) tx, err := ConvertIBCMessageToTx(msg) diff --git a/ibc/messages.go b/ibc/messages.go index c50912926..24bbe0a67 100644 --- a/ibc/messages.go +++ b/ibc/messages.go @@ -10,10 +10,10 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) -func CreateUpdateStoreMessage(key, value []byte) *types.IbcMessage { - return &types.IbcMessage{ - Event: &types.IbcMessage_Update{ - Update: &types.UpdateIbcStore{ +func CreateUpdateStoreMessage(key, value []byte) *types.IBCMessage { + return &types.IBCMessage{ + Event: &types.IBCMessage_Update{ + Update: &types.UpdateIBCStore{ Key: key, Value: value, }, @@ -21,23 +21,23 @@ func CreateUpdateStoreMessage(key, value []byte) *types.IbcMessage { } } -func CreatePruneStoreMessage(key []byte) *types.IbcMessage { - return &types.IbcMessage{ - Event: &types.IbcMessage_Prune{ - Prune: &types.PruneIbcStore{ +func CreatePruneStoreMessage(key []byte) *types.IBCMessage { + return &types.IBCMessage{ + Event: &types.IBCMessage_Prune{ + Prune: &types.PruneIBCStore{ Key: key, }, }, } } -func ConvertIBCMessageToTx(ibcMessage *types.IbcMessage) (*coreTypes.Transaction, error) { +func ConvertIBCMessageToTx(ibcMessage *types.IBCMessage) (*coreTypes.Transaction, error) { var anyMsg *anypb.Any var err error switch event := ibcMessage.Event.(type) { - case *types.IbcMessage_Update: + case *types.IBCMessage_Update: anyMsg, err = codec.GetCodec().ToAny(event.Update) - case *types.IbcMessage_Prune: + case *types.IBCMessage_Prune: anyMsg, err = codec.GetCodec().ToAny(event.Prune) default: return nil, coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", event)) diff --git a/ibc/module.go b/ibc/module.go index 9a10cd163..cf7e4c896 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -95,7 +95,7 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { if err != nil { return err } - ibcMessage, ok := msg.(*ibcTypes.IbcMessage) + ibcMessage, ok := msg.(*ibcTypes.IBCMessage) if !ok { return fmt.Errorf("failed to cast message to IBCMessage") } @@ -139,7 +139,7 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { if err := m.GetBus().GetUtilityModule().HandleTransaction(txBz); err != nil { return err } - m.logger.Info().Str("message_type", "IbcMessage").Msg("Successfully added a new message to the mempool!") + m.logger.Info().Str("message_type", "IBCMessage").Msg("Successfully added a new message to the mempool!") return nil } diff --git a/ibc/types/messages.go b/ibc/types/messages.go index e9163b2a1..917da4554 100644 --- a/ibc/types/messages.go +++ b/ibc/types/messages.go @@ -11,22 +11,22 @@ import ( // Implement the Message interface var ( - _ utilityTypes.Message = &UpdateIbcStore{} - _ utilityTypes.Message = &PruneIbcStore{} + _ utilityTypes.Message = &UpdateIBCStore{} + _ utilityTypes.Message = &PruneIBCStore{} ) -func (m *IbcMessage) ValidateBasic() coreTypes.Error { +func (m *IBCMessage) ValidateBasic() coreTypes.Error { switch msg := m.Event.(type) { - case *IbcMessage_Update: + case *IBCMessage_Update: return msg.Update.ValidateBasic() - case *IbcMessage_Prune: + case *IBCMessage_Prune: return msg.Prune.ValidateBasic() default: return coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", msg)) } } -func (m *UpdateIbcStore) ValidateBasic() coreTypes.Error { +func (m *UpdateIBCStore) ValidateBasic() coreTypes.Error { if m.Key == nil { return coreTypes.ErrNilField("key") } @@ -36,36 +36,36 @@ func (m *UpdateIbcStore) ValidateBasic() coreTypes.Error { return nil } -func (m *PruneIbcStore) ValidateBasic() coreTypes.Error { +func (m *PruneIBCStore) ValidateBasic() coreTypes.Error { if m.Key == nil { return coreTypes.ErrNilField("key") } return nil } -func (m *UpdateIbcStore) SetSigner(signer []byte) { m.Signer = signer } -func (m *PruneIbcStore) SetSigner(signer []byte) { m.Signer = signer } +func (m *UpdateIBCStore) SetSigner(signer []byte) { m.Signer = signer } +func (m *PruneIBCStore) SetSigner(signer []byte) { m.Signer = signer } -func (m *UpdateIbcStore) GetMessageName() string { +func (m *UpdateIBCStore) GetMessageName() string { return string(m.ProtoReflect().Descriptor().Name()) } -func (m *PruneIbcStore) GetMessageName() string { +func (m *PruneIBCStore) GetMessageName() string { return string(m.ProtoReflect().Descriptor().Name()) } -func (m *UpdateIbcStore) GetMessageRecipient() string { return "" } -func (m *PruneIbcStore) GetMessageRecipient() string { return "" } +func (m *UpdateIBCStore) GetMessageRecipient() string { return "" } +func (m *PruneIBCStore) GetMessageRecipient() string { return "" } -func (m *UpdateIbcStore) GetActorType() coreTypes.ActorType { +func (m *UpdateIBCStore) GetActorType() coreTypes.ActorType { return coreTypes.ActorType_ACTOR_TYPE_VAL } -func (m *PruneIbcStore) GetActorType() coreTypes.ActorType { +func (m *PruneIBCStore) GetActorType() coreTypes.ActorType { return coreTypes.ActorType_ACTOR_TYPE_VAL } -func (m *UpdateIbcStore) GetCanonicalBytes() []byte { +func (m *UpdateIBCStore) GetCanonicalBytes() []byte { bz, err := codec.GetCodec().Marshal(m) if err != nil { log.Fatalf("must marshal %v", err) @@ -73,7 +73,7 @@ func (m *UpdateIbcStore) GetCanonicalBytes() []byte { return bz // DISCUSS(#142): should we also sort the JSON like in V0? } -func (m *PruneIbcStore) GetCanonicalBytes() []byte { +func (m *PruneIBCStore) GetCanonicalBytes() []byte { bz, err := codec.GetCodec().Marshal(m) if err != nil { log.Fatalf("must marshal %v", err) diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto index 39b6c839c..6cc5e9efc 100644 --- a/ibc/types/proto/messages.proto +++ b/ibc/types/proto/messages.proto @@ -4,25 +4,25 @@ package types; option go_package = "github.com/pokt-network/pocket/ibc/types"; -// update_ibc_store defines a message representing the addition of a key/value pair to the IBC store +// UpdateIBCStore defines a message representing the addition of a key/value pair to the IBC store // the key field should be the full key - prefixed with the stores CommitmentPrefix -message update_ibc_store { +message UpdateIBCStore { bytes signer = 1; bytes key = 2; bytes value = 3; } -// prune_ibc_store defines a message representing the removal of a key from the IBC store +// PruneIBCStore defines a message representing the removal of a key from the IBC store // the key field should be the full key - prefixed with the stores CommitmentPrefix -message prune_ibc_store { +message PruneIBCStore { bytes signer = 1; bytes key = 2; } -// ibc_message defines the different types of IBC message that can be sent across the network -message ibc_message { +// IBCMessage defines the different types of IBC message that can be sent across the network +message IBCMessage { oneof event { - update_ibc_store update = 1; - prune_ibc_store prune = 2; + UpdateIBCStore update = 1; + PruneIBCStore prune = 2; } } diff --git a/utility/unit_of_work/gov.go b/utility/unit_of_work/gov.go index 19461e07b..0c71c9418 100644 --- a/utility/unit_of_work/gov.go +++ b/utility/unit_of_work/gov.go @@ -151,7 +151,7 @@ func (u *baseUtilityUnitOfWork) getParamOwner(paramName string) ([]byte, coreTyp func (u *baseUtilityUnitOfWork) getFee(msg typesUtil.Message, actorType coreTypes.ActorType) (amount *big.Int, err coreTypes.Error) { switch x := msg.(type) { // TECHDEBT(M6): Decide on IBC store change fees and move into a governance parameter - case *typesUtil.MessageSend, *ibcTypes.UpdateIbcStore, *ibcTypes.PruneIbcStore: + case *typesUtil.MessageSend, *ibcTypes.UpdateIBCStore, *ibcTypes.PruneIBCStore: return getGovParam[*big.Int](u, typesUtil.MessageSendFee) case *typesUtil.MessageStake: switch actorType { diff --git a/utility/unit_of_work/tx_message_handler.go b/utility/unit_of_work/tx_message_handler.go index 31e6fe46d..654f6d25f 100644 --- a/utility/unit_of_work/tx_message_handler.go +++ b/utility/unit_of_work/tx_message_handler.go @@ -27,10 +27,10 @@ func (u *baseUtilityUnitOfWork) handleMessage(msg typesUtil.Message) (err coreTy return u.handleUnpauseMessage(x) case *typesUtil.MessageChangeParameter: return u.handleMessageChangeParameter(x) - case *ibcTypes.UpdateIbcStore: - return u.handleUpdateIbcStore(x) - case *ibcTypes.PruneIbcStore: - return u.handlePruneIbcStore(x) + case *ibcTypes.UpdateIBCStore: + return u.handleUpdateIBCStore(x) + case *ibcTypes.PruneIBCStore: + return u.handlePruneIBCStore(x) default: return coreTypes.ErrUnknownMessage(x) } @@ -225,14 +225,14 @@ func (u *baseUtilityUnitOfWork) handleMessageChangeParameter(message *typesUtil. return u.updateParam(message.ParameterKey, v) } -func (u *baseUtilityUnitOfWork) handleUpdateIbcStore(message *ibcTypes.UpdateIbcStore) coreTypes.Error { +func (u *baseUtilityUnitOfWork) handleUpdateIBCStore(message *ibcTypes.UpdateIBCStore) coreTypes.Error { if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, message.Value); err != nil { return coreTypes.ErrUpdatingIBCStoreDB(err) } return nil } -func (u *baseUtilityUnitOfWork) handlePruneIbcStore(message *ibcTypes.PruneIbcStore) coreTypes.Error { +func (u *baseUtilityUnitOfWork) handlePruneIBCStore(message *ibcTypes.PruneIBCStore) coreTypes.Error { if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, nil); err != nil { return coreTypes.ErrUpdatingIBCStoreDB(err) } diff --git a/utility/unit_of_work/tx_message_signers.go b/utility/unit_of_work/tx_message_signers.go index 8e84bc133..602ef602a 100644 --- a/utility/unit_of_work/tx_message_signers.go +++ b/utility/unit_of_work/tx_message_signers.go @@ -25,10 +25,10 @@ func (u *baseUtilityUnitOfWork) getSignerCandidates(msg typesUtil.Message) ([][] return u.getMessageUnpauseSignerCandidates(x) case *typesUtil.MessageChangeParameter: return u.getMessageChangeParameterSignerCandidates(x) - case *ibcTypes.UpdateIbcStore: - return u.getUpdateIbcStoreSingerCandidates(x) - case *ibcTypes.PruneIbcStore: - return u.getPruneIbcStoreSingerCandidates(x) + case *ibcTypes.UpdateIBCStore: + return u.getUpdateIBCStoreSingerCandidates(x) + case *ibcTypes.PruneIBCStore: + return u.getPruneIBCStoreSingerCandidates(x) default: return nil, coreTypes.ErrUnknownMessage(x) } @@ -78,10 +78,10 @@ func (u *baseUtilityUnitOfWork) getMessageSendSignerCandidates(msg *typesUtil.Me return [][]byte{msg.FromAddress}, nil } -func (u *baseUtilityUnitOfWork) getUpdateIbcStoreSingerCandidates(msg *ibcTypes.UpdateIbcStore) ([][]byte, coreTypes.Error) { +func (u *baseUtilityUnitOfWork) getUpdateIBCStoreSingerCandidates(msg *ibcTypes.UpdateIBCStore) ([][]byte, coreTypes.Error) { return [][]byte{msg.Signer}, nil } -func (u *baseUtilityUnitOfWork) getPruneIbcStoreSingerCandidates(msg *ibcTypes.PruneIbcStore) ([][]byte, coreTypes.Error) { +func (u *baseUtilityUnitOfWork) getPruneIBCStoreSingerCandidates(msg *ibcTypes.PruneIBCStore) ([][]byte, coreTypes.Error) { return [][]byte{msg.Signer}, nil } From e54be28d43a3c105c1bc56073a469b41cca7cc0a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:52:24 +0100 Subject: [PATCH 57/74] Add signer comments --- ibc/types/proto/messages.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto index 6cc5e9efc..8c9ee95b1 100644 --- a/ibc/types/proto/messages.proto +++ b/ibc/types/proto/messages.proto @@ -5,17 +5,17 @@ package types; option go_package = "github.com/pokt-network/pocket/ibc/types"; // UpdateIBCStore defines a message representing the addition of a key/value pair to the IBC store -// the key field should be the full key - prefixed with the stores CommitmentPrefix +// the key field should be the full key - prefixed with the store's CommitmentPrefix message UpdateIBCStore { - bytes signer = 1; + bytes signer = 1; // signer should be the address of the node that sent the message bytes key = 2; bytes value = 3; } // PruneIBCStore defines a message representing the removal of a key from the IBC store -// the key field should be the full key - prefixed with the stores CommitmentPrefix +// the key field should be the full key - prefixed with the store's CommitmentPrefix message PruneIBCStore { - bytes signer = 1; + bytes signer = 1; // signer should be the address of the node that sent the message bytes key = 2; } From 9ccc19702f3a59e8c2e15fdc6d17c0735e2afaf4 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:52:35 +0100 Subject: [PATCH 58/74] Prefix errors with IBC --- shared/core/types/error.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index c317ce50a..b8c7595ed 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -320,14 +320,14 @@ const ( NegativeAmountError = "the amount is negative" UnknownActorTypeError = "the actor type is not recognized" UnknownMessageTypeError = "the message being by the utility message is not recognized" - HostAlreadyExistsError = "an ibc host already exists" - HostDoesNotExistError = "an ibc host does not exist" + IBCHostAlreadyExistsError = "an ibc host already exists" + IBCHostDoesNotExistError = "an ibc host does not exist" IBCInvalidIDError = "invalid ibc identifier" IBCInvalidPathError = "invalid ibc path" - CreatingProofError = "an error occurred creating the CommitmentProof" - UnknownIBCMessageTypeError = "the ibc message type is not recognized" + IBCCreatingProofError = "an error occurred creating the CommitmentProof" + IBCUnknownIBCMessageTypeError = "the ibc message type is not recognized" NilFieldError = "field cannot be nil" - UpdatingIBCStoreDBError = "an error occurred updating the ibc store postgres database" + IBCUpdatingIBCStoreDBError = "an error occurred updating the ibc store postgres database" ) func ErrUnknownParam(paramName string) Error { @@ -864,11 +864,11 @@ func ErrUnknownMessageType(messageType any) Error { } func ErrHostAlreadyExists() Error { - return NewError(CodeHostAlreadyExists, HostAlreadyExistsError) + return NewError(CodeHostAlreadyExists, IBCHostAlreadyExistsError) } func ErrHostDoesNotExist() Error { - return NewError(CodeHostDoesNotExist, HostDoesNotExistError) + return NewError(CodeHostDoesNotExist, IBCHostDoesNotExistError) } func ErrIBCInvalidID(identifier, msg string) Error { @@ -880,11 +880,11 @@ func ErrIBCInvalidPath(path string) Error { } func ErrCreatingProof(err error) Error { - return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", CreatingProofError, err.Error())) + return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", IBCCreatingProofError, err.Error())) } func ErrUnknownIBCMessageType(messageType string) Error { - return NewError(CodeUnknownIBCMessageTypeError, fmt.Sprintf("%s: %s", UnknownIBCMessageTypeError, messageType)) + return NewError(CodeUnknownIBCMessageTypeError, fmt.Sprintf("%s: %s", IBCUnknownIBCMessageTypeError, messageType)) } func ErrNilField(field string) Error { @@ -892,5 +892,5 @@ func ErrNilField(field string) Error { } func ErrUpdatingIBCStoreDB(err error) Error { - return NewError(CodeUpdatingIBCStoreDBError, fmt.Sprintf("%s: %s", UpdatingIBCStoreDBError, err.Error())) + return NewError(CodeUpdatingIBCStoreDBError, fmt.Sprintf("%s: %s", IBCUpdatingIBCStoreDBError, err.Error())) } From fbadb8d82614164b98279632ab4a07d5b4c8c73e Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:52:43 +0100 Subject: [PATCH 59/74] Update docs --- ibc/docs/ics24.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 82291862b..e32fdc65d 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -63,19 +63,19 @@ sequenceDiagram actor UB as User B R1->>R1: Watch(Chain A) R1->>R1: Watch(Chain B) - UA->>A1: Send 10$POKT to User B + UA->>+A1: Send 10$POKT to User B A1->>A1: Lock(10$POKT) - A1->>A2: Create(FungibleTokenPacketData) - A2->>A1: Commit(FungibleTokenPacketData) - A1->>A1: NewBlock() + A1->>+A2: Create(FungibleTokenPacketData) + A2->>-A1: Commit(FungibleTokenPacketData) + A1->>-A1: NewBlock() R1->>R1: CheckNewBlockForIBCPackets() - R1->>A2: QueryAndProve(FungibleTokenPacketData) + R1->>+A2: QueryAndProve(FungibleTokenPacketData) A2->>R1: FungibleTokenPacketData - A2->>R1: Proof(FungibleTokenPacketData) - R1->>B2: Validate(FungibleTokenPacketData, Proof(FungibleTokenPacketData)) - B2->>B3: Verify(Proof(FungibleTokenPacketData)) - B3->>B2: FoundInState(FungibleTokenPacketData) - B2->>B1: Send 10$POKT to User B + A2->>-R1: Proof(FungibleTokenPacketData) + R1->>+B2: Validate(FungibleTokenPacketData, Proof(FungibleTokenPacketData)) + B2->>+B3: Verify(Proof(FungibleTokenPacketData)) + B3->>-B2: FoundInState(FungibleTokenPacketData) + B2->>-B1: Send 10$POKT to User B B1->>B1: Mint(10$POKT) B1->>UB: Receive 10$POKT from User A ``` @@ -96,7 +96,7 @@ The `GetTimestamp()` function returns the current unix timestamp of the host mac ## IBC State -As mentioned [above](#persistence) the IBC store **must** be included in the consensus state of the network. As such the IBC store as defined in [ICS-24][ics24] has been implemented as a single IBC state tree. +As mentioned [above](#persistence) the IBC store **MUST** be included in the consensus state of the network. As such the IBC store as defined in [ICS-24][ics24] has been implemented as a single IBC state tree. ### IBC State Tree @@ -104,7 +104,9 @@ The IBC state tree is a non-value hashing `SMT` backed by a persistent `KVStore` ### IBC Messages -As the hosts make changes to this IBC store locally (creating light clients, opening connections, and sending packets for example); these changes must be propagated throughout the network to ensure the IBC state tree is consistent across all nodes. This is achieved by the new `IbcMessage` type defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: +Hosts maintain uncommitted changes in a local ephemeral IBC store while messages propagate through the mempool. + +These messages enable a variety of IBC related state changes such as creating light clients, opening connections, sending packets, etc... This is enabled by propagating `IbcMessage` types defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: - `UpdateIbcStore`: Updating the store with a key-value pair; adding a new or updating an existing element - `PruneIbcStore`: Pruning the store via its key; removal of an existing element From 65f647fe2fdb365d60bbfe9732a9e7be76adb242 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:59:41 +0100 Subject: [PATCH 60/74] Address comments --- consensus/e2e_tests/utils_test.go | 2 +- ibc/docs/ics24.md | 28 ++++++++++++------------- ibc/host.go | 2 +- ibc/{ibc.feature => ibc.feature.wip} | 9 +++++++- ibc/main_test.go | 10 ++++----- ibc/store/provable_store.go | 2 +- internal/testutil/ibc/mock.go | 8 +++---- persistence/db.go | 6 +++--- persistence/debug.go | 2 +- persistence/docs/PROTOCOL_STATE_HASH.md | 6 +++--- persistence/ibc.go | 2 +- persistence/sql/sql.go | 2 +- persistence/trees/trees.go | 14 ++++++------- persistence/types/ibc.go | 12 +++++------ shared/messaging/events.go | 2 +- shared/node.go | 2 +- 16 files changed, 58 insertions(+), 51 deletions(-) rename ibc/{ibc.feature => ibc.feature.wip} (57%) diff --git a/consensus/e2e_tests/utils_test.go b/consensus/e2e_tests/utils_test.go index 94294af26..21243c199 100644 --- a/consensus/e2e_tests/utils_test.go +++ b/consensus/e2e_tests/utils_test.go @@ -122,7 +122,7 @@ func CreateTestConsensusPocketNode( telemetryMock := baseTelemetryMock(t, eventsChannel) loggerMock := baseLoggerMock(t, eventsChannel) rpcMock := baseRpcMock(t, eventsChannel) - ibcMock := ibcUtils.IbcMockWithHost(t, eventsChannel) + ibcMock := ibcUtils.IBCMockWithHost(t, eventsChannel) for _, module := range []modules.Module{ p2pMock, diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index e32fdc65d..49edfddf3 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -106,20 +106,20 @@ The IBC state tree is a non-value hashing `SMT` backed by a persistent `KVStore` Hosts maintain uncommitted changes in a local ephemeral IBC store while messages propagate through the mempool. -These messages enable a variety of IBC related state changes such as creating light clients, opening connections, sending packets, etc... This is enabled by propagating `IbcMessage` types defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: +These messages enable a variety of IBC related state changes such as creating light clients, opening connections, sending packets, etc... This is enabled by propagating `IBCMessage` types defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: -- `UpdateIbcStore`: Updating the store with a key-value pair; adding a new or updating an existing element -- `PruneIbcStore`: Pruning the store via its key; removal of an existing element +- `UpdateIBCStore`: Updating the store with a key-value pair; adding a new or updating an existing element +- `PruneIBCStore`: Pruning the store via its key; removal of an existing element _Note: In both types described above the `key` field **must** already be prefixed with the `CommitmentPrefix` and should be a valid path in the store._ -When changes are made locally they are not committed to the IBC store itself but are instead used to create an `IbcMessage` which is broadcasted to the network. This is akin to a simple send transaction that has been propagated throughout the mempool but has not been committed to the on-chain state. +When changes are made locally they are not committed to the IBC store itself but are instead used to create an `IBCMessage` which is broadcasted to the network. This is akin to a simple send transaction that has been propagated throughout the mempool but has not been committed to the on-chain state. ### IBC Message Handling -Upon a node receiving an `IbcMessage` from the event bus it will use the `HandleMessage()` method of the `IBCModule` to add this message to the transactions mempool via the following steps: +Upon a node receiving an `IBCMessage` from the event bus it will use the `HandleMessage()` method of the `IBCModule` to add this message to the transactions mempool via the following steps: -1. Wrap the `IbcMessage` within a `Transaction` +1. Wrap the `IBCMessage` within a `Transaction` 2. Sign the `Transaction` using the `IBCModule`'s private key 3. Broadcast the `Transaction` throughout the mempool @@ -132,9 +132,9 @@ graph LR I1["HandleMessage(Message)"] end subgraph Handler - H1["ConvertIbcMessageToTransaction(IbcMessage)"] + H1["ConvertIBCMessageToTransaction(IBCMessage)"] subgraph Transaction - T1["coreTypes.Transaction{Msg: IbcMessage}"] + T1["coreTypes.Transaction{Msg: IBCMessage}"] end H2["SignTransaction(Transaction)"] end @@ -143,8 +143,8 @@ graph LR M2["AddToMempool(Transaction)"] end Bus--Message-->I - I--IbcMessage-->Handler - H1--IbcMessage-->Transaction + I--IBCMessage-->Handler + H1--IBCMessage-->Transaction Transaction--Transaction-->H2 Handler--Transaction-->Mempool M1--Transaction-->M2 @@ -154,14 +154,14 @@ See: [ibc/module.go](../module.go) for the specific implementation details. ### Mempool -With the `IbcMessage` now propagated through the network's mempool, when it is reaped (by the block proposer) the message's validity will be handled by first determining the type of the `IbcMessage`: +With the `IBCMessage` now propagated through the network's mempool, when it is reaped (by the block proposer) the message's validity will be handled by first determining the type of the `IBCMessage`: -- `UpdateIbcStore`: The `key` and `value` fields are tracked by persistence and used to update the `ibc` store state tree -- `PruneIbcStore`: The `key` field is tracked by persistence and marked for removal in the `ibc` store state tree +- `UpdateIBCStore`: The `key` and `value` fields are tracked by persistence and used to update the `ibc` store state tree +- `PruneIBCStore`: The `key` field is tracked by persistence and marked for removal in the `ibc` store state tree ### State Transition -See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-state-tree) for more information on how the persistence module uses the data it has tracked from the `IbcMessage` objects, in order to update the actual state trees and in turn the root hash. +See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-state-tree) for more information on how the persistence module uses the data it has tracked from the `IBCMessage` objects, in order to update the actual state trees and in turn the root hash. [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md diff --git a/ibc/host.go b/ibc/host.go index 442017b80..db2f791d3 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -1,7 +1,7 @@ // TECHDEBT(#854): The host should be a submodule with access to the bus, this allows for the // host to access the persistence module and thus the tree store in order to create local // copies of the IBC state tree with the GetProvableStore() function. Bus access also will -// allow for the host to send any local changes to these stores as IbcMessage types through +// allow for the host to send any local changes to these stores as IBCMessage types through // the P2P module's Broadcast() function to allow them to propagate through the network's mempool. package ibc diff --git a/ibc/ibc.feature b/ibc/ibc.feature.wip similarity index 57% rename from ibc/ibc.feature rename to ibc/ibc.feature.wip index e8c297bb2..cc5278341 100644 --- a/ibc/ibc.feature +++ b/ibc/ibc.feature.wip @@ -3,10 +3,17 @@ Tests to add: - Local IBC store change is propagated through the network - Check via P2P magic - Local IBC store change is included in PostgresDB - - Check via SQL query + - Check via SQL query/magic + - Duplicate IBC messages w/in the same block + - Check via SQL query/magic + - Shouldn't allow duplicates + - Invalid IBC messages + - Should error before going to the network - Add E2E tests to cover the following: - Local IBC store change gets included in the next block - Check via inclusion proof - Local IBC store change that is not included in next block stays in mempool - Check by querying mempool - Check via exclusion proof + - Duplicate IBC messages in subsequent blocks + - Changes that don't change the state should be noop and not included diff --git a/ibc/main_test.go b/ibc/main_test.go index a2f9abe7c..d96df0669 100644 --- a/ibc/main_test.go +++ b/ibc/main_test.go @@ -44,7 +44,7 @@ func newTestPersistenceModule(bus modules.Bus) modules.PersistenceModule { return persistenceMod.(modules.PersistenceModule) } -func newTestIbcModule(bus modules.Bus) modules.IBCModule { +func newTestIBCModule(bus modules.Bus) modules.IBCModule { ibcMod, err := Create(bus) if err != nil { log.Fatalf("Error creating ibc module: %s", err) @@ -77,8 +77,8 @@ func prepareEnvironment( err = testUtilityMod.Start() require.NoError(t, err) - testIbcMod := newTestIbcModule(bus) - err = testIbcMod.Start() + testIBCMod := newTestIBCModule(bus) + err = testIBCMod.Start() require.NoError(t, err) // Reset database to genesis before every test @@ -94,11 +94,11 @@ func prepareEnvironment( require.NoError(t, err) err = testUtilityMod.Stop() require.NoError(t, err) - err = testIbcMod.Stop() + err = testIBCMod.Stop() require.NoError(t, err) }) - return runtimeCfg, testUtilityMod, testPersistenceMod, testIbcMod + return runtimeCfg, testUtilityMod, testPersistenceMod, testIBCMod } //nolint:unparam // Test suite is not fully built out yet diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index 789685c21..98e85e067 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -14,7 +14,7 @@ var _ modules.ProvableStore = &provableStore{} // provableStore implements the ProvableStore interface and wraps an SMT // it operates in memory and thus cannot make any changes to the underlying -// database. All changes must be propagated through the `IbcMessage` type +// database. All changes must be propagated through the `IBCMessage` type // and added to the mempool for inclusion in the next block type provableStore struct { prefix coreTypes.CommitmentPrefix diff --git a/internal/testutil/ibc/mock.go b/internal/testutil/ibc/mock.go index b4a931d77..e44651125 100644 --- a/internal/testutil/ibc/mock.go +++ b/internal/testutil/ibc/mock.go @@ -9,8 +9,8 @@ import ( "github.com/regen-network/gocuke" ) -// BaseIbcMock returns a mock IBC module without a Host -func BaseIbcMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.MockIBCModule { +// BaseIBCMock returns a mock IBC module without a Host +func BaseIBCMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.MockIBCModule { ctrl := gomock.NewController(t) ibcMock := mockModules.NewMockIBCModule(ctrl) @@ -22,8 +22,8 @@ func BaseIbcMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.M return ibcMock } -// IbcMockWithHost returns a mock IBC module with a Host -func IbcMockWithHost(t gocuke.TestingT, _ modules.EventsChannel) *mockModules.MockIBCModule { +// IBCMockWithHost returns a mock IBC module with a Host +func IBCMockWithHost(t gocuke.TestingT, _ modules.EventsChannel) *mockModules.MockIBCModule { ctrl := gomock.NewController(t) ibcMock := mockModules.NewMockIBCModule(ctrl) diff --git a/persistence/db.go b/persistence/db.go index 8b420c0ff..27a0fd2c1 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -134,7 +134,7 @@ func initializeAllTables(ctx context.Context, db *pgxpool.Conn) error { } } - if err := initialiseIbcTables(ctx, db); err != nil { + if err := initialiseIBCTables(ctx, db); err != nil { return err } @@ -189,8 +189,8 @@ func initializeBlockTables(ctx context.Context, db *pgxpool.Conn) error { return nil } -func initialiseIbcTables(ctx context.Context, db *pgxpool.Conn) error { - if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IbcStoreTableName, types.IbcStoreTableSchema)); err != nil { +func initialiseIBCTables(ctx context.Context, db *pgxpool.Conn) error { + if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IBCStoreTableName, types.IBCStoreTableSchema)); err != nil { return err } return nil diff --git a/persistence/debug.go b/persistence/debug.go index 90baf0da8..38e69e221 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -14,7 +14,7 @@ var nonActorClearFunctions = []func() string{ types.ClearAllGovParamsQuery, types.ClearAllGovFlagsQuery, types.ClearAllBlocksQuery, - types.ClearAllIbcQuery, + types.ClearAllIBCQuery, } func (m *persistenceModule) HandleDebugMessage(debugMessage *messaging.DebugMessage) error { diff --git a/persistence/docs/PROTOCOL_STATE_HASH.md b/persistence/docs/PROTOCOL_STATE_HASH.md index 1d2e28c8e..d98040ccc 100644 --- a/persistence/docs/PROTOCOL_STATE_HASH.md +++ b/persistence/docs/PROTOCOL_STATE_HASH.md @@ -101,15 +101,15 @@ sequenceDiagram When the new state hash is computed, the different state trees read the updates from their respective Postgres tables and update the trees accordingly. -`IbcMessage` objects are inserted into the `ibc_message` table in two ways., depending on the IBC messages' type: 1. `UpdateIbcStore`: the `key` and `value` fields are inserted with the height into the table 2. `PruneIbcStore`: the `key` with a `nil` value is inserted into the table +`IBCMessage` objects are inserted into the `ibc_entries` table in two ways., depending on the IBC messages' type: 1. `UpdateIBCStore`: the `key` and `value` fields are inserted with the height into the table 2. `PruneIBCStore`: the `key` with a `nil` value is inserted into the table For each entry in the `ibc_message` table depending on the entries `value` field the tree will perform one of two operations: - `value == nil` - - This is a `PruneIbcStore` message and thus the tree will delete the entry with the given `key` + - This is a `PruneIBCStore` message and thus the tree will delete the entry with the given `key` - `ibcTree.Delete(key)` - `value != nil` - - This is an `UpdateIbcStore` message and thus the tree will update the entry with the given `key` to have the given `value` + - This is an `UpdateIBCStore` message and thus the tree will update the entry with the given `key` to have the given `value` - `ibcTree.Update(key, value)` _Note: Prior to insertion the `key` and `value` fields of the messages are hexadecimally encoded into strings._ diff --git a/persistence/ibc.go b/persistence/ibc.go index 7e7bdb71f..45ccf6740 100644 --- a/persistence/ibc.go +++ b/persistence/ibc.go @@ -7,7 +7,7 @@ import ( // SetIBCStoreEntry sets the key value pair in the IBC store postgres table func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { ctx, tx := p.getCtxAndTx() - if _, err := tx.Exec(ctx, pTypes.InsertIbcStoreEntryQuery(p.Height, key, value)); err != nil { + if _, err := tx.Exec(ctx, pTypes.InsertIBCStoreEntryQuery(p.Height, key, value)); err != nil { return err } return nil diff --git a/persistence/sql/sql.go b/persistence/sql/sql.go index 85b692c9e..80a1c8320 100644 --- a/persistence/sql/sql.go +++ b/persistence/sql/sql.go @@ -168,7 +168,7 @@ func GetParams(pgtx pgx.Tx, height uint64) ([]*coreTypes.Param, error) { // GetIBCStoreUpdates returns the set of key-value pairs updated at the current height for the IBC store func GetIBCStoreUpdates(pgtx pgx.Tx, height uint64) (keys, values [][]byte, err error) { fields := "key,value" - query := fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY key ASC", fields, ptypes.IbcStoreTableName, height) + query := fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY key ASC", fields, ptypes.IBCStoreTableName, height) rows, err := pgtx.Query(context.TODO(), query) if err != nil { return nil, nil, err diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index b06591969..e20b3f969 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -43,7 +43,7 @@ const ( TransactionsTreeName = "transactions" ParamsTreeName = "params" FlagsTreeName = "flags" - IbcTreeName = "ibc" + IBCTreeName = "ibc" ) var actorTypeToMerkleTreeName = map[coreTypes.ActorType]string{ @@ -66,7 +66,7 @@ var stateTreeNames = []string{ // Account Trees AccountTreeName, PoolTreeName, // Data Trees - TransactionsTreeName, ParamsTreeName, FlagsTreeName, IbcTreeName, + TransactionsTreeName, ParamsTreeName, FlagsTreeName, IBCTreeName, } // stateTree is a wrapper around the SMT that contains an identifying @@ -196,12 +196,12 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height if err := t.updateFlagsTree(flags); err != nil { return "", fmt.Errorf("failed to update flags tree - %w", err) } - case IbcTreeName: + case IBCTreeName: keys, values, err := sql.GetIBCStoreUpdates(pgtx, height) if err != nil { return "", fmt.Errorf("failed to get IBC store updates: %w", err) } - if err := t.updateIbcTree(keys, values); err != nil { + if err := t.updateIBCTree(keys, values); err != nil { return "", fmt.Errorf("failed to update IBC tree: %w", err) } // Default @@ -356,7 +356,7 @@ func (t *treeStore) updateFlagsTree(flags []*coreTypes.Flag) error { return nil } -func (t *treeStore) updateIbcTree(keys, values [][]byte) error { +func (t *treeStore) updateIBCTree(keys, values [][]byte) error { if len(keys) != len(values) { return fmt.Errorf("keys and values must be the same length") } @@ -364,11 +364,11 @@ func (t *treeStore) updateIbcTree(keys, values [][]byte) error { key := keys[i] value := values[i] if value == nil { - if err := t.merkleTrees[IbcTreeName].tree.Delete(key); err != nil { + if err := t.merkleTrees[IBCTreeName].tree.Delete(key); err != nil { return err } } else { - if err := t.merkleTrees[IbcTreeName].tree.Update(key, value); err != nil { + if err := t.merkleTrees[IBCTreeName].tree.Update(key, value); err != nil { return err } } diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go index 4d68688bd..cc5d05ed2 100644 --- a/persistence/types/ibc.go +++ b/persistence/types/ibc.go @@ -6,8 +6,8 @@ import ( ) const ( - IbcStoreTableName = "ibc_messages" - IbcStoreTableSchema = `( + IBCStoreTableName = "ibc_entries" + IBCStoreTableSchema = `( height BIGINT NOT NULL, key TEXT NOT NULL, value TEXT, @@ -15,16 +15,16 @@ const ( )` ) -func InsertIbcStoreEntryQuery(height int64, key, value []byte) string { +func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { return fmt.Sprintf( `INSERT INTO %s(height, key, value) VALUES(%d, '%s', '%s')`, - IbcStoreTableName, + IBCStoreTableName, height, hex.EncodeToString(key), hex.EncodeToString(value), ) } -func ClearAllIbcQuery() string { - return fmt.Sprintf(`DELETE FROM %s`, IbcStoreTableName) +func ClearAllIBCQuery() string { + return fmt.Sprintf(`DELETE FROM %s`, IBCStoreTableName) } diff --git a/shared/messaging/events.go b/shared/messaging/events.go index 68a5eddec..2d6f80a77 100644 --- a/shared/messaging/events.go +++ b/shared/messaging/events.go @@ -14,7 +14,7 @@ const ( TxGossipMessageContentType = "utility.TxGossipMessage" // IBC - IbcMessageContentType = "ibc.IbcMessage" + IBCMessageContentType = "ibc.IBCMessage" ) // Helper logger for state sync tranition events diff --git a/shared/node.go b/shared/node.go index c644cc793..f046caeda 100644 --- a/shared/node.go +++ b/shared/node.go @@ -189,7 +189,7 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) // TODO: Remove this lib once we move to Go 1.2 return multierr.Combine(err_consensus, err_p2p) - case messaging.IbcMessageContentType: + case messaging.IBCMessageContentType: return node.GetBus().GetIBCModule().HandleMessage(message.Content) default: logger.Global.Warn().Msgf("Unsupported message content type: %s", contentType) From 64140942faa739760f1d767768598bbdaddb0a4d Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:41:32 +0100 Subject: [PATCH 61/74] Update IBC errors --- ibc/messages.go | 2 +- ibc/module.go | 2 +- ibc/store/proofs_ics23.go | 4 +-- ibc/types/messages.go | 2 +- shared/core/types/error.go | 30 +++++++++++++--------- utility/unit_of_work/tx_message_handler.go | 4 +-- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/ibc/messages.go b/ibc/messages.go index 24bbe0a67..ae810c31f 100644 --- a/ibc/messages.go +++ b/ibc/messages.go @@ -40,7 +40,7 @@ func ConvertIBCMessageToTx(ibcMessage *types.IBCMessage) (*coreTypes.Transaction case *types.IBCMessage_Prune: anyMsg, err = codec.GetCodec().ToAny(event.Prune) default: - return nil, coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", event)) + return nil, coreTypes.ErrIBCUnknownMessageType(fmt.Sprintf("%T", event)) } if err != nil { return nil, err diff --git a/ibc/module.go b/ibc/module.go index cf7e4c896..d74766661 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -146,7 +146,7 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { // newHost returns a new IBC host instance if one is not already created func (m *ibcModule) newHost() error { if m.host != nil { - return coreTypes.ErrHostAlreadyExists() + return coreTypes.ErrIBCHostAlreadyExists() } host := &host{ logger: m.logger, diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 4f0d43fba..53e0b203d 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -64,7 +64,7 @@ func VerifyNonMembership(root ics23.CommitmentRoot, proof *ics23.CommitmentProof func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) if err != nil { - return nil, coreTypes.ErrCreatingProof(err) + return nil, coreTypes.ErrIBCCreatingProof(err) } return convertSMPToExistenceProof(proof, key, value), nil } @@ -73,7 +73,7 @@ func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentP func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) if err != nil { - return nil, coreTypes.ErrCreatingProof(err) + return nil, coreTypes.ErrIBCCreatingProof(err) } return convertSMPToExclusionProof(proof, key), nil diff --git a/ibc/types/messages.go b/ibc/types/messages.go index 917da4554..ac7eef3f4 100644 --- a/ibc/types/messages.go +++ b/ibc/types/messages.go @@ -22,7 +22,7 @@ func (m *IBCMessage) ValidateBasic() coreTypes.Error { case *IBCMessage_Prune: return msg.Prune.ValidateBasic() default: - return coreTypes.ErrUnknownIBCMessageType(fmt.Sprintf("%T", msg)) + return coreTypes.ErrIBCUnknownMessageType(fmt.Sprintf("%T", msg)) } } diff --git a/shared/core/types/error.go b/shared/core/types/error.go index b8c7595ed..31d1d9c07 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 146 +// NextCode: 147 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -182,9 +182,10 @@ const ( CodeIBCInvalidID Code = 140 CodeIBCInvalidPath Code = 141 CodeCreatingProofError Code = 142 - CodeUnknownIBCMessageTypeError Code = 143 + CodeIBCUnknownMessageTypeError Code = 143 CodeNilFieldError Code = 144 - CodeUpdatingIBCStoreDBError Code = 145 + CodeIBCUpdatingStoreError Code = 145 + CodeIBCStoreAlreadyExistsError Code = 146 ) const ( @@ -325,9 +326,10 @@ const ( IBCInvalidIDError = "invalid ibc identifier" IBCInvalidPathError = "invalid ibc path" IBCCreatingProofError = "an error occurred creating the CommitmentProof" - IBCUnknownIBCMessageTypeError = "the ibc message type is not recognized" + IBCUnknownMessageTypeError = "the ibc message type is not recognized" NilFieldError = "field cannot be nil" - IBCUpdatingIBCStoreDBError = "an error occurred updating the ibc store postgres database" + IBCUpdatingStoreError = "an error occurred updating the ibc store postgres database" + IBCStoreAlreadyExistsError = "ibc store already exists in the store manager" ) func ErrUnknownParam(paramName string) Error { @@ -863,11 +865,11 @@ func ErrUnknownMessageType(messageType any) Error { return NewError(CodeUnknownMessageType, fmt.Sprintf("%s: %v", UnknownMessageTypeError, messageType)) } -func ErrHostAlreadyExists() Error { +func ErrIBCHostAlreadyExists() Error { return NewError(CodeHostAlreadyExists, IBCHostAlreadyExistsError) } -func ErrHostDoesNotExist() Error { +func ErrIBCHostDoesNotExist() Error { return NewError(CodeHostDoesNotExist, IBCHostDoesNotExistError) } @@ -879,18 +881,22 @@ func ErrIBCInvalidPath(path string) Error { return NewError(CodeIBCInvalidPath, fmt.Sprintf("%s: %s", IBCInvalidPathError, path)) } -func ErrCreatingProof(err error) Error { +func ErrIBCCreatingProof(err error) Error { return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", IBCCreatingProofError, err.Error())) } -func ErrUnknownIBCMessageType(messageType string) Error { - return NewError(CodeUnknownIBCMessageTypeError, fmt.Sprintf("%s: %s", IBCUnknownIBCMessageTypeError, messageType)) +func ErrIBCUnknownMessageType(messageType string) Error { + return NewError(CodeIBCUnknownMessageTypeError, fmt.Sprintf("%s: %s", IBCUnknownMessageTypeError, messageType)) } func ErrNilField(field string) Error { return NewError(CodeNilFieldError, fmt.Sprintf("%s: %s", NilFieldError, field)) } -func ErrUpdatingIBCStoreDB(err error) Error { - return NewError(CodeUpdatingIBCStoreDBError, fmt.Sprintf("%s: %s", IBCUpdatingIBCStoreDBError, err.Error())) +func ErrIBCUpdatingStore(err error) Error { + return NewError(CodeIBCUpdatingStoreError, fmt.Sprintf("%s: %s", IBCUpdatingStoreError, err.Error())) +} + +func ErrIBCStoreAlreadyExists(name string) Error { + return NewError(CodeIBCStoreAlreadyExistsError, fmt.Sprintf("%s: %s", IBCStoreAlreadyExistsError, name)) } diff --git a/utility/unit_of_work/tx_message_handler.go b/utility/unit_of_work/tx_message_handler.go index 654f6d25f..6c696e14c 100644 --- a/utility/unit_of_work/tx_message_handler.go +++ b/utility/unit_of_work/tx_message_handler.go @@ -227,14 +227,14 @@ func (u *baseUtilityUnitOfWork) handleMessageChangeParameter(message *typesUtil. func (u *baseUtilityUnitOfWork) handleUpdateIBCStore(message *ibcTypes.UpdateIBCStore) coreTypes.Error { if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, message.Value); err != nil { - return coreTypes.ErrUpdatingIBCStoreDB(err) + return coreTypes.ErrIBCUpdatingStore(err) } return nil } func (u *baseUtilityUnitOfWork) handlePruneIBCStore(message *ibcTypes.PruneIBCStore) coreTypes.Error { if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, nil); err != nil { - return coreTypes.ErrUpdatingIBCStoreDB(err) + return coreTypes.ErrIBCUpdatingStore(err) } return nil } From 681a328403652be8be1a3167aae359a6353a0170 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:48:49 +0100 Subject: [PATCH 62/74] Remove no value hashing from state trees --- persistence/test/state_test.go | 6 +++--- persistence/trees/module.go | 4 ++-- persistence/trees/trees.go | 13 ++++--------- utility/session_test.go | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/persistence/test/state_test.go b/persistence/test/state_test.go index 37aeacf01..d2b67fcf6 100644 --- a/persistence/test/state_test.go +++ b/persistence/test/state_test.go @@ -42,9 +42,9 @@ func TestStateHash_DeterministicStateWhenUpdatingAppStake(t *testing.T) { // logic changes, these hashes will need to be updated based on the test output. // TODO: Add an explicit updateSnapshots flag to the test to make this more clear. stateHashes := []string{ - "00788e9a5133c31c869eb58d383f2fc30aac19b585947a8c82fe5808504ff84f", - "693450dee542f1455fdbfdeab22eb91e1888af50285330d45607b58c22f985f7", - "ad13b1646d02978f893919f413c7497b6f4d332011c40b6453499316d6705680", + "1e433a8905c7b1cf42222f8d01ba222038653f8ff35ae97cce1fd6a32d18b51e", + "4542dea3eedb99ad46b3c6e0ea901a9ee365b590b2f2ac7f12678ac369a2fe90", + "1524529fa827852e397adf1938eb058848209c4916cdacef929d050efc472c1d", } stakeAmount := initialStakeAmount diff --git a/persistence/trees/module.go b/persistence/trees/module.go index cb21c9913..9c0d5eff0 100644 --- a/persistence/trees/module.go +++ b/persistence/trees/module.go @@ -70,7 +70,7 @@ func (t *treeStore) setupTrees() error { } t.merkleTrees[stateTreeNames[i]] = &stateTree{ name: stateTreeNames[i], - tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher), + tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher), nodeStore: nodeStore, } } @@ -90,7 +90,7 @@ func (t *treeStore) setupInMemory() error { nodeStore := kvstore.NewMemKVStore() // For testing, `smt.NewSimpleMap()` can be used as well t.merkleTrees[stateTreeNames[i]] = &stateTree{ name: stateTreeNames[i], - tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher), + tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher), nodeStore: nodeStore, } } diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index e20b3f969..f12b0871c 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -23,14 +23,9 @@ import ( "github.com/pokt-network/smt" ) -var ( - // smtTreeHasher sets the hasher used by the tree SMT trees - // as a package level variable for visibility and internal use. - smtTreeHasher hash.Hash = sha256.New() - // smtValueHasher sets the hasher used by the tree SMT trees - // to be nil, which means that values are not prehashed - smtValueHasher smt.Option = smt.WithValueHasher(nil) -) +// smtTreeHasher sets the hasher used by the tree SMT trees +// as a package level variable for visibility and internal use. +var smtTreeHasher hash.Hash = sha256.New() const ( RootTreeName = "root" @@ -127,7 +122,7 @@ func (t *treeStore) DebugClearAll() error { if err := nodeStore.ClearAll(); err != nil { return fmt.Errorf("failed to clear %s node store: %w", treeName, err) } - stateTree.tree = smt.NewSparseMerkleTree(nodeStore, smtTreeHasher, smtValueHasher) + stateTree.tree = smt.NewSparseMerkleTree(nodeStore, smtTreeHasher) } return nil } diff --git a/utility/session_test.go b/utility/session_test.go index c05e5bbdf..319af206c 100644 --- a/utility/session_test.go +++ b/utility/session_test.go @@ -25,7 +25,7 @@ func TestSession_GetSession_SingleFishermanSingleServicerBaseCase(t *testing.T) numFishermen := 1 numServicers := 1 // needs to be manually updated if business logic changes - expectedSessionId := "3de7deb1c64b32cf313838f8124b2b123ae78cff02bf5eca1564d9f78a764608" + expectedSessionId := "b1e9791358aae070ac7f86fdb74e5a9d26fff025fb737a2114ccf9ad95b624bd" runtimeCfg, utilityMod, _ := prepareEnvironment(t, 5, numServicers, 1, numFishermen) From 128cee5dfa90cec18f5c4ae601ce6e49ebb00e97 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:00:53 +0100 Subject: [PATCH 63/74] Reword IBC stores --- ibc/store/provable_store.go | 202 +++++++++++++++++++++++---- ibc/store/store_manager.go | 111 +++++++++++++++ persistence/ibc.go | 13 +- persistence/types/ibc.go | 10 ++ shared/core/types/error.go | 8 +- shared/modules/persistence_module.go | 3 + 6 files changed, 315 insertions(+), 32 deletions(-) create mode 100644 ibc/store/store_manager.go diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index 98e85e067..36d7a24bd 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -2,65 +2,207 @@ package store import ( "bytes" + "fmt" + "strconv" + "strings" ics23 "github.com/cosmos/ics23/go" "github.com/pokt-network/pocket/ibc/host" + "github.com/pokt-network/pocket/persistence/kvstore" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/smt" ) -var _ modules.ProvableStore = &provableStore{} +// CachedEntry represents a local change made to the IBC store prior to it being +// committed to the state tree. These should be written to disk in the to prevent a +// loss of data and pruned when included in the state tree +// written to disk as follows: +// "{height}/{prefixedKey}" => value +type cachedEntry struct { + name string + height uint64 + prefixedKey []byte + value []byte +} -// provableStore implements the ProvableStore interface and wraps an SMT -// it operates in memory and thus cannot make any changes to the underlying -// database. All changes must be propagated through the `IBCMessage` type -// and added to the mempool for inclusion in the next block -type provableStore struct { - prefix coreTypes.CommitmentPrefix - tree *smt.SMT +// prepare returns the key and value to be written to disk +func (c *cachedEntry) prepare() ([]byte, []byte) { + return []byte(fmt.Sprintf("%s/%d/%s", c.name, c.height, string(c.prefixedKey))), c.value } -// GetCommitmentPrefix returns the commitment prefix of the provable store -func (p *provableStore) GetCommitmentPrefix() coreTypes.CommitmentPrefix { - return p.prefix +// provableStore is a struct that interfaces with the PostgresDB instance +// obtained from Persistence. It is used to Get/Set/Delete the keys in the +// IBC state tree, in doing so it will trigger the creation of +type provableStore struct { + bus modules.Bus // used to interact with persistence (passed from IBCHost) + name string // store name in storeManager + prefix coreTypes.CommitmentPrefix // []byte(name) + cache []*cachedEntry // in-memory cache of local changes to be written to disk } -// Get returns the value in the tree of the key prefixed with the CommitmentPrefix -func (p *provableStore) Get(key []byte) ([]byte, error) { - prefixed := applyPrefix(p.prefix, key) - return p.tree.Get(prefixed) +// newProvableStore returns a new instance of provableStore with the bus and prefix provided +func newProvableStore(bus modules.Bus, prefix coreTypes.CommitmentPrefix) *provableStore { + return &provableStore{ + bus: bus, + name: string(prefix), + prefix: prefix, + cache: make([]*cachedEntry, 0), + } } -// Set sets the value in the tree of the key prefixed with the CommitmentPrefix -func (p *provableStore) Set(key, value []byte) error { +// Get queries the persistence layer for the latest value stored in the IBC state tree +// keys are automatically prefixed with the CommitmentPrefix if not present +func (p *provableStore) Get(key []byte) ([]byte, error) { prefixed := applyPrefix(p.prefix, key) - return p.tree.Update(prefixed, value) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rCtx, err := p.bus.GetPersistenceModule().NewReadContext(currHeight) + if err != nil { + return nil, err + } + defer rCtx.Release() + return rCtx.GetIBCStoreEntry(prefixed, currHeight) // returns latest value stored } -// Delete deletes the value in the tree of the key prefixed with the CommitmentPrefix -func (p *provableStore) Delete(key []byte) error { +// Get queries the persistence layer for the latest value stored in the IBC state tree +// it then generates a proof by importing the IBC state tree from the TreeStore +// keys are automatically prefixed with the CommitmentPrefix if not present +func (p *provableStore) GetAndProve(key []byte, membership bool) ([]byte, *ics23.CommitmentProof, error) { prefixed := applyPrefix(p.prefix, key) - return p.tree.Delete(prefixed) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rCtx, err := p.bus.GetPersistenceModule().NewReadContext(currHeight) + if err != nil { + return nil, nil, err + } + value, err := rCtx.GetIBCStoreEntry(prefixed, currHeight) // returns latest value stored + if err != nil { + return nil, nil, err + } + defer rCtx.Release() + proof := new(ics23.CommitmentProof) + if membership { + proof, err = p.CreateMembershipProof(key, value) + } else { + proof, err = p.CreateNonMembershipProof(key) + } + if err != nil { + return nil, nil, err + } + return value, proof, nil } // CreateMembershipProof creates a membership proof for the key-value pair with the key -// prefixed with the CommitmentPrefix +// prefixed with the CommitmentPrefix, by importing the state tree from the TreeStore func (p *provableStore) CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) { + // import IBC state tree + // TODO(#854): Implement tree retrieval + /** prefixed := applyPrefix(p.prefix, key) - return createMembershipProof(p.tree, prefixed, value) + root, nodeStore := p.bus.GetTreeStore() + lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) + return createMembershipProof(lazy, prefixed, value) + **/ + return nil, nil } -// CreateNonMembershipProof creates a non-membership proof for the key prefixed with the CommitmentPrefix +// CreateNonMembershipProof creates a non-membership proof for the key prefixed with the +// CommitmentPrefix, by importing the state tree from the TreeStore func (p *provableStore) CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) { + // import IBC state tree + // TODO(#854): Implement tree retrieval + /** prefixed := applyPrefix(p.prefix, key) - return createNonMembershipProof(p.tree, prefixed) + root, nodeStore := p.bus.GetTreeStore() + lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) + return createNonMembershipProof(lazy, prefixed) + **/ + return nil, nil } -// Root returns the root of the entire tree -// NOTE: Root does not work on a per-prefix basis but returns the root of the entire tree -func (p *provableStore) Root() []byte { - return p.tree.Root() +// Set updates the persistence layer with the new key-value pair at the latest height and +// emits an UpdateIBCStore event to the bus for propogation throughout the network, to be +// included in each node's mempool and thus the next block +func (p *provableStore) Set(key, value []byte) error { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rwCtx, err := p.bus.GetPersistenceModule().NewRWContext(currHeight) + if err != nil { + return err + } + defer rwCtx.Release() + if err := rwCtx.SetIBCStoreEntry(prefixed, value); err != nil { + return err + } + // TODO(#854): Implement emit functions + // return emitUpdateStoreEvent(p.prefix, key, value) + return nil +} + +// Delete updates the persistence layer with the key and nil value pair at the latest height +// and emits an PruneIBCStore event to the bus for propogation throughout the network, to be +// included in each node's mempool and thus the next block +func (p *provableStore) Delete(key []byte) error { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rwCtx, err := p.bus.GetPersistenceModule().NewRWContext(currHeight) + if err != nil { + return err + } + defer rwCtx.Release() + if err := rwCtx.SetIBCStoreEntry(prefixed, nil); err != nil { + return err + } + // TODO(#854): Implement emit functions + // return emitPruneStoreEvent(p.prefix, key) + return nil +} + +// CacheEntries writes all local changes to disk and clears the in-memory cache +func (p *provableStore) CacheEntries(store kvstore.KVStore) error { + for _, entry := range p.cache { + key, value := entry.prepare() + if err := store.Set(key, value); err != nil { + return err + } + } + p.cache = make([]*cachedEntry, 0) + return nil +} + +// PruneCache removes all entries from the cache at the given height +func (p *provableStore) PruneCache(store kvstore.KVStore, height uint64) error { + keys, _, err := store.GetAll([]byte(fmt.Sprintf("%s/%d", p.name, height)), false) + if err != nil { + return err + } + for _, key := range keys { + if err := store.Delete(key); err != nil { + return err + } + } + return nil +} + +// RestoreCache loads all entries from disk into the cache +func (p *provableStore) RestoreCache(store kvstore.KVStore) error { + keys, values, err := store.GetAll(p.prefix, false) + if err != nil { + return err + } + for i := 0; i < len(keys); i++ { + parts := strings.SplitN(string(keys[i]), "/", 2) // name, heightStr, prefixedKeyStr + height, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return err + } + value := values[i] + p.cache = append(p.cache, &cachedEntry{ + name: parts[0], + height: height, + prefixedKey: []byte(parts[2]), + value: value, + }) + } + return nil } // applyPrefix will apply the CommitmentPrefix to the key provided if not already applied diff --git a/ibc/store/store_manager.go b/ibc/store/store_manager.go new file mode 100644 index 000000000..6e80f61b5 --- /dev/null +++ b/ibc/store/store_manager.go @@ -0,0 +1,111 @@ +package store + +import ( + "fmt" + "sync" + + "github.com/pokt-network/pocket/persistence/kvstore" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/modules" +) + +var cacheDirs = func(storesDir string) string { return fmt.Sprintf("%s/caches", storesDir) } + +// storeManager holds an in-memory map of all the provable stores in use +type storeManager struct { + m sync.Mutex + storesDir string + stores map[string]*provableStore +} + +// newStoreManager returns a new storeManager instance +func newStoreManager(storesDir string) *storeManager { + return &storeManager{ + m: sync.Mutex{}, + storesDir: storesDir, + stores: make(map[string]*provableStore, 0), + } +} + +// AddStore creates and adds a provableStore to the storeManager +// if one of the same name does not already exist +func (s *storeManager) AddStore(prefix coreTypes.CommitmentPrefix, bus modules.Bus) error { + s.m.Lock() + defer s.m.Unlock() + if _, ok := s.stores[string(prefix)]; ok { + return coreTypes.ErrIBCStoreAlreadyExists(string(prefix)) + } + store := newProvableStore(bus, prefix) + s.stores[store.name] = store + return nil +} + +// GetStore returns the provableStore with the given name +func (s *storeManager) GetStore(name string) (*provableStore, error) { + s.m.Lock() + defer s.m.Unlock() + store, ok := s.stores[name] + if !ok { + return nil, coreTypes.ErrIBCStoreDoesNotExist(name) + } + return store, nil +} + +// RemoveStore removes the provableStore with the given name +func (s *storeManager) RemoveStore(name string) error { + s.m.Lock() + defer s.m.Unlock() + if _, ok := s.stores[name]; !ok { + return coreTypes.ErrIBCStoreDoesNotExist(name) + } + delete(s.stores, name) + return nil +} + +// CacheAllEntries caches all the entries for all stores in the storeManager +func (s *storeManager) CacheAllEntries() error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.CacheEntries(disk); err != nil { + return err + } + } + return disk.Stop() +} + +// PruneCaches prunes the caches for all stores in the storeManager at the given height +func (s *storeManager) PruneCaches(height uint64) error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.PruneCache(disk, height); err != nil { + return err + } + } + return disk.Stop() +} + +// RestoreCaches restores the caches from disk for all stores in the storeManager +func (s *storeManager) RestoreCaches() error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.RestoreCache(disk); err != nil { + return err + } + } + return disk.Stop() +} diff --git a/persistence/ibc.go b/persistence/ibc.go index 45ccf6740..abfd76dba 100644 --- a/persistence/ibc.go +++ b/persistence/ibc.go @@ -4,7 +4,7 @@ import ( pTypes "github.com/pokt-network/pocket/persistence/types" ) -// SetIBCStoreEntry sets the key value pair in the IBC store postgres table +// SetIBCStoreEntry sets the key value pair in the IBC store postgres table at the current height func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { ctx, tx := p.getCtxAndTx() if _, err := tx.Exec(ctx, pTypes.InsertIBCStoreEntryQuery(p.Height, key, value)); err != nil { @@ -12,3 +12,14 @@ func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { } return nil } + +// GetIBCStoreEntry returns the stored value for the key at the height provided from the IBC store table +func (p *PostgresContext) GetIBCStoreEntry(key []byte, height int64) ([]byte, error) { + ctx, tx := p.getCtxAndTx() + row := tx.QueryRow(ctx, pTypes.GetIBCStoreEntryQuery(height, key)) + var value []byte + if err := row.Scan(&value); err != nil { + return nil, err + } + return value, nil +} diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go index cc5d05ed2..42bd096a8 100644 --- a/persistence/types/ibc.go +++ b/persistence/types/ibc.go @@ -25,6 +25,16 @@ func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { ) } +// Return the latest value for the key at the height provided or at the last updated height +func GetIBCStoreEntryQuery(height int64, key []byte) string { + return fmt.Sprintf( + `SELECT value FROM %s WHERE height <= %d AND key = '%s' ORDER BY height DESC LIMIT 1`, + IBCStoreTableName, + height, + hex.EncodeToString(key), + ) +} + func ClearAllIBCQuery() string { return fmt.Sprintf(`DELETE FROM %s`, IBCStoreTableName) } diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 31d1d9c07..76d7686f0 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 147 +// NextCode: 148 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -186,6 +186,7 @@ const ( CodeNilFieldError Code = 144 CodeIBCUpdatingStoreError Code = 145 CodeIBCStoreAlreadyExistsError Code = 146 + CodeIBCStoreDoesNotExistError Code = 147 ) const ( @@ -330,6 +331,7 @@ const ( NilFieldError = "field cannot be nil" IBCUpdatingStoreError = "an error occurred updating the ibc store postgres database" IBCStoreAlreadyExistsError = "ibc store already exists in the store manager" + IBCStoreDoesNotExistError = "ibc store does not exist in the store manager" ) func ErrUnknownParam(paramName string) Error { @@ -900,3 +902,7 @@ func ErrIBCUpdatingStore(err error) Error { func ErrIBCStoreAlreadyExists(name string) Error { return NewError(CodeIBCStoreAlreadyExistsError, fmt.Sprintf("%s: %s", IBCStoreAlreadyExistsError, name)) } + +func ErrIBCStoreDoesNotExist(name string) Error { + return NewError(CodeIBCStoreDoesNotExistError, fmt.Sprintf("%s: %s", IBCStoreDoesNotExistError, name)) +} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index d6c0d8a6b..c77288730 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -226,4 +226,7 @@ type PersistenceReadContext interface { GetIntFlag(paramName string, height int64) (int, bool, error) GetStringFlag(paramName string, height int64) (string, bool, error) GetBytesFlag(paramName string, height int64) ([]byte, bool, error) + + // IBC Queries + GetIBCStoreEntry(key []byte, height int64) ([]byte, error) } From ccea884ca543025a472ee0c64300a23ea0ec0294 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:32:04 +0100 Subject: [PATCH 64/74] Update interfaces --- ibc/store/provable_store.go | 45 ++++++++++++++++++++++++++++-------- ibc/store/store_manager.go | 15 +++++++----- shared/modules/ibc_module.go | 25 ++++++++++++++++---- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index 36d7a24bd..ac69698d8 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -13,21 +13,23 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) +var _ modules.ProvableStore = &provableStore{} + // CachedEntry represents a local change made to the IBC store prior to it being // committed to the state tree. These should be written to disk in the to prevent a // loss of data and pruned when included in the state tree // written to disk as follows: // "{height}/{prefixedKey}" => value type cachedEntry struct { - name string + storeName string height uint64 prefixedKey []byte value []byte } // prepare returns the key and value to be written to disk -func (c *cachedEntry) prepare() ([]byte, []byte) { - return []byte(fmt.Sprintf("%s/%d/%s", c.name, c.height, string(c.prefixedKey))), c.value +func (c *cachedEntry) prepare() (key []byte, value []byte) { + return []byte(fmt.Sprintf("%s/%d/%s", c.storeName, c.height, string(c.prefixedKey))), c.value } // provableStore is a struct that interfaces with the PostgresDB instance @@ -78,7 +80,7 @@ func (p *provableStore) GetAndProve(key []byte, membership bool) ([]byte, *ics23 return nil, nil, err } defer rCtx.Release() - proof := new(ics23.CommitmentProof) + var proof *ics23.CommitmentProof if membership { proof, err = p.CreateMembershipProof(key, value) } else { @@ -97,7 +99,7 @@ func (p *provableStore) CreateMembershipProof(key, value []byte) (*ics23.Commitm // TODO(#854): Implement tree retrieval /** prefixed := applyPrefix(p.prefix, key) - root, nodeStore := p.bus.GetTreeStore() + root, nodeStore := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) return createMembershipProof(lazy, prefixed, value) **/ @@ -111,7 +113,7 @@ func (p *provableStore) CreateNonMembershipProof(key []byte) (*ics23.CommitmentP // TODO(#854): Implement tree retrieval /** prefixed := applyPrefix(p.prefix, key) - root, nodeStore := p.bus.GetTreeStore() + root, nodeStore := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) return createNonMembershipProof(lazy, prefixed) **/ @@ -119,7 +121,7 @@ func (p *provableStore) CreateNonMembershipProof(key []byte) (*ics23.CommitmentP } // Set updates the persistence layer with the new key-value pair at the latest height and -// emits an UpdateIBCStore event to the bus for propogation throughout the network, to be +// emits an UpdateIBCStore event to the bus for propagation throughout the network, to be // included in each node's mempool and thus the next block func (p *provableStore) Set(key, value []byte) error { prefixed := applyPrefix(p.prefix, key) @@ -132,13 +134,19 @@ func (p *provableStore) Set(key, value []byte) error { if err := rwCtx.SetIBCStoreEntry(prefixed, value); err != nil { return err } + p.cache = append(p.cache, &cachedEntry{ + storeName: p.name, + height: uint64(currHeight), + prefixedKey: prefixed, + value: value, + }) // TODO(#854): Implement emit functions // return emitUpdateStoreEvent(p.prefix, key, value) return nil } // Delete updates the persistence layer with the key and nil value pair at the latest height -// and emits an PruneIBCStore event to the bus for propogation throughout the network, to be +// and emits an PruneIBCStore event to the bus for propagation throughout the network, to be // included in each node's mempool and thus the next block func (p *provableStore) Delete(key []byte) error { prefixed := applyPrefix(p.prefix, key) @@ -151,11 +159,30 @@ func (p *provableStore) Delete(key []byte) error { if err := rwCtx.SetIBCStoreEntry(prefixed, nil); err != nil { return err } + p.cache = append(p.cache, &cachedEntry{ + storeName: p.name, + height: uint64(currHeight), + prefixedKey: prefixed, + value: nil, + }) // TODO(#854): Implement emit functions // return emitPruneStoreEvent(p.prefix, key) return nil } +// GetCommitmentPrefix returns the CommitmentPrefix for the store +func (p *provableStore) GetCommitmentPrefix() coreTypes.CommitmentPrefix { return p.prefix } + +// Root returns the current root of the IBC state tree +func (p *provableStore) Root() ics23.CommitmentRoot { + // TODO(#854): Implement tree retrieval + /** + root, _ := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) + return root + **/ + return nil +} + // CacheEntries writes all local changes to disk and clears the in-memory cache func (p *provableStore) CacheEntries(store kvstore.KVStore) error { for _, entry := range p.cache { @@ -196,7 +223,7 @@ func (p *provableStore) RestoreCache(store kvstore.KVStore) error { } value := values[i] p.cache = append(p.cache, &cachedEntry{ - name: parts[0], + storeName: parts[0], height: height, prefixedKey: []byte(parts[2]), value: value, diff --git a/ibc/store/store_manager.go b/ibc/store/store_manager.go index 6e80f61b5..ba520652e 100644 --- a/ibc/store/store_manager.go +++ b/ibc/store/store_manager.go @@ -9,7 +9,10 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) -var cacheDirs = func(storesDir string) string { return fmt.Sprintf("%s/caches", storesDir) } +var ( + _ modules.IBCStoreManager = &storeManager{} + cacheDirs = func(storesDir string) string { return fmt.Sprintf("%s/caches", storesDir) } +) // storeManager holds an in-memory map of all the provable stores in use type storeManager struct { @@ -29,19 +32,19 @@ func newStoreManager(storesDir string) *storeManager { // AddStore creates and adds a provableStore to the storeManager // if one of the same name does not already exist -func (s *storeManager) AddStore(prefix coreTypes.CommitmentPrefix, bus modules.Bus) error { +func (s *storeManager) AddStore(name string, bus modules.Bus) error { s.m.Lock() defer s.m.Unlock() - if _, ok := s.stores[string(prefix)]; ok { - return coreTypes.ErrIBCStoreAlreadyExists(string(prefix)) + if _, ok := s.stores[name]; ok { + return coreTypes.ErrIBCStoreAlreadyExists(name) } - store := newProvableStore(bus, prefix) + store := newProvableStore(bus, coreTypes.CommitmentPrefix(name)) s.stores[store.name] = store return nil } // GetStore returns the provableStore with the given name -func (s *storeManager) GetStore(name string) (*provableStore, error) { +func (s *storeManager) GetStore(name string) (modules.ProvableStore, error) { s.m.Lock() defer s.m.Unlock() store, ok := s.stores[name] diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index 1ebc5b601..b03835467 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -2,11 +2,12 @@ package modules import ( ics23 "github.com/cosmos/ics23/go" + "github.com/pokt-network/pocket/persistence/kvstore" coreTypes "github.com/pokt-network/pocket/shared/core/types" "google.golang.org/protobuf/types/known/anypb" ) -//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler,ProvableStore +//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler,IBCStoreManager,ProvableStore const IBCModuleName = "ibc" @@ -206,12 +207,28 @@ type IBCHandler interface { **/ } +// IBCStoreManager manages the different ProvableStore instances created by the IBC host +type IBCStoreManager interface { + AddStore(name string, bus Bus) error + GetStore(name string) (ProvableStore, error) + RemoveStore(name string) error + CacheAllEntries() error + PruneCaches(height uint64) error + RestoreCaches() error +} + +// ProvableStore interacts with Persistence and the IBC state tree in order for the IBC host to +// be able to interact with the IBC store locally and propagate any changes throuhout the network type ProvableStore interface { Get(key []byte) ([]byte, error) + GetAndProve(key []byte, membership bool) ([]byte, *ics23.CommitmentProof, error) + CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) + CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) Set(key, value []byte) error Delete(key []byte) error - Root() []byte GetCommitmentPrefix() coreTypes.CommitmentPrefix - CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) - CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) + Root() ics23.CommitmentRoot + CacheEntries(kvstore.KVStore) error + PruneCache(store kvstore.KVStore, height uint64) error + RestoreCache(kvstore.KVStore) error } From d27211d0d4333b1f70613f896e6ddb7cd7952e36 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:42:34 +0100 Subject: [PATCH 65/74] Add storesDir to IBC config and pass to host and storemanger --- build/config/config.validator1.json | 3 ++- build/config/config.validator2.json | 3 ++- build/config/config.validator3.json | 3 ++- build/config/config.validator4.json | 3 ++- ibc/host.go | 4 +++- ibc/module.go | 5 ++++- ibc/store/store_manager.go | 9 +++++---- runtime/configs/config.go | 3 ++- runtime/configs/proto/ibc_config.proto | 1 + runtime/defaults/defaults.go | 3 ++- runtime/manager_test.go | 1 + shared/modules/ibc_module.go | 2 +- 12 files changed, 27 insertions(+), 13 deletions(-) diff --git a/build/config/config.validator1.json b/build/config/config.validator1.json index 746e74881..888d137bb 100644 --- a/build/config/config.validator1.json +++ b/build/config/config.validator1.json @@ -60,6 +60,7 @@ }, "ibc": { "enabled": true, - "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator2.json b/build/config/config.validator2.json index 052d8eeb7..aaf37c8de 100644 --- a/build/config/config.validator2.json +++ b/build/config/config.validator2.json @@ -53,6 +53,7 @@ }, "ibc": { "enabled": true, - "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator3.json b/build/config/config.validator3.json index 834a72ff2..86611bf12 100644 --- a/build/config/config.validator3.json +++ b/build/config/config.validator3.json @@ -53,6 +53,7 @@ }, "ibc": { "enabled": true, - "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator4.json b/build/config/config.validator4.json index df37cb69a..903a8dba0 100644 --- a/build/config/config.validator4.json +++ b/build/config/config.validator4.json @@ -53,6 +53,7 @@ }, "ibc": { "enabled": true, - "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/ibc/host.go b/ibc/host.go index db2f791d3..3218a554f 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -15,7 +15,9 @@ import ( var _ modules.IBCHost = &host{} type host struct { - logger *modules.Logger + logger *modules.Logger + storesDir string + storeManager modules.IBCStoreManager } // GetTimestamp returns the current unix timestamp diff --git a/ibc/module.go b/ibc/module.go index d74766661..0629716f4 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "github.com/pokt-network/pocket/ibc/store" ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/runtime/configs" @@ -149,7 +150,9 @@ func (m *ibcModule) newHost() error { return coreTypes.ErrIBCHostAlreadyExists() } host := &host{ - logger: m.logger, + logger: m.logger, + storesDir: m.cfg.StoresDir, + storeManager: store.NewStoreManager(m.cfg.StoresDir), } m.host = host return nil diff --git a/ibc/store/store_manager.go b/ibc/store/store_manager.go index ba520652e..62edc0caa 100644 --- a/ibc/store/store_manager.go +++ b/ibc/store/store_manager.go @@ -17,12 +17,13 @@ var ( // storeManager holds an in-memory map of all the provable stores in use type storeManager struct { m sync.Mutex + bus modules.Bus storesDir string stores map[string]*provableStore } -// newStoreManager returns a new storeManager instance -func newStoreManager(storesDir string) *storeManager { +// NewStoreManager returns a new storeManager instance +func NewStoreManager(storesDir string) *storeManager { return &storeManager{ m: sync.Mutex{}, storesDir: storesDir, @@ -32,13 +33,13 @@ func newStoreManager(storesDir string) *storeManager { // AddStore creates and adds a provableStore to the storeManager // if one of the same name does not already exist -func (s *storeManager) AddStore(name string, bus modules.Bus) error { +func (s *storeManager) AddStore(name string) error { s.m.Lock() defer s.m.Unlock() if _, ok := s.stores[name]; ok { return coreTypes.ErrIBCStoreAlreadyExists(name) } - store := newProvableStore(bus, coreTypes.CommitmentPrefix(name)) + store := newProvableStore(s.bus, coreTypes.CommitmentPrefix(name)) s.stores[store.name] = store return nil } diff --git a/runtime/configs/config.go b/runtime/configs/config.go index ee2f6f382..41df56173 100644 --- a/runtime/configs/config.go +++ b/runtime/configs/config.go @@ -160,7 +160,8 @@ func NewDefaultConfig(options ...func(*Config)) *Config { Servicer: &ServicerConfig{}, Fisherman: &FishermanConfig{}, IBC: &IBCConfig{ - Enabled: defaults.DefaultIBCEnabled, + Enabled: defaults.DefaultIBCEnabled, + StoresDir: defaults.DefaultIBCStoresDir, }, } diff --git a/runtime/configs/proto/ibc_config.proto b/runtime/configs/proto/ibc_config.proto index b83d4f406..a1e62686a 100644 --- a/runtime/configs/proto/ibc_config.proto +++ b/runtime/configs/proto/ibc_config.proto @@ -13,4 +13,5 @@ message IBCConfig { // to relay IBC packets using an IBC relayer binary bool enabled = 1; string private_key = 2; // hex encoded + string stores_dir = 3; } diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index ce02b2a05..660f49c22 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -73,7 +73,8 @@ var ( DefaultKeybaseVaultMountPath = "" // ibc - DefaultIBCEnabled = false + DefaultIBCEnabled = false + DefaultIBCStoresDir = "/var/ibc" ) var ( diff --git a/runtime/manager_test.go b/runtime/manager_test.go index 35fef8094..026ef1259 100644 --- a/runtime/manager_test.go +++ b/runtime/manager_test.go @@ -1826,6 +1826,7 @@ func TestNewManagerFromReaders(t *testing.T) { IBC: &configs.IBCConfig{ Enabled: true, PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + StoresDir: defaults.DefaultIBCStoresDir, }, }, genesisState: expectedGenesis, diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index b03835467..638e1f6dd 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -209,7 +209,7 @@ type IBCHandler interface { // IBCStoreManager manages the different ProvableStore instances created by the IBC host type IBCStoreManager interface { - AddStore(name string, bus Bus) error + AddStore(name string) error GetStore(name string) (ProvableStore, error) RemoveStore(name string) error CacheAllEntries() error From d47619102c5bf97538f510f2e17c68eabab24bb4 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:48:43 +0100 Subject: [PATCH 66/74] golint error --- ibc/docs/ics24.md | 2 +- ibc/store/provable_store.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 49edfddf3..a4e358aca 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -15,7 +15,7 @@ ## Overview -[ICS-24][ics24] details the requirements of the host chain, in order for it to be compatible with IBC. A host is defined as a node on a chain that runs the IBC software. A host has the ability to create connections with counterparty chains, open channels, and ports as well as commit proofs to the consensus state of its own chain for the relayer to submit to another chain. The host is responsible to managing and creating clients and all other aspects of the IBC module. +[ICS-24][ics24] details the requirements of the host chain, in order for it to be compatible with IBC. A host is defined as a node on a chain that runs the IBC software. A host has the ability to create connections with counterparty chains, open channels, expose ports, and commit proofs to the consensus state of its own chain for the relayer to submit to another chain. The host is responsible to managing and creating clients and all other aspects of the IBC module. As token transfers as defined in [ICS-20][ics20] work on a lock and mint pattern, any tokens sent from **chain A** to **chain B** will have a denomination unique to the connection/channel/port combination that the packet was sent over. This means that if a host where to shutdown a connection or channel without warning any tokens yet to be returned to the host chain would be lost. For this reason, only validator nodes are able to become hosts, as they provide the most reliability out of the different node types. diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index ac69698d8..f0866c85b 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -28,7 +28,7 @@ type cachedEntry struct { } // prepare returns the key and value to be written to disk -func (c *cachedEntry) prepare() (key []byte, value []byte) { +func (c *cachedEntry) prepare() (key, value []byte) { return []byte(fmt.Sprintf("%s/%d/%s", c.storeName, c.height, string(c.prefixedKey))), c.value } From 9cb2bddec81c6358c263ba3f10a4cb6a65e650c5 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:09:12 +0100 Subject: [PATCH 67/74] Reorganise --- ibc/handle_message_test.go | 18 +++++++++--------- ibc/module.go | 2 +- ibc/{host => path}/identifiers.go | 2 +- ibc/{host => path}/keys.go | 2 +- ibc/{host => path}/keys_ics02.go | 2 +- ibc/{host => path}/keys_ics03.go | 2 +- ibc/{host => path}/keys_ics04.go | 2 +- ibc/{host => path}/keys_ics05.go | 2 +- ibc/{host => path}/path_test.go | 2 +- ibc/{host => path}/prefix.go | 2 +- ibc/store/provable_store.go | 4 ++-- ibc/{messages.go => types/convert.go} | 25 ++++++++++++------------- 12 files changed, 32 insertions(+), 33 deletions(-) rename ibc/{host => path}/identifiers.go (99%) rename ibc/{host => path}/keys.go (99%) rename ibc/{host => path}/keys_ics02.go (99%) rename ibc/{host => path}/keys_ics03.go (99%) rename ibc/{host => path}/keys_ics04.go (99%) rename ibc/{host => path}/keys_ics05.go (98%) rename ibc/{host => path}/path_test.go (99%) rename ibc/{host => path}/prefix.go (98%) rename ibc/{messages.go => types/convert.go} (57%) diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go index b547d2a87..7bb81af20 100644 --- a/ibc/handle_message_test.go +++ b/ibc/handle_message_test.go @@ -51,27 +51,27 @@ func TestHandleMessage_BasicValidation_Message(t *testing.T) { }{ { name: "Valid Update Message", - msg: CreateUpdateStoreMessage([]byte("key"), []byte("value")), + msg: ibcTypes.CreateUpdateStoreMessage([]byte("key"), []byte("value")), expected: nil, }, { name: "Valid Prune Message", - msg: CreatePruneStoreMessage([]byte("key")), + msg: ibcTypes.CreatePruneStoreMessage([]byte("key")), expected: nil, }, { name: "Invalid Update Message: Empty Key", - msg: CreateUpdateStoreMessage(nil, []byte("value")), + msg: ibcTypes.CreateUpdateStoreMessage(nil, []byte("value")), expected: coreTypes.ErrNilField("key"), }, { name: "Invalid Update Message: Empty Value", - msg: CreateUpdateStoreMessage([]byte("key"), nil), + msg: ibcTypes.CreateUpdateStoreMessage([]byte("key"), nil), expected: coreTypes.ErrNilField("value"), }, { name: "Invalid Prune Message: Empty Key", - msg: CreatePruneStoreMessage(nil), + msg: ibcTypes.CreatePruneStoreMessage(nil), expected: coreTypes.ErrNilField("key"), }, } @@ -285,16 +285,16 @@ func TestHandleMessage_AddToMempool(t *testing.T) { func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { t.Helper() - msg := CreateUpdateStoreMessage(key, value) - tx, err := ConvertIBCMessageToTx(msg) + msg := ibcTypes.CreateUpdateStoreMessage(key, value) + tx, err := ibcTypes.ConvertIBCMessageToTx(msg) require.NoError(t, err) return msg, tx } func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { t.Helper() - msg := CreatePruneStoreMessage(key) - tx, err := ConvertIBCMessageToTx(msg) + msg := ibcTypes.CreatePruneStoreMessage(key) + tx, err := ibcTypes.ConvertIBCMessageToTx(msg) require.NoError(t, err) return msg, tx } diff --git a/ibc/module.go b/ibc/module.go index 0629716f4..e7a423636 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -105,7 +105,7 @@ func (m *ibcModule) HandleMessage(message *anypb.Any) error { } // Convert IBC message to a utility Transaction - tx, err := ConvertIBCMessageToTx(ibcMessage) + tx, err := ibcTypes.ConvertIBCMessageToTx(ibcMessage) if err != nil { return err } diff --git a/ibc/host/identifiers.go b/ibc/path/identifiers.go similarity index 99% rename from ibc/host/identifiers.go rename to ibc/path/identifiers.go index f6b4a046d..283edc8aa 100644 --- a/ibc/host/identifiers.go +++ b/ibc/path/identifiers.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys.go b/ibc/path/keys.go similarity index 99% rename from ibc/host/keys.go rename to ibc/path/keys.go index c5a62ff34..1cd8a4de8 100644 --- a/ibc/host/keys.go +++ b/ibc/path/keys.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys_ics02.go b/ibc/path/keys_ics02.go similarity index 99% rename from ibc/host/keys_ics02.go rename to ibc/path/keys_ics02.go index a114f2bf4..5cce681be 100644 --- a/ibc/host/keys_ics02.go +++ b/ibc/path/keys_ics02.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/keys_ics03.go b/ibc/path/keys_ics03.go similarity index 99% rename from ibc/host/keys_ics03.go rename to ibc/path/keys_ics03.go index 505d9e0e2..99e56a48a 100644 --- a/ibc/host/keys_ics03.go +++ b/ibc/path/keys_ics03.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/keys_ics04.go b/ibc/path/keys_ics04.go similarity index 99% rename from ibc/host/keys_ics04.go rename to ibc/path/keys_ics04.go index 1cabd8c7c..004ad3647 100644 --- a/ibc/host/keys_ics04.go +++ b/ibc/path/keys_ics04.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys_ics05.go b/ibc/path/keys_ics05.go similarity index 98% rename from ibc/host/keys_ics05.go rename to ibc/path/keys_ics05.go index 9ff1f1448..f6c807f9a 100644 --- a/ibc/host/keys_ics05.go +++ b/ibc/path/keys_ics05.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/path_test.go b/ibc/path/path_test.go similarity index 99% rename from ibc/host/path_test.go rename to ibc/path/path_test.go index 1c1c8d367..96d47f039 100644 --- a/ibc/host/path_test.go +++ b/ibc/path/path_test.go @@ -1,4 +1,4 @@ -package host +package path import ( "testing" diff --git a/ibc/host/prefix.go b/ibc/path/prefix.go similarity index 98% rename from ibc/host/prefix.go rename to ibc/path/prefix.go index 86510bbfd..ebeddab07 100644 --- a/ibc/host/prefix.go +++ b/ibc/path/prefix.go @@ -1,4 +1,4 @@ -package host +package path import ( coreTypes "github.com/pokt-network/pocket/shared/core/types" diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index f0866c85b..366013d06 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -7,7 +7,7 @@ import ( "strings" ics23 "github.com/cosmos/ics23/go" - "github.com/pokt-network/pocket/ibc/host" + "github.com/pokt-network/pocket/ibc/path" "github.com/pokt-network/pocket/persistence/kvstore" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" @@ -240,5 +240,5 @@ func applyPrefix(prefix coreTypes.CommitmentPrefix, key []byte) coreTypes.Commit if bytes.Equal(prefix[:len(slashed)], slashed) { return key } - return host.ApplyPrefix(prefix, string(key)) + return path.ApplyPrefix(prefix, string(key)) } diff --git a/ibc/messages.go b/ibc/types/convert.go similarity index 57% rename from ibc/messages.go rename to ibc/types/convert.go index ae810c31f..02f205f65 100644 --- a/ibc/messages.go +++ b/ibc/types/convert.go @@ -1,19 +1,18 @@ -package ibc +package types import ( "fmt" - "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" "google.golang.org/protobuf/types/known/anypb" ) -func CreateUpdateStoreMessage(key, value []byte) *types.IBCMessage { - return &types.IBCMessage{ - Event: &types.IBCMessage_Update{ - Update: &types.UpdateIBCStore{ +func CreateUpdateStoreMessage(key, value []byte) *IBCMessage { + return &IBCMessage{ + Event: &IBCMessage_Update{ + Update: &UpdateIBCStore{ Key: key, Value: value, }, @@ -21,23 +20,23 @@ func CreateUpdateStoreMessage(key, value []byte) *types.IBCMessage { } } -func CreatePruneStoreMessage(key []byte) *types.IBCMessage { - return &types.IBCMessage{ - Event: &types.IBCMessage_Prune{ - Prune: &types.PruneIBCStore{ +func CreatePruneStoreMessage(key []byte) *IBCMessage { + return &IBCMessage{ + Event: &IBCMessage_Prune{ + Prune: &PruneIBCStore{ Key: key, }, }, } } -func ConvertIBCMessageToTx(ibcMessage *types.IBCMessage) (*coreTypes.Transaction, error) { +func ConvertIBCMessageToTx(ibcMessage *IBCMessage) (*coreTypes.Transaction, error) { var anyMsg *anypb.Any var err error switch event := ibcMessage.Event.(type) { - case *types.IBCMessage_Update: + case *IBCMessage_Update: anyMsg, err = codec.GetCodec().ToAny(event.Update) - case *types.IBCMessage_Prune: + case *IBCMessage_Prune: anyMsg, err = codec.GetCodec().ToAny(event.Prune) default: return nil, coreTypes.ErrIBCUnknownMessageType(fmt.Sprintf("%T", event)) From 6b74a5fbee9712e65559dc6c15bb2185860ce633 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:40:48 +0100 Subject: [PATCH 68/74] Update docs on data retrieval --- ibc/docs/ics24.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index a4e358aca..0df83a951 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -8,6 +8,7 @@ - [Timestamps](#timestamps) - [IBC State](#ibc-state) - [IBC State Tree](#ibc-state-tree) + - [Data Retrieval](#data-retrieval) - [IBC Messages](#ibc-messages) - [IBC Message Handling](#ibc-message-handling) - [Mempool](#mempool) @@ -26,7 +27,8 @@ Only validators can be configured to be IBC hosts. If the IBC module, during its ```json "ibc": { "enabled": bool, - "private_key": string + "private_key": string, + "stores_dir": string } ``` @@ -86,9 +88,9 @@ See: [IBC State](#ibc-state) below for more details on the IBC state transition ### Paths and Identifiers -Paths are defined as bytestrings that are used to access the elements in the different stores. They are built with the function `ApplyPrefix()` which takes a store key as a prefix and a path string and will return the key to access an element in the specific store. The logic for paths can be found in [host/keys.go](../host/keys.go) and [host/prefix.go](../host/prefix.go) +Paths are defined as bytestrings that are used to access the elements in the different stores. They are built with the function `ApplyPrefix()` which takes a store key as a prefix and a path string and will return the key to access an element in the specific store. The logic for paths can be found in [path/keys.go](../path/keys.go) and [path/prefix.go](../path/prefix.go) -Identifiers are bytestrings constrained to specific characters and lengths depending on their usages. They are used to identify: channels, clients, connections and ports. Although the minimum length of the identifiers is much less we use a minimum length of 32 bytes and a maximum length that varies depending on the use case to randomly generate identifiers. This allows for an extremely low chance of collision between identifiers. Identifiers have no significance beyond their use to store different elements in the IBC stores and as such there is no need for non-random identifiers. The logic for identifiers can be found in [host/identifiers.go](../host/identifiers.go). +Identifiers are bytestrings constrained to specific characters and lengths depending on their usages. They are used to identify: channels, clients, connections and ports. Although the minimum length of the identifiers is much less we use a minimum length of 32 bytes and a maximum length that varies depending on the use case to randomly generate identifiers. This allows for an extremely low chance of collision between identifiers. Identifiers have no significance beyond their use to store different elements in the IBC stores and as such there is no need for non-random identifiers. The logic for identifiers can be found in [path/identifiers.go](../path/identifiers.go). ### Timestamps @@ -100,7 +102,15 @@ As mentioned [above](#persistence) the IBC store **MUST** be included in the con ### IBC State Tree -The IBC state tree is a non-value hashing `SMT` backed by a persistent `KVStore`, this is due to its need for data retrieval as well as proof generation/verification. The root hash of the IBC state tree is included in the `rootTree` which computes the networks state hash for any given block. This allows verifiers to not only verify the inclusion/exclusion of any element in the IBC state tree itself but also that the IBC state tree was used to compute the network's state hash, by utilising the `CommitmentProof` object defined in [ICS-23][ics23]. +The IBC state tree is an `SMT` backed by a persistent `KVStore`, this is used for proof generation/verification. Data retrieval uses the `peristence` layer, see the [data retrieval](#data-retrieval) section below for more details. + +The root hash of the IBC state tree is included in the `rootTree` which computes the network's state hash for any given block. This allows verifiers to not only verify the inclusion/exclusion of any element in the IBC state tree itself but also that the IBC state tree was used to compute the network's state hash, by utilising the `CommitmentProof` object defined in [ICS-23][ics23]. + +### Data Retrieval + +In order to query the IBC store the `persistence` layer is leveraged. All local changes to the IBC store are broadcasted as [IBC messages](#ibc-messages) and ultimately stored in each node's `peristence` layer. This allows for the efficient querying of the IBC store instead of having to query the IBC state tree directly. + +When attempting to generate a proof for a specific `key` in the IBC state tree the IBC host will import a local copy of the IBC state tree and use this to generate the proof. Otherwise all queries are handled by the `peristence` layer's underlying database. ### IBC Messages From 9ba683a0349f435b951bc6d4f0baf264b240b747 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:55:06 +0100 Subject: [PATCH 69/74] Update docs on provable stores and caching --- ibc/docs/README.md | 4 +++- ibc/docs/ics24.md | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ibc/docs/README.md b/ibc/docs/README.md index 01a04c4ed..28160a08b 100644 --- a/ibc/docs/README.md +++ b/ibc/docs/README.md @@ -95,7 +95,9 @@ _Note_: Connections, Channels and Ports in IBC are not the same as networking co [ICS24][ics24] defines the IBC stores and these must be a part of the Pocket networks consensus state. As such the `ibcTree` is defined as one of the state trees used to generate the root hash. This tree contains the relevant information the hosts/relayers need to be able to use IBC, in accordance with ICS-24 and the other ICS components. -TODO([#854](https://github.com/pokt-network/pocket/issues/854)): Add a local cache for changes to the state for use in the event of the node crashing. +In order to interact with the IBC store's the host must create a `ProvableStore` instance which can make local changes to the state and propagate these through the network. This store maintains a local cache that can be backed up to disk and restored. In the event of a node failure, or local changes being unable to be propagated, the cache can be restored from the disk backup and the host can attempt to propagate the changes again. + +See: [store/provable_store.go](../store/provable_store.go) and [ics24.md](ics24.md) for more details on the specifics of the IBC store implementation for Pocket. ## Components diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 0df83a951..256294deb 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -13,6 +13,8 @@ - [IBC Message Handling](#ibc-message-handling) - [Mempool](#mempool) - [State Transition](#state-transition) +- [Provable Stores](#provable-stores) + - [Caching](#caching) ## Overview @@ -173,6 +175,22 @@ With the `IBCMessage` now propagated through the network's mempool, when it is r See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-state-tree) for more information on how the persistence module uses the data it has tracked from the `IBCMessage` objects, in order to update the actual state trees and in turn the root hash. +## Provable Stores + +The `ProvableStore` interface defined in [shared/modules/ibc_module.go](../../shared/modules/ibc_module.go) is implemented by the [`provableStore`](../store/provable_store.go) type and managed by the [`StoreManager`](../store/store_manager.go). + +The provable stores are each assigned a `prefix`. This represents the specific sub-store that they are able to access and interact with in the IBC state tree. When doing any operation `get`/`set`/`delete` the `prefix` is applied to the `key` provided to generate the `CommitmentPath` to the element in the IBC state tree. + +The provable stores do not directly interface with the IBC state tree but instead utliise the `peristence` layer to query the data locally. This allows for the efficient querying of the IBC store instead of having to query the IBC state tree directly. Any changes made by the `ProvableStore` instance are broadcasted to the network for inclusion in the next block, being stored in their mempools. + +### Caching + +Every local change made to the IBC store (`update`/`delete`) is stored in an in-memory cache. These caches are written to disk by the [`StoreManager`](../store/store_manager.go). + +In the event of a node failure, or local changes not being propagated correctly. Any changes stored in the cache can be "replayed" by the node and broadcasted to the network for inclusion in the next block. + +_TODO: Implement this functionality_ + [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md [smt]: https://github.com/pokt-network/smt From 5bb0adf24c03dec7405fd323b33cc5e512f23a84 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 30 Jun 2023 09:50:03 +0100 Subject: [PATCH 70/74] Prefix errors --- shared/core/types/error.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 76d7686f0..162a9c42d 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -177,11 +177,11 @@ const ( CodeUnknownActorType Code = 130 CodeUnknownMessageType Code = 131 CodeProposalBlockNotSet Code = 133 - CodeHostAlreadyExists Code = 138 - CodeHostDoesNotExist Code = 139 + CodeIBCHostAlreadyExists Code = 138 + CodeIBCHostDoesNotExist Code = 139 CodeIBCInvalidID Code = 140 CodeIBCInvalidPath Code = 141 - CodeCreatingProofError Code = 142 + CodeIBCCreatingProofError Code = 142 CodeIBCUnknownMessageTypeError Code = 143 CodeNilFieldError Code = 144 CodeIBCUpdatingStoreError Code = 145 @@ -868,11 +868,11 @@ func ErrUnknownMessageType(messageType any) Error { } func ErrIBCHostAlreadyExists() Error { - return NewError(CodeHostAlreadyExists, IBCHostAlreadyExistsError) + return NewError(CodeIBCHostAlreadyExists, IBCHostAlreadyExistsError) } func ErrIBCHostDoesNotExist() Error { - return NewError(CodeHostDoesNotExist, IBCHostDoesNotExistError) + return NewError(CodeIBCHostDoesNotExist, IBCHostDoesNotExistError) } func ErrIBCInvalidID(identifier, msg string) Error { @@ -884,7 +884,7 @@ func ErrIBCInvalidPath(path string) Error { } func ErrIBCCreatingProof(err error) Error { - return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", IBCCreatingProofError, err.Error())) + return NewError(CodeIBCCreatingProofError, fmt.Sprintf("%s: %s", IBCCreatingProofError, err.Error())) } func ErrIBCUnknownMessageType(messageType string) Error { From e94f4a97ec8f43598e08c192d9f0b4720748aa75 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 30 Jun 2023 10:13:18 +0100 Subject: [PATCH 71/74] linter error --- shared/modules/persistence_module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 4fef44f32..acfea1473 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -147,7 +147,7 @@ type PersistenceWriteContext interface { // IBC Operations SetIBCStoreEntry(key, value []byte) error - // Relay Operations + // Relay Operations RecordRelayService(applicationAddress string, key []byte, relay *coreTypes.Relay, response *coreTypes.RelayResponse) error } From e6b53fea1f2ae1312c32da716dd635303d6eb79c Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:37:20 +0100 Subject: [PATCH 72/74] Address comments --- ibc/host.go | 2 +- ibc/module.go | 2 +- ibc/store/provable_store.go | 52 +++++++++++++++++----------- ibc/store/store_manager.go | 6 ++-- persistence/trees/trees.go | 3 +- shared/modules/ibc_module.go | 4 +-- shared/modules/persistence_module.go | 4 +++ utility/unit_of_work/gov.go | 2 +- 8 files changed, 45 insertions(+), 30 deletions(-) diff --git a/ibc/host.go b/ibc/host.go index 3218a554f..b8a415d67 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -28,7 +28,7 @@ func (h *host) GetTimestamp() uint64 { // GetProvableStore returns a copy of the IBC state tree where all operations observed by // this specific ibc light client were applied to its ephemeral (in memory) state and have not // yet been included in the next block. The aggregation of all light client-provable stores -// propagated throughout the network are happen validated by the block proposer when reaping +// propagated throughout the network are validated by the block proposer when reaping // the mempool, and lead to a valid on-chain state transition when consensus is reached. // TODO(#854): Implement this func (h *host) GetProvableStore(prefix coreTypes.CommitmentPrefix) (modules.ProvableStore, error) { diff --git a/ibc/module.go b/ibc/module.go index e7a423636..d8685059c 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -86,7 +86,7 @@ func (m *ibcModule) GetModuleName() string { return modules.IBCModuleName } -// HandleMessage accepts a generic IBC message and routes it to the specific handler +// HandleMessage accepts a generic IBC message and routes it to the utility mempool func (m *ibcModule) HandleMessage(message *anypb.Any) error { m.m.Lock() defer m.m.Unlock() diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index 366013d06..c2dbd3390 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "sync" ics23 "github.com/cosmos/ics23/go" "github.com/pokt-network/pocket/ibc/path" @@ -15,11 +16,9 @@ import ( var _ modules.ProvableStore = &provableStore{} -// CachedEntry represents a local change made to the IBC store prior to it being -// committed to the state tree. These should be written to disk in the to prevent a +// cachedEntry represents a local change made to the IBC store prior to it being +// committed to the state tree. These are written to disk in the to prevent a // loss of data and pruned when included in the state tree -// written to disk as follows: -// "{height}/{prefixedKey}" => value type cachedEntry struct { storeName string height uint64 @@ -28,18 +27,21 @@ type cachedEntry struct { } // prepare returns the key and value to be written to disk +// "{height}/{prefixedKey}" => value func (c *cachedEntry) prepare() (key, value []byte) { return []byte(fmt.Sprintf("%s/%d/%s", c.storeName, c.height, string(c.prefixedKey))), c.value } -// provableStore is a struct that interfaces with the PostgresDB instance -// obtained from Persistence. It is used to Get/Set/Delete the keys in the -// IBC state tree, in doing so it will trigger the creation of +// provableStore is a struct that interfaces with the persistence layer to +// Get/Set/Delete the keys in the IBC state tree, in doing so it will trigger +// the creation of IBC messages that are broadcasted through the network and +// included in the mempool/next block to change the state of the IBC tree type provableStore struct { + m sync.Mutex bus modules.Bus // used to interact with persistence (passed from IBCHost) name string // store name in storeManager prefix coreTypes.CommitmentPrefix // []byte(name) - cache []*cachedEntry // in-memory cache of local changes to be written to disk + cache map[string]*cachedEntry // in-memory cache of local changes to be written to disk } // newProvableStore returns a new instance of provableStore with the bus and prefix provided @@ -48,7 +50,7 @@ func newProvableStore(bus modules.Bus, prefix coreTypes.CommitmentPrefix) *prova bus: bus, name: string(prefix), prefix: prefix, - cache: make([]*cachedEntry, 0), + cache: make(map[string]*cachedEntry, 0), } } @@ -134,12 +136,14 @@ func (p *provableStore) Set(key, value []byte) error { if err := rwCtx.SetIBCStoreEntry(prefixed, value); err != nil { return err } - p.cache = append(p.cache, &cachedEntry{ + p.m.Lock() + defer p.m.Unlock() + p.cache[string(prefixed)] = &cachedEntry{ storeName: p.name, height: uint64(currHeight), prefixedKey: prefixed, value: value, - }) + } // TODO(#854): Implement emit functions // return emitUpdateStoreEvent(p.prefix, key, value) return nil @@ -159,12 +163,14 @@ func (p *provableStore) Delete(key []byte) error { if err := rwCtx.SetIBCStoreEntry(prefixed, nil); err != nil { return err } - p.cache = append(p.cache, &cachedEntry{ + p.m.Lock() + defer p.m.Unlock() + p.cache[string(prefixed)] = &cachedEntry{ storeName: p.name, height: uint64(currHeight), prefixedKey: prefixed, value: nil, - }) + } // TODO(#854): Implement emit functions // return emitPruneStoreEvent(p.prefix, key) return nil @@ -183,20 +189,24 @@ func (p *provableStore) Root() ics23.CommitmentRoot { return nil } -// CacheEntries writes all local changes to disk and clears the in-memory cache -func (p *provableStore) CacheEntries(store kvstore.KVStore) error { +// FlushEntries writes all local changes to disk and clears the in-memory cache +func (p *provableStore) FlushEntries(store kvstore.KVStore) error { + p.m.Lock() + defer p.m.Unlock() for _, entry := range p.cache { key, value := entry.prepare() if err := store.Set(key, value); err != nil { return err } + delete(p.cache, string(entry.prefixedKey)) } - p.cache = make([]*cachedEntry, 0) return nil } // PruneCache removes all entries from the cache at the given height func (p *provableStore) PruneCache(store kvstore.KVStore, height uint64) error { + p.m.Lock() + defer p.m.Unlock() keys, _, err := store.GetAll([]byte(fmt.Sprintf("%s/%d", p.name, height)), false) if err != nil { return err @@ -211,23 +221,25 @@ func (p *provableStore) PruneCache(store kvstore.KVStore, height uint64) error { // RestoreCache loads all entries from disk into the cache func (p *provableStore) RestoreCache(store kvstore.KVStore) error { + p.m.Lock() + defer p.m.Unlock() keys, values, err := store.GetAll(p.prefix, false) if err != nil { return err } - for i := 0; i < len(keys); i++ { - parts := strings.SplitN(string(keys[i]), "/", 2) // name, heightStr, prefixedKeyStr + for i, key := range keys { + parts := strings.SplitN(string(key), "/", 2) // name, heightStr, prefixedKeyStr height, err := strconv.ParseUint(parts[1], 10, 64) if err != nil { return err } value := values[i] - p.cache = append(p.cache, &cachedEntry{ + p.cache[parts[2]] = &cachedEntry{ storeName: parts[0], height: height, prefixedKey: []byte(parts[2]), value: value, - }) + } } return nil } diff --git a/ibc/store/store_manager.go b/ibc/store/store_manager.go index 62edc0caa..e7b028e6b 100644 --- a/ibc/store/store_manager.go +++ b/ibc/store/store_manager.go @@ -66,8 +66,8 @@ func (s *storeManager) RemoveStore(name string) error { return nil } -// CacheAllEntries caches all the entries for all stores in the storeManager -func (s *storeManager) CacheAllEntries() error { +// FlushAllEntries caches all the entries for all stores in the storeManager +func (s *storeManager) FlushAllEntries() error { s.m.Lock() defer s.m.Unlock() disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) @@ -75,7 +75,7 @@ func (s *storeManager) CacheAllEntries() error { return err } for _, store := range s.stores { - if err := store.CacheEntries(disk); err != nil { + if err := store.FlushEntries(disk); err != nil { return err } } diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index f12b0871c..74fe6599c 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -355,8 +355,7 @@ func (t *treeStore) updateIBCTree(keys, values [][]byte) error { if len(keys) != len(values) { return fmt.Errorf("keys and values must be the same length") } - for i := 0; i < len(keys); i++ { - key := keys[i] + for i, key := range keys { value := values[i] if value == nil { if err := t.merkleTrees[IBCTreeName].tree.Delete(key); err != nil { diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index 638e1f6dd..fb8f76129 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -212,7 +212,7 @@ type IBCStoreManager interface { AddStore(name string) error GetStore(name string) (ProvableStore, error) RemoveStore(name string) error - CacheAllEntries() error + FlushAllEntries() error PruneCaches(height uint64) error RestoreCaches() error } @@ -228,7 +228,7 @@ type ProvableStore interface { Delete(key []byte) error GetCommitmentPrefix() coreTypes.CommitmentPrefix Root() ics23.CommitmentRoot - CacheEntries(kvstore.KVStore) error + FlushEntries(kvstore.KVStore) error PruneCache(store kvstore.KVStore, height uint64) error RestoreCache(kvstore.KVStore) error } diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index acfea1473..55e338fa5 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -145,6 +145,9 @@ type PersistenceWriteContext interface { SetFlag(paramName string, value any, enabled bool) error // IBC Operations + // SetIBCStoreEntry sets the key-value pair in the ibc_entries table at the current height the + // key-value pairs represent the same key-value pairings in the IBC state tree. This table is + // used for data retrieval purposes and to update the state tree from the mempool of IBC transactions. SetIBCStoreEntry(key, value []byte) error // Relay Operations @@ -241,6 +244,7 @@ type PersistenceReadContext interface { GetBytesFlag(paramName string, height int64) ([]byte, bool, error) // IBC Queries + // GetIBCStoreEntry returns the value of the key at the given height from the ibc_entries table GetIBCStoreEntry(key []byte, height int64) ([]byte, error) } diff --git a/utility/unit_of_work/gov.go b/utility/unit_of_work/gov.go index 0c71c9418..29a561a47 100644 --- a/utility/unit_of_work/gov.go +++ b/utility/unit_of_work/gov.go @@ -150,7 +150,7 @@ func (u *baseUtilityUnitOfWork) getParamOwner(paramName string) ([]byte, coreTyp func (u *baseUtilityUnitOfWork) getFee(msg typesUtil.Message, actorType coreTypes.ActorType) (amount *big.Int, err coreTypes.Error) { switch x := msg.(type) { - // TECHDEBT(M6): Decide on IBC store change fees and move into a governance parameter + // TECHDEBT(M6): Decide on IBC store tx fees and move into a governance parameter case *typesUtil.MessageSend, *ibcTypes.UpdateIBCStore, *ibcTypes.PruneIBCStore: return getGovParam[*big.Int](u, typesUtil.MessageSendFee) case *typesUtil.MessageStake: From 733f53ba4188339f28d92fca04e8dcf063215e0b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:58:37 +0100 Subject: [PATCH 73/74] merge: squash and merge main --- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/golangci-lint.yml | 40 ++ .github/workflows/main.yml | 10 - .golangci.yml | 4 +- Makefile | 14 +- README.md | 2 +- app/client/cli/consensus.go | 1 + app/client/cli/debug.go | 4 +- app/client/cli/helpers/setup.go | 20 +- build/linters/blockers.go | 19 + build/linters/errs.go | 2 +- build/linters/tests.go | 2 +- build/localnet/Tiltfile | 7 +- consensus/events.go | 2 +- consensus/leader_election/module.go | 2 +- consensus/module.go | 4 +- consensus/pacemaker/module.go | 2 +- docs/devlog/devlog10.md | 154 +++++ ibc/module.go | 2 +- ibc/store/proofs_ics23.go | 1 - logger/module.go | 2 +- p2p/README.md | 575 +++++++++++++++--- p2p/background/router.go | 370 +++++++++-- p2p/background/router_test.go | 217 +++++-- p2p/bootstrap.go | 14 +- p2p/config/config.go | 18 +- p2p/event_handler.go | 35 +- p2p/module.go | 212 +++++-- p2p/module_raintree_test.go | 36 +- p2p/module_test.go | 8 +- p2p/protocol/protocol.go | 18 +- .../current_height_provider.go | 2 +- .../current_height_provider/rpc/provider.go | 4 +- .../peerstore_provider/peerstore_provider.go | 4 +- .../persistence/provider.go | 8 +- .../peerstore_provider/rpc/provider.go | 56 +- p2p/raintree/peers_manager_test.go | 12 +- p2p/raintree/router.go | 58 +- p2p/raintree/router_test.go | 2 + p2p/testutil.go | 48 ++ p2p/transport_encryption_test.go | 12 +- p2p/types/errors.go | 9 +- p2p/types/proto/background.proto | 13 + p2p/{raintree => }/types/proto/raintree.proto | 0 p2p/types/router.go | 3 +- p2p/unicast/router.go | 2 +- p2p/utils/host.go | 9 +- p2p/utils_test.go | 67 +- persistence/actor.go | 2 +- persistence/local/module.go | 4 +- persistence/module.go | 2 +- persistence/trees/trees.go | 7 +- rpc/module.go | 2 +- rpc/noop_module.go | 2 +- rpc/server.go | 6 +- runtime/bus.go | 4 +- runtime/manager.go | 2 +- runtime/modules_registry.go | 10 +- shared/messaging/envelope.go | 7 +- .../base_modules/integratable_module.go | 14 +- shared/modules/bus_module.go | 2 +- shared/modules/doc/README.md | 399 ++++++++++-- shared/modules/module.go | 30 +- shared/modules/modules_registry_module.go | 4 +- shared/modules/treestore_module.go | 2 +- shared/modules/utility_module.go | 2 +- state_machine/module.go | 4 +- telemetry/module.go | 2 +- telemetry/prometheus_module.go | 2 +- utility/doc/E2E_FEATURE_PATH_TEMPLATE.md | 18 +- utility/fisherman/module.go | 2 +- utility/module.go | 2 +- utility/servicer/module.go | 2 +- utility/unit_of_work/module.go | 2 +- utility/validator/module.go | 2 +- 75 files changed, 2155 insertions(+), 491 deletions(-) create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 build/linters/blockers.go create mode 100644 docs/devlog/devlog10.md create mode 100644 p2p/testutil.go create mode 100644 p2p/types/proto/background.proto rename p2p/{raintree => }/types/proto/raintree.proto (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 27adce6af..7acf0c407 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -32,7 +32,8 @@ Please mark the relevant option(s): ## List of changes - Change #1 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 000000000..12355c894 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,40 @@ +name: golangci-lint +# Copied from https://github.com/golangci/golangci-lint-action + +on: + pull_request: + types: [opened, reopened, synchronize] + +permissions: + contents: read + +env: + # Even though we can test against multiple versions, this one is considered a target version. + TARGET_GOLANG_VERSION: "1.19" + PROTOC_VERSION: "3.19.4" + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: ${{ env.TARGET_GOLANG_VERSION }} + cache: false + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: ${{ env.PROTOC_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install CI dependencies + run: make install_ci_deps + - name: generate protobufs, RPC server, RPC client and mocks + run: make protogen_clean && make protogen_local && make mockgen && make generate_rpc_openapi + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: latest + args: --timeout=10m --build-tags=test + skip-cache: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4afc72d67..9cfe65d92 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,6 @@ on: # - "docs/**" # - "**.md" - env: # Even though we can test against multiple versions, this one is considered a target version. TARGET_GOLANG_VERSION: "1.19" @@ -87,14 +86,6 @@ jobs: uses: codecov/codecov-action@v3 with: files: ./coverage.out - - name: golangci-lint - if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} - uses: golangci/golangci-lint-action@v3 - with: - version: latest - args: --timeout=10m --build-tags=test - skip-cache: true - only-new-issues: true # TODO(@okdas): reuse artifacts built by the previous job instead # of going through the build process in container build job again @@ -153,7 +144,6 @@ jobs: build-args: | TARGET_GOLANG_VERSION=${{ env.TARGET_GOLANG_VERSION }} - # Run e2e tests on devnet if the PR has a label "e2e-devnet-test" e2e-tests: runs-on: custom-runner diff --git a/.golangci.yml b/.golangci.yml index 8981308b0..bf024354e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,18 +7,16 @@ linters: - govet - ineffassign - staticcheck - - typecheck - unused - # Additional linters - gosec - misspell - promlinter - unparam - goimports - # Gocritic allows to use ruleguard; custom linting rules - gocritic + linters-settings: # Gocritic settings; https://go-critic.com/overview gocritic: diff --git a/Makefile b/Makefile index 0db3d4fd0..1d1831bb0 100644 --- a/Makefile +++ b/Makefile @@ -120,13 +120,17 @@ go_fmt: ## Format all the .go files in the project in place. gofmt -w -s . .PHONY: install_cli_deps -install_cli_deps: ## Installs `protoc-gen-go`, `mockgen`, 'protoc-go-inject-tag' and other tooling +install_cli_deps: ## Installs `helm`, `tilt` and the underlying `ci_deps` + make install_ci_deps + curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +.PHONY: install_ci_deps +install_ci_deps: ## Installs `protoc-gen-go`, `mockgen`, 'protoc-go-inject-tag' and other tools necessary for CI go install "google.golang.org/protobuf/cmd/protoc-gen-go@v1.28" && protoc-gen-go --version go install "github.com/golang/mock/mockgen@v1.6.0" && mockgen --version go install "github.com/favadi/protoc-go-inject-tag@latest" go install "github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.11.0" - curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash - curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash .PHONY: develop_start develop_start: ## Run all of the make commands necessary to develop on the project @@ -297,7 +301,7 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./consensus/types/proto --go_out=./consensus/types ./consensus/types/proto/*.proto # P2P - $(PROTOC_SHARED) -I=./p2p/raintree/types/proto --go_out=./p2p/types ./p2p/raintree/types/proto/*.proto + $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto @@ -591,4 +595,4 @@ ggshield_secrets_scan: ## Scans the project for secrets using ggshield .PHONY: ggshield_secrets_add ggshield_secrets_add: ## A helper that adds the last results from `make ggshield_secrets_scan`, store in `.cache_ggshield` to `.gitguardian.yaml`. See `ggshield for more configuratiosn` - ggshield secret ignore --last-found \ No newline at end of file + ggshield secret ignore --last-found diff --git a/README.md b/README.md index 57ac746ec..5c324ec3a 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ If you'd like to contribute to the Pocket V1 Protocol, start by: - [Shared Architecture](shared/README.md) - [Utility Architecture](utility/doc/README.md) -- _Coming Soon: Consensus Architecture_ // TODO(olshansky): needs a README file with proper code structure +- [Consensus Architecture](consensus/README.md) - [Persistence Architecture](persistence/docs/README.md) - [P2P Architecture](p2p/README.md) - [APP Architecture](app/client/doc/README.md) diff --git a/app/client/cli/consensus.go b/app/client/cli/consensus.go index a74876949..3cb69b32d 100644 --- a/app/client/cli/consensus.go +++ b/app/client/cli/consensus.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "github.com/spf13/cobra" "github.com/pokt-network/pocket/app/client/cli/flags" diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 07ab0cfb2..23b647933 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -228,7 +228,9 @@ func fetchPeerstore(cmd *cobra.Command) (typesP2P.Peerstore, error) { return nil, errors.New("retrieving bus from CLI context") } modulesRegistry := bus.GetModulesRegistry() - pstoreProvider, err := modulesRegistry.GetModule(peerstore_provider.ModuleName) + // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // is retrievable as a proper submodule + pstoreProvider, err := modulesRegistry.GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) if err != nil { return nil, errors.New("retrieving peerstore provider") } diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 376e9348e..5f5546a91 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -27,24 +27,26 @@ func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { bus := runtimeMgr.GetBus() SetValueInCLIContext(cmd, BusCLICtxKey, bus) - setupPeerstoreProvider(*runtimeMgr, flags.RemoteCLIURL) + if err := setupPeerstoreProvider(*runtimeMgr, flags.RemoteCLIURL); err != nil { + return err + } setupCurrentHeightProvider(*runtimeMgr, flags.RemoteCLIURL) setupAndStartP2PModule(*runtimeMgr) return nil } -func setupPeerstoreProvider(rm runtime.Manager, rpcURL string) { - bus := rm.GetBus() - modulesRegistry := bus.GetModulesRegistry() - pstoreProvider := rpcPSP.Create( - rpcPSP.WithP2PConfig(rm.GetConfig().P2P), - rpcPSP.WithCustomRPCURL(rpcURL), - ) - modulesRegistry.RegisterModule(pstoreProvider) +func setupPeerstoreProvider(rm runtime.Manager, rpcURL string) error { + // Ensure `PeerstoreProvider` exists in the modules registry. + if _, err := rpcPSP.Create(rm.GetBus(), rpcPSP.WithCustomRPCURL(rpcURL)); err != nil { + return err + } + return nil } func setupCurrentHeightProvider(rm runtime.Manager, rpcURL string) { + // TECHDEBT(#810): simplify after current height provider is refactored as + // a submodule. bus := rm.GetBus() modulesRegistry := bus.GetModulesRegistry() currentHeightProvider := rpcCHP.NewRPCCurrentHeightProvider( diff --git a/build/linters/blockers.go b/build/linters/blockers.go new file mode 100644 index 000000000..b139447bd --- /dev/null +++ b/build/linters/blockers.go @@ -0,0 +1,19 @@ +//nolint // it's not a Go code file +//go:build !codeanalysis +// +build !codeanalysis + +// This file includes our custom linters. +// If you want to add/modify an existing linter, please check out ruleguard's documentation: https://github.com/quasilyte/go-ruleguard#documentation + +package gorules + +import ( + "github.com/quasilyte/go-ruleguard/dsl" +) + +// Blocks merge if IN_THIS_ comments are present +func BlockInThisCommitPRComment(m dsl.Matcher) { + m.MatchComment(`//.*IN_THIS_.*`). + Where(!m.File().Name.Matches(`Makefile`) && !m.File().Name.Matches(`blockers.go`)). + Report("'IN_THIS_' comments must be addressed before merging to main") +} diff --git a/build/linters/errs.go b/build/linters/errs.go index bb14ee54d..f63b363fa 100644 --- a/build/linters/errs.go +++ b/build/linters/errs.go @@ -11,7 +11,7 @@ import ( "github.com/quasilyte/go-ruleguard/dsl" ) -// This is a custom linter that checks ensures a use of inline error checks +// This is a custom linter that ensures a use of inline error checks func InlineErrCheck(m dsl.Matcher) { m.Match(`$err := $x; if $err != nil { $*_ }`). Where(m["err"].Type.Is(`error`)). diff --git a/build/linters/tests.go b/build/linters/tests.go index 3b95049db..9b0ee71a2 100644 --- a/build/linters/tests.go +++ b/build/linters/tests.go @@ -11,7 +11,7 @@ import ( "github.com/quasilyte/go-ruleguard/dsl" ) -// This is a custom linter that checks ensures a use of require.Equal +// This is a custom linter that ensures the use of require.Equal func EqualInsteadOfTrue(m dsl.Matcher) { m.Match(`require.True($t, $x == $y, $*args)`). Suggest(`require.Equal($t, $x, $y, $args)`). diff --git a/build/localnet/Tiltfile b/build/localnet/Tiltfile index 31e867fdf..9f5a36b4c 100644 --- a/build/localnet/Tiltfile +++ b/build/localnet/Tiltfile @@ -120,8 +120,11 @@ docker_build_with_restart( "client-image", root_dir, dockerfile_contents="""FROM debian:bullseye -RUN apt-get update && apt-get install -y procps -WORKDIR / +RUN apt-get update && apt-get install -y procps bash-completion jq +RUN echo "source /etc/bash_completion" >> ~/.bashrc +# tail -n +2 removes the first line of the completion script since the CLI spits out some logs +RUN echo "source <(p1 completion bash | tail -n +2)" >> ~/.bashrc +WORKDIR /root COPY bin/p1-linux /usr/local/bin/p1 """, only=["bin/p1-linux"], diff --git a/consensus/events.go b/consensus/events.go index 0e31f8d72..5da26aba8 100644 --- a/consensus/events.go +++ b/consensus/events.go @@ -4,7 +4,7 @@ import ( "github.com/pokt-network/pocket/shared/messaging" ) -// publishNewHeightEvent publishes a new height event to the bus so that other interested IntegratableModules can react to it if necessary +// publishNewHeightEvent publishes a new height event to the bus so that other interested IntegrableModules can react to it if necessary func (m *consensusModule) publishNewHeightEvent(height uint64) { newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) if err != nil { diff --git a/consensus/leader_election/module.go b/consensus/leader_election/module.go index db419c800..25a38024c 100644 --- a/consensus/leader_election/module.go +++ b/consensus/leader_election/module.go @@ -14,7 +14,7 @@ type LeaderElectionModule interface { var _ LeaderElectionModule = &leaderElectionModule{} type leaderElectionModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule } diff --git a/consensus/module.go b/consensus/module.go index 339378096..2b32b62eb 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -31,7 +31,7 @@ const ( ) type consensusModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule privateKey cryptoPocket.Ed25519PrivateKey @@ -209,7 +209,7 @@ func (m *consensusModule) GetModuleName() string { } func (m *consensusModule) SetBus(pocketBus modules.Bus) { - m.IntegratableModule.SetBus(pocketBus) + m.IntegrableModule.SetBus(pocketBus) if m.paceMaker != nil { m.paceMaker.SetBus(pocketBus) } diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index 2219d82aa..3a9a16db1 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -45,7 +45,7 @@ type Pacemaker interface { } type pacemaker struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule pacemakerCfg *configs.PacemakerConfig diff --git a/docs/devlog/devlog10.md b/docs/devlog/devlog10.md new file mode 100644 index 000000000..9de0f019f --- /dev/null +++ b/docs/devlog/devlog10.md @@ -0,0 +1,154 @@ +# Pocket V1 DevLog #10 + +**Date Published**: July 5th, 2023 + +We have kept the goals and details in this document short, but feel free to reach out to @Olshansk in the [core-dev-chat](https://discord.com/channels/553741558869131266/986789914379186226) for additional details, links & resources. + +## Table of Contents + +- [Iteration 20 Goals \& Results](#iteration-20-goals--results) + - [V0](#v0) + - [M1: PoS](#m1-pos) + - [M2: DoS](#m2-dos) + - [M3: RoS](#m3-ros) + - [M7: IBC](#m7-ibc) +- [Contribute to V1 🧑‍💻](#contribute-to-v1-) + - [Links \& References](#links--references) +- [ScreenShots](#screenshots) + - [Iteration 19 - Completed](#iteration-19---completed) + - [Iteration 20 - Planned](#iteration-20---planned) + +## Iteration 20 Goals & Results + +**Iterate Dates**: June 15th - June 30th, 2023 + +🔥 The team frikkin killed it 🔥 + +```bash +git diff 2074a1b0c27ec2c73168b02852cc6145b657c0af --stat +# 107 files changed, 9032 insertions(+), 10508 deletions(-) +``` + +Note that this exclude the work we did on infrastructure support, internal documentation, v0 work, SMT repo, collaboration with other projects and a lot more that happens behind the scenes! + +### V0 + +- 🟢 **TestNet Rehearsal** + + - 100% completeness of TestNet Rehearsal + - **Grade**: 10 / 10 + - Completed + +- 🟢 **MainNet Rehearsal** + - Have 5% of MainNet run the latest beta (w/o protocol upgrade) + - **Grade**: 10 / 10 + - Completed and found some bugs along the way too when synching from scratch + +### M1: PoS + +- 🔴 Consensus + + - Attempt #1: Remove State Sync dependency on FSM + - Attempt #3: finish minimum viable state sync + - **Grade**: 1 / 10 + - Very little time left to work on this + +- 🟢 Persistence + + - MVP of the full commit & rollback DEMO + - **Grade**: 7 / 10 + - The test which was going to be the demo has fought me more than expected but good progress has been made, there’s a design document ready, and the test harness is there, the mocks and the submodule interactions have been the problem. + +- 🟢 P2P + - Attempt #N: Finishing off and merging in everything related to gossip and background + - **Grade**: 8.5 / 10 + +### M2: DoS + +- 🔴 **Primary focus: observability** + - Open question: need to identify issues w/ metric access + - Streamlining logging: Make structured logging system easily available to new devs w/ documentation part of LocalNet instructions + - Attach smaller tickets in a separate repo to V2 + - **Grade**: 0 / 10 + - Other infrastructure related maintenance issues took away time from being able to focus on observability + +### M3: RoS + +- 🟢 **Trustless Relay** + + - Session caching on the client + - Finish all the PRs in flight (review, merge in) + - Provide an E2E test that works, blocks CI if it breaks, documented and visible (DEMO) + - **Grade**: 8 / 10 + +- 🟡 **Feature Flags** + - Scope out the work necessary and create an E2E Feature Path github ticket using the template we created + - **Grade**: 5 / 10 + - Research and design doc made good progress w/ support from bigBoss bus still a lot to do. + +### M7: IBC + +- 🟢 **SMST** + + - Get it reviewed & merged in + - Clean up the documentation & merge it in + - Visualizers: create a visualizer for the tree + - Present: Finish off the SMT presentation + - Stretch goal: potentially start storing trustless relays in it + - **Grade**: 8.5 / 10 + - SMST merged (wrapper around SMT option) + - Visualiser is accurate but not pretty could do with some more work + - Presentation went well but definietly could improve on some packed slides + - Need to work closer with @Arash Deshmeh to get it in prod with M3 + +- 🟢 **ICS23** + + - Put up the github ticket and PR for reivew to merge in the proof mechanisms + - Up to cosmos on ETA to review/merge + - **Grade**: 9 / 10 + - ICS23 merged in our repo using my fork of `cosmos/ics23` as a dependency + - My explanations on why the exclusion proof is needed can improve as others find it hard to understand + - Cosmos PR is ready to merge pending review (probably take a while) + +- 🟡 **ICS24** + + - Put up event logging for review; stretch goal is to merge + - **Grade**: 6 / 10 + - ICS-24 stores have made great progress + - Event logging unfortunately didnt make this fortnight + - Message onto/off of bus as transactions works well 👍🏻 + +- 🟡 **Light client spike** + - Start knowing where to head with research + - **Grade**: 5 / 10 + - ICS-02 specced out well + - ICS-08 needs more work into its design + - Need to learn more about CosmWasm and WasmVM + - WIP document needs to be converted to ticket epic + - https://hackmd.io/0WVMarGpSIGqEyzvnygWpw + +## Contribute to V1 🧑‍💻 + +### Links & References + +- [V1 Specifications](https://github.com/pokt-network/pocket-network-protocol) +- [V1 Repo](https://github.com/pokt-network/pocket) +- [V1 Wiki](https://github.com/pokt-network/pocket/wiki) +- [V1 Project Dashboard](https://github.com/pokt-network/pocket/projects?query=is%3Aopen) + +## ScreenShots + +Please note that everything that was not `Done` in iteration19 is moving over to iteration20. + +### Iteration 19 - Completed + +![Iteration19_1](https://github.com/pokt-network/pocket/assets/1892194/93f033e9-a408-49bf-9531-9f84cc1bc254) +![Iteration19_2](https://github.com/pokt-network/pocket/assets/1892194/2c600d90-fe4c-496b-a4e4-66ef0afb4771) + +### Iteration 20 - Planned + +_tl;dr Aim to demo as much of the work from the previous iteration in action_ + +![Iteration20](![Screenshot 2023-07-05 at 1 35 07 PM](https://github.com/pokt-network/pocket-core/assets/1892194/8ae047ee-f186-4e1a-8ced-14764ec83886)) + + diff --git a/ibc/module.go b/ibc/module.go index d8685059c..d1382c4db 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -20,7 +20,7 @@ import ( var _ modules.IBCModule = &ibcModule{} type ibcModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule m sync.Mutex diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 53e0b203d..9d0e22961 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -75,7 +75,6 @@ func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof if err != nil { return nil, coreTypes.ErrIBCCreatingProof(err) } - return convertSMPToExclusionProof(proof, key), nil } diff --git a/logger/module.go b/logger/module.go index 091c4d5b5..961b1c4a2 100644 --- a/logger/module.go +++ b/logger/module.go @@ -14,7 +14,7 @@ import ( var _ modules.Module = &loggerModule{} type loggerModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule zerolog.Logger diff --git a/p2p/README.md b/p2p/README.md index 8263cd563..cde1a155c 100644 --- a/p2p/README.md +++ b/p2p/README.md @@ -5,11 +5,15 @@ This document is meant to be a supplement to the living specification of [1.0 Po ## Table of Contents - [Definitions](#definitions) -- [Interface](#interface) -- [Implementation](#implementation) - - [Code Architecture - P2P Module](#code-architecture---p2p-module) - - [Code Architecture - Network Module](#code-architecture---network-module) - - [Code Organization](#code-organization) +- [Interface & Integration](#interface--integration) +- [Module Architecture](#module-architecture) + - [Architecture Design Language](#architecture-design-language) + - [Legends](#legends) + - [P2P Module / Router Decoupling](#p2p-module--router-decoupling) + - [Message Propagation & Handling](#message-propagation--handling) + - [Message Deduplication](#message-deduplication) + - [Peer Discovery](#peer-discovery) + - [Code Organization](#code-organization) - [Testing](#testing) - [Running Unit Tests](#running-unit-tests) - [RainTree testing framework](#raintree-testing-framework) @@ -34,14 +38,11 @@ A structured "gossip" protocol (and implementation) which uses the raintree algo A "gossip" protocol (implementation TBD) which facilitates "gossip" to all P2P participants, including non-staked actors (e.g. full-nodes). -## Interface +## Interface & Integration -This module aims to implement the interface specified in `pocket/shared/modules/p2p_module.go` using the specification above. - -## Implementation - -### P2P Module Architecture +This module aims to implement the interface specified in [`pocket/shared/modules/p2p_module.go`](../shared/modules/p2p_module.go). +_(TODO: diagram legend)_ ```mermaid flowchart TD subgraph P2P["P2P Module"] @@ -67,105 +68,508 @@ flowchart TD class PN pocket_network ``` -`Routers` is where [RainTree](https://github.com/pokt-network/pocket/files/9853354/raintree.pdf) (or the simpler basic approach) is implemented. See `raintree/router.go` for the specific implementation of RainTree, but please refer to the [specifications](https://github.com/pokt-network/pocket-network-protocol/tree/main/p2p) for more details. +`Routers` is where [RainTree](https://github.com/pokt-network/pocket/files/9853354/raintree.pdf) is implemented. +See [`raintree/router.go`](./raintree/router.go) for the specific implementation of RainTree, but please refer to the [specifications](https://github.com/pokt-network/pocket-network-protocol/tree/main/p2p) for more details. -### Raintree Router Architecture +## Module Architecture + +_(TODO: move "arch. design lang." & "legends" sections into `shared` to support common usage)_ -_DISCUSS(team): If you feel this needs a diagram, please reach out to the team for additional details._ -_TODO(olshansky, BenVan): Link to RainTree visualizations once it is complete._ +### Architecture Design Language -### Message Propagation +The architecture design language expressed in this documentation is based on [UML](https://www.uml-diagrams.org/). +Due to limitations in the current version of mermaid, class diagrams are much more adherant to the UML component specification. +Component diagrams however are much more loosely inspired by their UML counterparts. -Given `Local P2P Module` has a message that it needs to propagate: +Regardless, each architecture diagram should be accompanied by a legend which covers all the design language features used to provide disambiguation. - +References: +- [Class Diagrams](https://www.uml-diagrams.org/class-diagrams-overview.html) +- [Component Diagrams](https://www.uml-diagrams.org/component-diagrams.html) + + _NOTE: mermaid does not support ports, interfaces, ... in component diagrams ("flowcharts)._ + +### Legends ```mermaid -flowchart TD - subgraph lMod[Local P2P Module] - subgraph lHost[Libp2p `Host`] +flowchart +subgraph Legend + m[[`Method`]] + c[Component] + + m -- "unconditional usage" --> c + m -. "conditional usage" .-> c + m -. "ignored" .-x c +end +``` + +```mermaid +classDiagram +class ConcreteType { + +ExportedField + -unexportedField + +ExportedMethod(...argTypes) (...returnTypes) + -unexportedMethod(...argTypes) (...returnTypes) +} + +class InterfaceType { + <> + +Method(...argTypes) (...returnTypes) +} + +ConcreteType --|> InterfaceType : Interface realization + +ConcreteType --> OtherType : Direct usage +ConcreteType --o OtherType : Composition +ConcreteType --* OtherType : Aggregatation +ConcreteType ..* "(cardinality)" OtherType : Indirect (via interface) +``` + +#### Interface Realization + +_TL;DR An instance (i.e. client) implements the associated interface (i.e. supplierl)._ + +> Realization is a specialized abstraction relationship between two sets of model elements, one representing a specification (the supplier) and the other represents an implementation of the latter (the client). + +> Realization can be used to model stepwise refinement, optimizations, transformations, templates, model synthesis, framework composition, etc. + +_(see: [UML Realization](https://www.uml-diagrams.org/realization.html))_ + +#### Direct Usage + +_TL;DR one instance (i.e. client) is dependent the associated instance(s) (i.e. supplier) to function properly._ + +> Dependency is a directed relationship which is used to show that some UML element or a set of elements requires, needs or depends on other model elements for specification or implementation. Because of this, dependency is called a supplier - client relationship, where supplier provides something to the client, and thus the client is in some sense incomplete while semantically or structurally dependent on the supplier element(s). Modification of the supplier may impact the client elements. + +> Usage is a dependency in which one named element (client) requires another named element (supplier) for its full definition or implementation. + +_(see: [UML Dependency](https://www.uml-diagrams.org/dependency.html))_ + +#### Composition + +_TL;DR deleting an instance also deletes the associated instance(s)._ + +> A "strong" form of aggregation + +> If a composite (whole) is deleted, all of its composite parts are "normally" deleted with it. + +_(see: [UML Shared composition](https://www.uml-diagrams.org/composition.html))_ + +#### Aggregation + + +_TL;DR deleting an instance does not necessarily delete the associated instance(s)._ + +> A "weak" form of aggregation + +> Shared part could be included in several composites, and if some or all of the composites are deleted, shared part may still exist. + +_(see: [UML Shared aggregation](https://www.uml-diagrams.org/aggregation.html))_ + +#### Cardinality + +_TL;DR indicates a number, or range of instances associated (i.e. supplier(s))_ + +Cardinality indicates the number or range of simultaneous instances of supplier that are associated with the client. +Applicable to multiple association types. +Can be expressed arbitrarily (e.g. wildcards, variable, equation, etc.) + +_(see: [UML Association](https://www.uml-diagrams.org/association.html#association-end))_ + + +### P2P Module / Router Decoupling + +The P2P module encapsulates the `RainTreeRouter` and `BackgroundRouter` submodules. +The P2P module internally refers to these as the `stakedActorRouter` and `unstakedActorRouter`, respectively. + +Depending on the necessary routing scheme (unicast / broadcast) and whether the peers involved are staked actors, a node will use one or both of these routers. + +**Unicast** + +| Sender | Receiver | Router | Example Usage | +|----------------|----------------|-----------------|----------------------------------------------------------------------| +| Staked Actor | Staked Actor | Raintree only | Consensus hotstuff messages (validators only) & state sync responses | +| Staked Actor | Untaked Actor | Background only | Consensus state sync responses | +| Unstaked Actor | Staked Actor | Background only | Consensus state sync responses, debug messages | +| Unstaked Actor | Unstaked Actor | Background only | Consensus state sync responses, debug messages | + +**Broadcast** + +| Broadcaster | Receiver | Router | Example Usage | +|----------------|----------------|-----------------------|-----------------------------------------------------------------| +| Staked Actor | Staked Actor | Raintree + Background | Utility tx messages, consensus state sync requests | +| Staked Actor | Untaked Actor | Background only | Utility tx messages (redundancy), consensus state sync requests | +| Unstaked Actor | Staked Actor | Background only | Utility tx messages (redundancy), consensus state sync requests | +| Unstaked Actor | Unstaked Actor | Background only | Utility tx messages, consensus state sync requests | + +Both router submodule implementations embed a `UnicastRouter` which enables them to send and receive messages directly to/from a single peer. + +**Class Diagram** + +```mermaid +classDiagram + class p2pModule { + -stakedActorRouter Router + -unstakedActorRouter Router + -handlePocketEnvelope([]byte) error + } + + class P2PModule { + <> + GetAddress() (Address, error) + HandleEvent(*anypb.Any) error + Send([]byte, Address) error + Broadcast([]byte) error + } + p2pModule --|> P2PModule + + class RainTreeRouter { + UnicastRouter + -handler MessageHandler + +Broadcast([]byte) error + -handleRainTreeMsg([]byte) error + } + + class BackgroundRouter { + UnicastRouter + -handler MessageHandler + +Broadcast([]byte) error + -handleBackgroundMsg([]byte) error + -readSubscription(subscription *pubsub.Subscription) + } + + class UnicastRouter { + -messageHandler MessageHandler + -peerHandler PeerHandler + +Send([]byte, Address) error + -handleStream(libp2pNetwork.Stream) + -readStream(libp2pNetwork.Stream) + } + RainTreeRouter --* UnicastRouter : (embedded) + BackgroundRouter --* UnicastRouter : (embedded) + + p2pModule --o "2" Router + p2pModule ..* RainTreeRouter : (`stakedActorRouter`) + p2pModule ..* BackgroundRouter : (`unstakedActorRouter`) + + class Router { + <> + +Send([]byte, Address) error + +Broadcast([]byte) error + } + BackgroundRouter --|> Router + RainTreeRouter --|> Router +``` + +### Message Propagation & Handling + +**Unicast** + +```mermaid +flowchart + subgraph lp2p["Local P2P Module (outgoing)"] + lps[[`Send`]] + lps -. "(iff local & remote peer are staked)" ..-> lrtu + lps -. "(if local or remote peer are not staked)" .-> lbgu + + lbgu -- "opens stream\nto target peer" ---> lhost + + lhost[Libp2p Host] + + subgraph lrt[RainTree Router] + subgraph lRTPS[Raintree Peerstore] + lStakedPS([staked actors only]) + end + + lrtu[UnicastRouter] + + lrtu -- "network address lookup" --> lRTPS + end + + lrtu -- "opens a stream\nto target peer" ---> lhost + + subgraph lbg[Background Router] + lbgu[UnicastRouter] + subgraph lBGPS[Background Peerstore] + lNetPS([all P2P participants]) + end + + lbgu -- "network address lookup" --> lBGPS + end + end + + subgraph rp2p["Remote P2P Module (incoming)"] + rhost[Libp2p Host] + + subgraph rrt[RainTree Router] + rrth[[`RainTreeMessage` Handler]] + rrtu[UnicastRouter] + end + + subgraph rbg[Background Router] + rbgh[[`BackgroundMessage` Handler]] + rbgu[UnicastRouter] + rbgu --> rbgh + end + + rp2ph[[`PocketEnvelope` Handler]] + rbus[bus] + rhost -. "new stream" .-> rrtu + rhost -- "new subscription message" --> rbgu + rrtu --> rrth + + rnd[Nonce Deduper] + rp2ph -- "deduplicate msg mempool" --> rnd end - subgraph lRT[Raintree Router] + + + rp2ph -. "(iff not duplicate msg)\npublish event" .-> rbus + + rrth --> rp2ph + rbgh --> rp2ph + + lhost --> rhost +``` + +**Broadcast** + +```mermaid +flowchart + subgraph lp2p["Local P2P Module (outgoing)"] + lpb[[`Broadcast`]] + lpb -. "(iff local & remote peer are staked)" ..-> lrtu + lpb -- "(always)" --> lbggt + + lbggt -- "msg published\n(gossipsub protocol)" ---> lhost + + lhost[Libp2p Host] + + subgraph lrt[RainTree Router] subgraph lRTPS[Raintree Peerstore] lStakedPS([staked actors only]) end - - subgraph lPM[PeerManager] - end - lPM --> lRTPS + + lrtu[UnicastRouter] + + lrtu -- "network address lookup" --> lRTPS end + + lrtu -- "opens a stream\nto target peer" ---> lhost - subgraph lBG[Background Router] + subgraph lbg[Background Router] + lbggt[Gossipsub Topic] subgraph lBGPS[Background Peerstore] lNetPS([all P2P participants]) end + + lbggt -- "network address lookup" --> lBGPS + end + end - subgraph lGossipSub[GossipSub] - end + subgraph rp2p["Remote P2P Module (incoming)"] + rhost[Libp2p Host] - subgraph lDHT[Kademlia DHT] - end + subgraph rrt[RainTree Router] + rrth[[`RainTreeMessage` Handler]] + rrtu[UnicastRouter] + end - lGossipSub --> lBGPS - lDHT --> lBGPS + subgraph rbg[Background Router] + rbgh[[`BackgroundMessage` Handler]] + rbgg[Gossipsub Subscription] + rbggt[Gossipsub Topic] + rbgg --> rbgh + rbgh -- "(background msg\npropagation cont.)" ---> rbggt end - lRT --1a--> lHost - lBG --1b--> lHost + rp2ph[[`PocketEnvelope` Handler]] + rbus[bus] + rhost -. "new stream" ..-> rrtu + rhost -- "new subscription message" --> rbgg + rbggt -- "(background msg\npropagation cont.)" --> rhost + rrtu --> rrth + rrth -. "(iff level > 0)\n(raintree msg\npropagation cont.)" .-> rrtu + rrtu -- "(raintree msg\npropagation cont.)" --> rhost + + rnd[Nonce Deduper] + rp2ph -- "deduplicate msg mempool" --> rnd end -subgraph rMod[Remote P2P Module] -subgraph rHost[Libp2p `Host`] -end -subgraph rRT[Raintree Router] -subgraph rPS[Raintree Peerstore] -rStakedPS([staked actors only]) -end -subgraph rPM[PeerManager] -end + rp2ph -. "(iff not duplicate msg)\npublish event" .-> rbus -rPM --> rStakedPS -end + rrth --> rp2ph + rbgh --> rp2ph -subgraph rBG[Background Router] -subgraph rBGPS[Background Peerstore] -rNetPS([all P2P participants]) -end + lhost --> rhost +``` -subgraph rGossipSub[GossipSub] -end +### Message Deduplication -subgraph rDHT[Kademlia DHT] -end +Messages MUST be deduplicated before broadcasting their respective event over the bus since it is expected that nodes will receive duplicate messages (for multiple reasons). -rGossipSub --> rBGPS -rDHT --> rBGPS -end +The responsibility of deduplication is encapsulated by the P2P module, As such duplicate messages may come from multiple routers in some of these scenarios. -rHost -. "3 (setStreamHandler())" .-> hs[[handleStream]] +The `NondeDeduper` state is not persisted outside of memory and therefore is cleared during node restarts. -hs --4a--> rRT -hs --4b--> rBG -rBG --"5a (cont. propagation)"--> rHost -linkStyle 11 stroke:#ff3 -rRT --"5b (cont. propagation)"--> rHost -linkStyle 12 stroke:#ff3 -end +```mermaid +classDiagram + class RainTreeMessage { + <> + +Level uint32 + +Data []byte + } + + class BackgroundMessage { + <> + +Data []byte + } + + class PocketEnvelope { + <> + +Content *anypb.Any + +Nonce uint64 + } + + RainTreeMessage --* PocketEnvelope : serialized as `Data` + BackgroundMessage --* PocketEnvelope : serialized as `Data` + + + class p2pModule { + -handlePocketEnvelope([]byte) error + } + + class P2PModule { + <> + GetAddress() (Address, error) + HandleEvent(*anypb.Any) error + Send([]byte, address Address) error + Broadcast([]byte) error + } + p2pModule --|> P2PModule + + class RainTreeRouter { + UnicastRouter + -handler MessageHandler + +Broadcast([]byte) error + -handleRainTreeMsg([]byte) error + } + + class NonceDeduper { + Push(Nonce) error + Contains(Nonce) bool + } + + class Bus { + <> + PublishEventToBus(PocketEnvelope) + GetBusEvent() PocketEnvelope + } + p2pModule --> Bus + + class BackgroundRouter { + UnicastRouter + -handler MessageHandler + +Broadcast([]byte) error + -handleBackgroundMsg([]byte) error + -readSubscription(subscription *pubsub.Subscription) + } + + class UnicastRouter { + -messageHandler MessageHandler + -peerHandler PeerHandler + +Send([]byte, address Address) error + -handleStream(stream libp2pNetwork.Stream) + -readStream(stream libp2pNetwork.Stream) + } + RainTreeRouter --* UnicastRouter : (embedded) + BackgroundRouter --* UnicastRouter : (embedded) + + p2pModule ..* RainTreeRouter + RainTreeRouter --o RainTreeMessage + + p2pModule ..* BackgroundRouter + BackgroundRouter --o BackgroundMessage + + p2pModule --o PocketEnvelope + p2pModule --* NonceDeduper +``` + +#### Configuration + +The size of the `NonceDeduper` queue is configurable via the `P2PConfig.MaxNonces` field. -lHost --2--> rHost +### Peer Discovery + +Peer discovery involves pairing peer IDs to their network addresses (multiaddr). +This pairing always has an associated TTL (time-to-live), near the end of which it must +be refreshed. + +In the background gossip overlay network (`backgroundRouter`), peers will re-advertise themselves every 3 hours through their TTL (see: [`RoutingDiscovery#Advertise()`](https://github.com/libp2p/go-libp2p/blob/87c2561238cb0340ddb182c61be8dbbc7a12a780/p2p/discovery/routing/routing.go#L34) and [`ProviderManager#AddProvider()`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.24.2/providers/providers_manager.go#L255)). +This refreshes the libp2p peerstore automatically. + +In the raintree gossip overlay network (`raintreeRouter`), the libp2p peerstore is **NOT** currently refreshed _(TODO: [#859](https://github.com/pokt-network/network/isues/859))_. + +```mermaid +flowchart TD + subgraph bus + end + + subgraph pers[Persistence Module] + end + + subgraph cons[Consensus Module] + end + + cons -- "(staked actor set changed)\npublish event" --> bus + bus --> rPM + rPM -- "get staked actors\nat current height" --> pers + + subgraph p2p["P2P Module"] + host[Libp2p Host] + host -- "incoming\nraintree message" --> rtu + host -- "incoming\nbackground message" --> bgu + host -- "incoming\ntopic message" --> bgr + host -- "DHT peer discovery" --> rDHT + + subgraph rt[RainTree Router] + subgraph rPS[Raintree Peerstore] + rStakedPS([staked actors only]) + end + + subgraph rPM[PeerManager] + end + + rtu[UnicastRouter] + + rPM -- "synchronize\n(add/remove)" --> rPS + rtu -. "(no discovery)" .-x rPS + end + + subgraph bg[Background Router] + subgraph rBGPS[Background Peerstore] + rNetPS([all P2P participants]) + end + + subgraph bgr[GossipSub Topic\nSubscription] + end + + subgraph rDHT[Kademlia DHT] + end + + bgu -- "add if new" --> rBGPS + bgr -- "add if new" --> rBGPS + rDHT -- "continuous import" --> rBGPS + + bgu[UnicastRouter] + end + + end ``` -The `Network Module` is where [RainTree](https://github.com/pokt-network/pocket/files/9853354/raintree.pdf) (or the simpler basic approach) is implemented. See `raintree/network.go` for the specific implementation of RainTree, but please refer to the [specifications](https://github.com/pokt-network/pocket-network-protocol/tree/main/p2p) for more details. +### Raintree Router Architecture + +_NOTE: If you (the reader) feel this needs a diagram, please reach out to the team for additional details._ ### Code Organization @@ -177,6 +581,8 @@ p2p │   └── router_test.go # `BackgroundRouter` functional tests ├── bootstrap.go # `p2pModule` bootstrap related method(s) ├── CHANGELOG.md +├── config +│   └── config.go ├── event_handler.go ├── module.go # `p2pModule` definition ├── module_raintree_test.go # `p2pModule` & `RainTreeRouter` functional tests (routing) @@ -189,21 +595,18 @@ p2p │   ├── peerstore_provider │   └── providers.go ├── raintree -│   ├── nonce_deduper.go -│   ├── nonce_deduper_test.go │   ├── peers_manager.go # `rainTreePeersManager` implementation of `PeersManager` interface │   ├── peers_manager_test.go │   ├── peerstore_utils.go # Raintree routing helpers │   ├── router.go # `RainTreeRouter` implementation of `Router` interface │   ├── router_test.go # `RainTreeRouter` functional tests │   ├── target.go # `target` definition -│   ├── types -│   │   └── proto -│   │   └── raintree.proto +│   ├── testutil.go │   └── utils_test.go -├── README.md +├── testutil.go ├── transport_encryption_test.go # Libp2p transport security integration test ├── types +│   ├── background.pb.go │   ├── errors.go │   ├── libp2p_mocks.go │   ├── mocks @@ -213,12 +616,18 @@ p2p │   ├── peerstore.go # `Peerstore` interface & `PeerAddrMap` implementation definitions │   ├── peers_view.go # `PeersView` interface & `sortedPeersView` implementation definitions │   ├── peers_view_test.go +│   ├── proto │   ├── raintree.pb.go │   └── router.go # `Router` interface definition +├── unicast +│   ├── logging.go +│   ├── router.go +│   └── testutil.go ├── utils -│   ├── config.go # `RouterConfig` definition │   ├── host.go # Helpers for working with libp2p hosts │   ├── logging.go # Helpers for logging +│   ├── nonce_deduper.go +│   ├── nonce_deduper_test.go │   ├── peer_conversion.go # Helpers for converting between "native" and libp2p peer representations │   ├── url_conversion.go # Helpers for converting between "native" and libp2p network address representations │   └── url_conversion_test.go diff --git a/p2p/background/router.go b/p2p/background/router.go index 199190b21..d5119b395 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -9,31 +9,51 @@ import ( dht "github.com/libp2p/go-libp2p-kad-dht" pubsub "github.com/libp2p/go-libp2p-pubsub" libp2pHost "github.com/libp2p/go-libp2p/core/host" + libp2pPeer "github.com/libp2p/go-libp2p/core/peer" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/config" "github.com/pokt-network/pocket/p2p/protocol" + "github.com/pokt-network/pocket/p2p/providers" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/unicast" "github.com/pokt-network/pocket/p2p/utils" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" ) var ( - _ typesP2P.Router = &backgroundRouter{} - _ modules.IntegratableModule = &backgroundRouter{} + _ typesP2P.Router = &backgroundRouter{} + _ modules.IntegrableModule = &backgroundRouter{} + _ backgroundRouterFactory = &backgroundRouter{} ) +type backgroundRouterFactory = modules.FactoryWithConfig[typesP2P.Router, *config.BackgroundConfig] + // backgroundRouter implements `typesP2P.Router` for use with all P2P participants. type backgroundRouter struct { - base_modules.IntegratableModule + base_modules.IntegrableModule + unicast.UnicastRouter logger *modules.Logger + // handler is the function to call when a message is received. + handler typesP2P.MessageHandler // host represents a libp2p network node, it encapsulates a libp2p peerstore // & connection manager. `libp2p.New` configures and starts listening // according to options. // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) host libp2pHost.Host + // cancelReadSubscription is the cancel function for the context which is + // monitored in the `#readSubscription()` go routine. Call to terminate it. + // only one read subscription exists per router at any point in time + cancelReadSubscription context.CancelFunc + + // Fields below are assigned during creation via `#setupDependencies()`. + // gossipSub is used for broadcast communication // (i.e. multiple, unidentified receivers) // TECHDEBT: investigate diff between randomSub and gossipSub @@ -47,96 +67,106 @@ type backgroundRouter struct { kadDHT *dht.IpfsDHT // TECHDEBT: `pstore` will likely be removed in future refactoring / simplification // of the `Router` interface. - // pstore is the background router's peerstore. + // pstore is the background router's peerstore. Assigned in `backgroundRouter#setupPeerstore()`. pstore typesP2P.Peerstore } -// NewBackgroundRouter returns a `backgroundRouter` as a `typesP2P.Router` +// Create returns a `backgroundRouter` as a `typesP2P.Router` // interface using the given configuration. -func NewBackgroundRouter(bus modules.Bus, cfg *config.BackgroundConfig) (typesP2P.Router, error) { - // TECHDEBT(#595): add ctx to interface methods and propagate down. - ctx := context.TODO() +func Create(bus modules.Bus, cfg *config.BackgroundConfig) (typesP2P.Router, error) { + return new(backgroundRouter).Create(bus, cfg) +} - networkLogger := logger.Global.CreateLoggerForModule("backgroundRouter") - networkLogger.Info().Msg("Initializing background router") +func (*backgroundRouter) Create(bus modules.Bus, cfg *config.BackgroundConfig) (typesP2P.Router, error) { + bgRouterLogger := logger.Global.CreateLoggerForModule("backgroundRouter") - // seed initial peerstore with current on-chain peer info (i.e. staked actors) - pstore, err := cfg.PeerstoreProvider.GetStakedPeerstoreAtHeight( - cfg.CurrentHeightProvider.CurrentHeight(), - ) - if err != nil { + if err := cfg.IsValid(); err != nil { return nil, err } - // CONSIDERATION: If switching to `NewRandomSub`, there will be a max size - gossipSub, err := pubsub.NewGossipSub(ctx, cfg.Host) - if err != nil { - return nil, fmt.Errorf("creating gossip pubsub: %w", err) - } + // TECHDEBT(#595): add ctx to interface methods and propagate down. + ctx, cancel := context.WithCancel(context.TODO()) - dhtMode := dht.ModeAutoServer - // NB: don't act as a bootstrap node in peer discovery in client debug mode - if isClientDebugMode(bus) { - dhtMode = dht.ModeClient + rtr := &backgroundRouter{ + logger: bgRouterLogger, + handler: cfg.Handler, + host: cfg.Host, + cancelReadSubscription: cancel, } + rtr.SetBus(bus) - kadDHT, err := dht.New(ctx, cfg.Host, dht.Mode(dhtMode)) - if err != nil { - return nil, fmt.Errorf("creating DHT: %w", err) - } + bgRouterLogger.Info().Fields(map[string]any{ + "host_id": cfg.Host.ID(), + "unicast_protocol_id": protocol.BackgroundProtocolID, + "broadcast_pubsub_topic": protocol.BackgroundTopicStr, + }).Msg("initializing background router") - topic, err := gossipSub.Join(protocol.BackgroundTopicStr) - if err != nil { - return nil, fmt.Errorf("joining background topic: %w", err) + if err := rtr.setupDependencies(ctx, cfg); err != nil { + return nil, err } - // INVESTIGATE: `WithBufferSize` `SubOpt`: - // > WithBufferSize is a Subscribe option to customize the size of the subscribe - // > output buffer. The default length is 32 but it can be configured to avoid - // > dropping messages if the consumer is not reading fast enough. - // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p-pubsub#WithBufferSize) - subscription, err := topic.Subscribe() - if err != nil { - return nil, fmt.Errorf("subscribing to background topic: %w", err) - } + go rtr.readSubscription(ctx) - rtr := &backgroundRouter{ - host: cfg.Host, - gossipSub: gossipSub, - kadDHT: kadDHT, - topic: topic, - subscription: subscription, - logger: networkLogger, - pstore: pstore, + return rtr, nil +} + +func (rtr *backgroundRouter) Close() error { + rtr.logger.Debug().Msg("closing background router") + + rtr.cancelReadSubscription() + rtr.subscription.Cancel() + + var topicCloseErr error + if err := rtr.topic.Close(); err != context.Canceled { + topicCloseErr = err } - return rtr, nil + return multierr.Append( + topicCloseErr, + rtr.kadDHT.Close(), + ) } // Broadcast implements the respective `typesP2P.Router` interface method. -func (rtr *backgroundRouter) Broadcast(data []byte) error { +func (rtr *backgroundRouter) Broadcast(pocketEnvelopeBz []byte) error { + backgroundMsg := &typesP2P.BackgroundMessage{ + Data: pocketEnvelopeBz, + } + backgroundMsgBz, err := proto.Marshal(backgroundMsg) + if err != nil { + return err + } + // TECHDEBT(#595): add ctx to interface methods and propagate down. - return rtr.topic.Publish(context.TODO(), data) + return rtr.topic.Publish(context.TODO(), backgroundMsgBz) } // Send implements the respective `typesP2P.Router` interface method. -func (rtr *backgroundRouter) Send(data []byte, address cryptoPocket.Address) error { +func (rtr *backgroundRouter) Send(pocketEnvelopeBz []byte, address cryptoPocket.Address) error { + backgroundMessage := &typesP2P.BackgroundMessage{ + Data: pocketEnvelopeBz, + } + backgroundMessageBz, err := proto.Marshal(backgroundMessage) + if err != nil { + return fmt.Errorf("marshalling background message: %w", err) + } + peer := rtr.pstore.GetPeer(address) if peer == nil { return fmt.Errorf("peer with address %s not in peerstore", address) } - if err := utils.Libp2pSendToPeer(rtr.host, data, peer); err != nil { + if err := utils.Libp2pSendToPeer( + rtr.host, + protocol.BackgroundProtocolID, + backgroundMessageBz, + peer, + ); err != nil { return err } return nil } -// HandleNetworkData implements the respective `typesP2P.Router` interface method. -func (rtr *backgroundRouter) HandleNetworkData(data []byte) ([]byte, error) { - return data, nil // intentional passthrough -} - // GetPeerstore implements the respective `typesP2P.Router` interface method. func (rtr *backgroundRouter) GetPeerstore() typesP2P.Peerstore { return rtr.pstore @@ -166,6 +196,230 @@ func (rtr *backgroundRouter) RemovePeer(peer typesP2P.Peer) error { return rtr.pstore.RemovePeer(peer.GetAddress()) } +// setupUnicastRouter configures and assigns `rtr.UnicastRouter`. +func (rtr *backgroundRouter) setupUnicastRouter() error { + unicastRouterCfg := config.UnicastRouterConfig{ + Logger: rtr.logger, + Host: rtr.host, + ProtocolID: protocol.BackgroundProtocolID, + MessageHandler: rtr.handleBackgroundMsg, + PeerHandler: rtr.AddPeer, + } + + unicastRouter, err := unicast.Create(rtr.GetBus(), &unicastRouterCfg) + if err != nil { + return fmt.Errorf("setting up unicast router: %w", err) + } + + rtr.UnicastRouter = *unicastRouter + return nil +} + +func (rtr *backgroundRouter) setupDependencies(ctx context.Context, cfg *config.BackgroundConfig) error { + // NB: The order in which the internal components are setup below is important + if err := rtr.setupUnicastRouter(); err != nil { + return err + } + + if err := rtr.setupPeerDiscovery(ctx); err != nil { + return fmt.Errorf("setting up peer discovery: %w", err) + } + + if err := rtr.setupPubsub(ctx); err != nil { + return fmt.Errorf("setting up pubsub: %w", err) + } + + if err := rtr.setupTopic(); err != nil { + return fmt.Errorf("setting up topic: %w", err) + } + + if err := rtr.setupSubscription(); err != nil { + return fmt.Errorf("setting up subscription: %w", err) + } + + if err := rtr.setupPeerstore( + ctx, + cfg.PeerstoreProvider, + cfg.CurrentHeightProvider, + ); err != nil { + return fmt.Errorf("setting up peerstore: %w", err) + } + return nil +} + +func (rtr *backgroundRouter) setupPeerstore( + ctx context.Context, + pstoreProvider providers.PeerstoreProvider, + currentHeightProvider providers.CurrentHeightProvider, +) (err error) { + // seed initial peerstore with current on-chain peer info (i.e. staked actors) + rtr.pstore, err = pstoreProvider.GetStakedPeerstoreAtHeight( + currentHeightProvider.CurrentHeight(), + ) + if err != nil { + return err + } + + // TECHDEBT(#859): integrate with `p2pModule#bootstrap()`. + if err := rtr.bootstrap(ctx); err != nil { + return fmt.Errorf("bootstrapping peerstore: %w", err) + } + + return nil +} + +// setupPeerDiscovery sets up the Kademlia Distributed Hash Table (DHT) +func (rtr *backgroundRouter) setupPeerDiscovery(ctx context.Context) (err error) { + dhtMode := dht.ModeAutoServer + // NB: don't act as a bootstrap node in peer discovery in client debug mode + if isClientDebugMode(rtr.GetBus()) { + dhtMode = dht.ModeClient + } + + rtr.kadDHT, err = dht.New(ctx, rtr.host, dht.Mode(dhtMode)) + return err +} + +// setupPubsub sets up a new gossip sub topic using libp2p +func (rtr *backgroundRouter) setupPubsub(ctx context.Context) (err error) { + // TECHDEBT(#730): integrate libp2p tracing via `pubsub.WithEventTracer()`. + + // CONSIDERATION: If switching to `NewRandomSub`, there will be a max size + rtr.gossipSub, err = pubsub.NewGossipSub(ctx, rtr.host) + return err +} + +func (rtr *backgroundRouter) setupTopic() (err error) { + if err := rtr.gossipSub.RegisterTopicValidator( + protocol.BackgroundTopicStr, + rtr.topicValidator, + ); err != nil { + return fmt.Errorf( + "registering topic validator for topic: %q: %w", + protocol.BackgroundTopicStr, err, + ) + } + + if rtr.topic, err = rtr.gossipSub.Join(protocol.BackgroundTopicStr); err != nil { + return fmt.Errorf( + "joining background topic: %q: %w", + protocol.BackgroundTopicStr, err, + ) + } + return nil +} + +func (rtr *backgroundRouter) setupSubscription() (err error) { + // INVESTIGATE: `WithBufferSize` `SubOpt`: + // > WithBufferSize is a Subscribe option to customize the size of the subscribe + // > output buffer. The default length is 32 but it can be configured to avoid + // > dropping messages if the consumer is not reading fast enough. + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p-pubsub#WithBufferSize) + rtr.subscription, err = rtr.topic.Subscribe() + return err +} + +// TECHDEBT(#859): integrate with `p2pModule#bootstrap()`. +func (rtr *backgroundRouter) bootstrap(ctx context.Context) error { + // CONSIDERATION: add `GetPeers` method, which returns a map, + // to the `PeerstoreProvider` interface to simplify this loop. + for _, peer := range rtr.pstore.GetPeerList() { + if err := utils.AddPeerToLibp2pHost(rtr.host, peer); err != nil { + return err + } + + libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) + if err != nil { + return fmt.Errorf( + "converting peer info, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + + // don't attempt to connect to self + if rtr.host.ID() == libp2pAddrInfo.ID { + return nil + } + + if err := rtr.host.Connect(ctx, libp2pAddrInfo); err != nil { + return fmt.Errorf("connecting to peer: %w", err) + } + } + return nil +} + +// topicValidator is used in conjunction with libp2p-pubsub's notion of "topic +// validaton". It is used for arbitrary and concurrent pre-propagation validation +// of messages. +// +// (see: https://github.com/libp2p/specs/tree/master/pubsub#topic-validation +// and https://pkg.go.dev/github.com/libp2p/go-libp2p-pubsub#PubSub.RegisterTopicValidator) +// +// Also note: https://pkg.go.dev/github.com/libp2p/go-libp2p-pubsub#BasicSeqnoValidator +func (rtr *backgroundRouter) topicValidator(_ context.Context, _ libp2pPeer.ID, msg *pubsub.Message) bool { + var backgroundMsg typesP2P.BackgroundMessage + if err := proto.Unmarshal(msg.Data, &backgroundMsg); err != nil { + rtr.logger.Error().Err(err).Msg("unmarshalling Background message") + return false + } + + if backgroundMsg.Data == nil { + rtr.logger.Debug().Msg("no data in Background message") + return false + } + + poktEnvelope := messaging.PocketEnvelope{} + if err := proto.Unmarshal(backgroundMsg.Data, &poktEnvelope); err != nil { + rtr.logger.Error().Err(err).Msg("Error decoding Background message") + return false + } + + return true +} + +// readSubscription is a while loop for receiving and handling messages from the +// subscription. It is intended to be called as a goroutine. +func (rtr *backgroundRouter) readSubscription(ctx context.Context) { + for { + if err := ctx.Err(); err != nil { + if err != context.Canceled { + rtr.logger.Error().Err(err). + Msg("context error while reading subscription") + } + return + } + msg, err := rtr.subscription.Next(ctx) + + if err != nil { + rtr.logger.Error().Err(err). + Msg("error reading from background topic subscription") + continue + } + + // TECHDEBT/DISCUSS: telemetry + if err := rtr.handleBackgroundMsg(msg.Data); err != nil { + rtr.logger.Error().Err(err).Msg("error handling background message") + continue + } + } +} + +func (rtr *backgroundRouter) handleBackgroundMsg(backgroundMsgBz []byte) error { + var backgroundMsg typesP2P.BackgroundMessage + if err := proto.Unmarshal(backgroundMsgBz, &backgroundMsg); err != nil { + return err + } + + // There was no error, but we don't need to forward this to the app-specific bus. + // For example, the message has already been handled by the application. + if backgroundMsg.Data == nil { + return nil + } + + return rtr.handler(backgroundMsg.Data) +} + // isClientDebugMode returns the value of `ClientDebugMode` in the base config func isClientDebugMode(bus modules.Bus) bool { return bus.GetRuntimeMgr().GetConfig().ClientDebugMode diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index a1a0fe40b..61cb5f153 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -8,32 +8,44 @@ import ( "time" "github.com/golang/mock/gomock" + pubsub "github.com/libp2p/go-libp2p-pubsub" libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" libp2pPeer "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "github.com/pokt-network/pocket/internal/testutil" "github.com/pokt-network/pocket/p2p/config" + "github.com/pokt-network/pocket/p2p/protocol" typesP2P "github.com/pokt-network/pocket/p2p/types" mock_types "github.com/pokt-network/pocket/p2p/types/mocks" "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/defaults" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/messaging" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - "github.com/stretchr/testify/require" ) // https://www.rfc-editor.org/rfc/rfc3986#section-3.2.2 -const testIP6ServiceURL = "[2a00:1450:4005:802::2004]:8080" +const ( + testIP6ServiceURL = "[2a00:1450:4005:802::2004]:8080" + invalidReceiveTimeout = time.Millisecond * 500 +) // TECHDEBT(#609): move & de-dup. -var testLocalServiceURL = fmt.Sprintf("127.0.0.1:%d", defaults.DefaultP2PPort) +var ( + testLocalServiceURL = fmt.Sprintf("127.0.0.1:%d", defaults.DefaultP2PPort) + noopHandler = func(data []byte) error { return nil } +) func TestBackgroundRouter_AddPeer(t *testing.T) { - testRouter := newTestRouter(t, nil) + testRouter := newTestRouter(t, nil, nil) libp2pPStore := testRouter.host.Peerstore() // NB: assert initial state @@ -81,7 +93,7 @@ func TestBackgroundRouter_AddPeer(t *testing.T) { } func TestBackgroundRouter_RemovePeer(t *testing.T) { - testRouter := newTestRouter(t, nil) + testRouter := newTestRouter(t, nil, nil) peerstore := testRouter.host.Peerstore() // NB: assert initial state @@ -114,6 +126,116 @@ func TestBackgroundRouter_RemovePeer(t *testing.T) { require.Len(t, existingPeerstoreAddrs, 1) } +func TestBackgroundRouter_Validation(t *testing.T) { + invalidProtoMessage := anypb.Any{ + TypeUrl: "/notADefinedProtobufType", + Value: []byte("not a serialized protobuf"), + } + + testCases := []struct { + name string + msgBz []byte + }{ + { + name: "invalid BackgroundMessage", + // NB: `msgBz` would normally be a serialized `BackgroundMessage`. + msgBz: mustMarshal(t, &invalidProtoMessage), + }, + { + name: "empty PocketEnvelope", + msgBz: mustMarshal(t, &typesP2P.BackgroundMessage{ + // NB: `Data` is normally a serialized `PocketEnvelope`. + Data: nil, + }), + }, + { + name: "invalid PoketEnvelope", + msgBz: mustMarshal(t, &typesP2P.BackgroundMessage{ + // NB: `Data` is normally a serialized `PocketEnvelope`. + Data: mustMarshal(t, &invalidProtoMessage), + }), + }, + } + + // Set up test router as the receiver. + ctx := context.Background() + libp2pMockNet := mocknet.New() + + receivedChan := make(chan []byte, 1) + receiverPrivKey, receiverPeer := newTestPeer(t) + receiverHost := newTestHost(t, libp2pMockNet, receiverPrivKey) + receiverRouter := newRouterWithSelfPeerAndHost( + t, receiverPeer, + receiverHost, + func(data []byte) error { + receivedChan <- data + return nil + }, + ) + + t.Cleanup(func() { + err := receiverRouter.Close() + require.NoError(t, err) + }) + + // Wrap `receiverRouter#topicValidator` to make assertions by. + // Existing topic validator must be unregistered first. + err := receiverRouter.gossipSub.UnregisterTopicValidator(protocol.BackgroundTopicStr) + require.NoError(t, err) + + // Register topic validator wrapper. + err = receiverRouter.gossipSub.RegisterTopicValidator( + protocol.BackgroundTopicStr, + func(ctx context.Context, peerID libp2pPeer.ID, msg *pubsub.Message) bool { + msgIsValid := receiverRouter.topicValidator(ctx, peerID, msg) + require.Falsef(t, msgIsValid, "expected message to be invalid") + + return msgIsValid + }, + ) + require.NoError(t, err) + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + senderPrivKey, _ := newTestPeer(t) + senderHost := newTestHost(t, libp2pMockNet, senderPrivKey) + gossipPubsub, err := pubsub.NewGossipSub(ctx, senderHost) + require.NoError(t, err) + + err = libp2pMockNet.LinkAll() + require.NoError(t, err) + + receiverAddrInfo, err := utils.Libp2pAddrInfoFromPeer(receiverPeer) + require.NoError(t, err) + + err = senderHost.Connect(ctx, receiverAddrInfo) + require.NoError(t, err) + + topic, err := gossipPubsub.Join(protocol.BackgroundTopicStr) + require.NoError(t, err) + + err = topic.Publish(ctx, testCase.msgBz) + require.NoError(t, err) + + // Destroy previous topic and sender instances to start with new ones + // for each test case. + t.Cleanup(func() { + _ = topic.Close() + _ = senderHost.Close() + }) + + // Ensure no messages were handled at the end of each test case for + // async errors. + select { + case <-receivedChan: + t.Fatal("no messages should have been handled by receiver router") + case <-time.After(invalidReceiveTimeout): + // no error, continue + } + }) + } +} + func TestBackgroundRouter_Broadcast(t *testing.T) { const ( numPeers = 4 @@ -138,17 +260,31 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { libp2pMockNet = mocknet.New() ) - // setup 4 libp2p hosts to listen for incoming streams from the test backgroundRouter + testPocketEnvelope, err := messaging.PackMessage(&anypb.Any{ + TypeUrl: "/test", + Value: []byte(testMsg), + }) + require.NoError(t, err) + + testPocketEnvelopeBz, err := proto.Marshal(testPocketEnvelope) + require.NoError(t, err) + + // setup 4 receiver routers to listen for incoming messages from the sender router for i := 0; i < numPeers; i++ { broadcastWaitgroup.Add(1) bootstrapWaitgroup.Add(1) - privKey, selfPeer := newTestPeer(t) + privKey, peer := newTestPeer(t) host := newTestHost(t, libp2pMockNet, privKey) testHosts = append(testHosts, host) expectedPeerIDs[i] = host.ID().String() - rtr := newRouterWithSelfPeerAndHost(t, selfPeer, host) - go readSubscription(t, ctx, &broadcastWaitgroup, rtr, &seenMessagesMutext, seenMessages) + newRouterWithSelfPeerAndHost(t, peer, host, func(data []byte) error { + seenMessagesMutext.Lock() + defer seenMessagesMutext.Unlock() + seenMessages[host.ID().String()] = struct{}{} + broadcastWaitgroup.Done() + return nil + }) } // bootstrap off of arbitrary testHost @@ -156,12 +292,12 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { // set up a test backgroundRouter testRouterHost := newTestHost(t, libp2pMockNet, privKey) - testRouter := newRouterWithSelfPeerAndHost(t, selfPeer, testRouterHost) + testRouter := newRouterWithSelfPeerAndHost(t, selfPeer, testRouterHost, nil) testHosts = append(testHosts, testRouterHost) // simulate network links between each to every other // (i.e. fully-connected network) - err := libp2pMockNet.LinkAll() + err = libp2pMockNet.LinkAll() require.NoError(t, err) // setup notifee/notify BEFORE bootstrapping @@ -189,7 +325,7 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { // broadcast message t.Log("broadcasting...") - err := testRouter.Broadcast([]byte(testMsg)) + err := testRouter.Broadcast(testPocketEnvelopeBz) require.NoError(t, err) // wait for broadcast to be received by all peers @@ -241,7 +377,11 @@ func bootstrap(t *testing.T, ctx context.Context, testHosts []libp2pHost.Host) { } // TECHDEBT(#609): move & de-duplicate -func newTestRouter(t *testing.T, libp2pMockNet mocknet.Mocknet) *backgroundRouter { +func newTestRouter( + t *testing.T, + libp2pMockNet mocknet.Mocknet, + handler typesP2P.MessageHandler, +) *backgroundRouter { t.Helper() privKey, selfPeer := newTestPeer(t) @@ -256,10 +396,15 @@ func newTestRouter(t *testing.T, libp2pMockNet mocknet.Mocknet) *backgroundRoute require.NoError(t, err) }) - return newRouterWithSelfPeerAndHost(t, selfPeer, host) + return newRouterWithSelfPeerAndHost(t, selfPeer, host, handler) } -func newRouterWithSelfPeerAndHost(t *testing.T, selfPeer typesP2P.Peer, host libp2pHost.Host) *backgroundRouter { +func newRouterWithSelfPeerAndHost( + t *testing.T, + selfPeer typesP2P.Peer, + host libp2pHost.Host, + handler typesP2P.MessageHandler, +) *backgroundRouter { t.Helper() ctrl := gomock.NewController(t) @@ -268,7 +413,7 @@ func newRouterWithSelfPeerAndHost(t *testing.T, selfPeer typesP2P.Peer, host lib P2P: &configs.P2PConfig{ IsClientOnly: false, }, - }) + }).AnyTimes() consensusMock := mockModules.NewMockConsensusModule(ctrl) consensusMock.EXPECT().CurrentHeight().Return(uint64(1)).AnyTimes() @@ -284,11 +429,16 @@ func newRouterWithSelfPeerAndHost(t *testing.T, selfPeer typesP2P.Peer, host lib err := pstore.AddPeer(selfPeer) require.NoError(t, err) - router, err := NewBackgroundRouter(busMock, &config.BackgroundConfig{ + if handler == nil { + handler = noopHandler + } + + router, err := Create(busMock, &config.BackgroundConfig{ Addr: selfPeer.GetAddress(), PeerstoreProvider: pstoreProviderMock, CurrentHeightProvider: consensusMock, Host: host, + Handler: handler, }) require.NoError(t, err) @@ -332,7 +482,11 @@ func newMockNetHostFromPeer( return host } -func newTestHost(t *testing.T, mockNet mocknet.Mocknet, privKey cryptoPocket.PrivateKey) libp2pHost.Host { +func newTestHost( + t *testing.T, + mockNet mocknet.Mocknet, + privKey cryptoPocket.PrivateKey, +) libp2pHost.Host { t.Helper() // listen on random port on loopback interface @@ -346,30 +500,11 @@ func newTestHost(t *testing.T, mockNet mocknet.Mocknet, privKey cryptoPocket.Pri return newMockNetHostFromPeer(t, mockNet, privKey, peer) } -func readSubscription( - t *testing.T, - ctx context.Context, - broadcastWaitGroup *sync.WaitGroup, - rtr *backgroundRouter, - mu *sync.Mutex, - seenMsgs map[string]struct{}, -) { +func mustMarshal(t *testing.T, msg proto.Message) []byte { t.Helper() - for { - if err := ctx.Err(); err != nil { - if err != context.Canceled || err != context.DeadlineExceeded { - require.NoError(t, err) - } - return - } - - _, err := rtr.subscription.Next(ctx) - require.NoError(t, err) + msgBz, err := proto.Marshal(msg) + require.NoError(t, err) - mu.Lock() - broadcastWaitGroup.Done() - seenMsgs[rtr.host.ID().String()] = struct{}{} - mu.Unlock() - } + return msgBz } diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 75198ee0e..58b923a5d 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -42,6 +42,7 @@ func (m *p2pModule) configureBootstrapNodes() error { } // bootstrap attempts to bootstrap from a bootstrap node +// TECHDEBT(#859): refactor bootstrapping. func (m *p2pModule) bootstrap() error { var pstore typesP2P.Peerstore @@ -58,12 +59,13 @@ func (m *p2pModule) bootstrap() error { continue } - pstoreProvider := rpcABP.Create( - rpcABP.WithP2PConfig( - m.GetBus().GetRuntimeMgr().GetConfig().P2P, - ), + pstoreProvider, err := rpcABP.Create( + m.GetBus(), rpcABP.WithCustomRPCURL(bootstrapNode), ) + if err != nil { + return fmt.Errorf("creating RPC peerstore provider: %w", err) + } currentHeightProvider := rpcCHP.NewRPCCurrentHeightProvider(rpcCHP.WithCustomRPCURL(bootstrapNode)) @@ -76,14 +78,14 @@ func (m *p2pModule) bootstrap() error { for _, peer := range pstore.GetPeerList() { m.logger.Debug().Str("address", peer.GetAddress().String()).Msg("Adding peer to router") - if err := m.router.AddPeer(peer); err != nil { + if err := m.stakedActorRouter.AddPeer(peer); err != nil { m.logger.Error().Err(err). Str("pokt_address", peer.GetAddress().String()). Msg("adding peer") } } - if m.router.GetPeerstore().Size() == 0 { + if m.stakedActorRouter.GetPeerstore().Size() == 0 { return fmt.Errorf("bootstrap failed") } return nil diff --git a/p2p/config/config.go b/p2p/config/config.go index d98f4b969..60361f662 100644 --- a/p2p/config/config.go +++ b/p2p/config/config.go @@ -16,6 +16,7 @@ import ( var ( _ typesP2P.RouterConfig = &baseConfig{} _ typesP2P.RouterConfig = &UnicastRouterConfig{} + _ typesP2P.RouterConfig = &BackgroundConfig{} _ typesP2P.RouterConfig = &RainTreeConfig{} ) @@ -30,6 +31,7 @@ type baseConfig struct { Addr crypto.Address CurrentHeightProvider providers.CurrentHeightProvider PeerstoreProvider providers.PeerstoreProvider + Handler func(data []byte) error } type UnicastRouterConfig struct { @@ -75,7 +77,11 @@ func (cfg *baseConfig) IsValid() (err error) { if cfg.PeerstoreProvider == nil { err = multierr.Append(err, fmt.Errorf("peerstore provider not configured")) } - return nil + + if cfg.Handler == nil { + err = multierr.Append(err, fmt.Errorf("handler not configured")) + } + return err } // IsValid implements the respective member of the `RouterConfig` interface. @@ -103,23 +109,25 @@ func (cfg *UnicastRouterConfig) IsValid() (err error) { } // IsValid implements the respective member of the `RouterConfig` interface. -func (cfg *BackgroundConfig) IsValid() (err error) { +func (cfg *BackgroundConfig) IsValid() error { baseCfg := baseConfig{ Host: cfg.Host, Addr: cfg.Addr, CurrentHeightProvider: cfg.CurrentHeightProvider, PeerstoreProvider: cfg.PeerstoreProvider, + Handler: cfg.Handler, } - return multierr.Append(err, baseCfg.IsValid()) + return baseCfg.IsValid() } // IsValid implements the respective member of the `RouterConfig` interface. -func (cfg *RainTreeConfig) IsValid() (err error) { +func (cfg *RainTreeConfig) IsValid() error { baseCfg := baseConfig{ Host: cfg.Host, Addr: cfg.Addr, CurrentHeightProvider: cfg.CurrentHeightProvider, PeerstoreProvider: cfg.PeerstoreProvider, + Handler: cfg.Handler, } - return multierr.Append(err, baseCfg.IsValid()) + return baseCfg.IsValid() } diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 48e1a7d73..ba0885839 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -3,10 +3,11 @@ package p2p import ( "fmt" + "google.golang.org/protobuf/types/known/anypb" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/messaging" - "google.golang.org/protobuf/types/known/anypb" ) // CONSIDERATION(#576): making this part of some new `ConnManager`. @@ -23,7 +24,13 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { return fmt.Errorf("failed to cast event to ConsensusNewHeightEvent") } - oldPeerList := m.router.GetPeerstore().GetPeerList() + if isStaked, err := m.isStakedActor(); err != nil { + return err + } else if !isStaked { + return nil // unstaked actors do not use RainTree and therefore do not need to update this router + } + + oldPeerList := m.stakedActorRouter.GetPeerstore().GetPeerList() updatedPeerstore, err := m.pstoreProvider.GetStakedPeerstoreAtHeight(consensusNewHeightEvent.Height) if err != nil { return err @@ -31,12 +38,12 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { added, removed := oldPeerList.Delta(updatedPeerstore.GetPeerList()) for _, add := range added { - if err := m.router.AddPeer(add); err != nil { + if err := m.stakedActorRouter.AddPeer(add); err != nil { return err } } for _, rm := range removed { - if err := m.router.RemovePeer(rm); err != nil { + if err := m.stakedActorRouter.RemovePeer(rm); err != nil { return err } } @@ -50,13 +57,25 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { m.logger.Debug().Fields(messaging.TransitionEventToMap(stateMachineTransitionEvent)).Msg("Received state machine transition event") if stateMachineTransitionEvent.NewState == string(coreTypes.StateMachineState_P2P_Bootstrapping) { - if m.router.GetPeerstore().Size() == 0 { - m.logger.Warn().Msg("No peers in addrbook, bootstrapping") + staked, err := m.isStakedActor() + if err != nil { + return err + } + if staked { + // TECHDEBT(#859): this will never happen as the peerstore is + // seeded from consensus during P2P module construction. + if m.stakedActorRouter.GetPeerstore().Size() == 0 { + m.logger.Warn().Msg("No peers in peerstore, bootstrapping") - if err := m.bootstrap(); err != nil { - return err + if err := m.bootstrap(); err != nil { + return err + } } } + + // TECHDEBT(#859): for unstaked actors, unstaked actor (background) + // router bootstrapping SHOULD complete before the event below is sent. + m.logger.Info().Bool("TODO", true).Msg("Advertise self to network") if err := m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_P2P_IsBootstrapped); err != nil { return err diff --git a/p2p/module.go b/p2p/module.go index 5f87d5b6e..d488afbc0 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -3,14 +3,17 @@ package p2p import ( "errors" "fmt" + "sync/atomic" "github.com/libp2p/go-libp2p" libp2pHost "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multiaddr" + "go.uber.org/multierr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/background" "github.com/pokt-network/pocket/p2p/config" "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/current_height_provider" @@ -33,8 +36,9 @@ import ( var _ modules.P2PModule = &p2pModule{} type p2pModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule + started atomic.Bool address cryptoPocket.Address logger *modules.Logger options []modules.ModuleOption @@ -55,8 +59,9 @@ type p2pModule struct { // holding a reference in the module struct. This will improve testability. // // Assigned during `#Start()`. TLDR; `host` listens on instantiation. - // and `router` depends on `host`. - router typesP2P.Router + // `stakedActorRouter` and `unstakedActorRouter` depends on `host`. + stakedActorRouter typesP2P.Router + unstakedActorRouter typesP2P.Router // host represents a libp2p network node, it encapsulates a libp2p peerstore // & connection manager. `libp2p.New` configures and starts listening // according to options. Assigned via `#Start()` (starts on instantiation). @@ -68,19 +73,6 @@ func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, e return new(p2pModule).Create(bus, options...) } -// WithHostOption associates an existing (i.e. "started") libp2p `host.Host` -// with this module, instead of creating a new one on `#Start()`. -// Primarily intended for testing. -func WithHostOption(host libp2pHost.Host) modules.ModuleOption { - return func(m modules.InitializableModule) { - mod, ok := m.(*p2pModule) - if ok { - mod.host = host - mod.logger.Debug().Msg("using host provided via `WithHostOption`") - } - } -} - func (m *p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { logger.Global.Debug().Msg("Creating P2P module") *m = p2pModule{ @@ -143,8 +135,12 @@ func (m *p2pModule) GetModuleName() string { } // Start instantiates and assigns `m.host`, unless one already exists, and -// `m.router` (which depends on `m.host` as a required config field). +// `m.stakedActorRouter` (which depends on `m.host` as a required config field). func (m *p2pModule) Start() (err error) { + if !m.started.CompareAndSwap(false, true) { + return fmt.Errorf("p2p module already started") + } + m.GetBus(). GetTelemetryModule(). GetTimeSeriesAgent(). @@ -153,7 +149,7 @@ func (m *p2pModule) Start() (err error) { telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_DESCRIPTION, ) - // Return early if host has already been started (e.g. via `WithHostOption`) + // Return early if host has already been started (e.g. via `WithHost`) if m.host == nil { // Libp2p hosts provided via `WithHost()` option are destroyed when // `#Stop()`ing the module. Therefore, a new one must be created. @@ -168,8 +164,8 @@ func (m *p2pModule) Start() (err error) { } } - if err := m.setupRouter(); err != nil { - return fmt.Errorf("setting up router: %w", err) + if err := m.setupRouters(); err != nil { + return fmt.Errorf("setting up routers: %w", err) } m.GetBus(). @@ -180,38 +176,102 @@ func (m *p2pModule) Start() (err error) { } func (m *p2pModule) Stop() error { - err := m.host.Close() + m.logger.Debug().Msg("stopping P2P module") + + if !m.started.CompareAndSwap(true, false) { + return fmt.Errorf("p2p module already stopped") + } + + var stakedActorRouterCloseErr error + if m.stakedActorRouter != nil { + stakedActorRouterCloseErr = m.stakedActorRouter.Close() + } + + routerCloseErrs := multierr.Append( + m.unstakedActorRouter.Close(), + stakedActorRouterCloseErr, + ) + + err := multierr.Append( + routerCloseErrs, + m.host.Close(), + ) // Don't reuse closed host, `#Start()` will re-create. m.host = nil + m.stakedActorRouter = nil + m.unstakedActorRouter = nil return err } func (m *p2pModule) Broadcast(msg *anypb.Any) error { - c := &messaging.PocketEnvelope{ + isStaked, err := m.isStakedActor() + if err != nil { + return err + } + + if isStaked { + if m.stakedActorRouter == nil { + return fmt.Errorf("broadcasting: staked actor router not started") + } + } + + if m.unstakedActorRouter == nil { + return fmt.Errorf("broadcasting: unstaked actor router not started") + } + + poktEnvelope := &messaging.PocketEnvelope{ Content: msg, Nonce: cryptoPocket.GetNonce(), } - data, err := codec.GetCodec().Marshal(c) + poktEnvelopeBz, err := codec.GetCodec().Marshal(poktEnvelope) if err != nil { return err } - return m.router.Broadcast(data) + var stakedBroadcastErr error + if isStaked { + stakedBroadcastErr = m.stakedActorRouter.Broadcast(poktEnvelopeBz) + } + + unstakedBroadcastErr := m.unstakedActorRouter.Broadcast(poktEnvelopeBz) + + return multierr.Append(stakedBroadcastErr, unstakedBroadcastErr) } func (m *p2pModule) Send(addr cryptoPocket.Address, msg *anypb.Any) error { - c := &messaging.PocketEnvelope{ + poktEnvelope := &messaging.PocketEnvelope{ Content: msg, Nonce: cryptoPocket.GetNonce(), } - data, err := codec.GetCodec().Marshal(c) + poktEnvelopeBz, err := codec.GetCodec().Marshal(poktEnvelope) if err != nil { return err } - return m.router.Send(data, addr) + isStaked, err := m.isStakedActor() + if err != nil { + return err + } + + // Send via the staked actor router both if this node and the peer are staked + // actors; otherwise, send via the unstaked actor router. + if !isStaked { + return m.unstakedActorRouter.Send(poktEnvelopeBz, addr) + } + + stakedActorSendErr := m.stakedActorRouter.Send(poktEnvelopeBz, addr) + + // Peer is not a staked actor. + if errors.Is(stakedActorSendErr, typesP2P.ErrUnknownPeer) { + m.logger.Warn(). + Str("address", addr.String()). + Msgf("attempting to send to unstaked actor") + + return m.unstakedActorRouter.Send(poktEnvelopeBz, addr) + } + return nil } // TECHDEBT(#348): Define what the node identity is throughout the codebase @@ -240,8 +300,11 @@ func (m *p2pModule) setupDependencies() error { func (m *p2pModule) setupPeerstoreProvider() error { m.logger.Debug().Msg("setupPeerstoreProvider") - // TECHDEBT(#810): simplify once submodules are more convenient to retrieve. - pstoreProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(peerstore_provider.ModuleName) + // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // is retrievable as a proper submodule + pstoreProviderModule, err := m.GetBus(). + GetModulesRegistry(). + GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) if err != nil { m.logger.Debug().Msg("creating new persistence peerstore...") pstoreProvider, err := persPSP.Create(m.GetBus()) @@ -273,6 +336,9 @@ func (m *p2pModule) setupCurrentHeightProvider() error { m.logger.Debug().Msg("setupCurrentHeightProvider") currentHeightProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(current_height_provider.ModuleName) if err != nil { + // TECHDEBT(#810): add a `consensusCurrentHeightProvider` submodule to wrap + // the consensus module usage (similar to how `persistencePeerstoreProvider` + // wraps persistence). currentHeightProviderModule = m.GetBus().GetConsensusModule() } @@ -305,11 +371,37 @@ func (m *p2pModule) setupNonceDeduper() error { return nil } -// setupRouter instantiates the configured router implementation. -func (m *p2pModule) setupRouter() (err error) { - // TECHDEBT(#810): register the router to the module registry instead of +// setupRouters instantiates the configured router implementations. +func (m *p2pModule) setupRouters() (err error) { + // TECHDEBT(#810): register the routers to the module registry instead of // holding a reference in the module struct. This will improve testability. - m.router, err = raintree.NewRainTreeRouter( + if err := m.setupStakedRouter(); err != nil { + return err + } + + if err := m.setupUnstakedRouter(); err != nil { + return err + } + return nil +} + +// setupStakedRouter initializes the staked actor router ONLY IF this node is +// a staked actor, exclusively for use between staked actors. +func (m *p2pModule) setupStakedRouter() (err error) { + // `nstakedActorRouter` may already be initialized via a `ModuleOption`. + if m.stakedActorRouter != nil { + m.logger.Debug().Msg("staked actor router already initialized") + return nil + } + + if isStaked, err := m.isStakedActor(); err != nil { + return err + } else if !isStaked { + return nil + } + + m.logger.Debug().Msg("setting up staked actor router") + m.stakedActorRouter, err = raintree.NewRainTreeRouter( m.GetBus(), &config.RainTreeConfig{ Addr: m.address, @@ -319,10 +411,39 @@ func (m *p2pModule) setupRouter() (err error) { Handler: m.handlePocketEnvelope, }, ) - return err + if err != nil { + return fmt.Errorf("setting up staked actor router: %w", err) + } + return nil } -// setupHost creates a new libp2p host and assignes it to `m.host`. Libp2p host +// setupUnstakedRouter initializes the unstaked actor router for use with the +// entire P2P network. +func (m *p2pModule) setupUnstakedRouter() (err error) { + // `unstakedActorRouter` may already be initialized via a `ModuleOption`. + if m.unstakedActorRouter != nil { + m.logger.Debug().Msg("unstaked actor router already initialized") + return nil + } + + m.logger.Debug().Msg("setting up unstaked actor router") + m.unstakedActorRouter, err = background.Create( + m.GetBus(), + &config.BackgroundConfig{ + Addr: m.address, + CurrentHeightProvider: m.currentHeightProvider, + PeerstoreProvider: m.pstoreProvider, + Host: m.host, + Handler: m.handlePocketEnvelope, + }, + ) + if err != nil { + return fmt.Errorf("unstaked actor router: %w", err) + } + return nil +} + +// setupHost creates a new libp2p host and assigns it to `m.host`. Libp2p host // starts listening upon instantiation. func (m *p2pModule) setupHost() (err error) { m.logger.Debug().Msg("creating new libp2p host") @@ -437,3 +558,24 @@ func (m *p2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { "%s:%d", m.cfg.Hostname, m.cfg.Port, )) } + +func (m *p2pModule) getStakedPeerstore() (typesP2P.Peerstore, error) { + return m.pstoreProvider.GetStakedPeerstoreAtHeight( + m.currentHeightProvider.CurrentHeight(), + ) +} + +// isStakedActor returns whether the current node is a staked actor at the current height. +// Return an error if a peerstore can't be provided. +func (m *p2pModule) isStakedActor() (bool, error) { + pstore, err := m.getStakedPeerstore() + if err != nil { + return false, fmt.Errorf("getting staked peerstore: %w", err) + } + + // Ensure self address is present in current height's staked actor set. + if self := pstore.GetPeer(m.address); self != nil { + return true, nil + } + return false, nil +} diff --git a/p2p/module_raintree_test.go b/p2p/module_raintree_test.go index 9bd873913..3dc98a988 100644 --- a/p2p/module_raintree_test.go +++ b/p2p/module_raintree_test.go @@ -9,18 +9,14 @@ import ( "regexp" "sort" "strconv" - "strings" "sync" "testing" - libp2pNetwork "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/internal/testutil" - "github.com/pokt-network/pocket/p2p/protocol" - "github.com/pokt-network/pocket/p2p/raintree" ) // TODO(#314): Add the tooling and instructions on how to generate unit tests in this file. @@ -220,11 +216,13 @@ func TestRainTreeNetworkCompleteTwentySevenNodes(t *testing.T) { // 1. It creates and configures a "real" P2P module where all the other components of the node are mocked. // 2. It then triggers a single message and waits for all of the expected messages transmission to complete before announcing failure. func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig TestNetworkSimulationConfig) { + var readWriteWaitGroup sync.WaitGroup + // Configure & prepare test module numValidators := len(networkSimulationConfig) runtimeConfigs := createMockRuntimeMgrs(t, numValidators) genesisMock := runtimeConfigs[0].GetGenesis() - busMocks := createMockBuses(t, runtimeConfigs) + busMocks := createMockBuses(t, runtimeConfigs, &readWriteWaitGroup) valIds := make([]string, 0, numValidators) for valId := range networkSimulationConfig { @@ -241,7 +239,6 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te // Create connection and bus mocks along with a shared WaitGroup to track the number of expected // reads and writes throughout the mocked local network - var wg sync.WaitGroup for i, valId := range valIds { expectedCall := networkSimulationConfig[valId] expectedReads := expectedCall.numNetworkReads @@ -249,50 +246,41 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te log.Printf("[valId: %s] expected reads: %d\n", valId, expectedReads) log.Printf("[valId: %s] expected writes: %d\n", valId, expectedWrites) - wg.Add(expectedReads) - wg.Add(expectedWrites) + readWriteWaitGroup.Add(expectedReads) + readWriteWaitGroup.Add(expectedWrites) persistenceMock := preparePersistenceMock(t, busMocks[i], genesisMock) consensusMock := prepareConsensusMock(t, busMocks[i]) - telemetryMock := prepareTelemetryMock(t, busMocks[i], valId, &wg, expectedWrites) + telemetryMock := prepareTelemetryMock(t, busMocks[i], valId, &readWriteWaitGroup, expectedWrites) prepareBusMock(busMocks[i], persistenceMock, consensusMock, telemetryMock) } libp2pMockNet := mocknet.New() - defer func() { - err := libp2pMockNet.Close() - require.NoError(t, err) - }() // Inject the connection and bus mocks into the P2P modules p2pModules := createP2PModules(t, busMocks, libp2pMockNet) - for serviceURL, p2pMod := range p2pModules { + for _, p2pMod := range p2pModules { err := p2pMod.Start() require.NoError(t, err) - - sURL := strings.Clone(serviceURL) - mod := *p2pMod - p2pMod.host.SetStreamHandler(protocol.PoktProtocolID, func(stream libp2pNetwork.Stream) { - log.Printf("[valID: %s] Read\n", sURL) - (&mod).router.(*raintree.RainTreeRouter).HandleStream(stream) - wg.Done() - }) } // Wait for completion - defer waitForNetworkSimulationCompletion(t, &wg) + defer waitForNetworkSimulationCompletion(t, &readWriteWaitGroup) t.Cleanup(func() { // Stop all p2p modules for _, p2pMod := range p2pModules { err := p2pMod.Stop() require.NoError(t, err) } + + err := libp2pMockNet.Close() + require.NoError(t, err) }) // Send the first message (by the originator) to trigger a RainTree broadcast - p := &anypb.Any{} + p := &anypb.Any{TypeUrl: "test"} p2pMod := p2pModules[origNode] require.NoError(t, p2pMod.Broadcast(p)) } diff --git a/p2p/module_test.go b/p2p/module_test.go index 79cd17066..45477bd1d 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -111,7 +111,7 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) - mockBus := createMockBus(t, mockRuntimeMgr) + mockBus := createMockBus(t, mockRuntimeMgr, nil) genesisStateMock := createMockGenesisState(keys) persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) @@ -137,7 +137,7 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { } host := newLibp2pMockNetHost(t, privKey, peer) - p2pMod, err := Create(mockBus, WithHostOption(host)) + p2pMod, err := Create(mockBus, WithHost(host)) if (err != nil) != tt.wantErr { t.Errorf("p2pModule.Create() error = %v, wantErr %v", err, tt.wantErr) } @@ -155,7 +155,7 @@ func TestP2pModule_WithHostOption_Restart(t *testing.T) { privKey := cryptoPocket.GetPrivKeySeed(1) mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) - mockBus := createMockBus(t, mockRuntimeMgr) + mockBus := createMockBus(t, mockRuntimeMgr, nil) genesisStateMock := createMockGenesisState(nil) persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) @@ -184,7 +184,7 @@ func TestP2pModule_WithHostOption_Restart(t *testing.T) { } mockNetHost := newLibp2pMockNetHost(t, privKey, peer) - p2pMod, err := Create(mockBus, WithHostOption(mockNetHost)) + p2pMod, err := Create(mockBus, WithHost(mockNetHost)) require.NoError(t, err) mod, ok := p2pMod.(*p2pModule) diff --git a/p2p/protocol/protocol.go b/p2p/protocol/protocol.go index 81737e9a8..c4463bb45 100644 --- a/p2p/protocol/protocol.go +++ b/p2p/protocol/protocol.go @@ -3,15 +3,17 @@ package protocol import "github.com/libp2p/go-libp2p/core/protocol" const ( - // PoktProtocolID is the libp2p protocol ID used when opening a new stream - // to a remote peer and setting the stream handler for the local peer. - // Libp2p APIs use this to distinguish which multiplexed protocols/streams to consider. - PoktProtocolID = protocol.ID("pokt/v1.0.0") + // RaintreeProtocolID is the libp2p protocol ID used in the Raintree router + // when opening a new stream to a remote peer and setting the stream handler + // for the local peer. Libp2p APIs use this to distinguish which multiplexed + // protocols/streams to consider. + RaintreeProtocolID = protocol.ID("pokt/raintree/v1.0.0") + // BackgroundProtocolID is the libp2p protocol ID used in the Background router + // when opening a new stream to a remote peer and setting the stream handler + // for the local peer. Libp2p APIs use this to distinguish which multiplexed + // protocols/streams to consider. + BackgroundProtocolID = protocol.ID("pokt/background/v1.0.0") // BackgroundTopicStr is a "default" pubsub topic string used when // subscribing and broadcasting. BackgroundTopicStr = "pokt/background" - // PeerDiscoveryNamespace used by both advertiser and discoverer to rendezvous - // during peer discovery. Advertiser(s) and discoverer(s) MUST have matching - // discovery namespaces to find one another. - PeerDiscoveryNamespace = "pokt/peer_discovery" ) diff --git a/p2p/providers/current_height_provider/current_height_provider.go b/p2p/providers/current_height_provider/current_height_provider.go index d20a48676..953577fa2 100644 --- a/p2p/providers/current_height_provider/current_height_provider.go +++ b/p2p/providers/current_height_provider/current_height_provider.go @@ -7,7 +7,7 @@ import "github.com/pokt-network/pocket/shared/modules" const ModuleName = "current_height_provider" type CurrentHeightProvider interface { - modules.IntegratableModule + modules.IntegrableModule modules.InterruptableModule CurrentHeight() uint64 diff --git a/p2p/providers/current_height_provider/rpc/provider.go b/p2p/providers/current_height_provider/rpc/provider.go index 4e88de6c2..9c569c988 100644 --- a/p2p/providers/current_height_provider/rpc/provider.go +++ b/p2p/providers/current_height_provider/rpc/provider.go @@ -16,7 +16,7 @@ import ( var _ current_height_provider.CurrentHeightProvider = &rpcCurrentHeightProvider{} type rpcCurrentHeightProvider struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule rpcURL string @@ -80,7 +80,7 @@ func (rchp *rpcCurrentHeightProvider) initRPCClient() { // WithCustomRPCURL allows to specify a custom RPC URL func WithCustomRPCURL(rpcURL string) modules.ModuleOption { - return func(rabp modules.InitializableModule) { + return func(rabp modules.InjectableModule) { rabp.(*rpcCurrentHeightProvider).rpcURL = rpcURL } } diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index e947c6712..bbf57746a 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -11,11 +11,11 @@ import ( "go.uber.org/multierr" ) -const ModuleName = "peerstore_provider" +const PeerstoreProviderSubmoduleName = "peerstore_provider" // PeerstoreProvider is an interface that provides Peerstore accessors type PeerstoreProvider interface { - modules.IntegratableModule + modules.Submodule // GetStakedPeerstoreAtHeight returns a peerstore containing all staked peers // at a given height. These peers communicate via the p2p module's staked actor diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index ce43dcf3d..cbd1ec82c 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -15,9 +15,8 @@ var ( type persistencePStoreProviderOption func(*persistencePeerstoreProvider) type persistencePStoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, persistencePStoreProviderOption] -// TECHDEBT(#810): refactor to implement `Submodule` interface. type persistencePeerstoreProvider struct { - base_modules.IntegratableModule + base_modules.IntegrableModule } func Create(bus modules.Bus, options ...persistencePStoreProviderOption) (peerstore_provider.PeerstoreProvider, error) { @@ -26,8 +25,9 @@ func Create(bus modules.Bus, options ...persistencePStoreProviderOption) (peerst func (*persistencePeerstoreProvider) Create(bus modules.Bus, options ...persistencePStoreProviderOption) (peerstore_provider.PeerstoreProvider, error) { persistencePSP := &persistencePeerstoreProvider{ - IntegratableModule: *base_modules.NewIntegratableModule(bus), + IntegrableModule: *base_modules.NewIntegrableModule(bus), } + bus.RegisterModule(persistencePSP) for _, o := range options { o(persistencePSP) @@ -37,7 +37,7 @@ func (*persistencePeerstoreProvider) Create(bus modules.Bus, options ...persiste } func (*persistencePeerstoreProvider) GetModuleName() string { - return peerstore_provider.ModuleName + return peerstore_provider.PeerstoreProviderSubmoduleName } // GetStakedPeerstoreAtHeight implements the respective `PeerstoreProvider` interface method. diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index ce8210fc7..b0da88f69 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -11,46 +11,53 @@ import ( "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/rpc" - "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" ) -var _ peerstore_provider.PeerstoreProvider = &rpcPeerstoreProvider{} +var ( + _ peerstore_provider.PeerstoreProvider = &rpcPeerstoreProvider{} + _ rpcPeerstoreProviderFactory = &rpcPeerstoreProvider{} +) + +type rpcPeerstoreProviderOption func(*rpcPeerstoreProvider) +type rpcPeerstoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, rpcPeerstoreProviderOption] -// TECHDEBT(#810): refactor to implement `Submodule` interface. type rpcPeerstoreProvider struct { - // TECHDEBT(#810): simplify once submodules are more convenient to retrieve. - base_modules.IntegratableModule - base_modules.InterruptableModule + base_modules.IntegrableModule rpcURL string - p2pCfg *configs.P2PConfig rpcClient *rpc.ClientWithResponses } -func Create(options ...modules.ModuleOption) *rpcPeerstoreProvider { - rabp := &rpcPeerstoreProvider{ +func Create( + bus modules.Bus, + options ...rpcPeerstoreProviderOption, +) (peerstore_provider.PeerstoreProvider, error) { + return new(rpcPeerstoreProvider).Create(bus, options...) +} + +func (*rpcPeerstoreProvider) Create( + bus modules.Bus, + options ...rpcPeerstoreProviderOption, +) (peerstore_provider.PeerstoreProvider, error) { + rpcPSP := &rpcPeerstoreProvider{ rpcURL: flags.RemoteCLIURL, } + bus.RegisterModule(rpcPSP) for _, o := range options { - o(rabp) + o(rpcPSP) } - rabp.initRPCClient() + rpcPSP.initRPCClient() - return rabp -} - -// TECHDEBT(#810): refactor to implement `Submodule` interface. -func (*rpcPeerstoreProvider) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - return Create(options...), nil + return rpcPSP, nil } func (*rpcPeerstoreProvider) GetModuleName() string { - return peerstore_provider.ModuleName + return peerstore_provider.PeerstoreProviderSubmoduleName } func (rpcPSP *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) { @@ -98,16 +105,9 @@ func (rpcPSP *rpcPeerstoreProvider) initRPCClient() { // options -// WithP2PConfig allows to specify a custom P2P config -func WithP2PConfig(p2pCfg *configs.P2PConfig) modules.ModuleOption { - return func(rabp modules.InitializableModule) { - rabp.(*rpcPeerstoreProvider).p2pCfg = p2pCfg - } -} - // WithCustomRPCURL allows to specify a custom RPC URL -func WithCustomRPCURL(rpcURL string) modules.ModuleOption { - return func(rabp modules.InitializableModule) { - rabp.(*rpcPeerstoreProvider).rpcURL = rpcURL +func WithCustomRPCURL(rpcURL string) rpcPeerstoreProviderOption { + return func(rpcPSP *rpcPeerstoreProvider) { + rpcPSP.rpcURL = rpcURL } } diff --git a/p2p/raintree/peers_manager_test.go b/p2p/raintree/peers_manager_test.go index 7fa01a3d3..9e3fa2502 100644 --- a/p2p/raintree/peers_manager_test.go +++ b/p2p/raintree/peers_manager_test.go @@ -9,8 +9,11 @@ import ( "github.com/foxcpp/go-mockdns" "github.com/golang/mock/gomock" + libp2pPeer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" + "github.com/pokt-network/pocket/internal/testutil" "github.com/pokt-network/pocket/p2p/config" typesP2P "github.com/pokt-network/pocket/p2p/types" @@ -18,7 +21,6 @@ import ( "github.com/pokt-network/pocket/runtime/configs" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - "github.com/stretchr/testify/require" ) const ( @@ -26,6 +28,8 @@ const ( addrAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ[" ) +var noopHandler = func(_ []byte) error { return nil } + type ExpectedRainTreeRouterConfig struct { numNodes int numExpectedLevels int @@ -101,6 +105,7 @@ func TestRainTree_Peerstore_HandleUpdate(t *testing.T) { Addr: pubKey.Address(), PeerstoreProvider: pstoreProviderMock, CurrentHeightProvider: currentHeightProviderMock, + Handler: noopHandler, } router, err := NewRainTreeRouter(mockBus, rtCfg) @@ -168,6 +173,7 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { Addr: pubKey.Address(), PeerstoreProvider: pstoreProviderMock, CurrentHeightProvider: currentHeightProviderMock, + Handler: noopHandler, } router, err := NewRainTreeRouter(mockBus, rtCfg) @@ -286,13 +292,15 @@ func testRainTreeMessageTargets(t *testing.T, expectedMsgProp *ExpectedRainTreeM hostMock := mocksP2P.NewMockHost(ctrl) hostMock.EXPECT().Peerstore().Return(libp2pPStore).AnyTimes() - hostMock.EXPECT().SetStreamHandler(gomock.Any(), gomock.Any()).Times(1) + hostMock.EXPECT().SetStreamHandler(gomock.Any(), gomock.Any()).AnyTimes() + hostMock.EXPECT().ID().Return(libp2pPeer.ID("")).AnyTimes() rtCfg := &config.RainTreeConfig{ Host: hostMock, Addr: []byte{expectedMsgProp.orig}, PeerstoreProvider: pstoreProviderMock, CurrentHeightProvider: currentHeightProviderMock, + Handler: noopHandler, } router, err := NewRainTreeRouter(busMock, rtCfg) diff --git a/p2p/raintree/router.go b/p2p/raintree/router.go index bd0656ef7..a96c52665 100644 --- a/p2p/raintree/router.go +++ b/p2p/raintree/router.go @@ -4,7 +4,6 @@ import ( "fmt" libp2pHost "github.com/libp2p/go-libp2p/core/host" - "github.com/pokt-network/pocket/p2p/unicast" "google.golang.org/protobuf/proto" "github.com/pokt-network/pocket/logger" @@ -13,10 +12,10 @@ import ( "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/unicast" "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/shared/codec" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/mempool" "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" @@ -24,15 +23,15 @@ import ( ) var ( - _ typesP2P.Router = &rainTreeRouter{} - _ modules.IntegratableModule = &rainTreeRouter{} - _ rainTreeFactory = &rainTreeRouter{} + _ typesP2P.Router = &rainTreeRouter{} + _ modules.IntegrableModule = &rainTreeRouter{} + _ rainTreeFactory = &rainTreeRouter{} ) type rainTreeFactory = modules.FactoryWithConfig[typesP2P.Router, *config.RainTreeConfig] type rainTreeRouter struct { - base_modules.IntegratableModule + base_modules.IntegrableModule unicast.UnicastRouter logger *modules.Logger @@ -48,7 +47,6 @@ type rainTreeRouter struct { peersManager *rainTreePeersManager pstoreProvider peerstore_provider.PeerstoreProvider currentHeightProvider providers.CurrentHeightProvider - nonceDeduper *mempool.GenericFIFOSet[uint64, uint64] } func NewRainTreeRouter(bus modules.Bus, cfg *config.RainTreeConfig) (typesP2P.Router, error) { @@ -56,9 +54,7 @@ func NewRainTreeRouter(bus modules.Bus, cfg *config.RainTreeConfig) (typesP2P.Ro } func (*rainTreeRouter) Create(bus modules.Bus, cfg *config.RainTreeConfig) (typesP2P.Router, error) { - routerLogger := logger.Global.CreateLoggerForModule("router") - routerLogger.Info().Msg("Initializing rainTreeRouter") - + rainTreeLogger := logger.Global.CreateLoggerForModule("rainTreeRouter") if err := cfg.IsValid(); err != nil { return nil, err } @@ -68,11 +64,24 @@ func (*rainTreeRouter) Create(bus modules.Bus, cfg *config.RainTreeConfig) (type selfAddr: cfg.Addr, pstoreProvider: cfg.PeerstoreProvider, currentHeightProvider: cfg.CurrentHeightProvider, - logger: routerLogger, + logger: rainTreeLogger, handler: cfg.Handler, } rtr.SetBus(bus) + height := rtr.currentHeightProvider.CurrentHeight() + pstore, err := rtr.pstoreProvider.GetStakedPeerstoreAtHeight(height) + if err != nil { + return nil, fmt.Errorf("getting staked peerstore at height %d: %w", height, err) + } + rainTreeLogger.Info().Fields(map[string]any{ + "address": cfg.Addr, + "host_id": cfg.Host.ID(), + "protocol_id": protocol.BackgroundProtocolID, + "current_height": height, + "peerstore_size": pstore.Size(), + }).Msg("initializing raintree router") + if err := rtr.setupDependencies(); err != nil { return nil, err } @@ -80,6 +89,10 @@ func (*rainTreeRouter) Create(bus modules.Bus, cfg *config.RainTreeConfig) (type return typesP2P.Router(rtr), nil } +func (rtr *rainTreeRouter) Close() error { + return nil +} + // NetworkBroadcast implements the respective member of `typesP2P.Router`. func (rtr *rainTreeRouter) Broadcast(data []byte) error { return rtr.broadcastAtLevel(data, rtr.peersManager.GetMaxNumLevels()) @@ -153,15 +166,14 @@ func (rtr *rainTreeRouter) sendInternal(data []byte, address cryptoPocket.Addres peer := rtr.peersManager.GetPeerstore().GetPeer(address) if peer == nil { - return fmt.Errorf("no known peer with pokt address %s", address) + return fmt.Errorf("%w: with pokt address %s", typesP2P.ErrUnknownPeer, address) } // debug logging hostname := rtr.getHostname() utils.LogOutgoingMsg(rtr.logger, hostname, peer) - if err := utils.Libp2pSendToPeer(rtr.host, data, peer); err != nil { - rtr.logger.Debug().Err(err).Msg("from libp2pSendInternal") + if err := utils.Libp2pSendToPeer(rtr.host, protocol.RaintreeProtocolID, data, peer); err != nil { return err } @@ -202,14 +214,13 @@ func (rtr *rainTreeRouter) handleRainTreeMsg(rainTreeMsgBz []byte) error { var rainTreeMsg typesP2P.RainTreeMessage if err := proto.Unmarshal(rainTreeMsgBz, &rainTreeMsg); err != nil { + // TECHDEBT: add telemetry return err } - // TECHDEBT(#763): refactor as "pre-propagation validation" - networkMessage := messaging.PocketEnvelope{} - if err := proto.Unmarshal(rainTreeMsg.Data, &networkMessage); err != nil { - rtr.logger.Error().Err(err).Msg("Error decoding network message") - return err + if err := rtr.validateRainTreeMsg(&rainTreeMsg); err != nil { + // TECHDEBT: add telemetry + return fmt.Errorf("validating raintree message: %w", err) } // Continue RainTree propagation @@ -233,6 +244,13 @@ func (rtr *rainTreeRouter) handleRainTreeMsg(rainTreeMsgBz []byte) error { return nil } +// validateRainTreeMsg ensures that the `data` contained within the RainTree message +// is a valid `PocketEnvelope` by attempting to deserialize it. +func (rtr *rainTreeRouter) validateRainTreeMsg(rainTreeMsg *typesP2P.RainTreeMessage) error { + networkMessage := messaging.PocketEnvelope{} + return proto.Unmarshal(rainTreeMsg.Data, &networkMessage) +} + // GetPeerstore implements the respective member of `typesP2P.Router`. func (rtr *rainTreeRouter) GetPeerstore() typesP2P.Peerstore { return rtr.peersManager.GetPeerstore() @@ -286,7 +304,7 @@ func (rtr *rainTreeRouter) setupUnicastRouter() error { unicastRouterCfg := config.UnicastRouterConfig{ Logger: rtr.logger, Host: rtr.host, - ProtocolID: protocol.PoktProtocolID, + ProtocolID: protocol.RaintreeProtocolID, MessageHandler: rtr.handleRainTreeMsg, PeerHandler: rtr.AddPeer, } diff --git a/p2p/raintree/router_test.go b/p2p/raintree/router_test.go index 1ef093a20..2865a584b 100644 --- a/p2p/raintree/router_test.go +++ b/p2p/raintree/router_test.go @@ -57,6 +57,7 @@ func TestRainTreeRouter_AddPeer(t *testing.T) { Addr: selfAddr, PeerstoreProvider: peerstoreProviderMock, CurrentHeightProvider: currentHeightProviderMock, + Handler: noopHandler, } router, err := NewRainTreeRouter(busMock, rtCfg) @@ -119,6 +120,7 @@ func TestRainTreeRouter_RemovePeer(t *testing.T) { Addr: selfAddr, PeerstoreProvider: peerstoreProviderMock, CurrentHeightProvider: currentHeightProviderMock, + Handler: noopHandler, } router, err := NewRainTreeRouter(busMock, rtCfg) diff --git a/p2p/testutil.go b/p2p/testutil.go new file mode 100644 index 000000000..c0c42704a --- /dev/null +++ b/p2p/testutil.go @@ -0,0 +1,48 @@ +//go:build test + +package p2p + +import ( + libp2pHost "github.com/libp2p/go-libp2p/core/host" + typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/shared/modules" +) + +// WithHost associates an existing (i.e. "started") libp2p `host.Host` +// with this module, instead of creating a new one on `#Start()`. +// Primarily intended for testing. +func WithHost(host libp2pHost.Host) modules.ModuleOption { + return func(m modules.InjectableModule) { + mod, ok := m.(*p2pModule) + if ok { + mod.host = host + mod.logger.Debug().Msg("using host provided via `WithHost`") + } + } +} + +// WithUnstakedActorRouter assigns the given router to the P2P modules +// `#unstakedActor` field, used to communicate between unstaked actors +// and the rest of the network, plus as a redundancy to the staked actor +// router when broadcasting. +func WithUnstakedActorRouter(router typesP2P.Router) modules.ModuleOption { + return func(m modules.InjectableModule) { + mod, ok := m.(*p2pModule) + if ok { + mod.unstakedActorRouter = router + mod.logger.Debug().Msg("using unstaked actor router provided via `WithUnstakeActorRouter`") + } + } +} + +// WithStakedActorRouter assigns the given router to the P2P modules' +// `#stakedActor` field, exclusively used to communicate between staked actors. +func WithStakedActorRouter(router typesP2P.Router) modules.ModuleOption { + return func(m modules.InjectableModule) { + mod, ok := m.(*p2pModule) + if ok { + mod.stakedActorRouter = router + mod.logger.Debug().Msg("using staked actor router provided via `WithStakeActorRouter`") + } + } +} diff --git a/p2p/transport_encryption_test.go b/p2p/transport_encryption_test.go index d95cb6496..799340a17 100644 --- a/p2p/transport_encryption_test.go +++ b/p2p/transport_encryption_test.go @@ -14,6 +14,7 @@ import ( "github.com/pokt-network/pocket/internal/testutil" "github.com/pokt-network/pocket/p2p/protocol" typesP2P "github.com/pokt-network/pocket/p2p/types" + mock_types "github.com/pokt-network/pocket/p2p/types/mocks" "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/configs/types" @@ -23,7 +24,7 @@ import ( mockModules "github.com/pokt-network/pocket/shared/modules/mocks" ) -func TestP2pModule_Insecure_Error(t *testing.T) { +func TestP2pModule_RainTreeRouter_Insecure_Error(t *testing.T) { // TECHDEBT(#609): refactor mock setup with similar test utilities. ctrl := gomock.NewController(t) hostname := "127.0.0.1" @@ -54,7 +55,7 @@ func TestP2pModule_Insecure_Error(t *testing.T) { telemetryMock.EXPECT().GetEventMetricsAgent().Return(eventMetricsAgentMock).AnyTimes() telemetryMock.EXPECT().GetModuleName().Return(modules.TelemetryModuleName).AnyTimes() - busMock := createMockBus(t, runtimeMgrMock) + busMock := createMockBus(t, runtimeMgrMock, nil) busMock.EXPECT().GetConsensusModule().Return(mockConsensusModule).AnyTimes() busMock.EXPECT().GetRuntimeMgr().Return(runtimeMgrMock).AnyTimes() busMock.EXPECT().GetTelemetryModule().Return(telemetryMock).AnyTimes() @@ -73,7 +74,10 @@ func TestP2pModule_Insecure_Error(t *testing.T) { dnsDone := testutil.PrepareDNSMockFromServiceURLs(t, serviceURLs) t.Cleanup(dnsDone) - p2pMod, err := Create(busMock) + routerMock := mock_types.NewMockRouter(ctrl) + routerMock.EXPECT().Close().Times(1) + + p2pMod, err := Create(busMock, WithUnstakedActorRouter(routerMock)) require.NoError(t, err) err = p2pMod.Start() @@ -114,6 +118,6 @@ func TestP2pModule_Insecure_Error(t *testing.T) { require.NoError(t, err) ctx := context.Background() - _, err = clearNode.NewStream(ctx, libp2pPeerInfo.ID, protocol.PoktProtocolID) + _, err = clearNode.NewStream(ctx, libp2pPeerInfo.ID, protocol.RaintreeProtocolID) require.ErrorContains(t, err, "failed to negotiate security protocol: protocols not supported:") } diff --git a/p2p/types/errors.go b/p2p/types/errors.go index 0688e5757..632bdd534 100644 --- a/p2p/types/errors.go +++ b/p2p/types/errors.go @@ -1,6 +1,13 @@ package types -import "fmt" +import ( + "errors" + "fmt" +) + +var ( + ErrUnknownPeer = errors.New("unknown peer") +) func ErrUnknownEventType(msg any) error { return fmt.Errorf("unknown event type: %v", msg) diff --git a/p2p/types/proto/background.proto b/p2p/types/proto/background.proto new file mode 100644 index 000000000..da1e138e7 --- /dev/null +++ b/p2p/types/proto/background.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package background; + +option go_package = "github.com/pokt-network/pocket/p2p/types"; + +// BackgroundMessage is intended to be used with the background router for +// communication with unstaked actors. For unstaked actors, this is the only +// means of communication with the network. For staked actors, this functions +// as a redundancy for broadcast propagation (in addition to the staked actor +// router broadcast message - i.e. `RainTreeMessage`). +message BackgroundMessage { + bytes data = 1; +} diff --git a/p2p/raintree/types/proto/raintree.proto b/p2p/types/proto/raintree.proto similarity index 100% rename from p2p/raintree/types/proto/raintree.proto rename to p2p/types/proto/raintree.proto diff --git a/p2p/types/router.go b/p2p/types/router.go index 37080acbd..c3776a114 100644 --- a/p2p/types/router.go +++ b/p2p/types/router.go @@ -10,10 +10,11 @@ import ( // TECHDEBT(olshansky): When we delete `stdnetwork` and only go with `raintree`, this interface // can be simplified greatly. type Router interface { - modules.IntegratableModule + modules.IntegrableModule Broadcast(data []byte) error Send(data []byte, address cryptoPocket.Address) error + Close() error // GetPeerstore is used by the P2P module to update the staked actor router's // (`RainTreeRouter`) peerstore. diff --git a/p2p/unicast/router.go b/p2p/unicast/router.go index 18a41fc32..7b6a9ea88 100644 --- a/p2p/unicast/router.go +++ b/p2p/unicast/router.go @@ -25,7 +25,7 @@ var _ unicastRouterFactory = &UnicastRouter{} type unicastRouterFactory = modules.FactoryWithConfig[*UnicastRouter, *config.UnicastRouterConfig] type UnicastRouter struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger host libp2pHost.Host diff --git a/p2p/utils/host.go b/p2p/utils/host.go index e9c6e130b..3597856b7 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -6,10 +6,11 @@ import ( "time" libp2pHost "github.com/libp2p/go-libp2p/core/host" + libp2pProtocol "github.com/libp2p/go-libp2p/core/protocol" + "go.uber.org/multierr" + "github.com/pokt-network/pocket/logger" - "github.com/pokt-network/pocket/p2p/protocol" typesP2P "github.com/pokt-network/pocket/p2p/types" - "go.uber.org/multierr" ) const ( @@ -76,7 +77,7 @@ func RemovePeerFromLibp2pHost(host libp2pHost.Host, peer typesP2P.Peer) error { } // Libp2pSendToPeer sends data to the given pocket peer from the given libp2p host. -func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) error { +func Libp2pSendToPeer(host libp2pHost.Host, protocolID libp2pProtocol.ID, data []byte, peer typesP2P.Peer) error { // TECHDEBT(#595): add ctx to interface methods and propagate down. ctx := context.TODO() @@ -94,7 +95,7 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err logger.Global.Debug().Err(err).Msg("logging resource scope stats") } - stream, err := host.NewStream(ctx, peerInfo.ID, protocol.PoktProtocolID) + stream, err := host.NewStream(ctx, peerInfo.ID, protocolID) if err != nil { return fmt.Errorf("opening stream: %w", err) } diff --git a/p2p/utils_test.go b/p2p/utils_test.go index 633ea1425..678f2f2b8 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -20,6 +20,7 @@ import ( "github.com/pokt-network/pocket/p2p/providers/current_height_provider" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" + mock_types "github.com/pokt-network/pocket/p2p/types/mocks" "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/configs" @@ -29,6 +30,7 @@ import ( "github.com/pokt-network/pocket/runtime/test_artifacts" coreTypes "github.com/pokt-network/pocket/shared/core/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" "github.com/pokt-network/pocket/telemetry" @@ -107,10 +109,20 @@ func waitForNetworkSimulationCompletion(t *testing.T, wg *sync.WaitGroup) { func createP2PModules(t *testing.T, busMocks []*mockModules.MockBus, netMock mocknet.Mocknet) (p2pModules map[string]*p2pModule) { peerIDs := setupMockNetPeers(t, netMock, len(busMocks)) + ctrl := gomock.NewController(t) + noopBackgroundRouterMock := mock_types.NewMockRouter(ctrl) + noopBackgroundRouterMock.EXPECT().Broadcast(gomock.Any()).Times(1) + noopBackgroundRouterMock.EXPECT().Close().Times(len(busMocks)) + p2pModules = make(map[string]*p2pModule, len(busMocks)) for i := range busMocks { host := netMock.Host(peerIDs[i]) - p2pMod, err := Create(busMocks[i], WithHostOption(host)) + p2pMod, err := Create( + busMocks[i], + WithHost(host), + // mock background router to prevent & ignore background message propagation. + WithUnstakedActorRouter(noopBackgroundRouterMock), + ) require.NoError(t, err) p2pModules[validatorId(i+1)] = p2pMod.(*p2pModule) } @@ -180,25 +192,41 @@ func createMockRuntimeMgrs(t *testing.T, numValidators int) []modules.RuntimeMgr return mockRuntimeMgrs } -func createMockBuses(t *testing.T, runtimeMgrs []modules.RuntimeMgr) []*mockModules.MockBus { +func createMockBuses( + t *testing.T, + runtimeMgrs []modules.RuntimeMgr, + readWriteWaitGroup *sync.WaitGroup, +) []*mockModules.MockBus { mockBuses := make([]*mockModules.MockBus, len(runtimeMgrs)) for i := range mockBuses { - mockBuses[i] = createMockBus(t, runtimeMgrs[i]) + mockBuses[i] = createMockBus(t, runtimeMgrs[i], readWriteWaitGroup) } return mockBuses } -func createMockBus(t *testing.T, runtimeMgr modules.RuntimeMgr) *mockModules.MockBus { +func createMockBus( + t *testing.T, + runtimeMgr modules.RuntimeMgr, + readWriteWaitGroup *sync.WaitGroup, +) *mockModules.MockBus { ctrl := gomock.NewController(t) mockBus := mockModules.NewMockBus(ctrl) mockBus.EXPECT().GetRuntimeMgr().Return(runtimeMgr).AnyTimes() - mockBus.EXPECT().RegisterModule(gomock.Any()).DoAndReturn(func(m modules.Module) { + mockBus.EXPECT().RegisterModule(gomock.Any()).DoAndReturn(func(m modules.Submodule) { m.SetBus(mockBus) }).AnyTimes() mockModulesRegistry := mockModules.NewMockModulesRegistry(ctrl) - mockModulesRegistry.EXPECT().GetModule(peerstore_provider.ModuleName).Return(nil, runtime.ErrModuleNotRegistered(peerstore_provider.ModuleName)).AnyTimes() + mockModulesRegistry.EXPECT().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName).Return(nil, runtime.ErrModuleNotRegistered(peerstore_provider.PeerstoreProviderSubmoduleName)).AnyTimes() mockModulesRegistry.EXPECT().GetModule(current_height_provider.ModuleName).Return(nil, runtime.ErrModuleNotRegistered(current_height_provider.ModuleName)).AnyTimes() mockBus.EXPECT().GetModulesRegistry().Return(mockModulesRegistry).AnyTimes() + mockBus.EXPECT().PublishEventToBus(gomock.AssignableToTypeOf(&messaging.PocketEnvelope{})). + Do(func(envelope *messaging.PocketEnvelope) { + fmt.Println("[valId: unknown] Read") + fmt.Printf("content type: %s\n", envelope.Content.GetTypeUrl()) + if readWriteWaitGroup != nil { + readWriteWaitGroup.Done() + } + }).AnyTimes() // TECHDEBT: assert number of times. Consider `waitForEventsInternal` or similar as in consensus. mockBus.EXPECT().PublishEventToBus(gomock.Any()).AnyTimes() return mockBus } @@ -313,11 +341,32 @@ func prepareEventMetricsAgentMock(t *testing.T, valId string, wg *sync.WaitGroup ctrl := gomock.NewController(t) eventMetricsAgentMock := mockModules.NewMockEventMetricsAgent(ctrl) - eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() - eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Eq(telemetry.P2P_RAINTREE_MESSAGE_EVENT_METRIC_SEND_LABEL), gomock.Any()).Do(func(n, e any, l ...any) { + // TECHDEBT(#886): The number of times each telemetry event is expected + // (below) is dependent on the number of redundant messages all validators see, + // which is a function of the network size. Until this function is derived and + // implemented, we cannot predict the number of times each event is expected. + _ = expectedNumNetworkWrites + + eventMetricsAgentMock.EXPECT().EmitEvent( + gomock.Any(), + gomock.Any(), + gomock.Eq(telemetry.P2P_RAINTREE_MESSAGE_EVENT_METRIC_SEND_LABEL), + gomock.Any(), + ).Do(func(n, e any, l ...any) { + log.Printf("[valId: %s] Write\n", valId) + wg.Done() + }).AnyTimes() // TECHDEBT: expect specific number of non-redundant writes once known. + eventMetricsAgentMock.EXPECT().EmitEvent( + gomock.Any(), + gomock.Eq(telemetry.P2P_BROADCAST_MESSAGE_REDUNDANCY_PER_BLOCK_EVENT_METRIC_NAME), + gomock.Any(), + gomock.Any(), // nonce + gomock.Any(), + gomock.Any(), // blockHeight + ).Do(func(n, e any, l ...any) { log.Printf("[valId: %s] Write\n", valId) wg.Done() - }).Times(expectedNumNetworkWrites) + }).AnyTimes() // TECHDEBT: expect specific number of redundant writes once known. eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Not(telemetry.P2P_RAINTREE_MESSAGE_EVENT_METRIC_SEND_LABEL), gomock.Any()).AnyTimes() return eventMetricsAgentMock diff --git a/persistence/actor.go b/persistence/actor.go index ed9079b1f..de96548ae 100644 --- a/persistence/actor.go +++ b/persistence/actor.go @@ -17,7 +17,7 @@ func (p *PostgresContext) GetActor(actorType coreTypes.ActorType, address []byte case types.FishermanActor.GetActorType(): schema = types.FishermanActor case types.ValidatorActor.GetActorType(): - schema = types.FishermanActor + schema = types.ValidatorActor default: return nil, fmt.Errorf("invalid actor type: %s", actorType) } diff --git a/persistence/local/module.go b/persistence/local/module.go index ced14fbcc..3c0930cc6 100644 --- a/persistence/local/module.go +++ b/persistence/local/module.go @@ -16,14 +16,14 @@ const ( var _ modules.PersistenceLocalContext = &persistenceLocalContext{} type persistenceLocalContext struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger databasePath string } func WithLocalDatabasePath(databasePath string) modules.ModuleOption { - return func(m modules.InitializableModule) { + return func(m modules.InjectableModule) { if plc, ok := m.(*persistenceLocalContext); ok { plc.databasePath = databasePath } diff --git a/persistence/module.go b/persistence/module.go index 31727796a..dec15112b 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -25,7 +25,7 @@ var ( // TODO: convert address and public key to string not bytes in all account and actor functions // TODO: remove address parameter from all pool operations type persistenceModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 74fe6599c..b960e4b71 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -80,7 +80,7 @@ var _ modules.TreeStoreModule = &treeStore{} // of the underlying trees by utilizing the lazy loading // functionality provided by the underlying smt library. type treeStore struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger treeStoreDir string @@ -127,6 +127,11 @@ func (t *treeStore) DebugClearAll() error { return nil } +// GetModuleName implements the respective `TreeStoreModule` interface method. +func (t *treeStore) GetModuleName() string { + return modules.TreeStoreModuleName +} + // updateMerkleTrees updates all of the merkle trees in order defined by `numMerkleTrees` // * it returns the new state hash capturing the state of all the trees or an error if one occurred func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height uint64) (string, error) { diff --git a/rpc/module.go b/rpc/module.go index 48a715521..37505a560 100644 --- a/rpc/module.go +++ b/rpc/module.go @@ -12,7 +12,7 @@ import ( var _ modules.RPCModule = &rpcModule{} type rpcModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule logger *modules.Logger diff --git a/rpc/noop_module.go b/rpc/noop_module.go index 1c7401b80..0063d423b 100644 --- a/rpc/noop_module.go +++ b/rpc/noop_module.go @@ -10,7 +10,7 @@ import ( var _ modules.RPCModule = &noopRpcModule{} type noopRpcModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule } diff --git a/rpc/server.go b/rpc/server.go index 696c78a58..20dfdfa79 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -11,14 +11,14 @@ import ( ) type rpcServer struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger modules.Logger } var ( - _ ServerInterface = &rpcServer{} - _ modules.IntegratableModule = &rpcServer{} + _ ServerInterface = &rpcServer{} + _ modules.IntegrableModule = &rpcServer{} ) func NewRPCServer(bus modules.Bus) *rpcServer { diff --git a/runtime/bus.go b/runtime/bus.go index 0b343a8a7..53df65752 100644 --- a/runtime/bus.go +++ b/runtime/bus.go @@ -45,7 +45,7 @@ func (m *bus) GetModulesRegistry() modules.ModulesRegistry { return m.modulesRegistry } -func (m *bus) RegisterModule(module modules.Module) { +func (m *bus) RegisterModule(module modules.Submodule) { module.SetBus(m) m.modulesRegistry.RegisterModule(module) } @@ -124,7 +124,7 @@ func (m *bus) GetIBCModule() modules.IBCModule { } // getModuleFromRegistry is a helper function to get a module from the registry that handles errors and casting via generics -func getModuleFromRegistry[T modules.Module](m *bus, moduleName string) T { +func getModuleFromRegistry[T modules.Submodule](m *bus, moduleName string) T { mod, err := m.modulesRegistry.GetModule(moduleName) if err != nil { logger.Global.Logger.Fatal(). diff --git a/runtime/manager.go b/runtime/manager.go index 8c5c245b7..151f2c198 100644 --- a/runtime/manager.go +++ b/runtime/manager.go @@ -17,7 +17,7 @@ import ( var _ modules.RuntimeMgr = &Manager{} type Manager struct { - base_modules.IntegratableModule + base_modules.IntegrableModule config *configs.Config genesisState *genesis.GenesisState diff --git a/runtime/modules_registry.go b/runtime/modules_registry.go index 85b9c2c75..ac7a6eb42 100644 --- a/runtime/modules_registry.go +++ b/runtime/modules_registry.go @@ -10,22 +10,22 @@ var _ modules.ModulesRegistry = &modulesRegistry{} type modulesRegistry struct { m sync.Mutex - registry map[string]modules.Module + registry map[string]modules.InjectableModule } -func NewModulesRegistry() *modulesRegistry { +func NewModulesRegistry() modules.ModulesRegistry { return &modulesRegistry{ - registry: make(map[string]modules.Module), + registry: make(map[string]modules.InjectableModule), } } -func (m *modulesRegistry) RegisterModule(module modules.Module) { +func (m *modulesRegistry) RegisterModule(module modules.InjectableModule) { m.m.Lock() defer m.m.Unlock() m.registry[module.GetModuleName()] = module } -func (m *modulesRegistry) GetModule(moduleName string) (modules.Module, error) { +func (m *modulesRegistry) GetModule(moduleName string) (modules.InjectableModule, error) { m.m.Lock() defer m.m.Unlock() if mod, ok := m.registry[moduleName]; ok { diff --git a/shared/messaging/envelope.go b/shared/messaging/envelope.go index 4150fbfba..f65304007 100644 --- a/shared/messaging/envelope.go +++ b/shared/messaging/envelope.go @@ -4,6 +4,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/anypb" + + cryptoPocket "github.com/pokt-network/pocket/shared/crypto" ) // PackMessage returns a *PocketEnvelope after having packed the message supplied as an argument @@ -12,7 +14,10 @@ func PackMessage(message proto.Message) (*PocketEnvelope, error) { if err != nil { return nil, err } - return &PocketEnvelope{Content: anyMsg}, nil + return &PocketEnvelope{ + Content: anyMsg, + Nonce: cryptoPocket.GetNonce(), + }, nil } // UnpackMessage extracts the message inside the PocketEnvelope decorating it with typing information diff --git a/shared/modules/base_modules/integratable_module.go b/shared/modules/base_modules/integratable_module.go index 4b24254c4..5b79b10ed 100644 --- a/shared/modules/base_modules/integratable_module.go +++ b/shared/modules/base_modules/integratable_module.go @@ -2,24 +2,24 @@ package base_modules import "github.com/pokt-network/pocket/shared/modules" -var _ modules.IntegratableModule = &IntegratableModule{} +var _ modules.IntegrableModule = &IntegrableModule{} -// IntegratableModule is a base struct that is meant to be embedded in module structs that implement the interface `modules.IntegratableModule`. +// IntegrableModule is a base struct that is meant to be embedded in module structs that implement the interface `modules.IntegrableModule`. // // It provides the basic logic for the `SetBus` and `GetBus` methods and allows the implementer to reduce boilerplate code keeping the code // DRY (Don't Repeat Yourself) while preserving the ability to override the methods if needed. -type IntegratableModule struct { +type IntegrableModule struct { bus modules.Bus } -func NewIntegratableModule(bus modules.Bus) *IntegratableModule { - return &IntegratableModule{bus: bus} +func NewIntegrableModule(bus modules.Bus) *IntegrableModule { + return &IntegrableModule{bus: bus} } -func (m *IntegratableModule) GetBus() modules.Bus { +func (m *IntegrableModule) GetBus() modules.Bus { return m.bus } -func (m *IntegratableModule) SetBus(bus modules.Bus) { +func (m *IntegrableModule) SetBus(bus modules.Bus) { m.bus = bus } diff --git a/shared/modules/bus_module.go b/shared/modules/bus_module.go index 195d13dfd..6afa1a56f 100644 --- a/shared/modules/bus_module.go +++ b/shared/modules/bus_module.go @@ -21,7 +21,7 @@ type Bus interface { // Dependency Injection / Service Discovery GetModulesRegistry() ModulesRegistry - RegisterModule(module Module) + RegisterModule(module Submodule) // Pocket modules GetPersistenceModule() PersistenceModule diff --git a/shared/modules/doc/README.md b/shared/modules/doc/README.md index 431e47a3b..28a2c249b 100644 --- a/shared/modules/doc/README.md +++ b/shared/modules/doc/README.md @@ -2,48 +2,174 @@ This document outlines how we structured the code by splitting it into modules, what a module is and how to create one. - - ## Contents +- [tl;dr Just show me an example](#tldr-just-show-me-an-example) - [Definitions](#definitions) - - [Shared module interface](#shared-module-interface) - - [Module](#module) - - [Module mock](#module-mock) - - [Base module](#base-module) + - [Requirement Level Keywords](#requirement-level-keywords) + - [Module](#module) + - [Module mock](#module-mock) + - [Shared module interfaces](#shared-module-interfaces) + - [Base module](#base-module) + - [Submodule](#submodule) + - [Shared submodule](#shared-submodule) + - [Class Diagram Legend](#class-diagram-legend) + - [Module, Submodule \& Shared Interfaces](#module-submodule--shared-interfaces) + - [Factory interfaces](#factory-interfaces) - [Code Organization](#code-organization) -- [Modules in detail](#modules-in-detail) - - [Module creation](#module-creation) - - [Interacting \& Registering with the `bus`](#interacting--registering-with-the-bus) - - [Modules Registry](#modules-registry) - - [Modules Registry Example](#modules-registry-example) - - [Start the module](#start-the-module) - - [Add a logger to the module](#add-a-logger-to-the-module) - - [Get the module `bus`](#get-the-module-bus) - - [Stop the module](#stop-the-module) +- [(Sub)Modules in detail](#submodules-in-detail) - [Shared (sub)module interfaces](#shared-submodule-interfaces) - [Construction parameters \& non-(sub)module dependencies](#construction-parameters--non-submodule-dependencies) + - [Module creation](#module-creation) + - [Module configs \& options](#module-configs--options) + - [Submodule creation](#submodule-creation) + - [Submodule configs \& options](#submodule-configs--options) + - [Configs](#configs) + - [Options](#options) + - [Comprehensive Submodule Example:](#comprehensive-submodule-example) + - [Interacting \& Registering with the `bus`](#interacting--registering-with-the-bus) + - [Modules Registry](#modules-registry) + - [Modules Registry Example](#modules-registry-example) + - [Start the module](#start-the-module) + - [Add a logger to the module](#add-a-logger-to-the-module) + - [Get the module `bus`](#get-the-module-bus) + - [Stop the module](#stop-the-module) + +## tl;dr Just show me an example + +If you're just interested in an example PR that introduced a new module to the codebase, see [#842](https://github.com/pokt-network/pocket/pull/842) which added the first iteration of the IBC module ## Definitions -### Shared module interface +### Requirement Level Keywords + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +### Module + +A module is an abstraction (go interface) of a self-contained unit of functionality that carries out a specific task/set of tasks with the idea of being reusable, modular, replacable, mockable, testable and exposing a clear and concise API. + +A module **MAY** implement 0..N common interfaces. + +Each _shared_ module **MUST** be registered with the module registry such that it can be retrieved via the bus. + +You can find additional details in the [Modules in detail](#submodules-in-detail) section below. + +### Module mock + +A mock is a stand-in, a fake or simplified implementation of a module that is used for testing purposes. + +It is used to simulate the behaviour of the module and to verify that the module is interacting with other modules correctly. + +Mocks are generated using `go:generate` directives together with the [`mockgen` tool](https://pkg.go.dev/github.com/golang/mock#readme-running-mockgen). + +A global search of `mockgen` in the codebase will provide examples of where and how it's used. + +### Shared module interfaces + +A shared module interface is an interface that defines the methods that we modelled as common to multiple modules. + +**For example**: the ability to start/stop a module is pretty common and for that we defined the `InterruptableModule` interface. -A shared module interface is an interface that defines the methods that we modelled as common to multiple modules. For example: the ability to start/stop a module is pretty common and for that we defined the `InterruptableModule` interface. There are some interfaces that are common to multiple modules and we followed the [Interface segregation principle](https://en.wikipedia.org/wiki/Interface_segregation_principle) and also [Rob Pike's Go Proverb](https://youtu.be/PAAkCSZUG1c?t=317): > The bigger the interface, the weaker the abstraction. These interfaces that can be embedded in modules are defined in `shared/modules/module.go`. GoDoc comments will provide you with more information about each interface. -### Module +### Base module -A module is an abstraction (go interface) of a self-contained unit of functionality that carries out a specific task/set of tasks with the idea of being reusable, modular, testable and having a clear and concise API. A module might implement 0..N common interfaces. You can find additional details in the [Modules in detail](#modules-in-detail) section below. +A base module is a module that implements a common interface, exposing the most basic logic. -### Module mock +Base modules are meant to be **embedded** in module structs which implement this common interface **and** don't need to override the respective interface member(s). -A mock is a stand-in, a fake or simplified implementation of a module that is used for testing purposes. It is used to simulate the behaviour of the module and to verify that the module is interacting with other modules correctly. Mocks are generated using `go:generate` directives together with the [`mockgen` tool](https://pkg.go.dev/github.com/golang/mock#readme-running-mockgen). +The intention being to improve **DRYness (Don't Repeat Yourself)** and to reduce boilerplate code. -### Base module +You can find the base modules in the `shared/modules/base_modules` package. + +### Submodule + +A submodule is a self-contained unit of functionality which is composed by a module and is necessary for that module to function. + +Each module **SHOULD** be registered with the module registry such that it can be retrieved via the bus. + +### Shared submodule + +A shared submodule is any submodule which is a dependency of more than one module. + +Shared submodules **MUST** define their respective interface type in the `shared/modules` package to avoid import cycles. + +_NOTE: "interface types" of "shared submodules" are distinctly different from "shared module interfaces"._ -A base module is a module that implements a common interface, exposing the most basic logic. Base modules are meant to be **embedded** in module structs which implement this common interface **and** don't need to override the respective interface member(s). The intention being to improve DRYness (Don't Repeat Yourself) and to reduce boilerplate code. You can find the base modules in the `shared/modules/base_modules` package. +### Class Diagram Legend + +```mermaid +classDiagram + direction LR + + class concreteType + + class InterfaceType { + <> + InterfaceMethod() + } + + concreteType --|> InterfaceType : realizes interface + concreteType ..|> InterfaceType : realizes one of (this) + concreteType ..|> InterfaceType : realizes one of (or this) +``` + +### Module, Submodule & Shared Interfaces + +```mermaid +classDiagram + direction TB + + class IntegrableModule { + <> + GetBus() Bus + SetBus(bus Bus) + } + + class ModuleFactoryWithOptions { + <> + Create(bus Bus, options ...ModuleOption) (Module, error) + } + + class Module { + <> + } + class InjectableModule { + <> + GetModuleName() string + } + class InterruptableModule { + <> + Start() error + Stop() error + } + + Module --|> InjectableModule + Module --|> IntegrableModule + Module --|> InterruptableModule + Module --|> ModuleFactoryWithOptions + + class Submodule { + <> + } + + Submodule --|> InjectableModule + Submodule --|> IntegrableModule +``` + +### Factory interfaces + +In order to formalize and support the generalization of modules and submodule while still providing flexibility, we defined a set of generic factory interfaces which can be embedded or used directly to enforce consistent constructor semantics. + +These interfaces are [defined in `shared/modules/factory.go`](../factory.go) and outlined convenience: + +- `ModuleFactoryWithOptions`: Specialized factory interface for modules conforming to the "typical" signature above. Useful when creating modules that only require optional runtime configurations. +- `FactoryWithConfig`: A generic type, used to create a module with a specific configuration type. +- `FactoryWithOptions`: The generic form used to define `ModuleFactoryWithConfig`. +- `FactoryWithConfigAndOptions`: Another generic type which combines the capabilities of the previous two. Suitable for creating modules that require both specific configuration and optional runtime configurations. ## Code Organization @@ -57,36 +183,222 @@ A base module is a module that implements a common interface, exposing the most └── types # Common types for the modules ``` -## Modules in detail +## (Sub)Modules in detail + +#### Shared (sub)module interfaces + +Each module and submodule defines its module interface (and supporting interfaces, if any) in the `shared/modules` package, where the file containing the module interface follows the naming convention `[moduleName]_[sub]module.go`. + +You should start by looking at the interfaces of the modules we already implemented to get a better idea of what a module is and how it should be structured. -We structured the code so that each module has its interface (and supporting interfaces, if any) defined in the `shared/modules` package, where the file containing the module interface follows the naming convention `[moduleName]_module.go`. -You can start by looking at the interfaces of the modules we already implemented to get a better idea of what a module is and how it should be structured. You might notice that these files include `go:generate` directives, these are used to generate the module mocks for testing purposes. +#### Construction parameters & non-(sub)module dependencies + +Modules and submodules MAY have zero or more REQUIRED and/or OPTIONAL construction parameters. Often (sub)modules are defined in terms of such parameters to allow for flexibility in their creation and/or usage in different scenarios (e.g. identity, actor type, etc.). + +_NOTE: Would-be dependencies/parameters which are modules or submodules should be retrieved via the [module registry](#modules-registry) instead._ + +| | Module | Submodule | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| **has no construction parameters** | Constructor MUST receive and SHOULD call option function arguments. | Submodule SHOULD use a factory interface type based on `Factory`. | +| **has REQUIRED construction parameters** | Parameter arguments MUST be represented as primitive/serializable types & consolidated into a protobuf type to be included in the runtime module's config. Subsequently referenced via the bus. High-level objects MAY be derived in the module constructor. | Parameter arguments MUST be consolidated and passed as a single "config" type using a factory interface type based on `FactoryWithConfig`. | +| **has OPTIONAL construction parameters** | Constructor MUST receive and SHOULD call option function arguments. | Constructor MUST receive option function arguments using a factory interface type based on `FactoryWithOptions`. | + ### Module creation -Module creation uses a typical constructor pattern signature `Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error)` where `options ...modules.ModuleOption` is an optional variadic argument that allows for the passing of options to the module. -This is useful to configure the module at creation time and it's usually used during prototyping and in "sub-modules" that don't have a specific configuration file and where adding it would add unnecessary complexity and overhead. If a module has a lot of `ModuleOption`s, at that point a configuration file might be advisable. +Modules **MUST** implement the `ModuleFactoryWithConfig` flavor of [factory interfaces](#factory-interfaces), enforcing the following constructor signature: -To formalize the module creation convention, we've introduced a set of factory interfaces in [`shared/modules/factory.go`](https://github.com/pokt-network/pocket/tree/main/shared/modules/factory.go). -These interfaces allow for more flexible module (or "sub-module"; see: [`RainTreeFactory`](https://github.com/pokt-network/pocket/tree/main/p2p/raintree/network.go)) creation and configuration, while maintaining a clear and concise API. +```go +Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) +``` -- `ModuleFactoryWithOptions`: Specialized factory interface for modules conforming to the "typical" signature above. Useful when creating modules that only require optional runtime configurations. -- `FactoryWithConfig`: A generic type, used to create a module with a specific configuration type. -- `FactoryWithOptions`: The generic form used to define `ModuleFactoryWithConfig`. -- `FactoryWithConfigAndOptions`: Another generic type which combines the capabilities of the previous two. Suitable for creating modules that require both specific configuration and optional runtime configurations. +Where `options` is an (optional) variadic argument that allows for the passing of an arbitrary number options to the module. +This is useful to configure the module at creation time. -Module creation utilizes the factory interfaces defined above. -Depending on your module's requirements, you can choose the most suitable factory interface for your module (or "sub-module"). +Each module constructor **MUST** receive this variadic option argument as all modules share this interface. -For examples of (sub-)modules that implement `ModuleFactoryWithConfig`, see: +Additionally, each module **SHOULD** consider these options (i.e. loop over and call them) as support may be added for new options at any time. -- [`p2pModule`](https://github.com/pokt-network/pocket/tree/main/p2p/module.go) - `WithHostOption` -- [`rpcCurrentHeightProvider`](https://github.com/pokt-network/pocket/tree/main/p2p/providers/current_height_provider/rpc/provider.go) - `WithCustomRPCURL` +See `ModuleFactoryWithOptions`, also defined in `shared/modules/factory.go`. +### Module configs & options -Essentially the `ModuleOption` sets a custom RPC URL for the module at runtime. +_(see: [constructor parameters](#construction-parameters--non-submodule-dependencies))_ +Modules **MAY** depend on **optional** values; i.e. not necessary in order for a module to function properly. + +An option function type MUST be defined for each optional value dependency. +Such option functions receive the module for option assignment by implementing `ModuleOption`: + +```go +type WithFooOption func(foo string) ModuleOption { + return func(m InjectableModule) { + m.(*FooModule).SetFoo(foo) + } +} +``` + +Module **MAY** depend on some **required** values; i.e. necessary in order for a module to function properly. + +Any such required values **MUST** be composed into a single "config" struct which can be received via the respective factory interface type. + +Such a config type **SHOULD** implement a `#IsValid()` method which is likely called in the module constructor. + +_(TODO: consider if it would be appropriate to enforce this via an interface)_ + +For modules, this config type MUST be defined as a protobuf type and incorporated into the [`runtime.Config`](../../../runtime/configs/config.go). +This automatically implies the following: + +- config values can be set via the node's config file and overriden via environment variables +- config values MUST be serializable (i.e. no references to high-level objects in memory) + +This serializable constraint on modules' configs will likely foster the creation of submodules through refactoring anywhere high-level, non-serializable, and non-submodule dependencies arise. + +For examples of modules and options see: + +- `Module`: [`p2pModule`](../../../p2p/module.go) +- `P2PConfig`: [`P2PConfig`](../../../runtime/configs/proto/p2p_config.proto) +- `ModuleOption`: [`WithHost`](../../../p2p/testutil.go) +- `ModuleOption`: [`WithStakedActorRouter`](../../../p2p/testutil.go) +- `ModuleOption`: [`WithUnstakedActorRouter`](../../../p2p/testutil.go) + +### Submodule creation + +Submodules **MUST** define their own concrete factory interface type derived from the most applicable [generic factory interface type](#factory-interfaces). + +Depending on your submodule's requirements, you should choose the most suitable factory interface. + +Each submodule factory interface type SHOULD return an interface type for the submodule; however it MAY be appropriate to return a concrete type in some cases. + +```mermaid +classDiagram + direction TB + + class IntegrableModule { + <> + +GetBus() Bus + +SetBus(bus Bus) + } + class InjectableModule { + <> + GetModuleName() string + } + + class Submodule { + <> + } + + Submodule --|> InjectableModule + Submodule --|> IntegrableModule + + class ExampleSubmoduleInterface { + <> + ... + } + + class exampleSubmodule + class exampleSubmoduleFactory { + <> + Create(...) (ExampleSubmoduleInterface, error) + } + + exampleSubmodule --|> ExampleSubmoduleInterface + ExampleSubmoduleInterface --|> Submodule : (embed) + ExampleSubmoduleInterface --|> exampleSubmoduleFactory :(embed) + exampleSubmoduleFactory ..|> Factory + exampleSubmoduleFactory ..|> FactoryWithConfig + exampleSubmoduleFactory ..|> FactoryWithOptions + exampleSubmoduleFactory ..|> FactoryWithConfigAndOptions +``` + +### Submodule configs & options + +_(see: [constructor parameters](#construction-parameters--non-submodule-dependencies))._ + +#### Configs + +Submodules **MAY** also depend on some **required** values; i.e. necessary in order for a submodule to function properly. + +Any such values **MUST** be composed in to a single config struct in accordance with the respective factory interface type. + +#### Options + +Submodules **MAY** depend on **optional** values; i.e. not necessary in order for a submodule to function properly. + +An option function type **MUST** be defined for each optional value dependency. + +Such option functions **MUST** receive the submodule for option assignment (similar to `ModuleOption`) in accordance with the respective factory interface type. + +Option functions **SHOULD** be written in terms of interface submodule types but MAY instead be written in terms of concrete submodule types if they are only intended or only make sense to be used with a specific concrete submodule type. + +Submodules **SHOULD NOT** consider options when none exist, this should be reflected in the respective factory interface type applied (i.e. no variadic options argument). + +### Comprehensive Submodule Example: + +```go +// Enforce the submodule interface type +var _ ExampleSubmoduleInterface = &exampleSubmodule{} + +// Submodule interface type, embedding the submodule & +// submodule factory interface types. +type ExampleSubmoduleInterface interface { + modules.Submodule + exampleSubmoduleFactory +} + +// "Config" struct for the submodule. +type exampleSubmoduleConfig struct { + someRequiredValue func(data []byte) error +} + +// Option function type for the submodule. +type exampleSubmoduleOption func(*exampleSubmodule) + +// Submodule factory interface type. +type exampleSubmoduleFactory = FactoryWithConfigAndOptions[ + ExampleSubmoduleInterface, + exampleSubmoduleConfig, + exampleSubmoduleOption +] + +// Concrete submodule type. +type exampleSubmodule struct { + someRequiredValue func(data []byte) error + someOptionalValue string +} + +// Create implements exampleSubmoduleFactory. +func (*exampleSubmodule) Create( + bus modules.Bus, + cfg exampleSubmoduleConfig, + opts ...exampleSubmoduleOption +) (ExampleSubmoduleInterface, error) { + // Consider config values + sub := &exampleSubmodule{ + someRequiredValue: cfg.someRequiredValue, + } + + // Apply option functions. + for _, opt := range opts { + opt(sub) + } + return sub, nil +} + +// WithSomeOption returns an option function which assigns +// `someOption` to the submodule. +func WithSomeOption(someOption string) exampleSubmoduleOption { + return func(s *exampleSubmodule) { + s.someOptionalValue = someOption + } +} +``` + +For more / real examples of submodules, see: + +- `Submodule`: [`persistencePeerstoreProvider`](https://github.com/pokt-network/pocket/tree/main/p2p/providers/current_height_provider/rpc/provider.go) +- `Submodule`: [`rpcPeerstoreProvider`](https://github.com/pokt-network/pocket/tree/main/p2p/providers/current_height_provider/rpc/provider.go) ### Interacting & Registering with the `bus` @@ -98,13 +410,12 @@ When a module is constructed via the `Create(bus modules.Bus, options ...modules tl;dr Pocket module's version of dependency injection. -We implemented a `ModulesRegistry` module [here](https://github.com/pokt-network/pocket/blob/19bf4d3f6507f5d406d9fafdb69b81359bccf110/runtime/modules_registry.go) that takes care of the module registration and retrieval. +We implemented a [`ModulesRegistry` module](https://github.com/pokt-network/pocket/blob/19bf4d3f6507f5d406d9fafdb69b81359bccf110/runtime/modules_registry.go) that takes care of the module registration and retrieval. This module is registered with the `bus` at the application level, it is accessible to all modules via the `bus` interface and it's also mockable as you would expect. Modules register themselves with the `bus` by calling `bus.RegisterModule(module)`. This is done in the `Create` function of the module. (For example, in the [consensus module](https://github.com/pokt-network/pocket/blob/19bf4d3f6507f5d406d9fafdb69b81359bccf110/consensus/module.go#L146)) - What the `bus` does is setting its reference to the module instance and delegating the registration to the `ModulesRegistry`. ```golang @@ -116,11 +427,13 @@ func (m *bus) RegisterModule(module modules.Module) { Under the hood, the module name is used to map to the instance of the module so that it's possible to retrieve a module by its name. -This is quite **important** because it unlock a powerful concept **Dependency Injection**. +This is quite **important** because it unlocks a powerful concept **Dependency Injection**. This enables the developer to define different implementations of a module and to register the one that is needed at runtime. This is because we can only have one module registered with a unique name and also because, by convention, we keep module names defined as constants. This is useful not only for prototyping but also for different use cases such as the `p1` CLI and the `pocket` binary where different implementations of the same module are necessary due to the fact that the `p1` CLI doesn't have a persistence module but still needs to know what's going on in the network. +Submodules can be registered the same way full Modules can be, by passing the Submodule to the RegisterModule function. Submodules should typically be registered to the bus for dependency injection reasons. + ##### Modules Registry Example For example, see the `peerstore_provider` ([here](https://github.com/pokt-network/pocket/tree/19bf4d3f6507f5d406d9fafdb69b81359bccf110/p2p/providers/peerstore_provider)). diff --git a/shared/modules/module.go b/shared/modules/module.go index 0550b4704..a9fe527a6 100644 --- a/shared/modules/module.go +++ b/shared/modules/module.go @@ -5,15 +5,20 @@ import ( ) type Module interface { - InitializableModule - IntegratableModule + InjectableModule + IntegrableModule InterruptableModule ModuleFactoryWithOptions } -// IntegratableModule is a module that integrates with the bus. +type Submodule interface { + InjectableModule + IntegrableModule +} + +// IntegrableModule is a module that integrates with the bus. // Essentially it's a module that is capable of communicating with the `bus` (see `shared/modules/bus_module.go`) for additional details. -type IntegratableModule interface { +type IntegrableModule interface { // SetBus sets the bus for the module. // // Generally it is called by the `bus` itself whenever a module is registered via `bus.RegisterModule(module modules.Module)` @@ -23,7 +28,7 @@ type IntegratableModule interface { GetBus() Bus } -// InitializableModule is a module that has some basic lifecycle logic. Specifically, it can be started and stopped. +// InjectableModule is a module that has some basic lifecycle logic. Specifically, it can be started and stopped. type InterruptableModule interface { // Start starts the module and executes any logic that is required at the beginning of the module's lifecycle. Start() error @@ -41,13 +46,13 @@ type InterruptableModule interface { // where there is no configuration, which is often the case for sub-modules that are used // and configured at runtime. // -// It accepts an InitializableModule as a parameter, because in order to create a module with these options, -// at a minimum, the module must implement the InitializableModule interface. +// It accepts an InjectableModule as a parameter, because in order to create a module with these options, +// at a minimum, the module must implement the InjectableModule interface. // // Example: // // func WithFoo(foo string) ModuleOption { -// return func(m InitializableModule) { +// return func(m InjectableModule) { // m.(*MyModule).foo = foo // } // } @@ -59,16 +64,13 @@ type InterruptableModule interface { // } // return m, nil // } -type ModuleOption func(InitializableModule) +type ModuleOption func(InjectableModule) -// InitializableModule is a module that can be created via the standardized `Create` method and that has a name +// InjectableModule is a module that can be created via the standardized `Create` method and that has a name // that can be used to identify it (see `shared\modules\modules_registry_module.go`) for additional details. -type InitializableModule interface { +type InjectableModule interface { // GetModuleName returns the name of the module. GetModuleName() string - - // Create creates a new instance of the module. - Create(bus Bus, options ...ModuleOption) (Module, error) } // KeyholderModule is a module that can provide a private key. diff --git a/shared/modules/modules_registry_module.go b/shared/modules/modules_registry_module.go index b2f286f8e..f50c63cae 100644 --- a/shared/modules/modules_registry_module.go +++ b/shared/modules/modules_registry_module.go @@ -4,7 +4,7 @@ package modules type ModulesRegistry interface { // RegisterModule registers a Module with the ModuleRegistry - RegisterModule(module Module) + RegisterModule(module InjectableModule) // GetModule returns a Module by name or nil if not found in the ModuleRegistry - GetModule(moduleName string) (Module, error) + GetModule(moduleName string) (InjectableModule, error) } diff --git a/shared/modules/treestore_module.go b/shared/modules/treestore_module.go index f187b18f2..9af58c71d 100644 --- a/shared/modules/treestore_module.go +++ b/shared/modules/treestore_module.go @@ -16,7 +16,7 @@ type TreeStoreFactory = FactoryWithOptions[TreeStoreModule, TreeStoreOption] // TreeStoreModules defines the interface for atomic updates and rollbacks to the internal // merkle trees that compose the state hash of pocket. type TreeStoreModule interface { - IntegratableModule + Submodule // Update returns the new state hash for a given height. // * Height is passed through to the Update function and is used to query the TxIndexer for transactions diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index c52bbac3c..720a4fa09 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -88,7 +88,7 @@ type UnstakingActor interface { // Conversely, use Local context from persistence module to track off-chain data that will eventually lead to a state transition, // e.g. reward claim for a session that spans multiple blocks. type UtilityUnitOfWork interface { - IntegratableModule + IntegrableModule // SetProposalBlock updates the utility unit of work with the proposed state transition. // It does not apply, validate or commit the changes. diff --git a/state_machine/module.go b/state_machine/module.go index b86550846..5ed8531af 100644 --- a/state_machine/module.go +++ b/state_machine/module.go @@ -14,7 +14,7 @@ import ( var _ modules.StateMachineModule = &stateMachineModule{} type stateMachineModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule *fsm.FSM @@ -70,7 +70,7 @@ func (m *stateMachineModule) SendEvent(event coreTypes.StateMachineEvent, args . // options func WithCustomStateMachine(stateMachine *fsm.FSM) modules.ModuleOption { - return func(m modules.InitializableModule) { + return func(m modules.InjectableModule) { if m, ok := m.(*stateMachineModule); ok { m.FSM = stateMachine } diff --git a/telemetry/module.go b/telemetry/module.go index ba4f4f2aa..87cb764ce 100644 --- a/telemetry/module.go +++ b/telemetry/module.go @@ -14,7 +14,7 @@ var ( ) type telemetryModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule } diff --git a/telemetry/prometheus_module.go b/telemetry/prometheus_module.go index c5b4b221b..988f16381 100644 --- a/telemetry/prometheus_module.go +++ b/telemetry/prometheus_module.go @@ -24,7 +24,7 @@ var ( // DISCUSS(team): Should the warning logs in this module be handled differently? type PrometheusTelemetryModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule config *configs.TelemetryConfig diff --git a/utility/doc/E2E_FEATURE_PATH_TEMPLATE.md b/utility/doc/E2E_FEATURE_PATH_TEMPLATE.md index 946cc99b0..e173c3540 100644 --- a/utility/doc/E2E_FEATURE_PATH_TEMPLATE.md +++ b/utility/doc/E2E_FEATURE_PATH_TEMPLATE.md @@ -1,6 +1,6 @@ # E2E Feature Path -_IMPROVE(olshansky): Once we've completed the entire process at least once, we'll add links to each step._ +_IMPROVE(olshansky): Once we've completed the entire process at least once, we'll add links to each step. [A potential example to use](https://github.com/pokt-network/pocket/pull/869#issuecomment-1618484939)._ - [Introduction \& Goals](#introduction--goals) - [Developer Journey](#developer-journey) @@ -168,10 +168,26 @@ Open a [new issue](https://github.com/pokt-network/pocket/issues/new?assignees=& ## E2E Feature Implementation +### TL;DR + +1. Open a [POC PR](https://github.com/pokt-network/pocket/blob/main/utility/doc/E2E_FEATURE_PATH_TEMPLATE.md#poc-proof-of-concept). + +2. List [the PRs needed for making the E2E test pass](https://github.com/pokt-network/pocket/pull/869#issuecomment-1618484939) + +3. Make sure the reviewers of the POC PR are in agreement wih the proposed PR list + +4. Work through the set of proposed PRs to make the E2E test pass + ### POC: Proof of Concept Create a single PR where you _"do everything"_ with the knowledge that it'll be closed without being merged in. This is an opportunity to get your hands dirty, understand the problem more deeply and have some fun. This PR may be split into multiple smaller PRs or just refactored altogether. +A reasonable set of goals for this PR would be: + - A) Get a review on the approach + - B) Breakup the work into a list of upcoming PRs. + +As an example, here is a PR adding [a `.feature` file](https://github.com/pokt-network/pocket/pull/869/files#diff-30c6c02e8594cf72662ab975e75a810a5bbd702f274e2eaf160c97ec14f5e642) and [the updated `.go` steps file](https://github.com/pokt-network/pocket/pull/869/files#diff-01dec4121ae8acb7a1f4bb72a6c2104827d2c2d2197eb5f45fa5c032ffba32cd). + _TODO(example): Link to Alessandro's KISS or Bryan's P2P._ ### MVC: Minimum Viable Change diff --git a/utility/fisherman/module.go b/utility/fisherman/module.go index 54948d2d7..ff2b89788 100644 --- a/utility/fisherman/module.go +++ b/utility/fisherman/module.go @@ -11,7 +11,7 @@ const ( ) type fisherman struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger } diff --git a/utility/module.go b/utility/module.go index 184f8074e..c8b027b9e 100644 --- a/utility/module.go +++ b/utility/module.go @@ -23,7 +23,7 @@ var ( ) type utilityModule struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger diff --git a/utility/servicer/module.go b/utility/servicer/module.go index eb4ec9c26..dcd4c48f1 100644 --- a/utility/servicer/module.go +++ b/utility/servicer/module.go @@ -48,7 +48,7 @@ type sessionTokens struct { } type servicer struct { - base_modules.IntegratableModule + base_modules.IntegrableModule base_modules.InterruptableModule logger *modules.Logger diff --git a/utility/unit_of_work/module.go b/utility/unit_of_work/module.go index 0136e99bf..2624e5bac 100644 --- a/utility/unit_of_work/module.go +++ b/utility/unit_of_work/module.go @@ -17,7 +17,7 @@ const ( var _ modules.UtilityUnitOfWork = &baseUtilityUnitOfWork{} type baseUtilityUnitOfWork struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger diff --git a/utility/validator/module.go b/utility/validator/module.go index 818280d49..68bf36eba 100644 --- a/utility/validator/module.go +++ b/utility/validator/module.go @@ -11,7 +11,7 @@ const ( ) type validator struct { - base_modules.IntegratableModule + base_modules.IntegrableModule logger *modules.Logger } From 57c6b9224cb761b1dcf7414250c4ddbdb36ea89b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:44:52 +0100 Subject: [PATCH 74/74] address comments --- ibc/module.go | 3 +++ ibc/store/provable_store.go | 2 ++ persistence/trees/trees.go | 8 ++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ibc/module.go b/ibc/module.go index d1382c4db..27d89ec79 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -87,6 +87,9 @@ func (m *ibcModule) GetModuleName() string { } // HandleMessage accepts a generic IBC message and routes it to the utility mempool +// TECHDEBT(#868): This SHOULD NOT be this way and is actually just a temporary workaround. +// The correct thing to do is sign transactions before broadcasting them and pass them +// directly into the utility module. This function will be removed in #868 func (m *ibcModule) HandleMessage(message *anypb.Any) error { m.m.Lock() defer m.m.Unlock() diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index c2dbd3390..1b7298374 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -19,6 +19,8 @@ var _ modules.ProvableStore = &provableStore{} // cachedEntry represents a local change made to the IBC store prior to it being // committed to the state tree. These are written to disk in the to prevent a // loss of data and pruned when included in the state tree +// The cache will be used in the event of a node failure or failure to propagate +// an IBC message, in order for the changes to be "replayed" if needed type cachedEntry struct { storeName string height uint64 diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index b960e4b71..64f210122 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -366,10 +366,10 @@ func (t *treeStore) updateIBCTree(keys, values [][]byte) error { if err := t.merkleTrees[IBCTreeName].tree.Delete(key); err != nil { return err } - } else { - if err := t.merkleTrees[IBCTreeName].tree.Update(key, value); err != nil { - return err - } + continue + } + if err := t.merkleTrees[IBCTreeName].tree.Update(key, value); err != nil { + return err } } return nil