diff --git a/Makefile b/Makefile index 888559ad4..1d1831bb0 100644 --- a/Makefile +++ b/Makefile @@ -303,6 +303,9 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # P2P $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto + # IBC + $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto + # echo "View generated proto files by running: make protogen_show" # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following diff --git a/build/config/config.validator1.json b/build/config/config.validator1.json index 702ad4232..888d137bb 100644 --- a/build/config/config.validator1.json +++ b/build/config/config.validator1.json @@ -59,6 +59,8 @@ "chains": ["0001"] }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator2.json b/build/config/config.validator2.json index 3e2ef6f6e..aaf37c8de 100644 --- a/build/config/config.validator2.json +++ b/build/config/config.validator2.json @@ -52,6 +52,8 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator3.json b/build/config/config.validator3.json index 5cd45e544..86611bf12 100644 --- a/build/config/config.validator3.json +++ b/build/config/config.validator3.json @@ -52,6 +52,8 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/build/config/config.validator4.json b/build/config/config.validator4.json index 2eb7747d7..903a8dba0 100644 --- a/build/config/config.validator4.json +++ b/build/config/config.validator4.json @@ -52,6 +52,8 @@ "use_cors": false }, "ibc": { - "enabled": true + "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + "stores_dir": "/var/ibc" } } diff --git a/consensus/e2e_tests/utils_test.go b/consensus/e2e_tests/utils_test.go index 94294af26..21243c199 100644 --- a/consensus/e2e_tests/utils_test.go +++ b/consensus/e2e_tests/utils_test.go @@ -122,7 +122,7 @@ func CreateTestConsensusPocketNode( telemetryMock := baseTelemetryMock(t, eventsChannel) loggerMock := baseLoggerMock(t, eventsChannel) rpcMock := baseRpcMock(t, eventsChannel) - ibcMock := ibcUtils.IbcMockWithHost(t, eventsChannel) + ibcMock := ibcUtils.IBCMockWithHost(t, eventsChannel) for _, module := range []modules.Module{ p2pMock, diff --git a/ibc/docs/README.md b/ibc/docs/README.md index 01a04c4ed..28160a08b 100644 --- a/ibc/docs/README.md +++ b/ibc/docs/README.md @@ -95,7 +95,9 @@ _Note_: Connections, Channels and Ports in IBC are not the same as networking co [ICS24][ics24] defines the IBC stores and these must be a part of the Pocket networks consensus state. As such the `ibcTree` is defined as one of the state trees used to generate the root hash. This tree contains the relevant information the hosts/relayers need to be able to use IBC, in accordance with ICS-24 and the other ICS components. -TODO([#854](https://github.com/pokt-network/pocket/issues/854)): Add a local cache for changes to the state for use in the event of the node crashing. +In order to interact with the IBC store's the host must create a `ProvableStore` instance which can make local changes to the state and propagate these through the network. This store maintains a local cache that can be backed up to disk and restored. In the event of a node failure, or local changes being unable to be propagated, the cache can be restored from the disk backup and the host can attempt to propagate the changes again. + +See: [store/provable_store.go](../store/provable_store.go) and [ics24.md](ics24.md) for more details on the specifics of the IBC store implementation for Pocket. ## Components diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index 8834c4b88..256294deb 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -1,32 +1,196 @@ # ICS-24 Host Requirements - [Overview](#overview) +- [Host Configuration](#host-configuration) - [Implementation](#implementation) + - [Persistence](#persistence) - [Paths and Identifiers](#paths-and-identifiers) - [Timestamps](#timestamps) +- [IBC State](#ibc-state) + - [IBC State Tree](#ibc-state-tree) + - [Data Retrieval](#data-retrieval) + - [IBC Messages](#ibc-messages) + - [IBC Message Handling](#ibc-message-handling) + - [Mempool](#mempool) + - [State Transition](#state-transition) +- [Provable Stores](#provable-stores) + - [Caching](#caching) ## Overview -[ICS-24][ics24] details the requirements of the host chain, in order for it to be compatible with IBC. A host is defined as a node on a chain that runs the IBC software. A host has the ability to create connections with counterparty chains, open channels, and ports as well as commit proofs to the consensus state of its own chain for the relayer to submit to another chain. The host is responsible to managing and creating clients and all other aspects of the IBC module. +[ICS-24][ics24] details the requirements of the host chain, in order for it to be compatible with IBC. A host is defined as a node on a chain that runs the IBC software. A host has the ability to create connections with counterparty chains, open channels, expose ports, and commit proofs to the consensus state of its own chain for the relayer to submit to another chain. The host is responsible to managing and creating clients and all other aspects of the IBC module. As token transfers as defined in [ICS-20][ics20] work on a lock and mint pattern, any tokens sent from **chain A** to **chain B** will have a denomination unique to the connection/channel/port combination that the packet was sent over. This means that if a host where to shutdown a connection or channel without warning any tokens yet to be returned to the host chain would be lost. For this reason, only validator nodes are able to become hosts, as they provide the most reliability out of the different node types. +## Host Configuration + +Only validators can be configured to be IBC hosts. If the IBC module, during its creation, detects the node is a validator (and the IBC `enabled` field in the config is `true`) it will automatically create a host. + +```json +"ibc": { + "enabled": bool, + "private_key": string, + "stores_dir": string +} +``` + +The `PrivateKey` field of the configuration is used to sign IBC store related messages and state transitions for inclusion in the block. + ## Implementation **Note**: The ICS-24 implementation is still a work in progress and is not yet fully implemented. ICS-24 has numerous sub components that must be implemented in order for the host to be fully functional. These range from type definitions for identifiers, paths and stores as well as the methods to interact with them. Alongside these ICS-24 also defines the Event Logging system which is used to store the packet data and timeouts for the relayers to read, as only the `CommitmentProof` objects are committed to the chain state. In addition to these numerous other features are part of ICS-24 that are closely linked to other ICS components such as consensus state introspection and client state validation. +### Persistence + +The IBC stores must be included in the networks consensus state as one of the many state trees. This is to ensure the IBC light clients verifying Pocket network's state can verify the inclusion or exclusion of IBC related information from the block headers. + +The following is a simplified sequence diagram of an IBC fungible token transfer. This requires **Chain A** to commit to its state the packet data related to the transfer, so that **Chain B** can verify the inclusion of this packet data with the light client of **Chain A** it runs. + +```mermaid +sequenceDiagram + actor UA as User A + box Transparent Chain A + participant A1 as Validator A + participant A2 as IBC Host A + participant A3 as Light Client B + end + box Transparent Relayer + actor R1 as Relayer + end + box Transparent Chain B + participant B1 as Validator B + participant B2 as IBC Host B + participant B3 as Light Client A + end + actor UB as User B + R1->>R1: Watch(Chain A) + R1->>R1: Watch(Chain B) + UA->>+A1: Send 10$POKT to User B + A1->>A1: Lock(10$POKT) + A1->>+A2: Create(FungibleTokenPacketData) + A2->>-A1: Commit(FungibleTokenPacketData) + A1->>-A1: NewBlock() + R1->>R1: CheckNewBlockForIBCPackets() + R1->>+A2: QueryAndProve(FungibleTokenPacketData) + A2->>R1: FungibleTokenPacketData + A2->>-R1: Proof(FungibleTokenPacketData) + R1->>+B2: Validate(FungibleTokenPacketData, Proof(FungibleTokenPacketData)) + B2->>+B3: Verify(Proof(FungibleTokenPacketData)) + B3->>-B2: FoundInState(FungibleTokenPacketData) + B2->>-B1: Send 10$POKT to User B + B1->>B1: Mint(10$POKT) + B1->>UB: Receive 10$POKT from User A +``` + +As the IBC host will make changes to the IBC store locally, in response to functions being called by relayers, they require these changes to be propagated throughout the network (i.e. the mempool) and included in all other node's IBC stores so that during block production these changes are reflected in the state transition. This is done by utilizing the existing transaction workflow, adding the IBC store change messages to the mempool and then handling them as a new message type in block production/application logic. + +See: [IBC State](#ibc-state) below for more details on the IBC state transition process. + ### Paths and Identifiers -Paths are defined as bytestrings that are used to access the elements in the different stores. They are built with the function `ApplyPrefix()` which takes a store key as a prefix and a path string and will return the key to access an element in the specific store. The logic for paths can be found in [host/keys.go](../host/keys.go) and [host/prefix.go](../host/prefix.go) +Paths are defined as bytestrings that are used to access the elements in the different stores. They are built with the function `ApplyPrefix()` which takes a store key as a prefix and a path string and will return the key to access an element in the specific store. The logic for paths can be found in [path/keys.go](../path/keys.go) and [path/prefix.go](../path/prefix.go) -Identifiers are bytestrings constrained to specific characters and lengths depending on their usages. They are used to identify: channels, clients, connections and ports. Although the minimum length of the identifiers is much less we use a minimum length of 32 bytes and a maximum length that varies depending on the use case to randomly generate identifiers. This allows for an extremely low chance of collision between identifiers. Identifiers have no significance beyond their use to store different elements in the IBC stores and as such there is no need for non-random identifiers. The logic for identifiers can be found in [host/identifiers.go](../host/identifiers.go). +Identifiers are bytestrings constrained to specific characters and lengths depending on their usages. They are used to identify: channels, clients, connections and ports. Although the minimum length of the identifiers is much less we use a minimum length of 32 bytes and a maximum length that varies depending on the use case to randomly generate identifiers. This allows for an extremely low chance of collision between identifiers. Identifiers have no significance beyond their use to store different elements in the IBC stores and as such there is no need for non-random identifiers. The logic for identifiers can be found in [path/identifiers.go](../path/identifiers.go). ### Timestamps The `GetTimestamp()` function returns the current unix timestamp of the host machine and is used to calculate timeout periods for packets +## IBC State + +As mentioned [above](#persistence) the IBC store **MUST** be included in the consensus state of the network. As such the IBC store as defined in [ICS-24][ics24] has been implemented as a single IBC state tree. + +### IBC State Tree + +The IBC state tree is an `SMT` backed by a persistent `KVStore`, this is used for proof generation/verification. Data retrieval uses the `peristence` layer, see the [data retrieval](#data-retrieval) section below for more details. + +The root hash of the IBC state tree is included in the `rootTree` which computes the network's state hash for any given block. This allows verifiers to not only verify the inclusion/exclusion of any element in the IBC state tree itself but also that the IBC state tree was used to compute the network's state hash, by utilising the `CommitmentProof` object defined in [ICS-23][ics23]. + +### Data Retrieval + +In order to query the IBC store the `persistence` layer is leveraged. All local changes to the IBC store are broadcasted as [IBC messages](#ibc-messages) and ultimately stored in each node's `peristence` layer. This allows for the efficient querying of the IBC store instead of having to query the IBC state tree directly. + +When attempting to generate a proof for a specific `key` in the IBC state tree the IBC host will import a local copy of the IBC state tree and use this to generate the proof. Otherwise all queries are handled by the `peristence` layer's underlying database. + +### IBC Messages + +Hosts maintain uncommitted changes in a local ephemeral IBC store while messages propagate through the mempool. + +These messages enable a variety of IBC related state changes such as creating light clients, opening connections, sending packets, etc... This is enabled by propagating `IBCMessage` types defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events: + +- `UpdateIBCStore`: Updating the store with a key-value pair; adding a new or updating an existing element +- `PruneIBCStore`: Pruning the store via its key; removal of an existing element + +_Note: In both types described above the `key` field **must** already be prefixed with the `CommitmentPrefix` and should be a valid path in the store._ + +When changes are made locally they are not committed to the IBC store itself but are instead used to create an `IBCMessage` which is broadcasted to the network. This is akin to a simple send transaction that has been propagated throughout the mempool but has not been committed to the on-chain state. + +### IBC Message Handling + +Upon a node receiving an `IBCMessage` from the event bus it will use the `HandleMessage()` method of the `IBCModule` to add this message to the transactions mempool via the following steps: + +1. Wrap the `IBCMessage` within a `Transaction` +2. Sign the `Transaction` using the `IBCModule`'s private key +3. Broadcast the `Transaction` throughout the mempool + +```mermaid +graph LR + subgraph Bus + A[Events] + end + subgraph I[IBC Host] + I1["HandleMessage(Message)"] + end + subgraph Handler + H1["ConvertIBCMessageToTransaction(IBCMessage)"] + subgraph Transaction + T1["coreTypes.Transaction{Msg: IBCMessage}"] + end + H2["SignTransaction(Transaction)"] + end + subgraph Mempool + M1["ValidateTransaction(Transaction)"] + M2["AddToMempool(Transaction)"] + end + Bus--Message-->I + I--IBCMessage-->Handler + H1--IBCMessage-->Transaction + Transaction--Transaction-->H2 + Handler--Transaction-->Mempool + M1--Transaction-->M2 +``` + +See: [ibc/module.go](../module.go) for the specific implementation details. + +### Mempool + +With the `IBCMessage` now propagated through the network's mempool, when it is reaped (by the block proposer) the message's validity will be handled by first determining the type of the `IBCMessage`: + +- `UpdateIBCStore`: The `key` and `value` fields are tracked by persistence and used to update the `ibc` store state tree +- `PruneIBCStore`: The `key` field is tracked by persistence and marked for removal in the `ibc` store state tree + +### State Transition + +See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-state-tree) for more information on how the persistence module uses the data it has tracked from the `IBCMessage` objects, in order to update the actual state trees and in turn the root hash. + +## Provable Stores + +The `ProvableStore` interface defined in [shared/modules/ibc_module.go](../../shared/modules/ibc_module.go) is implemented by the [`provableStore`](../store/provable_store.go) type and managed by the [`StoreManager`](../store/store_manager.go). + +The provable stores are each assigned a `prefix`. This represents the specific sub-store that they are able to access and interact with in the IBC state tree. When doing any operation `get`/`set`/`delete` the `prefix` is applied to the `key` provided to generate the `CommitmentPath` to the element in the IBC state tree. + +The provable stores do not directly interface with the IBC state tree but instead utliise the `peristence` layer to query the data locally. This allows for the efficient querying of the IBC store instead of having to query the IBC state tree directly. Any changes made by the `ProvableStore` instance are broadcasted to the network for inclusion in the next block, being stored in their mempools. + +### Caching + +Every local change made to the IBC store (`update`/`delete`) is stored in an in-memory cache. These caches are written to disk by the [`StoreManager`](../store/store_manager.go). + +In the event of a node failure, or local changes not being propagated correctly. Any changes stored in the cache can be "replayed" by the node and broadcasted to the network for inclusion in the next block. + +_TODO: Implement this functionality_ + [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md [smt]: https://github.com/pokt-network/smt diff --git a/ibc/handle_message_test.go b/ibc/handle_message_test.go new file mode 100644 index 000000000..7bb81af20 --- /dev/null +++ b/ibc/handle_message_test.go @@ -0,0 +1,325 @@ +package ibc + +import ( + "errors" + "fmt" + "testing" + + ibcTypes "github.com/pokt-network/pocket/ibc/types" + "github.com/pokt-network/pocket/persistence/indexer" + "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" + "github.com/stretchr/testify/require" +) + +func TestHandleMessage_ErrorAlreadyInMempool(t *testing.T) { + // Prepare test data + _, tx := prepareUpdateMessage(t, []byte("key"), []byte("value")) + txProtoBytes, err := codec.GetCodec().Marshal(tx) + require.NoError(t, err) + + // Prepare the environment + _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) + + // Manually add the tx to the mempool + err = utilityMod.GetMempool().AddTx(txProtoBytes) + require.NoError(t, err) + + // Error on having a duplciate transaction + err = utilityMod.HandleTransaction(txProtoBytes) + require.Error(t, err) + require.EqualError(t, err, coreTypes.ErrDuplicateTransaction().Error()) +} + +func TestHandleMessage_ErrorAlreadyCommitted(t *testing.T) { + // Prepare the environment + _, utilityMod, persistenceMod, _ := prepareEnvironment(t, 0, 0, 0, 0) + idxTx := prepareIndexedMessage(t, persistenceMod.GetTxIndexer()) + + // Error on having an indexed transaction + err := utilityMod.HandleTransaction(idxTx.Tx) + require.Error(t, err) + require.EqualError(t, err, coreTypes.ErrTransactionAlreadyCommitted().Error()) +} + +func TestHandleMessage_BasicValidation_Message(t *testing.T) { + testCases := []struct { + name string + msg *ibcTypes.IBCMessage + expected error + }{ + { + name: "Valid Update Message", + msg: ibcTypes.CreateUpdateStoreMessage([]byte("key"), []byte("value")), + expected: nil, + }, + { + name: "Valid Prune Message", + msg: ibcTypes.CreatePruneStoreMessage([]byte("key")), + expected: nil, + }, + { + name: "Invalid Update Message: Empty Key", + msg: ibcTypes.CreateUpdateStoreMessage(nil, []byte("value")), + expected: coreTypes.ErrNilField("key"), + }, + { + name: "Invalid Update Message: Empty Value", + msg: ibcTypes.CreateUpdateStoreMessage([]byte("key"), nil), + expected: coreTypes.ErrNilField("value"), + }, + { + name: "Invalid Prune Message: Empty Key", + msg: ibcTypes.CreatePruneStoreMessage(nil), + expected: coreTypes.ErrNilField("key"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.msg.ValidateBasic() + if tc.expected != nil { + require.EqualError(t, err, tc.expected.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestHandleMessage_BasicValidation_Transaction(t *testing.T) { + // Prepare the environment + _, utilityMod, _, _ := prepareEnvironment(t, 1, 0, 0, 0) + + privKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + falseKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + + validUpdateMsg, validUpdateTx := prepareUpdateMessage(t, []byte("key"), []byte("value")) + require.NoError(t, err) + err = validUpdateTx.Sign(privKey) + require.NoError(t, err) + updateAny, err := codec.GetCodec().ToAny(validUpdateMsg.GetUpdate()) + require.NoError(t, err) + bz, err := validUpdateTx.SignableBytes() + require.NoError(t, err) + falseUpdateSig, err := falseKey.Sign(bz) + require.NoError(t, err) + + validPruneMsg, validPruneTx := preparePruneMessage(t, []byte("key")) + require.NoError(t, err) + err = validPruneTx.Sign(privKey) + require.NoError(t, err) + pruneAny, err := codec.GetCodec().ToAny(validPruneMsg.GetPrune()) + require.NoError(t, err) + bz, err = validPruneTx.SignableBytes() + require.NoError(t, err) + falsePruneSig, err := falseKey.Sign(bz) + require.NoError(t, err) + + testCases := []struct { + name string + tx *coreTypes.Transaction + expected error + }{ + { + name: "Valid Update Transaction", + tx: validUpdateTx, + expected: nil, + }, + { + name: "Valid Prune Transaction", + tx: validPruneTx, + expected: nil, + }, + { + name: "Invalid Update Transaction: Empty Nonce", + tx: &coreTypes.Transaction{ + Msg: updateAny, + }, + expected: coreTypes.ErrEmptyNonce(), + }, + { + name: "Invalid Prune Transaction: Empty Nonce", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + }, + expected: coreTypes.ErrEmptyNonce(), + }, + { + name: "Invalid Update Transaction: Empty Signature", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, + expected: coreTypes.ErrEmptySignatureStructure(), + }, + { + name: "Invalid Prune Transaction: Empty Signature", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, + expected: coreTypes.ErrEmptySignatureStructure(), + }, + { + name: "Invalid Update Transaction: Bad Key", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: []byte("bad key"), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrNewPublicKeyFromBytes(errors.New("the public key length is not valid, expected length 32, actual length: 7")), + }, + { + name: "Invalid Prune Transaction: Bad Signature", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: []byte("bad key"), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrNewPublicKeyFromBytes(errors.New("the public key length is not valid, expected length 32, actual length: 7")), + }, + { + name: "Invalid Update Transaction: Bad Signature", + tx: &coreTypes.Transaction{ + Msg: updateAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: privKey.PublicKey().Bytes(), + Signature: falseUpdateSig, + }, + }, + expected: coreTypes.ErrSignatureVerificationFailed(), + }, + { + name: "Invalid Prune Transaction: Bad Key", + tx: &coreTypes.Transaction{ + Msg: pruneAny, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + Signature: &coreTypes.Signature{ + PublicKey: privKey.PublicKey().Bytes(), + Signature: falsePruneSig, + }, + }, + expected: coreTypes.ErrSignatureVerificationFailed(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + txProtoBytes, err := codec.GetCodec().Marshal(tc.tx) + require.NoError(t, err) + err = utilityMod.HandleTransaction(txProtoBytes) + if tc.expected != nil { + require.EqualError(t, err, tc.expected.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestHandleMessage_GetIndexedMessage(t *testing.T) { + // Prepare the environment + _, utilityMod, persistenceMod, _ := prepareEnvironment(t, 1, 0, 0, 0) + idxTx := prepareIndexedMessage(t, persistenceMod.GetTxIndexer()) + + tests := []struct { + name string + txProtoBytes []byte + txExists bool + expectErr error + }{ + {"returns indexed transaction when it exists", idxTx.Tx, true, nil}, + {"returns error when transaction doesn't exist", []byte("Does not exist"), false, coreTypes.ErrTransactionNotCommitted()}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + idTx, err := utilityMod.GetIndexedTransaction(test.txProtoBytes) + if test.expectErr != nil { + require.EqualError(t, err, test.expectErr.Error()) + require.Nil(t, idTx) + } else { + require.NoError(t, err) + require.NotNil(t, idTx) + } + }) + } +} + +func TestHandleMessage_AddToMempool(t *testing.T) { + // prepare the environment + _, _, _, ibcMod := prepareEnvironment(t, 1, 0, 0, 0) + require.Len(t, ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll(), 0) + msg, _ := prepareUpdateMessage(t, []byte("key"), []byte("value")) + anyMsg, err := codec.GetCodec().ToAny(msg) + require.NoError(t, err) + require.NoError(t, ibcMod.HandleMessage(anyMsg)) + mempoolTxs := ibcMod.GetBus().GetUtilityModule().GetMempool().GetAll() + + require.Len(t, mempoolTxs, 1) + mTx := mempoolTxs[0] + require.NotNil(t, mTx) + txProto := &coreTypes.Transaction{} + err = codec.GetCodec().Unmarshal(mTx, txProto) + require.NoError(t, err) + ibcMsg := &ibcTypes.UpdateIBCStore{} + require.NoError(t, txProto.GetMsg().UnmarshalTo(ibcMsg)) + + // Compare messages are equal + origBz, err := codec.GetCodec().Marshal(msg.GetUpdate()) + require.NoError(t, err) + ibcMsgBz, err := codec.GetCodec().Marshal(ibcMsg) + require.NoError(t, err) + require.Equal(t, origBz, ibcMsgBz) +} + +func prepareUpdateMessage(t *testing.T, key, value []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { + t.Helper() + msg := ibcTypes.CreateUpdateStoreMessage(key, value) + tx, err := ibcTypes.ConvertIBCMessageToTx(msg) + require.NoError(t, err) + return msg, tx +} + +func preparePruneMessage(t *testing.T, key []byte) (*ibcTypes.IBCMessage, *coreTypes.Transaction) { + t.Helper() + msg := ibcTypes.CreatePruneStoreMessage(key) + tx, err := ibcTypes.ConvertIBCMessageToTx(msg) + require.NoError(t, err) + return msg, tx +} + +func prepareIndexedMessage(t *testing.T, txIndexer indexer.TxIndexer) *coreTypes.IndexedTransaction { + t.Helper() + _, tx := preparePruneMessage(t, []byte{}) + txProtoBytes, err := codec.GetCodec().Marshal(tx) + require.NoError(t, err) + + // Test data - Prepare IndexedTransaction + idxTx := &coreTypes.IndexedTransaction{ + Tx: txProtoBytes, + Height: 0, + Index: 0, + ResultCode: 0, + Error: "h5law", + SignerAddr: "h5law", + RecipientAddr: "h5law", + MessageType: "h5law", + } + + // Index a test transaction + err = txIndexer.Index(idxTx) + require.NoError(t, err) + + return idxTx +} diff --git a/ibc/host.go b/ibc/host.go index 603580cac..b8a415d67 100644 --- a/ibc/host.go +++ b/ibc/host.go @@ -1,18 +1,36 @@ +// TECHDEBT(#854): The host should be a submodule with access to the bus, this allows for the +// host to access the persistence module and thus the tree store in order to create local +// copies of the IBC state tree with the GetProvableStore() function. Bus access also will +// allow for the host to send any local changes to these stores as IBCMessage types through +// the P2P module's Broadcast() function to allow them to propagate through the network's mempool. package ibc import ( "time" + coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" ) var _ modules.IBCHost = &host{} type host struct { - logger *modules.Logger + logger *modules.Logger + storesDir string + storeManager modules.IBCStoreManager } // GetTimestamp returns the current unix timestamp func (h *host) GetTimestamp() uint64 { return uint64(time.Now().Unix()) } + +// GetProvableStore returns a copy of the IBC state tree where all operations observed by +// this specific ibc light client were applied to its ephemeral (in memory) state and have not +// yet been included in the next block. The aggregation of all light client-provable stores +// propagated throughout the network are validated by the block proposer when reaping +// the mempool, and lead to a valid on-chain state transition when consensus is reached. +// TODO(#854): Implement this +func (h *host) GetProvableStore(prefix coreTypes.CommitmentPrefix) (modules.ProvableStore, error) { + return nil, nil +} diff --git a/ibc/ibc.feature.wip b/ibc/ibc.feature.wip new file mode 100644 index 000000000..cc5278341 --- /dev/null +++ b/ibc/ibc.feature.wip @@ -0,0 +1,19 @@ +Tests to add: + - Add integration tests to cover the following: + - Local IBC store change is propagated through the network + - Check via P2P magic + - Local IBC store change is included in PostgresDB + - Check via SQL query/magic + - Duplicate IBC messages w/in the same block + - Check via SQL query/magic + - Shouldn't allow duplicates + - Invalid IBC messages + - Should error before going to the network + - Add E2E tests to cover the following: + - Local IBC store change gets included in the next block + - Check via inclusion proof + - Local IBC store change that is not included in next block stays in mempool + - Check by querying mempool + - Check via exclusion proof + - Duplicate IBC messages in subsequent blocks + - Changes that don't change the state should be noop and not included diff --git a/ibc/main_test.go b/ibc/main_test.go new file mode 100644 index 000000000..d96df0669 --- /dev/null +++ b/ibc/main_test.go @@ -0,0 +1,147 @@ +package ibc + +import ( + "log" + "os" + "testing" + + "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/runtime/test_artifacts/keygen" + "github.com/pokt-network/pocket/shared/messaging" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/utility" + "github.com/stretchr/testify/require" +) + +var dbURL string + +// NB: `TestMain` serves all tests in the immediate `ibc` package and not its children +func TestMain(m *testing.M) { + pool, resource, url := test_artifacts.SetupPostgresDocker() + dbURL = url + + exitCode := m.Run() + test_artifacts.CleanupPostgresDocker(m, pool, resource) + os.Exit(exitCode) +} + +func newTestUtilityModule(bus modules.Bus) modules.UtilityModule { + utilityMod, err := utility.Create(bus) + if err != nil { + log.Fatalf("Error creating utility module: %s", err) + } + return utilityMod.(modules.UtilityModule) +} + +func newTestPersistenceModule(bus modules.Bus) modules.PersistenceModule { + persistenceMod, err := persistence.Create(bus) + if err != nil { + log.Fatalf("Error creating persistence module: %s", err) + } + return persistenceMod.(modules.PersistenceModule) +} + +func newTestIBCModule(bus modules.Bus) modules.IBCModule { + ibcMod, err := Create(bus) + if err != nil { + log.Fatalf("Error creating ibc module: %s", err) + } + return ibcMod.(modules.IBCModule) +} + +// Prepares a runtime environment for testing along with a genesis state, a persistence module and a utility module +// +//nolint:unparam // Test suite is not fully built out yet +func prepareEnvironment( + t *testing.T, + numValidators, // nolint:unparam // we are not currently modifying parameter but want to keep it modifiable in the future + numServicers, + numApplications, + numFisherman int, + genesisOpts ...test_artifacts.GenesisOption, +) (*runtime.Manager, modules.UtilityModule, modules.PersistenceModule, modules.IBCModule) { + teardownDeterministicKeygen := keygen.GetInstance().SetSeed(42) + + runtimeCfg := newTestRuntimeConfig(numValidators, numServicers, numApplications, numFisherman, genesisOpts...) + bus, err := runtime.CreateBus(runtimeCfg) + require.NoError(t, err) + + testPersistenceMod := newTestPersistenceModule(bus) + err = testPersistenceMod.Start() + require.NoError(t, err) + + testUtilityMod := newTestUtilityModule(bus) + err = testUtilityMod.Start() + require.NoError(t, err) + + testIBCMod := newTestIBCModule(bus) + err = testIBCMod.Start() + require.NoError(t, err) + + // Reset database to genesis before every test + err = testPersistenceMod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + }) + require.NoError(t, err) + + t.Cleanup(func() { + teardownDeterministicKeygen() + err := testPersistenceMod.Stop() + require.NoError(t, err) + err = testUtilityMod.Stop() + require.NoError(t, err) + err = testIBCMod.Stop() + require.NoError(t, err) + }) + + return runtimeCfg, testUtilityMod, testPersistenceMod, testIBCMod +} + +//nolint:unparam // Test suite is not fully built out yet +func newTestRuntimeConfig( + numValidators, + numServicers, + numApplications, + numFisherman int, + genesisOpts ...test_artifacts.GenesisOption, +) *runtime.Manager { + cfg, err := configs.CreateTempConfig(&configs.Config{ + Utility: &configs.UtilityConfig{ + MaxMempoolTransactionBytes: 1000000, + MaxMempoolTransactions: 1000, + }, + Persistence: &configs.PersistenceConfig{ + PostgresUrl: dbURL, + NodeSchema: "test_schema", + BlockStorePath: ":memory:", + TxIndexerPath: ":memory:", + TreesStoreDir: ":memory:", + MaxConnsCount: 50, + MinConnsCount: 1, + MaxConnLifetime: "5m", + MaxConnIdleTime: "1m", + HealthCheckPeriod: "30s", + }, + Validator: &configs.ValidatorConfig{Enabled: true}, + IBC: &configs.IBCConfig{ + Enabled: true, + PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + }, + }) + if err != nil { + log.Fatalf("Error creating config: %s", err) + } + genesisState, _ := test_artifacts.NewGenesisState( + numValidators, + numServicers, + numApplications, + numFisherman, + genesisOpts..., + ) + runtimeCfg := runtime.NewManager(cfg, genesisState) + return runtimeCfg +} diff --git a/ibc/module.go b/ibc/module.go index bc6634858..27d89ec79 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -1,11 +1,20 @@ package ibc import ( + "encoding/hex" + "fmt" + "sync" + + "github.com/pokt-network/pocket/ibc/store" + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" + "google.golang.org/protobuf/types/known/anypb" ) var _ modules.IBCModule = &ibcModule{} @@ -13,6 +22,8 @@ var _ modules.IBCModule = &ibcModule{} type ibcModule struct { base_modules.IntegrableModule + m sync.Mutex + cfg *configs.IBCConfig logger *modules.Logger @@ -75,13 +86,76 @@ func (m *ibcModule) GetModuleName() string { return modules.IBCModuleName } +// HandleMessage accepts a generic IBC message and routes it to the utility mempool +// TECHDEBT(#868): This SHOULD NOT be this way and is actually just a temporary workaround. +// The correct thing to do is sign transactions before broadcasting them and pass them +// directly into the utility module. This function will be removed in #868 +func (m *ibcModule) HandleMessage(message *anypb.Any) error { + m.m.Lock() + defer m.m.Unlock() + + // Check the message is actually a valid IBC message + msg, err := codec.GetCodec().FromAny(message) + if err != nil { + return err + } + ibcMessage, ok := msg.(*ibcTypes.IBCMessage) + if !ok { + return fmt.Errorf("failed to cast message to IBCMessage") + } + if err := ibcMessage.ValidateBasic(); err != nil { + return err + } + + // Convert IBC message to a utility Transaction + tx, err := ibcTypes.ConvertIBCMessageToTx(ibcMessage) + if err != nil { + return err + } + + // Sign the transaction + pkBz, err := hex.DecodeString(m.cfg.PrivateKey) + if err != nil { + return err + } + pk, err := crypto.NewPrivateKeyFromBytes(pkBz) + if err != nil { + return err + } + signableBz, err := tx.SignableBytes() + if err != nil { + return err + } + signature, err := pk.Sign(signableBz) + if err != nil { + return err + } + tx.Signature = &coreTypes.Signature{ + Signature: signature, + PublicKey: pk.PublicKey().Bytes(), + } + + // Marshall the Transaction and send it to the utility module + txBz, err := codec.GetCodec().Marshal(tx) + if err != nil { + return err + } + if err := m.GetBus().GetUtilityModule().HandleTransaction(txBz); err != nil { + return err + } + m.logger.Info().Str("message_type", "IBCMessage").Msg("Successfully added a new message to the mempool!") + return nil +} + // newHost returns a new IBC host instance if one is not already created func (m *ibcModule) newHost() error { if m.host != nil { - return coreTypes.ErrHostAlreadyExists() + return coreTypes.ErrIBCHostAlreadyExists() } host := &host{ - logger: m.logger, + logger: m.logger, + storesDir: m.cfg.StoresDir, + storeManager: store.NewStoreManager(m.cfg.StoresDir), } m.host = host return nil diff --git a/ibc/host/identifiers.go b/ibc/path/identifiers.go similarity index 99% rename from ibc/host/identifiers.go rename to ibc/path/identifiers.go index f6b4a046d..283edc8aa 100644 --- a/ibc/host/identifiers.go +++ b/ibc/path/identifiers.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys.go b/ibc/path/keys.go similarity index 99% rename from ibc/host/keys.go rename to ibc/path/keys.go index c5a62ff34..1cd8a4de8 100644 --- a/ibc/host/keys.go +++ b/ibc/path/keys.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys_ics02.go b/ibc/path/keys_ics02.go similarity index 99% rename from ibc/host/keys_ics02.go rename to ibc/path/keys_ics02.go index a114f2bf4..5cce681be 100644 --- a/ibc/host/keys_ics02.go +++ b/ibc/path/keys_ics02.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/keys_ics03.go b/ibc/path/keys_ics03.go similarity index 99% rename from ibc/host/keys_ics03.go rename to ibc/path/keys_ics03.go index 505d9e0e2..99e56a48a 100644 --- a/ibc/host/keys_ics03.go +++ b/ibc/path/keys_ics03.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/keys_ics04.go b/ibc/path/keys_ics04.go similarity index 99% rename from ibc/host/keys_ics04.go rename to ibc/path/keys_ics04.go index 1cabd8c7c..004ad3647 100644 --- a/ibc/host/keys_ics04.go +++ b/ibc/path/keys_ics04.go @@ -1,4 +1,4 @@ -package host +package path import ( "fmt" diff --git a/ibc/host/keys_ics05.go b/ibc/path/keys_ics05.go similarity index 98% rename from ibc/host/keys_ics05.go rename to ibc/path/keys_ics05.go index 9ff1f1448..f6c807f9a 100644 --- a/ibc/host/keys_ics05.go +++ b/ibc/path/keys_ics05.go @@ -1,4 +1,4 @@ -package host +package path import "fmt" diff --git a/ibc/host/path_test.go b/ibc/path/path_test.go similarity index 99% rename from ibc/host/path_test.go rename to ibc/path/path_test.go index 1c1c8d367..96d47f039 100644 --- a/ibc/host/path_test.go +++ b/ibc/path/path_test.go @@ -1,4 +1,4 @@ -package host +package path import ( "testing" diff --git a/ibc/host/prefix.go b/ibc/path/prefix.go similarity index 98% rename from ibc/host/prefix.go rename to ibc/path/prefix.go index 86510bbfd..ebeddab07 100644 --- a/ibc/host/prefix.go +++ b/ibc/path/prefix.go @@ -1,4 +1,4 @@ -package host +package path import ( coreTypes "github.com/pokt-network/pocket/shared/core/types" diff --git a/ibc/store/proofs_ics23.go b/ibc/store/proofs_ics23.go index 9716ec6db..9d0e22961 100644 --- a/ibc/store/proofs_ics23.go +++ b/ibc/store/proofs_ics23.go @@ -63,8 +63,8 @@ func VerifyNonMembership(root ics23.CommitmentRoot, proof *ics23.CommitmentProof // in the SMT provided func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) - if err != nil || proof == nil { - return nil, coreTypes.ErrCreatingProof(err) + if err != nil { + return nil, coreTypes.ErrIBCCreatingProof(err) } return convertSMPToExistenceProof(proof, key, value), nil } @@ -72,8 +72,8 @@ func createMembershipProof(tree *smt.SMT, key, value []byte) (*ics23.CommitmentP // createNonMembershipProof generates a CommitmentProof object verifying the membership of an unrealted key at the given key in the SMT provided func createNonMembershipProof(tree *smt.SMT, key []byte) (*ics23.CommitmentProof, error) { proof, err := tree.Prove(key) - if err != nil || proof == nil { - return nil, coreTypes.ErrCreatingProof(err) + if err != nil { + return nil, coreTypes.ErrIBCCreatingProof(err) } return convertSMPToExclusionProof(proof, key), nil } diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go new file mode 100644 index 000000000..1b7298374 --- /dev/null +++ b/ibc/store/provable_store.go @@ -0,0 +1,258 @@ +package store + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "sync" + + ics23 "github.com/cosmos/ics23/go" + "github.com/pokt-network/pocket/ibc/path" + "github.com/pokt-network/pocket/persistence/kvstore" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/modules" +) + +var _ modules.ProvableStore = &provableStore{} + +// cachedEntry represents a local change made to the IBC store prior to it being +// committed to the state tree. These are written to disk in the to prevent a +// loss of data and pruned when included in the state tree +// The cache will be used in the event of a node failure or failure to propagate +// an IBC message, in order for the changes to be "replayed" if needed +type cachedEntry struct { + storeName string + height uint64 + prefixedKey []byte + value []byte +} + +// prepare returns the key and value to be written to disk +// "{height}/{prefixedKey}" => value +func (c *cachedEntry) prepare() (key, value []byte) { + return []byte(fmt.Sprintf("%s/%d/%s", c.storeName, c.height, string(c.prefixedKey))), c.value +} + +// provableStore is a struct that interfaces with the persistence layer to +// Get/Set/Delete the keys in the IBC state tree, in doing so it will trigger +// the creation of IBC messages that are broadcasted through the network and +// included in the mempool/next block to change the state of the IBC tree +type provableStore struct { + m sync.Mutex + bus modules.Bus // used to interact with persistence (passed from IBCHost) + name string // store name in storeManager + prefix coreTypes.CommitmentPrefix // []byte(name) + cache map[string]*cachedEntry // in-memory cache of local changes to be written to disk +} + +// newProvableStore returns a new instance of provableStore with the bus and prefix provided +func newProvableStore(bus modules.Bus, prefix coreTypes.CommitmentPrefix) *provableStore { + return &provableStore{ + bus: bus, + name: string(prefix), + prefix: prefix, + cache: make(map[string]*cachedEntry, 0), + } +} + +// Get queries the persistence layer for the latest value stored in the IBC state tree +// keys are automatically prefixed with the CommitmentPrefix if not present +func (p *provableStore) Get(key []byte) ([]byte, error) { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rCtx, err := p.bus.GetPersistenceModule().NewReadContext(currHeight) + if err != nil { + return nil, err + } + defer rCtx.Release() + return rCtx.GetIBCStoreEntry(prefixed, currHeight) // returns latest value stored +} + +// Get queries the persistence layer for the latest value stored in the IBC state tree +// it then generates a proof by importing the IBC state tree from the TreeStore +// keys are automatically prefixed with the CommitmentPrefix if not present +func (p *provableStore) GetAndProve(key []byte, membership bool) ([]byte, *ics23.CommitmentProof, error) { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rCtx, err := p.bus.GetPersistenceModule().NewReadContext(currHeight) + if err != nil { + return nil, nil, err + } + value, err := rCtx.GetIBCStoreEntry(prefixed, currHeight) // returns latest value stored + if err != nil { + return nil, nil, err + } + defer rCtx.Release() + var proof *ics23.CommitmentProof + if membership { + proof, err = p.CreateMembershipProof(key, value) + } else { + proof, err = p.CreateNonMembershipProof(key) + } + if err != nil { + return nil, nil, err + } + return value, proof, nil +} + +// CreateMembershipProof creates a membership proof for the key-value pair with the key +// prefixed with the CommitmentPrefix, by importing the state tree from the TreeStore +func (p *provableStore) CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) { + // import IBC state tree + // TODO(#854): Implement tree retrieval + /** + prefixed := applyPrefix(p.prefix, key) + root, nodeStore := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) + lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) + return createMembershipProof(lazy, prefixed, value) + **/ + return nil, nil +} + +// CreateNonMembershipProof creates a non-membership proof for the key prefixed with the +// CommitmentPrefix, by importing the state tree from the TreeStore +func (p *provableStore) CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) { + // import IBC state tree + // TODO(#854): Implement tree retrieval + /** + prefixed := applyPrefix(p.prefix, key) + root, nodeStore := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) + lazy := smt.ImportSparseMerkleTree(nodeStore, root, sha256.New()) + return createNonMembershipProof(lazy, prefixed) + **/ + return nil, nil +} + +// Set updates the persistence layer with the new key-value pair at the latest height and +// emits an UpdateIBCStore event to the bus for propagation throughout the network, to be +// included in each node's mempool and thus the next block +func (p *provableStore) Set(key, value []byte) error { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rwCtx, err := p.bus.GetPersistenceModule().NewRWContext(currHeight) + if err != nil { + return err + } + defer rwCtx.Release() + if err := rwCtx.SetIBCStoreEntry(prefixed, value); err != nil { + return err + } + p.m.Lock() + defer p.m.Unlock() + p.cache[string(prefixed)] = &cachedEntry{ + storeName: p.name, + height: uint64(currHeight), + prefixedKey: prefixed, + value: value, + } + // TODO(#854): Implement emit functions + // return emitUpdateStoreEvent(p.prefix, key, value) + return nil +} + +// Delete updates the persistence layer with the key and nil value pair at the latest height +// and emits an PruneIBCStore event to the bus for propagation throughout the network, to be +// included in each node's mempool and thus the next block +func (p *provableStore) Delete(key []byte) error { + prefixed := applyPrefix(p.prefix, key) + currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) + rwCtx, err := p.bus.GetPersistenceModule().NewRWContext(currHeight) + if err != nil { + return err + } + defer rwCtx.Release() + if err := rwCtx.SetIBCStoreEntry(prefixed, nil); err != nil { + return err + } + p.m.Lock() + defer p.m.Unlock() + p.cache[string(prefixed)] = &cachedEntry{ + storeName: p.name, + height: uint64(currHeight), + prefixedKey: prefixed, + value: nil, + } + // TODO(#854): Implement emit functions + // return emitPruneStoreEvent(p.prefix, key) + return nil +} + +// GetCommitmentPrefix returns the CommitmentPrefix for the store +func (p *provableStore) GetCommitmentPrefix() coreTypes.CommitmentPrefix { return p.prefix } + +// Root returns the current root of the IBC state tree +func (p *provableStore) Root() ics23.CommitmentRoot { + // TODO(#854): Implement tree retrieval + /** + root, _ := p.bus.GetTreeStore().GetTree(trees.IBCStateTree) + return root + **/ + return nil +} + +// FlushEntries writes all local changes to disk and clears the in-memory cache +func (p *provableStore) FlushEntries(store kvstore.KVStore) error { + p.m.Lock() + defer p.m.Unlock() + for _, entry := range p.cache { + key, value := entry.prepare() + if err := store.Set(key, value); err != nil { + return err + } + delete(p.cache, string(entry.prefixedKey)) + } + return nil +} + +// PruneCache removes all entries from the cache at the given height +func (p *provableStore) PruneCache(store kvstore.KVStore, height uint64) error { + p.m.Lock() + defer p.m.Unlock() + keys, _, err := store.GetAll([]byte(fmt.Sprintf("%s/%d", p.name, height)), false) + if err != nil { + return err + } + for _, key := range keys { + if err := store.Delete(key); err != nil { + return err + } + } + return nil +} + +// RestoreCache loads all entries from disk into the cache +func (p *provableStore) RestoreCache(store kvstore.KVStore) error { + p.m.Lock() + defer p.m.Unlock() + keys, values, err := store.GetAll(p.prefix, false) + if err != nil { + return err + } + for i, key := range keys { + parts := strings.SplitN(string(key), "/", 2) // name, heightStr, prefixedKeyStr + height, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return err + } + value := values[i] + p.cache[parts[2]] = &cachedEntry{ + storeName: parts[0], + height: height, + prefixedKey: []byte(parts[2]), + value: value, + } + } + return nil +} + +// applyPrefix will apply the CommitmentPrefix to the key provided if not already applied +func applyPrefix(prefix coreTypes.CommitmentPrefix, key []byte) coreTypes.CommitmentPath { + slashed := make([]byte, 0, len(key)+1) + slashed = append(slashed, key...) + slashed = append(slashed, []byte("/")...) + if bytes.Equal(prefix[:len(slashed)], slashed) { + return key + } + return path.ApplyPrefix(prefix, string(key)) +} diff --git a/ibc/store/store_manager.go b/ibc/store/store_manager.go new file mode 100644 index 000000000..e7b028e6b --- /dev/null +++ b/ibc/store/store_manager.go @@ -0,0 +1,115 @@ +package store + +import ( + "fmt" + "sync" + + "github.com/pokt-network/pocket/persistence/kvstore" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/modules" +) + +var ( + _ modules.IBCStoreManager = &storeManager{} + cacheDirs = func(storesDir string) string { return fmt.Sprintf("%s/caches", storesDir) } +) + +// storeManager holds an in-memory map of all the provable stores in use +type storeManager struct { + m sync.Mutex + bus modules.Bus + storesDir string + stores map[string]*provableStore +} + +// NewStoreManager returns a new storeManager instance +func NewStoreManager(storesDir string) *storeManager { + return &storeManager{ + m: sync.Mutex{}, + storesDir: storesDir, + stores: make(map[string]*provableStore, 0), + } +} + +// AddStore creates and adds a provableStore to the storeManager +// if one of the same name does not already exist +func (s *storeManager) AddStore(name string) error { + s.m.Lock() + defer s.m.Unlock() + if _, ok := s.stores[name]; ok { + return coreTypes.ErrIBCStoreAlreadyExists(name) + } + store := newProvableStore(s.bus, coreTypes.CommitmentPrefix(name)) + s.stores[store.name] = store + return nil +} + +// GetStore returns the provableStore with the given name +func (s *storeManager) GetStore(name string) (modules.ProvableStore, error) { + s.m.Lock() + defer s.m.Unlock() + store, ok := s.stores[name] + if !ok { + return nil, coreTypes.ErrIBCStoreDoesNotExist(name) + } + return store, nil +} + +// RemoveStore removes the provableStore with the given name +func (s *storeManager) RemoveStore(name string) error { + s.m.Lock() + defer s.m.Unlock() + if _, ok := s.stores[name]; !ok { + return coreTypes.ErrIBCStoreDoesNotExist(name) + } + delete(s.stores, name) + return nil +} + +// FlushAllEntries caches all the entries for all stores in the storeManager +func (s *storeManager) FlushAllEntries() error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.FlushEntries(disk); err != nil { + return err + } + } + return disk.Stop() +} + +// PruneCaches prunes the caches for all stores in the storeManager at the given height +func (s *storeManager) PruneCaches(height uint64) error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.PruneCache(disk, height); err != nil { + return err + } + } + return disk.Stop() +} + +// RestoreCaches restores the caches from disk for all stores in the storeManager +func (s *storeManager) RestoreCaches() error { + s.m.Lock() + defer s.m.Unlock() + disk, err := kvstore.NewKVStore(cacheDirs(s.storesDir)) + if err != nil { + return err + } + for _, store := range s.stores { + if err := store.RestoreCache(disk); err != nil { + return err + } + } + return disk.Stop() +} diff --git a/ibc/types/convert.go b/ibc/types/convert.go new file mode 100644 index 000000000..02f205f65 --- /dev/null +++ b/ibc/types/convert.go @@ -0,0 +1,51 @@ +package types + +import ( + "fmt" + + "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" + "google.golang.org/protobuf/types/known/anypb" +) + +func CreateUpdateStoreMessage(key, value []byte) *IBCMessage { + return &IBCMessage{ + Event: &IBCMessage_Update{ + Update: &UpdateIBCStore{ + Key: key, + Value: value, + }, + }, + } +} + +func CreatePruneStoreMessage(key []byte) *IBCMessage { + return &IBCMessage{ + Event: &IBCMessage_Prune{ + Prune: &PruneIBCStore{ + Key: key, + }, + }, + } +} + +func ConvertIBCMessageToTx(ibcMessage *IBCMessage) (*coreTypes.Transaction, error) { + var anyMsg *anypb.Any + var err error + switch event := ibcMessage.Event.(type) { + case *IBCMessage_Update: + anyMsg, err = codec.GetCodec().ToAny(event.Update) + case *IBCMessage_Prune: + anyMsg, err = codec.GetCodec().ToAny(event.Prune) + default: + return nil, coreTypes.ErrIBCUnknownMessageType(fmt.Sprintf("%T", event)) + } + if err != nil { + return nil, err + } + return &coreTypes.Transaction{ + Msg: anyMsg, + Nonce: fmt.Sprintf("%d", crypto.GetNonce()), + }, nil +} diff --git a/ibc/types/messages.go b/ibc/types/messages.go new file mode 100644 index 000000000..ac7eef3f4 --- /dev/null +++ b/ibc/types/messages.go @@ -0,0 +1,82 @@ +package types + +import ( + "fmt" + "log" + + "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + utilityTypes "github.com/pokt-network/pocket/utility/types" +) + +// Implement the Message interface +var ( + _ utilityTypes.Message = &UpdateIBCStore{} + _ utilityTypes.Message = &PruneIBCStore{} +) + +func (m *IBCMessage) ValidateBasic() coreTypes.Error { + switch msg := m.Event.(type) { + case *IBCMessage_Update: + return msg.Update.ValidateBasic() + case *IBCMessage_Prune: + return msg.Prune.ValidateBasic() + default: + return coreTypes.ErrIBCUnknownMessageType(fmt.Sprintf("%T", msg)) + } +} + +func (m *UpdateIBCStore) ValidateBasic() coreTypes.Error { + if m.Key == nil { + return coreTypes.ErrNilField("key") + } + if m.Value == nil { + return coreTypes.ErrNilField("value") + } + return nil +} + +func (m *PruneIBCStore) ValidateBasic() coreTypes.Error { + if m.Key == nil { + return coreTypes.ErrNilField("key") + } + return nil +} + +func (m *UpdateIBCStore) SetSigner(signer []byte) { m.Signer = signer } +func (m *PruneIBCStore) SetSigner(signer []byte) { m.Signer = signer } + +func (m *UpdateIBCStore) GetMessageName() string { + return string(m.ProtoReflect().Descriptor().Name()) +} + +func (m *PruneIBCStore) GetMessageName() string { + return string(m.ProtoReflect().Descriptor().Name()) +} + +func (m *UpdateIBCStore) GetMessageRecipient() string { return "" } +func (m *PruneIBCStore) GetMessageRecipient() string { return "" } + +func (m *UpdateIBCStore) GetActorType() coreTypes.ActorType { + return coreTypes.ActorType_ACTOR_TYPE_VAL +} + +func (m *PruneIBCStore) GetActorType() coreTypes.ActorType { + return coreTypes.ActorType_ACTOR_TYPE_VAL +} + +func (m *UpdateIBCStore) GetCanonicalBytes() []byte { + bz, err := codec.GetCodec().Marshal(m) + if err != nil { + log.Fatalf("must marshal %v", err) + } + return bz // DISCUSS(#142): should we also sort the JSON like in V0? +} + +func (m *PruneIBCStore) GetCanonicalBytes() []byte { + bz, err := codec.GetCodec().Marshal(m) + if err != nil { + log.Fatalf("must marshal %v", err) + } + return bz // DISCUSS(#142): should we also sort the JSON like in V0? +} diff --git a/ibc/types/proto/messages.proto b/ibc/types/proto/messages.proto new file mode 100644 index 000000000..8c9ee95b1 --- /dev/null +++ b/ibc/types/proto/messages.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package types; + +option go_package = "github.com/pokt-network/pocket/ibc/types"; + +// UpdateIBCStore defines a message representing the addition of a key/value pair to the IBC store +// the key field should be the full key - prefixed with the store's CommitmentPrefix +message UpdateIBCStore { + bytes signer = 1; // signer should be the address of the node that sent the message + bytes key = 2; + bytes value = 3; +} + +// PruneIBCStore defines a message representing the removal of a key from the IBC store +// the key field should be the full key - prefixed with the store's CommitmentPrefix +message PruneIBCStore { + bytes signer = 1; // signer should be the address of the node that sent the message + bytes key = 2; +} + +// IBCMessage defines the different types of IBC message that can be sent across the network +message IBCMessage { + oneof event { + UpdateIBCStore update = 1; + PruneIBCStore prune = 2; + } +} diff --git a/internal/testutil/ibc/mock.go b/internal/testutil/ibc/mock.go index b4a931d77..e44651125 100644 --- a/internal/testutil/ibc/mock.go +++ b/internal/testutil/ibc/mock.go @@ -9,8 +9,8 @@ import ( "github.com/regen-network/gocuke" ) -// BaseIbcMock returns a mock IBC module without a Host -func BaseIbcMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.MockIBCModule { +// BaseIBCMock returns a mock IBC module without a Host +func BaseIBCMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.MockIBCModule { ctrl := gomock.NewController(t) ibcMock := mockModules.NewMockIBCModule(ctrl) @@ -22,8 +22,8 @@ func BaseIbcMock(t gocuke.TestingT, busMock *mockModules.MockBus) *mockModules.M return ibcMock } -// IbcMockWithHost returns a mock IBC module with a Host -func IbcMockWithHost(t gocuke.TestingT, _ modules.EventsChannel) *mockModules.MockIBCModule { +// IBCMockWithHost returns a mock IBC module with a Host +func IBCMockWithHost(t gocuke.TestingT, _ modules.EventsChannel) *mockModules.MockIBCModule { ctrl := gomock.NewController(t) ibcMock := mockModules.NewMockIBCModule(ctrl) diff --git a/persistence/db.go b/persistence/db.go index 43bf14e91..27a0fd2c1 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -134,6 +134,10 @@ func initializeAllTables(ctx context.Context, db *pgxpool.Conn) error { } } + if err := initialiseIBCTables(ctx, db); err != nil { + return err + } + return nil } @@ -184,3 +188,10 @@ func initializeBlockTables(ctx context.Context, db *pgxpool.Conn) error { } return nil } + +func initialiseIBCTables(ctx context.Context, db *pgxpool.Conn) error { + if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IBCStoreTableName, types.IBCStoreTableSchema)); err != nil { + return err + } + return nil +} diff --git a/persistence/debug.go b/persistence/debug.go index 81b075365..38e69e221 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -14,6 +14,7 @@ var nonActorClearFunctions = []func() string{ types.ClearAllGovParamsQuery, types.ClearAllGovFlagsQuery, types.ClearAllBlocksQuery, + types.ClearAllIBCQuery, } func (m *persistenceModule) HandleDebugMessage(debugMessage *messaging.DebugMessage) error { diff --git a/persistence/docs/PROTOCOL_STATE_HASH.md b/persistence/docs/PROTOCOL_STATE_HASH.md index 75ffd4886..d98040ccc 100644 --- a/persistence/docs/PROTOCOL_STATE_HASH.md +++ b/persistence/docs/PROTOCOL_STATE_HASH.md @@ -10,6 +10,7 @@ Alternative implementation of the persistence module are free to choose their ow - [Block Proto](#block-proto) - [Trees](#trees) - [Compute State Hash](#compute-state-hash) + - [IBC State Tree](#ibc-state-tree) - [Store Block (Commit)](#store-block-commit) - [Failed Commitments](#failed-commitments) @@ -96,6 +97,23 @@ sequenceDiagram deactivate P ``` +### IBC State Tree + +When the new state hash is computed, the different state trees read the updates from their respective Postgres tables and update the trees accordingly. + +`IBCMessage` objects are inserted into the `ibc_entries` table in two ways., depending on the IBC messages' type: 1. `UpdateIBCStore`: the `key` and `value` fields are inserted with the height into the table 2. `PruneIBCStore`: the `key` with a `nil` value is inserted into the table + +For each entry in the `ibc_message` table depending on the entries `value` field the tree will perform one of two operations: + +- `value == nil` + - This is a `PruneIBCStore` message and thus the tree will delete the entry with the given `key` + - `ibcTree.Delete(key)` +- `value != nil` + - This is an `UpdateIBCStore` message and thus the tree will update the entry with the given `key` to have the given `value` + - `ibcTree.Update(key, value)` + +_Note: Prior to insertion the `key` and `value` fields of the messages are hexadecimally encoded into strings._ + ## Store Block (Commit) When the `Commit(proposer, quorumCert)` function is invoked, the current context is committed to disk. The `PersistenceContext` does the following: diff --git a/persistence/ibc.go b/persistence/ibc.go new file mode 100644 index 000000000..abfd76dba --- /dev/null +++ b/persistence/ibc.go @@ -0,0 +1,25 @@ +package persistence + +import ( + pTypes "github.com/pokt-network/pocket/persistence/types" +) + +// SetIBCStoreEntry sets the key value pair in the IBC store postgres table at the current height +func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { + ctx, tx := p.getCtxAndTx() + if _, err := tx.Exec(ctx, pTypes.InsertIBCStoreEntryQuery(p.Height, key, value)); err != nil { + return err + } + return nil +} + +// GetIBCStoreEntry returns the stored value for the key at the height provided from the IBC store table +func (p *PostgresContext) GetIBCStoreEntry(key []byte, height int64) ([]byte, error) { + ctx, tx := p.getCtxAndTx() + row := tx.QueryRow(ctx, pTypes.GetIBCStoreEntryQuery(height, key)) + var value []byte + if err := row.Scan(&value); err != nil { + return nil, err + } + return value, nil +} diff --git a/persistence/sql/sql.go b/persistence/sql/sql.go index 48bbec2d3..80a1c8320 100644 --- a/persistence/sql/sql.go +++ b/persistence/sql/sql.go @@ -165,6 +165,36 @@ func GetParams(pgtx pgx.Tx, height uint64) ([]*coreTypes.Param, error) { return paramSlice, nil } +// GetIBCStoreUpdates returns the set of key-value pairs updated at the current height for the IBC store +func GetIBCStoreUpdates(pgtx pgx.Tx, height uint64) (keys, values [][]byte, err error) { + fields := "key,value" + query := fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY key ASC", fields, ptypes.IBCStoreTableName, height) + rows, err := pgtx.Query(context.TODO(), query) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + var hexKey, hexValue string + for rows.Next() { + if err := rows.Scan(&hexKey, &hexValue); err != nil { + return nil, nil, err + } + key, err := hex.DecodeString(hexKey) + if err != nil { + return nil, nil, err + } + value, err := hex.DecodeString(hexValue) + if err != nil { + return nil, nil, err + } + keys = append(keys, key) + values = append(values, value) + } + + return keys, values, nil +} + func getActor(tx pgx.Tx, actorSchema ptypes.ProtocolActorSchema, address []byte, height int64) (actor *coreTypes.Actor, err error) { ctx := context.TODO() actor, height, err = getActorFromRow(actorSchema.GetActorType(), tx.QueryRow(ctx, actorSchema.GetQuery(hex.EncodeToString(address), height))) diff --git a/persistence/test/benchmark_state_test.go b/persistence/test/benchmark_state_test.go index 9084faf8a..2e40e2fbb 100644 --- a/persistence/test/benchmark_state_test.go +++ b/persistence/test/benchmark_state_test.go @@ -118,7 +118,7 @@ MethodLoop: case reflect.Slice: switch arg.Elem().Kind() { case reflect.Uint8: - v = reflect.ValueOf([]uint8{0}) + v = reflect.ValueOf([]uint8{uint8(rand.Intn(2 ^ 8 - 1))}) case reflect.String: v = reflect.ValueOf([]string{"abc"}) default: diff --git a/persistence/test/state_test.go b/persistence/test/state_test.go index d7a7555cd..d2b67fcf6 100644 --- a/persistence/test/state_test.go +++ b/persistence/test/state_test.go @@ -42,9 +42,9 @@ func TestStateHash_DeterministicStateWhenUpdatingAppStake(t *testing.T) { // logic changes, these hashes will need to be updated based on the test output. // TODO: Add an explicit updateSnapshots flag to the test to make this more clear. stateHashes := []string{ - "fb1c1b2da242eb6148884e1f11c184c673963b6fcb59feea4ea51c9840e4b56c", - "9ae40f9fd0864c01760c19d6688d1c7bba5ad746fc3f7123cf071b142c2a302a", - "5c13743e9d29e83cbff1301e3750c7583e365b2c3fd81b643ed8db985f46268f", + "1e433a8905c7b1cf42222f8d01ba222038653f8ff35ae97cce1fd6a32d18b51e", + "4542dea3eedb99ad46b3c6e0ea901a9ee365b590b2f2ac7f12678ac369a2fe90", + "1524529fa827852e397adf1938eb058848209c4916cdacef929d050efc472c1d", } stakeAmount := initialStakeAmount diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 2f397c781..64f210122 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -38,6 +38,7 @@ const ( TransactionsTreeName = "transactions" ParamsTreeName = "params" FlagsTreeName = "flags" + IBCTreeName = "ibc" ) var actorTypeToMerkleTreeName = map[coreTypes.ActorType]string{ @@ -60,7 +61,7 @@ var stateTreeNames = []string{ // Account Trees AccountTreeName, PoolTreeName, // Data Trees - TransactionsTreeName, ParamsTreeName, FlagsTreeName, + TransactionsTreeName, ParamsTreeName, FlagsTreeName, IBCTreeName, } // stateTree is a wrapper around the SMT that contains an identifying @@ -195,7 +196,15 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height if err := t.updateFlagsTree(flags); err != nil { return "", fmt.Errorf("failed to update flags tree - %w", err) } - // Default should not happen; panic and log the treeType - this is a strong code smell. + case IBCTreeName: + keys, values, err := sql.GetIBCStoreUpdates(pgtx, height) + if err != nil { + return "", fmt.Errorf("failed to get IBC store updates: %w", err) + } + if err := t.updateIBCTree(keys, values); err != nil { + return "", fmt.Errorf("failed to update IBC tree: %w", err) + } + // Default default: t.logger.Panic().Msgf("unhandled merkle tree type: %s", treeName) } @@ -346,3 +355,22 @@ func (t *treeStore) updateFlagsTree(flags []*coreTypes.Flag) error { return nil } + +func (t *treeStore) updateIBCTree(keys, values [][]byte) error { + if len(keys) != len(values) { + return fmt.Errorf("keys and values must be the same length") + } + for i, key := range keys { + value := values[i] + if value == nil { + if err := t.merkleTrees[IBCTreeName].tree.Delete(key); err != nil { + return err + } + continue + } + if err := t.merkleTrees[IBCTreeName].tree.Update(key, value); err != nil { + return err + } + } + return nil +} diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go new file mode 100644 index 000000000..42bd096a8 --- /dev/null +++ b/persistence/types/ibc.go @@ -0,0 +1,40 @@ +package types + +import ( + "encoding/hex" + "fmt" +) + +const ( + IBCStoreTableName = "ibc_entries" + IBCStoreTableSchema = `( + height BIGINT NOT NULL, + key TEXT NOT NULL, + value TEXT, + PRIMARY KEY (height, key) + )` +) + +func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { + return fmt.Sprintf( + `INSERT INTO %s(height, key, value) VALUES(%d, '%s', '%s')`, + IBCStoreTableName, + height, + hex.EncodeToString(key), + hex.EncodeToString(value), + ) +} + +// Return the latest value for the key at the height provided or at the last updated height +func GetIBCStoreEntryQuery(height int64, key []byte) string { + return fmt.Sprintf( + `SELECT value FROM %s WHERE height <= %d AND key = '%s' ORDER BY height DESC LIMIT 1`, + IBCStoreTableName, + height, + hex.EncodeToString(key), + ) +} + +func ClearAllIBCQuery() string { + return fmt.Sprintf(`DELETE FROM %s`, IBCStoreTableName) +} diff --git a/runtime/configs/config.go b/runtime/configs/config.go index 6e428652d..07b420074 100644 --- a/runtime/configs/config.go +++ b/runtime/configs/config.go @@ -161,7 +161,8 @@ func NewDefaultConfig(options ...func(*Config)) *Config { Servicer: &ServicerConfig{}, Fisherman: &FishermanConfig{}, IBC: &IBCConfig{ - Enabled: defaults.DefaultIBCEnabled, + Enabled: defaults.DefaultIBCEnabled, + StoresDir: defaults.DefaultIBCStoresDir, }, } @@ -182,6 +183,7 @@ func WithPK(pk string) func(*Config) { cfg.PrivateKey = pk cfg.Consensus.PrivateKey = pk cfg.P2P.PrivateKey = pk + cfg.IBC.PrivateKey = pk } } diff --git a/runtime/configs/proto/ibc_config.proto b/runtime/configs/proto/ibc_config.proto index 389e19528..a1e62686a 100644 --- a/runtime/configs/proto/ibc_config.proto +++ b/runtime/configs/proto/ibc_config.proto @@ -12,4 +12,6 @@ message IBCConfig { // 2. The node is a servicer and thus when IBC enabled is they are enabled // to relay IBC packets using an IBC relayer binary bool enabled = 1; + string private_key = 2; // hex encoded + string stores_dir = 3; } diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index ce02b2a05..660f49c22 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -73,7 +73,8 @@ var ( DefaultKeybaseVaultMountPath = "" // ibc - DefaultIBCEnabled = false + DefaultIBCEnabled = false + DefaultIBCStoresDir = "/var/ibc" ) var ( diff --git a/runtime/manager_test.go b/runtime/manager_test.go index 1eece78aa..12f4e1daa 100644 --- a/runtime/manager_test.go +++ b/runtime/manager_test.go @@ -1820,7 +1820,11 @@ func TestNewManagerFromReaders(t *testing.T) { Servicer: &configs.ServicerConfig{Enabled: true}, Validator: &configs.ValidatorConfig{Enabled: true}, Fisherman: defaultCfg.Fisherman, - IBC: &configs.IBCConfig{Enabled: true}, + IBC: &configs.IBCConfig{ + Enabled: true, + PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", + StoresDir: defaults.DefaultIBCStoresDir, + }, }, genesisState: expectedGenesis, clock: clock.New(), diff --git a/shared/core/types/error.go b/shared/core/types/error.go index 870b6b733..162a9c42d 100644 --- a/shared/core/types/error.go +++ b/shared/core/types/error.go @@ -38,7 +38,7 @@ func NewError(code Code, msg string) Error { } } -// NextCode: 142 +// NextCode: 148 type Code float64 // CONSIDERATION: Should these be a proto enum or a golang iota? //nolint:gosec // G101 - Not hard-coded credentials @@ -177,10 +177,16 @@ const ( CodeUnknownActorType Code = 130 CodeUnknownMessageType Code = 131 CodeProposalBlockNotSet Code = 133 - CodeHostAlreadyExists Code = 138 - CodeIBCInvalidID Code = 139 - CodeIBCInvalidPath Code = 140 - CodeCreatingProofError Code = 141 + CodeIBCHostAlreadyExists Code = 138 + CodeIBCHostDoesNotExist Code = 139 + CodeIBCInvalidID Code = 140 + CodeIBCInvalidPath Code = 141 + CodeIBCCreatingProofError Code = 142 + CodeIBCUnknownMessageTypeError Code = 143 + CodeNilFieldError Code = 144 + CodeIBCUpdatingStoreError Code = 145 + CodeIBCStoreAlreadyExistsError Code = 146 + CodeIBCStoreDoesNotExistError Code = 147 ) const ( @@ -316,10 +322,16 @@ const ( NegativeAmountError = "the amount is negative" UnknownActorTypeError = "the actor type is not recognized" UnknownMessageTypeError = "the message being by the utility message is not recognized" - HostAlreadyExistsError = "an ibc host already exists" + IBCHostAlreadyExistsError = "an ibc host already exists" + IBCHostDoesNotExistError = "an ibc host does not exist" IBCInvalidIDError = "invalid ibc identifier" IBCInvalidPathError = "invalid ibc path" - CreatingProofError = "an error occurred creating the CommitmentProof" + IBCCreatingProofError = "an error occurred creating the CommitmentProof" + IBCUnknownMessageTypeError = "the ibc message type is not recognized" + NilFieldError = "field cannot be nil" + IBCUpdatingStoreError = "an error occurred updating the ibc store postgres database" + IBCStoreAlreadyExistsError = "ibc store already exists in the store manager" + IBCStoreDoesNotExistError = "ibc store does not exist in the store manager" ) func ErrUnknownParam(paramName string) Error { @@ -855,8 +867,12 @@ func ErrUnknownMessageType(messageType any) Error { return NewError(CodeUnknownMessageType, fmt.Sprintf("%s: %v", UnknownMessageTypeError, messageType)) } -func ErrHostAlreadyExists() Error { - return NewError(CodeHostAlreadyExists, HostAlreadyExistsError) +func ErrIBCHostAlreadyExists() Error { + return NewError(CodeIBCHostAlreadyExists, IBCHostAlreadyExistsError) +} + +func ErrIBCHostDoesNotExist() Error { + return NewError(CodeIBCHostDoesNotExist, IBCHostDoesNotExistError) } func ErrIBCInvalidID(identifier, msg string) Error { @@ -867,6 +883,26 @@ func ErrIBCInvalidPath(path string) Error { return NewError(CodeIBCInvalidPath, fmt.Sprintf("%s: %s", IBCInvalidPathError, path)) } -func ErrCreatingProof(err error) Error { - return NewError(CodeCreatingProofError, fmt.Sprintf("%s: %s", CreatingProofError, err.Error())) +func ErrIBCCreatingProof(err error) Error { + return NewError(CodeIBCCreatingProofError, fmt.Sprintf("%s: %s", IBCCreatingProofError, err.Error())) +} + +func ErrIBCUnknownMessageType(messageType string) Error { + return NewError(CodeIBCUnknownMessageTypeError, fmt.Sprintf("%s: %s", IBCUnknownMessageTypeError, messageType)) +} + +func ErrNilField(field string) Error { + return NewError(CodeNilFieldError, fmt.Sprintf("%s: %s", NilFieldError, field)) +} + +func ErrIBCUpdatingStore(err error) Error { + return NewError(CodeIBCUpdatingStoreError, fmt.Sprintf("%s: %s", IBCUpdatingStoreError, err.Error())) +} + +func ErrIBCStoreAlreadyExists(name string) Error { + return NewError(CodeIBCStoreAlreadyExistsError, fmt.Sprintf("%s: %s", IBCStoreAlreadyExistsError, name)) +} + +func ErrIBCStoreDoesNotExist(name string) Error { + return NewError(CodeIBCStoreDoesNotExistError, fmt.Sprintf("%s: %s", IBCStoreDoesNotExistError, name)) } diff --git a/shared/messaging/events.go b/shared/messaging/events.go index a726a55a0..2d6f80a77 100644 --- a/shared/messaging/events.go +++ b/shared/messaging/events.go @@ -12,6 +12,9 @@ const ( // Utility TxGossipMessageContentType = "utility.TxGossipMessage" + + // IBC + IBCMessageContentType = "ibc.IBCMessage" ) // Helper logger for state sync tranition events diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index d006ee194..fb8f76129 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -1,6 +1,13 @@ package modules -//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler +import ( + ics23 "github.com/cosmos/ics23/go" + "github.com/pokt-network/pocket/persistence/kvstore" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "google.golang.org/protobuf/types/known/anypb" +) + +//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHost,IBCHandler,IBCStoreManager,ProvableStore const IBCModuleName = "ibc" @@ -9,6 +16,9 @@ type IBCModule interface { // GetHost returns the IBC host of the modules GetHost() IBCHost + + // HandleMessage handles the given IBC message + HandleMessage(*anypb.Any) error } // IBCHost is the interface used by the host machine (a Pocket node) to interact with the IBC module @@ -196,3 +206,29 @@ type IBCHandler interface { ) (Packet, error) **/ } + +// IBCStoreManager manages the different ProvableStore instances created by the IBC host +type IBCStoreManager interface { + AddStore(name string) error + GetStore(name string) (ProvableStore, error) + RemoveStore(name string) error + FlushAllEntries() error + PruneCaches(height uint64) error + RestoreCaches() error +} + +// ProvableStore interacts with Persistence and the IBC state tree in order for the IBC host to +// be able to interact with the IBC store locally and propagate any changes throuhout the network +type ProvableStore interface { + Get(key []byte) ([]byte, error) + GetAndProve(key []byte, membership bool) ([]byte, *ics23.CommitmentProof, error) + CreateMembershipProof(key, value []byte) (*ics23.CommitmentProof, error) + CreateNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) + Set(key, value []byte) error + Delete(key []byte) error + GetCommitmentPrefix() coreTypes.CommitmentPrefix + Root() ics23.CommitmentRoot + FlushEntries(kvstore.KVStore) error + PruneCache(store kvstore.KVStore, height uint64) error + RestoreCache(kvstore.KVStore) error +} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 0dbccfc6e..55e338fa5 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -144,6 +144,13 @@ type PersistenceWriteContext interface { InitFlags() error SetFlag(paramName string, value any, enabled bool) error + // IBC Operations + // SetIBCStoreEntry sets the key-value pair in the ibc_entries table at the current height the + // key-value pairs represent the same key-value pairings in the IBC state tree. This table is + // used for data retrieval purposes and to update the state tree from the mempool of IBC transactions. + SetIBCStoreEntry(key, value []byte) error + + // Relay Operations RecordRelayService(applicationAddress string, key []byte, relay *coreTypes.Relay, response *coreTypes.RelayResponse) error } @@ -235,6 +242,10 @@ type PersistenceReadContext interface { GetIntFlag(paramName string, height int64) (int, bool, error) GetStringFlag(paramName string, height int64) (string, bool, error) GetBytesFlag(paramName string, height int64) ([]byte, bool, error) + + // IBC Queries + // GetIBCStoreEntry returns the value of the key at the given height from the ibc_entries table + GetIBCStoreEntry(key []byte, height int64) ([]byte, error) } // PersistenceLocalContext defines the set of operations specific to local persistence. diff --git a/shared/node.go b/shared/node.go index ccdb806f7..f046caeda 100644 --- a/shared/node.go +++ b/shared/node.go @@ -189,6 +189,8 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) // TODO: Remove this lib once we move to Go 1.2 return multierr.Combine(err_consensus, err_p2p) + case messaging.IBCMessageContentType: + return node.GetBus().GetIBCModule().HandleMessage(message.Content) default: logger.Global.Warn().Msgf("Unsupported message content type: %s", contentType) } diff --git a/utility/session_test.go b/utility/session_test.go index ff544e508..319af206c 100644 --- a/utility/session_test.go +++ b/utility/session_test.go @@ -25,7 +25,7 @@ func TestSession_GetSession_SingleFishermanSingleServicerBaseCase(t *testing.T) numFishermen := 1 numServicers := 1 // needs to be manually updated if business logic changes - expectedSessionId := "7a915e89e4805095150e6826dff161ab2612e766b9e6893e6fe747e20a3abfa8" + expectedSessionId := "b1e9791358aae070ac7f86fdb74e5a9d26fff025fb737a2114ccf9ad95b624bd" runtimeCfg, utilityMod, _ := prepareEnvironment(t, 5, numServicers, 1, numFishermen) diff --git a/utility/unit_of_work/gov.go b/utility/unit_of_work/gov.go index 2e18914ff..29a561a47 100644 --- a/utility/unit_of_work/gov.go +++ b/utility/unit_of_work/gov.go @@ -4,6 +4,7 @@ import ( "math/big" "strings" + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/persistence" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -24,9 +25,7 @@ func init() { } } -var ( - govParamTypes map[string]int -) +var govParamTypes map[string]int const ( BIGINT int = iota @@ -151,7 +150,8 @@ func (u *baseUtilityUnitOfWork) getParamOwner(paramName string) ([]byte, coreTyp func (u *baseUtilityUnitOfWork) getFee(msg typesUtil.Message, actorType coreTypes.ActorType) (amount *big.Int, err coreTypes.Error) { switch x := msg.(type) { - case *typesUtil.MessageSend: + // TECHDEBT(M6): Decide on IBC store tx fees and move into a governance parameter + case *typesUtil.MessageSend, *ibcTypes.UpdateIBCStore, *ibcTypes.PruneIBCStore: return getGovParam[*big.Int](u, typesUtil.MessageSendFee) case *typesUtil.MessageStake: switch actorType { diff --git a/utility/unit_of_work/transaction.go b/utility/unit_of_work/transaction.go index 3e91c0c14..6e2d04c87 100644 --- a/utility/unit_of_work/transaction.go +++ b/utility/unit_of_work/transaction.go @@ -33,7 +33,7 @@ func (u *baseUtilityUnitOfWork) basicValidateTransaction(tx *coreTypes.Transacti // Get the address of the transaction signer pubKey, er := crypto.NewPublicKeyFromBytes(tx.Signature.PublicKey) - if err != nil { + if er != nil { return nil, coreTypes.ErrNewPublicKeyFromBytes(er) } address := pubKey.Address() diff --git a/utility/unit_of_work/tx_message_handler.go b/utility/unit_of_work/tx_message_handler.go index 9b3080af9..6c696e14c 100644 --- a/utility/unit_of_work/tx_message_handler.go +++ b/utility/unit_of_work/tx_message_handler.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "math/big" + ibcTypes "github.com/pokt-network/pocket/ibc/types" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" @@ -26,6 +27,10 @@ func (u *baseUtilityUnitOfWork) handleMessage(msg typesUtil.Message) (err coreTy return u.handleUnpauseMessage(x) case *typesUtil.MessageChangeParameter: return u.handleMessageChangeParameter(x) + case *ibcTypes.UpdateIBCStore: + return u.handleUpdateIBCStore(x) + case *ibcTypes.PruneIBCStore: + return u.handlePruneIBCStore(x) default: return coreTypes.ErrUnknownMessage(x) } @@ -220,6 +225,20 @@ func (u *baseUtilityUnitOfWork) handleMessageChangeParameter(message *typesUtil. return u.updateParam(message.ParameterKey, v) } +func (u *baseUtilityUnitOfWork) handleUpdateIBCStore(message *ibcTypes.UpdateIBCStore) coreTypes.Error { + if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, message.Value); err != nil { + return coreTypes.ErrIBCUpdatingStore(err) + } + return nil +} + +func (u *baseUtilityUnitOfWork) handlePruneIBCStore(message *ibcTypes.PruneIBCStore) coreTypes.Error { + if err := u.persistenceRWContext.SetIBCStoreEntry(message.Key, nil); err != nil { + return coreTypes.ErrIBCUpdatingStore(err) + } + return nil +} + func (u *baseUtilityUnitOfWork) checkBelowMaxChains(actorType coreTypes.ActorType, chains []string) coreTypes.Error { // validators don't have chains field if actorType == coreTypes.ActorType_ACTOR_TYPE_VAL { diff --git a/utility/unit_of_work/tx_message_signers.go b/utility/unit_of_work/tx_message_signers.go index 1da3b9acc..602ef602a 100644 --- a/utility/unit_of_work/tx_message_signers.go +++ b/utility/unit_of_work/tx_message_signers.go @@ -1,6 +1,7 @@ package unit_of_work import ( + ibcTypes "github.com/pokt-network/pocket/ibc/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" typesUtil "github.com/pokt-network/pocket/utility/types" @@ -24,6 +25,10 @@ func (u *baseUtilityUnitOfWork) getSignerCandidates(msg typesUtil.Message) ([][] return u.getMessageUnpauseSignerCandidates(x) case *typesUtil.MessageChangeParameter: return u.getMessageChangeParameterSignerCandidates(x) + case *ibcTypes.UpdateIBCStore: + return u.getUpdateIBCStoreSingerCandidates(x) + case *ibcTypes.PruneIBCStore: + return u.getPruneIBCStoreSingerCandidates(x) default: return nil, coreTypes.ErrUnknownMessage(x) } @@ -72,3 +77,11 @@ func (u *baseUtilityUnitOfWork) getMessageUnpauseSignerCandidates(msg *typesUtil func (u *baseUtilityUnitOfWork) getMessageSendSignerCandidates(msg *typesUtil.MessageSend) ([][]byte, coreTypes.Error) { return [][]byte{msg.FromAddress}, nil } + +func (u *baseUtilityUnitOfWork) getUpdateIBCStoreSingerCandidates(msg *ibcTypes.UpdateIBCStore) ([][]byte, coreTypes.Error) { + return [][]byte{msg.Signer}, nil +} + +func (u *baseUtilityUnitOfWork) getPruneIBCStoreSingerCandidates(msg *ibcTypes.PruneIBCStore) ([][]byte, coreTypes.Error) { + return [][]byte{msg.Signer}, nil +} diff --git a/utility/utility_message_handler.go b/utility/utility_message_handler.go index 4cb6ee2a7..46079d1c5 100644 --- a/utility/utility_message_handler.go +++ b/utility/utility_message_handler.go @@ -44,7 +44,6 @@ func (u *utilityModule) HandleUtilityMessage(message *anypb.Any) error { } u.logger.Info().Str("message_type", "TxGossipMessage").Msg("Successfully added a new message to the mempool!") - default: return coreTypes.ErrUnknownMessageType(message.MessageName()) }