diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 0000000..7c28dc1 --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,59 @@ +name: End-to-end Test + +on: + pull_request: + push: + branches: ["main"] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v5 + - name: Cache test dependencies + uses: actions/cache@v4 + id: cache-dependencies + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + /usr/local/bin + $GOPATH/bin + /lib + key: ${{ runner.os }}-${{ hashFiles('/usr/local/bin', '$GOPATH/bin', '/lib', '**/go.sum') }} + restore-keys: | + ${{ runner.os }}- + - uses: actions/checkout@v4.2.2 + if: steps.cache-dependencies.outputs.cache-it != 'true' + with: + repository: JackalLabs/canine-chain + ref: v4.6.2-alpha.1 + path: canine-chain + - name: Install canined + if: steps.cache-dependencies.outputs.cache-it != 'true' + working-directory: ./canine-chain + run: | + sudo wget -O /lib/libwasmvm.x86_64.so https://github.com/CosmWasm/wasmvm/raw/v1.2.6/internal/api/libwasmvm.x86_64.so + make install + - name: Install venom + if: steps.cache-dependencies.outputs.cache-it != 'true' + run: curl https://github.com/ovh/venom/releases/download/v1.2.0/venom.linux-amd64 -L -o /usr/local/bin/venom && chmod +x /usr/local/bin/venom + - uses: actions/checkout@v4.2.2 + with: + path: sequoia + - name: Get sequoia build pkgs + if: steps.cache-dependencies.outputs.cache-it != 'true' + working-directory: ./sequoia + run: go mod download + - name: Install sequoia + working-directory: ./sequoia + run: make install + - name: Test + shell: bash + working-directory: ./sequoia/test + run: venom run -vv e2e/init.yaml `find e2e/ -type f -name "*_test-*.yaml" | sort` + - name: Stop containers + working-directory: ./sequoia/test + run: docker compose down + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0d4a987 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.24 + +ARG RPC_ADDR="http://localhost:26657" +ARG GRPC_ADDR="localhost:9090" + +WORKDIR /usr/src/app + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN make install + +# RPC GRPC api ipfs +EXPOSE 26657 9090 3333 4005 + +WORKDIR /root/.sequoia + +RUN sequoia init + +RUN sed -i -e "s/http:\/\/localhost:26657/${RPC_ADDR}/" config.yaml +RUN sed -i -e "s/localhost:9090/${GRPC_ADDR}/" config.yaml + +CMD ["sequoia", "start"] diff --git a/Makefile b/Makefile index 8517452..3cf88d1 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMMIT := $(shell git log -1 --format='%H') ldflags = -X github.com/JackalLabs/sequoia/config.COMMIT=$(COMMIT) \ -X github.com/JackalLabs/sequoia/config.VERSION=$(VERSION) -all: lint test-unit +all: lint test install: @go install -ldflags '$(ldflags)' @@ -69,7 +69,21 @@ format: ### Tests & Simulation ### ############################################################################### +test: test-unit test-e2e + test-unit: @echo "Executing unit tests..." @go test -mod=readonly -v -coverprofile coverage.txt ./... -.PHONY: test-unit + +test-e2e: + @echo "Executing e2e tests..." + @venom run ./test/e2e/*.yml + +test-e2e-dev-mode: + @echo "Executing e2e tests..." + @venom run --var dev-mode=true ./test/e2e/*.yml + +test-clean: + rm venom*.log + +.PHONY: test-unit test-e2e test-e2e-dev-mode test test-clean diff --git a/go.mod b/go.mod index 124ad24..e90de7c 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/adlio/schema v1.3.6 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 5f39e6a..6d3f6d7 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= diff --git a/network/downloads.go b/network/downloads.go index 6afc688..9ad1f9f 100644 --- a/network/downloads.go +++ b/network/downloads.go @@ -2,6 +2,8 @@ package network import ( "bytes" + "compress/flate" + "compress/gzip" "context" "encoding/json" "errors" @@ -11,6 +13,8 @@ import ( "os" "time" + "github.com/andybalholm/brotli" + apiTypes "github.com/JackalLabs/sequoia/api/types" ipfslite "github.com/hsanjuan/ipfs-lite" @@ -77,7 +81,7 @@ func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start // Returns the number of bytes written, or an error if the download or write fails. Timeout and HTTP errors are // reported with detailed messages. func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, owner string, start int64, chunkSize int64, proofType int64, ipfsParams *ipfslite.AddParams, fileSize int64) (int, error) { - log.Info().Msg(fmt.Sprintf("Downloading %x from %s...", merkle, url)) + log.Info().Msgf("Downloading %x from %s...", merkle, url) // Calculate timeout based on file size // Base timeout + additional time for large files @@ -154,6 +158,29 @@ func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, o return 0, fmt.Errorf("could not get file, code: %d | msg: %s", resp.StatusCode, e.Error) } + var bodyReader io.Reader = resp.Body + contentEncoding := resp.Header.Get("Content-Encoding") + log.Info().Str("merkle", fmt.Sprintf("%x", merkle)).Msgf("Downloads content encoding: %s", contentEncoding) + switch contentEncoding { + case "gzip": + gz, err := gzip.NewReader(resp.Body) + if err != nil { + return 0, fmt.Errorf("failed to create gzip reader: %w", err) + } + //nolint:errcheck + defer gz.Close() + bodyReader = gz + case "deflate": + deflateReader := flate.NewReader(resp.Body) + //nolint:errcheck + defer deflateReader.Close() + bodyReader = deflateReader + case "br": + bodyReader = brotli.NewReader(resp.Body) + default: + // No compression or unsupported; use raw body + } + buff := bytes.NewBuffer([]byte{}) // Use TeeReader to monitor for context cancellation while copying @@ -161,7 +188,7 @@ func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, o errCh := make(chan error, 1) go func() { - _, err := io.Copy(buff, resp.Body) + _, err := io.Copy(buff, bodyReader) if err != nil { errCh <- err } diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..fdcdd04 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# End-to-End Testing + +### Requirements +[canined](https://github.com/JackalLabs/canine-chain/releases) +[venom](https://github.com/ovh/venom/tree/master?tab=readme-ov-file#installing) +docker +docker compose + +### Running Tests + +> Add `--var dev-mode=true` to skip tests that alter current setup files +> Add `-vv` flag to debug tests + +Setup canined and sequoia containers +```bash +venom run e2e/init.yaml +``` +Run tests: +```bash +venom run ./test/e2e/*_test-*.yaml +``` +Or all at once: +```bash +venom run e2e/init.yaml `find e2e/ -type f -name "*_test-*.yaml" | sort` +``` + +Stop and delete test containers +```bash +docker compose down +``` + +### Writing Tests + +If one test depends on the other, make the file name in [sort](https://en.wikipedia.org/wiki/Sort_(Unix))ed order. diff --git a/test/compose.yaml b/test/compose.yaml new file mode 100644 index 0000000..ef1039c --- /dev/null +++ b/test/compose.yaml @@ -0,0 +1,27 @@ +services: + chain: + build: ./resources/canine + image: e2e-test-chain + ports: + - "26656:26656" + - "26657:26657" + - "1317:1317" + - "9092:9092" + - "9090:9090" + + sequoia: + build: + context: ../ + args: + RPC_ADDR: 'http:\/\/chain:26657' + GRPC_ADDR: "chain:9090" + image: e2e-test-sequoia + volumes: + - $HOME/.sequoia/provider_wallet.json:/root/.sequoia/provider_wallet.json + depends_on: + - chain + entrypoint: ["/bin/sh", "-c", "sleep 10 && sequoia start"] + ports: + - "3333:3333" + - "4005:4005" + diff --git a/test/e2e/01_test-post-file.yaml b/test/e2e/01_test-post-file.yaml new file mode 100644 index 0000000..c40de1f --- /dev/null +++ b/test/e2e/01_test-post-file.yaml @@ -0,0 +1,21 @@ +name: Test Sequoia File Upload +description: Test file upload to sequoia +vars: + dev-mode: false + +testcases: +- name: Start sequoia and canine-chain containers + steps: + - script: docker compose up -d; sleep 5 + +- name: Upload file to sequoia + + steps: + - name: Upload file using canined + script: 'canined tx storage post ../resources/logo.png 500000 --dest http://localhost:3333 --max_proofs 1 --from j1 -y' + retry: 3 + delay: 10 + assertions: + - result.systemerr ShouldBeBlank + - result.systemout ShouldContainSubstring 1ab5cd31e2d001fedad0b92abb01cf36f16e1c395c7880a67e2d18d39d81040d7732fa5a33d81bb33578243c54d60c8d5a56de604b1827243dc1feab39b4b3a6 + - result.code ShouldEqual 0 diff --git a/test/e2e/init.yaml b/test/e2e/init.yaml new file mode 100644 index 0000000..96b8ecd --- /dev/null +++ b/test/e2e/init.yaml @@ -0,0 +1,58 @@ +name: Test Sequoia Initialization +description: Test provider Initialization on chain +vars: + dev-mode: false + faucet: jkl1hj5fveer5cjtn4wd6wstzugjfdxzl0xpljur4u + +testcases: +- name: Initialize canine-chain and sequoia configs + skip: + - dev-mode ShouldEqual false + + steps: + - name: Delete existing docker containers + script: "docker compose down" + + - name: Initialize sequoia config + script: "rm -rf $HOME/.sequoia; sequoia init" + + - name: Initialize canined config + script: "rm -rf $HOME/.canine; ../resources/canine/setup-chain.sh" + + - name: Build and start canined container + script: "docker compose up -d chain; sleep 5" # wait for the chain to open api + +- name: Fund sequoia provider wallet + skip: + - dev-mode ShouldEqual false + + steps: + - name: Get provider address + script: "sequoia wallet address | sed 's/Provider Address: //'" + vars: + address: + from: result.systemout + + - name: Fund provider + info: "provider address {{.address}}" + script: "canined tx bank send {{.faucet}} {{.address}} 100000000000ujkl -y" + retry: 3 + delay: 10 + assertions: + - result.code ShouldEqual 0 + +- name: Start sequoia + steps: + - name: Build and start sequoia container + script: "docker compose up -d sequoia" + assertions: + - result.code ShouldEqual 0 + +- name: Verify provider registration on chain + steps: + - script: "canined query storage show-providers {{.Fund-sequoia-provider-wallet.address}}" + retry: 3 + delay: 10 # bump this number if provider takes longer to start up + assertions: + - result.systemout ShouldContainSubstring {{.Fund-sequoia-provider-wallet.address}} + - result.code ShouldEqual 0 diff --git a/test/resources/canine/Dockerfile b/test/resources/canine/Dockerfile new file mode 100644 index 0000000..0bcf3cd --- /dev/null +++ b/test/resources/canine/Dockerfile @@ -0,0 +1,48 @@ +# syntax=docker/dockerfile:1 + +# Start from a base Golang image to build the binary +FROM golang:1.23 AS builder + +RUN apt-get update && apt-get install -y jq + +WORKDIR /lib + +COPY ./setup-chain.sh . + +RUN wget https://github.com/CosmWasm/wasmvm/raw/v1.2.6/internal/api/libwasmvm.x86_64.so + +WORKDIR /app + +RUN git clone https://github.com/JackalLabs/canine-chain.git . + +RUN git checkout $(git describe --abbrev=0 --tags) + +RUN make build + +# Use a smaller image to run the binary +FROM debian:bookworm-slim + +WORKDIR /root + +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + curl \ + bash \ + wget \ + jq + +COPY --from=builder /app/build/canined /usr/local/bin/canined +COPY --from=builder /lib/libwasmvm.x86_64.so /lib/libwasmvm.x86_64.so +COPY --from=builder /lib/setup-chain.sh /root/setup-chain.sh + +WORKDIR /root + +RUN ./setup-chain.sh + +# Expose default RPC, P2P, and REST ports +EXPOSE 26656 26657 1317 9092 9090 + +WORKDIR /root/.canine + +CMD ["canined", "start"] diff --git a/test/resources/canine/setup-chain.sh b/test/resources/canine/setup-chain.sh new file mode 100755 index 0000000..ba9ae6d --- /dev/null +++ b/test/resources/canine/setup-chain.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +set -eu + +KEY="j1" +KEY1="j2" +KEY2="charlie" +DEPOACCKEY="deposit_account" + +CHAINID="test-1" +MONIKER="localjack" +KEYALGO="secp256k1" +KEYRING="test" +LOGLEVEL="info" +BROADCASTMODE="block" + +command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; } + +add_keys () { + # jkl1hj5fveer5cjtn4wd6wstzugjfdxzl0xpljur4u '{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"ApZa31BR3NWLylRT6Qi5+f+zXtj2OpqtC76vgkUGLyww"}' + echo "decorate bright ozone fork gallery riot bus exhaust worth way bone indoor calm squirrel merry zero scheme cotton until shop any excess stage laundry" | canined keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO --recover + # j2 jkl1s00nvkagel9xe6luqmmd09jt6jgjl7qu57prct '{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"Ah3VzRghgXLn8IA2AH6qaoiuBwZv3ADg3gNPFTo92FwM"}' + echo "guess census arena parent ribbon among advice green electric almost wink muffin size unfold hedgehog gather warfare embrace float entry cargo ice fade best" | canined keys add $DEPOACCKEY --keyring-backend $KEYRING --algo $KEYALGO --recover + + echo "video pluck level diagram maximum grant make there clog tray enrich book hawk confirm spot you book vendor ensure theory sure jewel sort basket" | canined keys add $KEY1 --algo $KEYALGO --keyring-backend $KEYRING --recover + + echo "flock stereo dignity lawsuit mouse page faith exact mountain clinic hazard parent arrest face couch asset jump feed benefit upper hair scrap loud spirit" | canined keys add $KEY2 --algo $KEYALGO --keyring-backend $KEYRING --recover +} + +from_scratch () { + # remove existing daemon + rm -rf ~/.canine/* + + add_keys + + canined init $MONIKER --chain-id $CHAINID + + canined config keyring-backend $KEYRING + canined config chain-id $CHAINID + canined config broadcast-mode $BROADCASTMODE + + # Function updates the config based on a jq argument as a string + update_test_genesis () { + # EX: update_test_genesis '.consensus_params["block"]["max_gas"]="100000000"' + cat $HOME/.canine/config/genesis.json | jq "$1" > $HOME/.canine/config/tmp_genesis.json && mv $HOME/.canine/config/tmp_genesis.json $HOME/.canine/config/genesis.json + } + + # Set gas limit in genesis + update_test_genesis '.consensus_params["block"]["max_gas"]="100000000"' + update_test_genesis '.app_state["gov"]["voting_params"]["voting_period"]="15s"' + update_test_genesis '.app_state["staking"]["params"]["bond_denom"]="ujkl"' + update_test_genesis '.app_state["mint"]["params"]["mint_denom"]="ujkl"' + update_test_genesis '.app_state["gov"]["deposit_params"]["min_deposit"]=[{"denom": "ujkl","amount": "1000000"}]' + update_test_genesis '.app_state["crisis"]["constant_fee"]={"denom": "ujkl","amount": "1000"}' + + # Use jkl bech32 prefix account for storage and oracle modules + update_test_genesis '.app_state["storage"]["params"]["deposit_account"]="'"$(canined keys show -a $DEPOACCKEY)"'"' + update_test_genesis '.app_state["storage"]["params"]["chunk_size"]="'10240'"' + update_test_genesis '.app_state["storage"]["params"]["proof_window"]="'25'"' + update_test_genesis '.app_state["oracle"]["params"]["deposit"]="'"$(canined keys show -a $DEPOACCKEY)"'"' + update_test_genesis '.app_state["rns"]["params"]["deposit_account"]="'"$(canined keys show -a $DEPOACCKEY)"'"' + #'.app_state["storage"]["params"]["check_window"]="'10'"' + + #update_test_genesis '.app_state["jklmint"]["params"]["deposit_account"]="'"$(canined keys show -a $DEPOACCKEY)"'"' + + # adding providers to genesis + canined add-genesis-account jkl1xclg3utp4yuvaxa54r39xzrudc988s82ykve3f 1100000000000ujkl + canined add-genesis-account jkl1tcveayn80pe3d5wallj9kev3rfefctsmrqf6ks 1100000000000ujkl + canined add-genesis-account jkl1eg3gm3e3k4dypvvme26ejmajnyvtgwwlaaeu2y 1100000000000ujkl + canined add-genesis-account jkl1ga0348r8zhn8k4xy3fagwvkwzvyh5lynxr5kak 1100000000000ujkl + canined add-genesis-account jkl18encuf0esmxv3pxqjqvn0u4tgd6yzuc8urzlp0 1100000000000ujkl + canined add-genesis-account jkl1sqt9v0zwwx362szrek7pr3lpq29aygw06hgyza 1100000000000ujkl + canined add-genesis-account jkl1yu099xns2qpslvyrymxq3hwrqhevs7qxksvu8p 1100000000000ujkl + canined add-genesis-account jkl1su8xhm089rg2qs7hx4ngzt42mg45ma89g00e30 1100000000000ujkl + + + # Allocate genesis accounts + canined add-genesis-account $KEY 100000000000000ujkl --keyring-backend $KEYRING + canined add-genesis-account $DEPOACCKEY 1000000000000ujkl --keyring-backend $KEYRING + canined add-genesis-account $KEY1 1000000000000ujkl --keyring-backend $KEYRING + canined add-genesis-account $KEY2 1000000000000ujkl --keyring-backend $KEYRING + + canined gentx $KEY 1000000ujkl --keyring-backend $KEYRING --chain-id $CHAINID + + canined collect-gentxs + + canined validate-genesis +} + +fix_config() { + sed -i.bak -e 's/stake/ujkl/' $HOME/.canine/config/genesis.json + sed -i.bak -e 's/ujklr_ratio/staker_ratio/' $HOME/.canine/config/genesis.json + sed -i.bak -e 's/^minimum-gas-prices =""/minimum-gas-prices = \"0.0025ujkl\"/' $HOME/.canine/config/app.toml + sed -i.bak -e 's/enable = false/enable=true/' $HOME/.canine/config/app.toml + sed -i.bak -e 's/enable=false/enable=true/' $HOME/.canine/config/app.toml + sed -i.bak -e 's/enabled-unsafe-cors = false/enabled-unsafe-cors = true/' $HOME/.canine/config/app.toml + sed -i.bak -e 's/cors_allowed_origins = \[\]/cors_allowed_origins = \["*"\]/' $HOME/.canine/config/config.toml + sed -i.bak -e 's/laddr = "tcp:\/\/127.0.0.1:26657"/laddr = "tcp:\/\/0.0.0.0:26657"/' $HOME/.canine/config/config.toml + sed -i.bak -e 's/laddr = "tcp:\/\/127.0.0.1:26656"/laddr = "tcp:\/\/0.0.0.0:26656"/' $HOME/.canine/config/config.toml + sed -i.bak -e 's/chain-id = ""/chain-id = "canine-1"/' $HOME/.canine/config/client.toml + sed -i.bak -e 's/address = "0.0.0.0:9090"/address = "0.0.0.0:9090"/' $HOME/.canine/config/app.toml + sed -i.bak -e 's/grpc_laddr = ""/grpc_laddr = "tcp:\/\/0.0.0.0:9092"/' $HOME/.canine/config/config.toml + +} + +from_scratch +fix_config diff --git a/test/resources/logo.png b/test/resources/logo.png new file mode 100644 index 0000000..7dc23f0 Binary files /dev/null and b/test/resources/logo.png differ