diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index 173feef..0000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Test -description: Lints and tests explored - -runs: - using: composite - steps: - - name: Configure git # required for golangci-lint on Windows - shell: bash - run: git config --global core.autocrlf false - - name: Lint - uses: golangci/golangci-lint-action@v3 - with: - skip-cache: true - - name: Analyze - uses: SiaFoundation/action-golang-analysis@HEAD - with: - analyzers: | - go.sia.tech/jape.Analyzer - directories: | - api - - name: Test - uses: n8maninger/action-golang-test@v1 - with: - args: "-race;-tags=testing netgo" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cd88554..27b41fa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + groups: + all-dependencies: + patterns: + - "*" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37bd033..4679649 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,30 +1,25 @@ -name: Main +name: Lint & Test on: - workflow_dispatch: pull_request: push: branches: - master + env: CGO_ENABLED: 1 jobs: test: - runs-on: ${{ matrix.os }} - permissions: - contents: read - strategy: - matrix: - os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.21', '1.22' ] + uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master + analyze: + runs-on: ubuntu-latest steps: - - name: Configure git - run: git config --global core.autocrlf false # required on Windows - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - name: Checkout + uses: actions/checkout@v4 + - name: Jape Analyzer + uses: SiaFoundation/action-golang-analysis@HEAD with: - go-version: ${{ matrix.go-version }} - - name: Test - uses: ./.github/actions/test - - name: Build - run: go build -o bin/ ./cmd/explored + analyzers: | + go.sia.tech/jape.Analyzer@master + directories: | + api diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ae0bc80 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,25 @@ +name: Publish + +# Controls when the action will run. +on: + push: + branches: + - master + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-**" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + publish: + uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@d0721ade42a1811c199a3f5707c6980f1d8a2ad8 + secrets: inherit + with: + linux-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + windows-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + macos-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w' + cgo-enabled: 1 + project: explored diff --git a/.golangci.yml b/.golangci.yml index 0d11f13..050db11 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,28 +17,8 @@ run: # list of build tags, all linters use it. Default is empty list. build-tags: [] - # which dirs to skip: issues from them won't be reported; - # can use regexp here: generated.*, regexp is applied on full path; - # default value is empty list, but default dirs are skipped independently - # from this option's value (see skip-dirs-use-default). - skip-dirs: - - cover - - # default is true. Enables skipping of directories: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs-use-default: true - - # which files to skip: they will be analyzed, but issues from them - # won't be reported. Default value is empty list, but there is - # no need to include all autogenerated files, we confidently recognize - # autogenerated files. If it's not please let us know. - skip-files: [] - # output configuration options output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number - # print lines of code with issue, default is true print-issued-lines: true @@ -50,7 +30,6 @@ linters-settings: ## Enabled linters: govet: # report about shadowed variables - check-shadowing: false disable-all: false tagliatelle: @@ -65,23 +44,31 @@ linters-settings: # See https://go-critic.github.io/overview#checks-overview # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` # By default list of stable checks is used. - enabled-checks: - - argOrder # Diagnostic options - - badCond - - caseOrder - - dupArg - - dupBranchBody - - dupCase - - dupSubExpr - - nilValReturn - - offBy1 - - weakCond - - boolExprSimplify # Style options here and below. - - builtinShadow - - emptyFallthrough - - hexLiteral - - underef - - equalFold + enabled-tags: + - diagnostic + - style + disabled-checks: + # diagnostic + - appendAssign + - commentedOutCode + - uncheckedInlineErr + # style + - httpNoBody + - exitAfterDefer + - ifElseChain + - importShadow + - initClause + - nestingReduce + - octalLiteral + - paramTypeCombine + - ptrToRefParam + - stringsCompare + - tooManyResultsChecker + - typeDefFirst + - typeUnparen + - unlabelStmt + - unnamedResult + - whyNoLint revive: ignore-generated-header: true rules: @@ -90,7 +77,7 @@ linters-settings: - name: bool-literal-in-expr disabled: false - name: confusing-naming - disabled: true + disabled: false - name: confusing-results disabled: false - name: constant-logical-expr @@ -108,7 +95,7 @@ linters-settings: - name: increment-decrement disabled: false - name: modifies-value-receiver - disabled: false + disabled: true - name: optimize-operands-order disabled: false - name: range-val-in-closure @@ -154,12 +141,10 @@ issues: # But independently from this option we use default exclude patterns, # it can be disabled by `exclude-use-default: false`. To list all # excluded by default patterns execute `golangci-lint run --help` - exclude: - - "ifElseChain:.*" - - "exitAfterDefer:.*" + exclude: [] # Independently from option `exclude` we use default exclude patterns, # it can be disabled by this option. To list all # excluded by default patterns execute `golangci-lint run --help`. # Default value for this option is true. - exclude-use-default: false + exclude-use-default: false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a26b2f7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM docker.io/library/golang:1.23 AS builder + +WORKDIR /explored + +# Install dependencies +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source +COPY . . + +# Enable CGO for sqlite3 support +ENV CGO_ENABLED=1 + +RUN go generate ./... +RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/explored + +FROM docker.io/library/alpine:3 +LABEL maintainer="The Sia Foundation " \ + org.opencontainers.image.description.vendor="The Sia Foundation" \ + org.opencontainers.image.description="An explored container - indexes the state of the Sia blockchain" \ + org.opencontainers.image.source="https://github.com/SiaFoundation/explored" \ + org.opencontainers.image.licenses=MIT + +ENV PUID=0 +ENV PGID=0 + +# copy binary and prepare data dir. +COPY --from=builder /explored/bin/* /usr/bin/ +VOLUME [ "/data" ] + +# API port +EXPOSE 9980/tcp +# RPC port +EXPOSE 9981/tcp + +USER ${PUID}:${PGID} + +ENV EXPLORED_CONFIG_FILE=/data/explored.yml +ENTRYPOINT [ "explored", "--dir", "/data" ] \ No newline at end of file diff --git a/README.md b/README.md index 3b5c2d3..7ac77d5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ [![GoDoc](https://godoc.org/go.sia.tech/explored?status.svg)](https://godoc.org/go.sia.tech/explored) `explored` is an explorer for Sia. + +## Required Disclosure + +This product includes GeoLite2 data created by MaxMind, available at https://www.maxmind.com. It is provided under a [Creative Commons Corporation Attribution-ShareAlike 4.0 International License](https://www.maxmind.com/en/geolite2/eula). diff --git a/api/api.go b/api/api.go index bad8591..278bb26 100644 --- a/api/api.go +++ b/api/api.go @@ -4,9 +4,18 @@ import ( "time" "go.sia.tech/core/types" - "go.sia.tech/explored/explorer" ) +// A StateResponse returns information about the current state of the explored +// daemon. +type StateResponse struct { + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + BuildTime time.Time `json:"buildTime"` + StartTime time.Time `json:"startTime"` +} + // A GatewayPeer is a currently-connected peer. type GatewayPeer struct { Addr string `json:"addr"` @@ -31,12 +40,6 @@ type TxpoolTransactionsResponse struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } -// AddressUTXOsResponse is the response for /addresses/:address/utxos. -type AddressUTXOsResponse struct { - UnspentSiacoinOutputs []explorer.SiacoinOutput `json:"unspentSiacoinOutputs"` - UnspentSiafundOutputs []explorer.SiafundOutput `json:"unspentSiafundOutputs"` -} - // AddressBalanceResponse is the response for /addresses/:address/balance. type AddressBalanceResponse struct { UnspentSiacoins types.Currency `json:"unspentSiacoins"` diff --git a/api/api_test.go b/api/api_test.go new file mode 100644 index 0000000..6b70930 --- /dev/null +++ b/api/api_test.go @@ -0,0 +1,628 @@ +package api_test + +import ( + "context" + "math" + "net" + "net/http" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/explored/api" + "go.sia.tech/explored/build" + "go.sia.tech/explored/config" + "go.sia.tech/explored/exchangerates" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" + "go.sia.tech/explored/persist/sqlite" + "go.uber.org/zap/zaptest" +) + +const testPassword = "password" + +func newExplorer(t *testing.T, network *consensus.Network, genesisBlock types.Block, scanCfg config.Scanner) (*explorer.Explorer, *chain.Manager, error) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + e, err := explorer.NewExplorer(cm, db, config.Index{ + BatchSize: 1000, + }, scanCfg, log) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + e.Shutdown(ctx) + + db.Close() + bdb.Close() + }) + + return e, cm, nil +} + +func newServer(t *testing.T, cm *chain.Manager, e *explorer.Explorer, listenAddr string) (*http.Server, error) { + ctx, cancel := context.WithCancel(context.Background()) + ex := exchangerates.NewKraken(map[string]string{ + exchangerates.CurrencyUSD: exchangerates.KrakenPairSiacoinUSD}, time.Second) + go ex.Start(ctx) + + api := api.NewServer(e, cm, &syncer.Syncer{}, ex, testPassword) + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api") { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") + api.ServeHTTP(w, r) + return + } + http.NotFound(w, r) + }), + ReadTimeout: 15 * time.Second, + } + + httpListener, err := net.Listen("tcp", listenAddr) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + server.Close() + httpListener.Close() + cancel() + }) + go func() { + server.Serve(httpListener) + }() + + return server, nil +} + +func TestAPI(t *testing.T) { + const netAddr1 = "127.0.0.1:1234" + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + contractFilesize := uint64(10) + + network, genesisBlock := ctestutil.Network() + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + giftSF := genesisBlock.Transactions[0].SiafundOutputs[0].Value + + scanCfg := config.Scanner{ + NumThreads: 100, + ScanTimeout: 30 * time.Second, + ScanFrequency: 100 * time.Millisecond, + ScanInterval: 3 * time.Hour, + MinLastAnnouncement: 90 * 24 * time.Hour, + } + e, cm, err := newExplorer(t, network, genesisBlock, scanCfg) + if err != nil { + t.Fatal(err) + } + + listenAddr := "127.0.0.1:9999" + _, err = newServer(t, cm, e, listenAddr) + if err != nil { + t.Fatal(err) + } + + scOutputID := genesisBlock.Transactions[0].SiacoinOutputID(0) + sfOutputID := genesisBlock.Transactions[0].SiafundOutputID(0) + unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) + + windowStart := cm.Tip().Height + 10 + windowEnd := windowStart + 10 + fc := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), windowStart, windowEnd, types.VoidAddress) + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: scOutputID, + UnlockConditions: unlockConditions, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: giftSC.Sub(fc.Payout), + }}, + FileContracts: []types.FileContract{fc}, + } + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + b1 := testutil.MineBlock(cm.TipState(), []types.Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + + txn2 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: sfOutputID, + UnlockConditions: unlockConditions, + }}, + SiafundOutputs: []types.SiafundOutput{ + { + Address: addr2, + Value: giftSF - 1, + }, + { + Address: addr1, + Value: 1, + }, + }, + ArbitraryData: [][]byte{ + testutil.CreateAnnouncement(pk1, netAddr1), + }, + } + testutil.SignTransaction(cm.TipState(), pk1, &txn2) + + fcID := txn1.FileContractID(0) + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{ + renterPublicKey.UnlockKey(), + hostPublicKey.UnlockKey(), + }, + SignaturesRequired: 2, + } + revFC := fc + revFC.RevisionNumber++ + reviseTxn := types.Transaction{ + FileContractRevisions: []types.FileContractRevision{{ + ParentID: fcID, + UnlockConditions: uc, + FileContract: revFC, + }}, + } + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn) + + b2 := testutil.MineBlock(cm.TipState(), []types.Transaction{txn2, reviseTxn}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + + // Unconfirmed transaction relevant to addr1 + txn3 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: txn1.SiacoinOutputID(0), + UnlockConditions: unlockConditions, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: types.VoidAddress, + Value: txn1.SiacoinOutputs[0].Value, + }}, + } + testutil.SignTransaction(cm.TipState(), pk1, &txn3) + + // Other unconfirmed transaction + txn4 := types.Transaction{} + if _, err := cm.AddPoolTransactions([]types.Transaction{txn3, txn4}); err != nil { + t.Fatal(err) + } + + // Ensure explorer has time to add blocks + time.Sleep(2 * time.Second) + + client := api.NewClient("http://"+listenAddr+"/api", testPassword) + badAuthClient := api.NewClient("http://"+listenAddr+"/api", "") + + subtests := []struct { + name string + test func(t *testing.T) + }{ + {"State", func(t *testing.T) { + resp, err := client.State() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "Version", build.Version(), resp.Version) + testutil.Equal(t, "Commit", build.Commit(), resp.Commit) + testutil.Equal(t, "OS", runtime.GOOS, resp.OS) + testutil.Equal(t, "BuildTime", build.Time().UTC(), resp.BuildTime.UTC()) + }}, + {"ConsensusTip", func(t *testing.T) { + resp, err := client.ConsensusTip() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "tip", cm.Tip(), resp) + }}, + {"BestIndex", func(t *testing.T) { + for i := uint64(0); i < cm.Tip().Height; i++ { + resp, err := client.BestIndex(i) + if err != nil { + t.Fatal(err) + } + tip, err := e.BestTip(i) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "tip", tip, resp) + } + }}, + {"ConsensusNetwork", func(t *testing.T) { + resp, err := client.ConsensusNetwork() + if err != nil { + t.Fatal(err) + } + + // fix because reflect.DeepEqual can't compare timestamps + n := cm.TipState().Network + n.HardforkOak.GenesisTimestamp = n.HardforkOak.GenesisTimestamp.UTC() + resp.HardforkOak.GenesisTimestamp = resp.HardforkOak.GenesisTimestamp.UTC() + + testutil.Equal(t, "network", n, resp) + }}, + {"ConsensusState", func(t *testing.T) { + resp, err := client.ConsensusState() + if err != nil { + t.Fatal(err) + } + cs := cm.TipState() + testutil.Equal(t, "index", cs.Index, resp.Index) + + // fix timestamps again + for i := range cs.PrevTimestamps { + cs.PrevTimestamps[i] = cs.PrevTimestamps[i].UTC() + } + for i := range resp.PrevTimestamps { + resp.PrevTimestamps[i] = resp.PrevTimestamps[i].UTC() + } + + testutil.Equal(t, "previous timestamps", cs.PrevTimestamps, resp.PrevTimestamps) + testutil.Equal(t, "depth", cs.Depth, resp.Depth) + testutil.Equal(t, "child target", cs.ChildTarget, resp.ChildTarget) + testutil.Equal(t, "siafund tax revenue", cs.SiafundTaxRevenue, resp.SiafundTaxRevenue) + }}, + {"Tip", func(t *testing.T) { + resp, err := client.Tip() + if err != nil { + t.Fatal(err) + } + tip, err := e.Tip() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "tip", tip, resp) + }}, + {"BlockMetrics", func(t *testing.T) { + resp, err := client.BlockMetrics() + if err != nil { + t.Fatal(err) + } + cs := cm.TipState() + testutil.Equal(t, "index", cs.Index, resp.Index) + testutil.Equal(t, "difficulty", cs.Difficulty, resp.Difficulty) + testutil.Equal(t, "siafund tax revenue", cs.SiafundTaxRevenue, resp.SiafundTaxRevenue) + testutil.Equal(t, "total hosts", 1, resp.TotalHosts) + testutil.Equal(t, "active contracts", 1, resp.ActiveContracts) + testutil.Equal(t, "failed contracts", 0, resp.FailedContracts) + testutil.Equal(t, "failed contracts", 0, resp.SuccessfulContracts) + testutil.Equal(t, "storage utilization", contractFilesize, resp.StorageUtilization) + testutil.Equal(t, "contract revenue", types.ZeroCurrency, resp.ContractRevenue) + }}, + {"BlockMetricsID", func(t *testing.T) { + // block before revision and host announcement + tip, err := e.BestTip(1) + if err != nil { + t.Fatal(err) + } + resp, err := client.BlockMetricsID(tip.ID) + if err != nil { + t.Fatal(err) + } + cs := cm.TipState() + testutil.Equal(t, "index", tip, resp.Index) + testutil.Equal(t, "difficulty", cs.Difficulty, resp.Difficulty) + testutil.Equal(t, "siafund tax revenue", cs.SiafundTaxRevenue, resp.SiafundTaxRevenue) + testutil.Equal(t, "total hosts", 0, resp.TotalHosts) + testutil.Equal(t, "active contracts", 1, resp.ActiveContracts) + testutil.Equal(t, "failed contracts", 0, resp.FailedContracts) + testutil.Equal(t, "failed contracts", 0, resp.SuccessfulContracts) + testutil.Equal(t, "storage utilization", contractFilesize, resp.StorageUtilization) + testutil.Equal(t, "contract revenue", types.ZeroCurrency, resp.ContractRevenue) + }}, + {"Block", func(t *testing.T) { + tip := cm.Tip() + parentIndex, err := e.BestTip(tip.Height - 1) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Block(tip.ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "height", tip.Height, resp.Height) + testutil.Equal(t, "parent ID", parentIndex.ID, resp.ParentID) + testutil.Equal(t, "nonce", b2.Nonce, resp.Nonce) + testutil.Equal(t, "timestamp", b2.Timestamp.UTC(), resp.Timestamp.UTC()) + testutil.Equal(t, "miner payout address", b2.MinerPayouts[0].Address, resp.MinerPayouts[0].SiacoinOutput.Address) + testutil.Equal(t, "miner payout value", b2.MinerPayouts[0].Value, resp.MinerPayouts[0].SiacoinOutput.Value) + testutil.Equal(t, "miner payout source", explorer.SourceMinerPayout, resp.MinerPayouts[0].Source) + testutil.Equal(t, "miner payout spent index", nil, resp.MinerPayouts[0].SpentIndex) + + testutil.Equal(t, "len(transactions)", len(b2.Transactions), len(resp.Transactions)) + for i := range b2.Transactions { + testutil.CheckTransaction(t, b2.Transactions[i], resp.Transactions[i]) + } + }}, + {"Transaction", func(t *testing.T) { + resp, err := client.Transaction(txn1.ID()) + if err != nil { + t.Fatal(err) + } + testutil.CheckTransaction(t, txn1, resp) + }}, + {"Transactions", func(t *testing.T) { + resp, err := client.Transactions([]types.TransactionID{txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckTransaction(t, txn2, resp[0]) + }}, + {"TransactionChainIndices", func(t *testing.T) { + resp, err := client.TransactionChainIndices(txn2.ID(), 0, 500) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(chainIndices)", 1, len(resp)) + testutil.Equal(t, "chain index", cm.Tip(), resp[0]) + }}, + {"AddressSiacoinUTXOs", func(t *testing.T) { + resp, err := client.AddressSiacoinUTXOs(addr1, 0, 500) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scos)", 1, len(resp)) + testutil.Equal(t, "output source", explorer.SourceTransaction, resp[0].Source) + testutil.Equal(t, "output spent index", nil, resp[0].SpentIndex) + testutil.Equal(t, "output address", txn1.SiacoinOutputs[0].Address, resp[0].SiacoinOutput.Address) + testutil.Equal(t, "output value", txn1.SiacoinOutputs[0].Value, resp[0].SiacoinOutput.Value) + }}, + {"AddressSiacoinUTXOs offset", func(t *testing.T) { + resp, err := client.AddressSiacoinUTXOs(addr1, 1, 500) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scos)", 0, len(resp)) + }}, + {"AddressSiacoinUTXOs limit", func(t *testing.T) { + resp, err := client.AddressSiacoinUTXOs(addr1, 0, 0) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scos)", 0, len(resp)) + }}, + {"AddressSiafundUTXOs", func(t *testing.T) { + resp, err := client.AddressSiafundUTXOs(addr1, 0, 500) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sfos)", 1, len(resp)) + testutil.Equal(t, "output spent index", nil, resp[0].SpentIndex) + testutil.Equal(t, "output address", txn2.SiafundOutputs[1].Address, resp[0].SiafundOutput.Address) + testutil.Equal(t, "output value", txn2.SiafundOutputs[1].Value, resp[0].SiafundOutput.Value) + }}, + {"AddressBalance", func(t *testing.T) { + resp, err := client.AddressBalance(addr1) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "unspent siacoins", txn1.SiacoinOutputs[0].Value, resp.UnspentSiacoins) + testutil.Equal(t, "immature siacoins", types.ZeroCurrency, resp.ImmatureSiacoins) + testutil.Equal(t, "unspent siafunds", txn2.SiafundOutputs[1].Value, resp.UnspentSiafunds) + }}, + {"AddressEvents", func(t *testing.T) { + resp, err := client.AddressEvents(addr1, 0, 500) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 3, len(resp)) + + ev0 := resp[0].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, txn2, ev0.Transaction) + + ev1 := resp[1].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, txn1, ev1.Transaction) + + ev2 := resp[2].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, genesisBlock.Transactions[0], ev2.Transaction) + }}, + {"Event", func(t *testing.T) { + resp, err := client.Event(types.Hash256(txn2.ID())) + if err != nil { + t.Fatal(err) + } + + ev := resp.Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, txn2, ev.Transaction) + }}, + {"Event unconfirmed", func(t *testing.T) { + resp, err := client.Event(types.Hash256(txn3.ID())) + if err != nil { + t.Fatal(err) + } + + ev := resp.Data.(explorer.EventV1Transaction) + ev.Transaction.SiacoinOutputs[0].Source = explorer.SourceTransaction + testutil.CheckTransaction(t, txn3, ev.Transaction) + }}, + {"Address event unconfirmed", func(t *testing.T) { + resp, err := client.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 1, len(resp)) + + ev := resp[0].Data.(explorer.EventV1Transaction) + ev.Transaction.SiacoinOutputs[0].Source = explorer.SourceTransaction + testutil.CheckTransaction(t, txn3, ev.Transaction) + }}, + {"Contract", func(t *testing.T) { + resp, err := client.Contract(txn1.FileContractID(0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckFC(t, true, false, false, revFC, resp) + }}, + {"Contracts", func(t *testing.T) { + resp, err := client.Contracts([]types.FileContractID{txn1.FileContractID(0)}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(contracts)", 1, len(resp)) + testutil.CheckFC(t, true, false, false, revFC, resp[0]) + }}, + {"ContractsKey", func(t *testing.T) { + resp, err := client.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(contracts)", 1, len(resp)) + testutil.CheckFC(t, true, false, false, revFC, resp[0]) + }}, + {"Search siacoin", func(t *testing.T) { + resp, err := client.Search(txn1.SiacoinOutputID(0).String()) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "search type", explorer.SearchTypeSiacoinElement, resp) + }}, + {"Search siafund", func(t *testing.T) { + resp, err := client.Search(txn2.SiafundOutputID(1).String()) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "search type", explorer.SearchTypeSiafundElement, resp) + }}, + {"Search contract", func(t *testing.T) { + resp, err := client.Search(txn1.FileContractID(0).String()) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "search type", explorer.SearchTypeContract, resp) + }}, + {"Search contract prefixed", func(t *testing.T) { + resp, err := client.Search("fcid:" + txn1.FileContractID(0).String()) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "search type", explorer.SearchTypeContract, resp) + }}, + {"Search invalid", func(t *testing.T) { + if _, err := client.Search(":"); err == nil || !strings.Contains(err.Error(), explorer.ErrSearchParse.Error()) { + t.Fatal("unparsable search should have failed") + } + if _, err := client.Search("nbcbsfdhjf"); err == nil || !strings.Contains(err.Error(), explorer.ErrSearchParse.Error()) { + t.Fatal("non hex search should have failed") + } + }}, + {"Search host", func(t *testing.T) { + resp, err := client.Search(pk1.PublicKey().String()) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "search type", explorer.SearchTypeHost, resp) + }}, + {"Exchange rate", func(t *testing.T) { + resp, err := client.ExchangeRate("USD") + if err != nil { + t.Fatal(err) + } + if resp <= 0 { + t.Fatal("exchange rate should be positive") + } + t.Logf("Exchange rate: %f", resp) + }}, + {"Search host by pubkey", func(t *testing.T) { + pubkey := pk1.PublicKey() + host, err := client.Host(pubkey) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "pubkey", pubkey, host.PublicKey) + testutil.Equal(t, "net address", netAddr1, host.NetAddress) + }}, + {"Search host by net address", func(t *testing.T) { + pubkey := pk1.PublicKey() + hosts, err := client.HostsList(explorer.HostQuery{ + NetAddresses: []string{netAddr1}, + }, explorer.HostSortPublicKey, explorer.HostSortAsc, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "len(hosts)", 1, len(hosts)) + + host := hosts[0] + testutil.Equal(t, "pubkey", pubkey, host.PublicKey) + testutil.Equal(t, "net address", netAddr1, host.NetAddress) + }}, + {"Manual scan", func(t *testing.T) { + pubkey := pk1.PublicKey() + scan, err := client.ScanHost(pubkey) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "pubkey", pubkey, scan.PublicKey) + testutil.Equal(t, "success", false, scan.Success) + if scan.Error == nil { + t.Fatal("got no error when scannning an invalid host") + } + }}, + {"Manual scan with invalid credential", func(t *testing.T) { + pubkey := pk1.PublicKey() + _, err := badAuthClient.ScanHost(pubkey) + if err == nil || !strings.Contains(err.Error(), api.ErrBadCredentials.Error()) { + t.Fatal("got wrong error when trying to use manual scan with bad auth", err) + } + }}, + {"Syncer connect with invalid credential", func(t *testing.T) { + err := badAuthClient.SyncerConnect("127.0.0.0.1:65535") + if err == nil || !strings.Contains(err.Error(), api.ErrBadCredentials.Error()) { + t.Fatal("got no error when trying to use syncer connect with bad auth", err) + } + }}, + } + + for _, subtest := range subtests { + t.Run(subtest.name, subtest.test) + } +} diff --git a/api/client.go b/api/client.go index 6f1d696..800aa4c 100644 --- a/api/client.go +++ b/api/client.go @@ -1,7 +1,10 @@ package api import ( + "context" "fmt" + "net/url" + "strconv" "go.sia.tech/core/consensus" "go.sia.tech/core/types" @@ -24,88 +27,278 @@ func NewClient(addr, password string) *Client { }} } +// State returns information about the current state of the explored daemon. +func (c *Client) State() (resp StateResponse, err error) { + err = c.c.GET(context.Background(), "/state", &resp) + return +} + // TxpoolBroadcast broadcasts a set of transaction to the network. func (c *Client) TxpoolBroadcast(txns []types.Transaction, v2txns []types.V2Transaction) (err error) { - err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) + err = c.c.POST(context.Background(), "/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) return } // TxpoolTransactions returns all transactions in the transaction pool. func (c *Client) TxpoolTransactions() (txns []types.Transaction, v2txns []types.V2Transaction, err error) { var resp TxpoolTransactionsResponse - err = c.c.GET("/txpool/transactions", &resp) + err = c.c.GET(context.Background(), "/txpool/transactions", &resp) return resp.Transactions, resp.V2Transactions, err } // TxpoolFee returns the recommended fee (per weight unit) to ensure a high // probability of inclusion in the next block. func (c *Client) TxpoolFee() (resp types.Currency, err error) { - err = c.c.GET("/txpool/fee", &resp) + err = c.c.GET(context.Background(), "/txpool/fee", &resp) return } // SyncerConnect adds the address as a peer of the syncer. func (c *Client) SyncerConnect(addr string) (err error) { - err = c.c.POST("/syncer/connect", addr, nil) + err = c.c.POST(context.Background(), "/syncer/connect", addr, nil) + return +} + +// SyncerPeers returns the peers of the syncer. +func (c *Client) SyncerPeers() (resp []string, err error) { + err = c.c.GET(context.Background(), "/syncer/peers", &resp) return } // SyncerBroadcastBlock broadcasts a block to all peers. func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { - err = c.c.POST("/syncer/broadcast/block", b, nil) + err = c.c.POST(context.Background(), "/syncer/broadcast/block", b, nil) + return +} + +// ConsensusTip returns the current tip of the chain manager. +func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { + err = c.c.GET(context.Background(), "/consensus/tip", &resp) return } // Tip returns the current tip of the explorer. func (c *Client) Tip() (resp types.ChainIndex, err error) { - err = c.c.GET("/explorer/tip", &resp) + err = c.c.GET(context.Background(), "/explorer/tip", &resp) + return +} + +// BestIndex returns the chain index at the specified height. +func (c *Client) BestIndex(height uint64) (resp types.ChainIndex, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/tip/%d", height), &resp) + return +} + +// ConsensusNetwork returns the network parameters of the consensus set. +func (c *Client) ConsensusNetwork() (n *consensus.Network, err error) { + err = c.c.GET(context.Background(), "/consensus/network", &n) return } -// TipByHeight returns the chain index at the specified height. -func (c *Client) TipByHeight(height uint64) (resp types.ChainIndex, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/tip/%d", height), &resp) +// ConsensusState returns the current state of the consensus set. +func (c *Client) ConsensusState() (state consensus.State, err error) { + if c.n == nil { + c.n, err = c.ConsensusNetwork() + if err != nil { + return + } + } + err = c.c.GET(context.Background(), "/consensus/state", &state) + state.Network = c.n return } // Block returns the block with the specified ID. func (c *Client) Block(id types.BlockID) (resp explorer.Block, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/block/%s", id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/blocks/%s", id), &resp) return } // Transaction returns the transaction with the specified ID. func (c *Client) Transaction(id types.TransactionID) (resp explorer.Transaction, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/transactions/%s", id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/transactions/%s", id), &resp) return } // Transactions returns the transactions with the specified IDs. func (c *Client) Transactions(ids []types.TransactionID) (resp []explorer.Transaction, err error) { - err = c.c.POST("/explorer/transactions", ids, &resp) + err = c.c.POST(context.Background(), "/transactions", ids, &resp) + return +} + +// TransactionChainIndices returns chain indices a transaction was +// included in. +func (c *Client) TransactionChainIndices(id types.TransactionID, offset, limit uint64) (resp []types.ChainIndex, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/transactions/%s/indices?offset=%d&limit=%d", id, offset, limit), &resp) + return +} + +// V2Transaction returns the v2 transaction with the specified ID. +func (c *Client) V2Transaction(id types.TransactionID) (resp explorer.V2Transaction, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/v2/transactions/%s", id), &resp) + return +} + +// V2Transactions returns the v2 transactions with the specified IDs. +func (c *Client) V2Transactions(ids []types.TransactionID) (resp []explorer.V2Transaction, err error) { + err = c.c.POST(context.Background(), "/v2/transactions", ids, &resp) + return +} + +// V2TransactionChainIndices returns chain indices a v2 transaction was +// included in. +func (c *Client) V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) (resp []types.ChainIndex, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/v2/transactions/%s/indices?offset=%d&limit=%d", id, offset, limit), &resp) + return +} + +// AddressSiacoinUTXOs returns the specified address' unspent outputs. +func (c *Client) AddressSiacoinUTXOs(address types.Address, offset, limit uint64) (resp []explorer.SiacoinOutput, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%s/utxos/siacoin?offset=%d&limit=%d", address, offset, limit), &resp) + return +} + +// AddressSiafundUTXOs returns the specified address' unspent outputs. +func (c *Client) AddressSiafundUTXOs(address types.Address, offset, limit uint64) (resp []explorer.SiafundOutput, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%s/utxos/siafund?offset=%d&limit=%d", address, offset, limit), &resp) return } -// AddressUTXOs returns the specified address' unspent outputs. -func (c *Client) AddressUTXOs(address types.Address, limit, offset uint64) (resp AddressUTXOsResponse, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/addresses/%s/utxos?limit=%d&offset=%d", address, limit, offset), &resp) +// OutputSiacoin returns the specified siacoin output. +func (c *Client) OutputSiacoin(id types.SiacoinOutputID) (resp explorer.SiacoinOutput, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/outputs/siacoin/%s", id), &resp) + return +} + +// OutputSiafund returns the specified siafund output. +func (c *Client) OutputSiafund(id types.SiafundOutputID) (resp explorer.SiafundOutput, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/outputs/siafund/%s", id), &resp) + return +} + +// AddressEvents returns the specified address' events. +func (c *Client) AddressEvents(address types.Address, offset, limit uint64) (resp []explorer.Event, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%s/events?offset=%d&limit=%d", address, offset, limit), &resp) + return +} + +// AddressUnconfirmedEvents returns the specified address' unconfirmed events. +func (c *Client) AddressUnconfirmedEvents(address types.Address) (resp []explorer.Event, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%s/events/unconfirmed", address), &resp) + return +} + +// Event returns the specified event. +func (c *Client) Event(id types.Hash256) (resp explorer.Event, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/events/%s", id), &resp) return } // AddressBalance returns the specified address' balance. func (c *Client) AddressBalance(address types.Address) (resp AddressBalanceResponse, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/addresses/%s/balance", address), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%s/balance", address), &resp) return } // Contract returns the file contract with the specified ID. -func (c *Client) Contract(id types.FileContractID) (resp explorer.FileContract, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/contracts/%s", id), &resp) +func (c *Client) Contract(id types.FileContractID) (resp explorer.ExtendedFileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/contracts/%s", id), &resp) + return +} + +// Contracts returns the contracts with the specified IDs. +func (c *Client) Contracts(ids []types.FileContractID) (resp []explorer.ExtendedFileContract, err error) { + err = c.c.POST(context.Background(), "/contracts", ids, &resp) + return +} + +// ContractsKey returns the contracts for a particular ed25519 key. +func (c *Client) ContractsKey(key types.PublicKey) (resp []explorer.ExtendedFileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/pubkey/%s/contracts", key), &resp) + return +} + +// ContractRevisions returns all the revisions of the contract with the +// specified ID. +func (c *Client) ContractRevisions(id types.FileContractID) (resp []explorer.ExtendedFileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/contracts/%s/revisions", id), &resp) + return +} + +// V2Contract returns the v2 file contract with the specified ID. +func (c *Client) V2Contract(id types.FileContractID) (resp explorer.V2FileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/v2/contracts/%s", id), &resp) + return +} + +// V2Contracts returns the v2 contracts with the specified IDs. +func (c *Client) V2Contracts(ids []types.FileContractID) (resp []explorer.V2FileContract, err error) { + err = c.c.POST(context.Background(), "/v2/contracts", ids, &resp) + return +} + +// V2ContractRevisions returns all the revisions of the contract with the +// specified ID. +func (c *Client) V2ContractRevisions(id types.FileContractID) (resp []explorer.V2FileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/v2/contracts/%s/revisions", id), &resp) + return +} + +// V2ContractsKey returns the v2 contracts for a particular ed25519 key. +func (c *Client) V2ContractsKey(key types.PublicKey) (resp []explorer.V2FileContract, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/v2/pubkey/%s/contracts", key), &resp) + return +} + +// Host returns information about the host with a given ed25519 key. +func (c *Client) Host(key types.PublicKey) (resp explorer.Host, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/hosts/%s", key), &resp) + return +} + +// ScanHost triggers a manual host scan. +func (c *Client) ScanHost(key types.PublicKey) (resp explorer.HostScan, err error) { + err = c.c.POST(context.Background(), fmt.Sprintf("/hosts/%s/scan", key), nil, &resp) + return +} + +// BlockMetrics returns the most recent metrics about the Sia blockchain. +func (c *Client) BlockMetrics() (resp explorer.Metrics, err error) { + err = c.c.GET(context.Background(), "/metrics/block", &resp) + return +} + +// BlockMetricsID returns various metrics about Sia at the given block ID. +func (c *Client) BlockMetricsID(id types.BlockID) (resp explorer.Metrics, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/metrics/block/%s", id), &resp) + return +} + +// HostMetrics returns various metrics about currently available hosts. +func (c *Client) HostMetrics() (resp explorer.HostMetrics, err error) { + err = c.c.GET(context.Background(), "/metrics/host", &resp) + return +} + +// Search returns what type of object an ID is. +func (c *Client) Search(id string) (resp explorer.SearchType, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/search/%s", id), &resp) + return +} + +// HostsList searches the hosts by the given criteria. +func (c *Client) HostsList(params explorer.HostQuery, sortBy explorer.HostSortColumn, dir explorer.HostSortDir, offset, limit uint64) (resp []explorer.Host, err error) { + v := url.Values{} + v.Add("sort", string(sortBy)) + v.Add("dir", string(dir)) + v.Add("offset", strconv.FormatUint(offset, 10)) + v.Add("limit", strconv.FormatUint(limit, 10)) + err = c.c.POST(context.Background(), "/hosts?"+v.Encode(), params, &resp) return } -// Contracts returns the transactions with the specified IDs. -func (c *Client) Contracts(ids []types.FileContractID) (resp []explorer.FileContract, err error) { - err = c.c.POST("/explorer/contracts", ids, &resp) +// ExchangeRate returns the value of 1 SC in the specified currency. +func (c *Client) ExchangeRate(currency string) (resp float64, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/exchange-rate/siacoin/%s", currency), &resp) return } diff --git a/api/server.go b/api/server.go index bc78feb..cc56891 100644 --- a/api/server.go +++ b/api/server.go @@ -5,6 +5,9 @@ import ( "errors" "fmt" "net/http" + "runtime" + "strings" + "time" "go.sia.tech/jape" @@ -12,12 +15,15 @@ import ( "go.sia.tech/core/gateway" "go.sia.tech/core/types" "go.sia.tech/coreutils/syncer" + "go.sia.tech/explored/build" + "go.sia.tech/explored/exchangerates" "go.sia.tech/explored/explorer" ) type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { + Tip() types.ChainIndex TipState() consensus.State AddBlocks([]types.Block) error RecommendedFee() types.Currency @@ -33,10 +39,10 @@ type ( Addr() string Peers() []*syncer.Peer Connect(ctx context.Context, addr string) (*syncer.Peer, error) - BroadcastHeader(bh gateway.BlockHeader) - BroadcastTransactionSet(txns []types.Transaction) - BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) - BroadcastV2BlockOutline(bo gateway.V2BlockOutline) + BroadcastHeader(bh types.BlockHeader) error + BroadcastTransactionSet(txns []types.Transaction) error + BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) error + BroadcastV2BlockOutline(bo gateway.V2BlockOutline) error } // Explorer implements a Sia explorer. @@ -44,11 +50,32 @@ type ( Tip() (types.ChainIndex, error) Block(id types.BlockID) (explorer.Block, error) BestTip(height uint64) (types.ChainIndex, error) + Metrics(id types.BlockID) (explorer.Metrics, error) + HostMetrics() (explorer.HostMetrics, error) Transactions(ids []types.TransactionID) ([]explorer.Transaction, error) + TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) + V2Transactions(ids []types.TransactionID) ([]explorer.V2Transaction, error) + V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) - UnspentSiacoinOutputs(address types.Address, limit, offset uint64) ([]explorer.SiacoinOutput, error) - UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]explorer.SiafundOutput, error) - Contracts(ids []types.FileContractID) (result []explorer.FileContract, err error) + SiacoinElements(ids []types.SiacoinOutputID) (result []explorer.SiacoinOutput, err error) + SiafundElements(ids []types.SiafundOutputID) (result []explorer.SiafundOutput, err error) + UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]explorer.SiacoinOutput, error) + UnspentSiafundOutputs(address types.Address, offset, limit uint64) ([]explorer.SiafundOutput, error) + AddressEvents(address types.Address, offset, limit uint64) (events []explorer.Event, err error) + AddressUnconfirmedEvents(address types.Address) ([]explorer.Event, error) + Events(ids []types.Hash256) ([]explorer.Event, error) + UnconfirmedEvents(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) ([]explorer.Event, error) + Contracts(ids []types.FileContractID) (result []explorer.ExtendedFileContract, err error) + ContractsKey(key types.PublicKey) (result []explorer.ExtendedFileContract, err error) + ContractRevisions(id types.FileContractID) (result []explorer.ExtendedFileContract, err error) + V2Contracts(ids []types.FileContractID) (result []explorer.V2FileContract, err error) + V2ContractsKey(key types.PublicKey) (result []explorer.V2FileContract, err error) + V2ContractRevisions(id types.FileContractID) (result []explorer.V2FileContract, err error) + Search(id string) (explorer.SearchType, error) + + ScanHosts(pks ...types.PublicKey) ([]explorer.HostScan, error) + Hosts(pks []types.PublicKey) ([]explorer.Host, error) + QueryHosts(params explorer.HostQuery, sortBy explorer.HostSortColumn, dir explorer.HostSortDir, offset, limit uint64) ([]explorer.Host, error) } ) @@ -56,17 +83,73 @@ const ( maxIDs = 5000 ) +const ( + defaultLimit uint64 = 100 + maxLimit uint64 = 500 +) + var ( - errTooManyIDs = fmt.Errorf("too many IDs provided (provide less than %d)", maxIDs) + // ErrBadCredentials is returned when the supplied credentials for a protected + // endpoint are invalid. + ErrBadCredentials = errors.New("bad credentials") + + // ErrTransactionNotFound is returned by /transactions/:id when we are + // unable to find the transaction with that `id`. + ErrTransactionNotFound = errors.New("no transaction found") + // ErrSiacoinOutputNotFound is returned by /outputs/siacoin/:id when we + // are unable to find the siacoin output with that `id`. + ErrSiacoinOutputNotFound = errors.New("no siacoin output found") + // ErrSiafundOutputNotFound is returned by /outputs/siafund/:id when we + // are unable to find the siafund output with that `id`. + ErrSiafundOutputNotFound = errors.New("no siafund output found") + // ErrHostNotFound is returned by /pubkey/:key/host when we are unable to + // find the host with the pubkey `key`. + ErrHostNotFound = errors.New("no host found") + // ErrEventNotFound is returned by /events/:id when we can't find the event + // with the id `id`. + ErrEventNotFound = errors.New("no event found") + + // ErrTooManyIDs is returned by the batch transaction and contract + // endpoints when more than maxIDs IDs are specified. + ErrTooManyIDs = fmt.Errorf("too many IDs provided (provide less than %d)", maxIDs) ) type server struct { - cm ChainManager - e Explorer - s Syncer + cm ChainManager + e Explorer + s Syncer + ex exchangerates.Source + apiPassword string + + startTime time.Time +} + +func (s *server) checkAuth(jc jape.Context) bool { + // We could use jape.BasicAuth when defining the route in the map, but it + // makes the jape linter think that the route is undefined, so we have some + // auth code here. + if _, p, ok := jc.Request.BasicAuth(); !ok || s.apiPassword == "" || p != s.apiPassword { + jc.Error(ErrBadCredentials, http.StatusUnauthorized) + return false + } + return true +} + +func (s *server) stateHandler(jc jape.Context) { + jc.Encode(StateResponse{ + Version: build.Version(), + Commit: build.Commit(), + OS: runtime.GOOS, + BuildTime: build.Time(), + StartTime: s.startTime, + }) } func (s *server) syncerConnectHandler(jc jape.Context) { + if !s.checkAuth(jc) { + return + } + var addr string if jc.Decode(&addr) != nil { return @@ -75,6 +158,16 @@ func (s *server) syncerConnectHandler(jc jape.Context) { jc.Check("couldn't connect to peer", err) } +func (s *server) syncerPeersHandler(jc jape.Context) { + peers := s.s.Peers() + + var result []string + for _, peer := range peers { + result = append(result, peer.ConnAddr) + } + jc.Encode(result) +} + func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { var b types.Block if jc.Decode(&b) != nil { @@ -83,14 +176,13 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { return } if b.V2 == nil { - s.s.BroadcastHeader(gateway.BlockHeader{ - ParentID: b.ParentID, - Nonce: b.Nonce, - Timestamp: b.Timestamp, - MerkleRoot: b.MerkleRoot(), - }) + if jc.Check("failed to broadcast header", s.s.BroadcastHeader(b.Header())) != nil { + return + } } else { - s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { + return + } } } @@ -120,17 +212,50 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Check("invalid transaction set", err) != nil { return } - s.s.BroadcastTransactionSet(tbr.Transactions) + if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { + return + } } if len(tbr.V2Transactions) != 0 { _, err := s.cm.AddV2PoolTransactions(tip, tbr.V2Transactions) if jc.Check("invalid v2 transaction set", err) != nil { return } - s.s.BroadcastV2TransactionSet(tip, tbr.V2Transactions) + if jc.Check("failed to broadcast v2 transaction set", s.s.BroadcastV2TransactionSet(tip, tbr.V2Transactions)) != nil { + return + } } } +func (s *server) consensusTipHandler(jc jape.Context) { + jc.Encode(s.cm.Tip()) +} + +func (s *server) consensusTipHeightHandler(jc jape.Context) { + var height uint64 + if jc.DecodeParam("height", &height) != nil { + return + } + + tip, err := s.e.BestTip(height) + if errors.Is(err, explorer.ErrNoTip) { + jc.Error(explorer.ErrNoTip, http.StatusNotFound) + return + } else if jc.Check("failed to get tip", err) != nil { + return + } + + jc.Encode(tip) +} + +func (s *server) consensusNetworkHandler(jc jape.Context) { + jc.Encode(s.cm.TipState().Network) +} + +func (s *server) consensusStateHandler(jc jape.Context) { + jc.Encode(s.cm.TipState()) +} + func (s *server) explorerTipHandler(jc jape.Context) { tip, err := s.e.Tip() if jc.Check("failed to get tip", err) != nil { @@ -139,33 +264,55 @@ func (s *server) explorerTipHandler(jc jape.Context) { jc.Encode(tip) } -func (s *server) explorerTipHeightHandler(jc jape.Context) { - var height uint64 - if jc.DecodeParam("height", &height) != nil { +func (s *server) blocksMetricsHandler(jc jape.Context) { + tip, err := s.e.Tip() + if jc.Check("failed to get tip", err) != nil { return } - tip, err := s.e.BestTip(height) - if jc.Check("failed to get block", err) != nil { + + metrics, err := s.e.Metrics(tip.ID) + if jc.Check("failed to get metrics", err) != nil { return } - jc.Encode(tip) + jc.Encode(metrics) +} + +func (s *server) blocksMetricsIDHandler(jc jape.Context) { + var id types.BlockID + if jc.DecodeParam("id", &id) != nil { + return + } + metrics, err := s.e.Metrics(id) + if jc.Check("failed to get metrics", err) != nil { + return + } + jc.Encode(metrics) +} + +func (s *server) hostMetricsHandler(jc jape.Context) { + metrics, err := s.e.HostMetrics() + if jc.Check("failed to get host metrics", err) != nil { + return + } + jc.Encode(metrics) } -func (s *server) explorerBlockHandler(jc jape.Context) { +func (s *server) blocksIDHandler(jc jape.Context) { var id types.BlockID if jc.DecodeParam("id", &id) != nil { return } block, err := s.e.Block(id) - if jc.Check("failed to get block", err) != nil { + if errors.Is(err, explorer.ErrNoBlock) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to get block", err) != nil { return } jc.Encode(block) } -func (s *server) explorerTransactionsIDHandler(jc jape.Context) { - errNotFound := errors.New("no transaction found") - +func (s *server) transactionsIDHandler(jc jape.Context) { var id types.TransactionID if jc.DecodeParam("id", &id) != nil { return @@ -174,18 +321,40 @@ func (s *server) explorerTransactionsIDHandler(jc jape.Context) { if jc.Check("failed to get transaction", err) != nil { return } else if len(txns) == 0 { - jc.Error(errNotFound, http.StatusNotFound) + jc.Error(ErrTransactionNotFound, http.StatusNotFound) return } jc.Encode(txns[0]) } -func (s *server) explorerTransactionsHandler(jc jape.Context) { +func (s *server) transactionsIDIndicesHandler(jc jape.Context) { + var id types.TransactionID + if jc.DecodeParam("id", &id) != nil { + return + } + + limit := defaultLimit + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + if limit > maxLimit { + limit = maxLimit + } + + indices, err := s.e.TransactionChainIndices(id, offset, limit) + if jc.Check("failed to get transaction indices", err) != nil { + return + } + jc.Encode(indices) +} + +func (s *server) transactionsBatchHandler(jc jape.Context) { var ids []types.TransactionID if jc.Decode(&ids) != nil { return } else if len(ids) > maxIDs { - jc.Error(errTooManyIDs, http.StatusBadRequest) + jc.Error(ErrTooManyIDs, http.StatusBadRequest) return } @@ -196,34 +365,104 @@ func (s *server) explorerTransactionsHandler(jc jape.Context) { jc.Encode(txns) } -func (s *server) explorerAddressessAddressUtxosHandler(jc jape.Context) { +func (s *server) v2TransactionsIDHandler(jc jape.Context) { + var id types.TransactionID + if jc.DecodeParam("id", &id) != nil { + return + } + txns, err := s.e.V2Transactions([]types.TransactionID{id}) + if jc.Check("failed to get transaction", err) != nil { + return + } else if len(txns) == 0 { + jc.Error(ErrTransactionNotFound, http.StatusNotFound) + return + } + jc.Encode(txns[0]) +} + +func (s *server) v2TransactionsIDIndicesHandler(jc jape.Context) { + var id types.TransactionID + if jc.DecodeParam("id", &id) != nil { + return + } + + limit := defaultLimit + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + if limit > maxLimit { + limit = maxLimit + } + + indices, err := s.e.V2TransactionChainIndices(id, offset, limit) + if jc.Check("failed to get transaction indices", err) != nil { + return + } + jc.Encode(indices) +} + +func (s *server) v2TransactionsBatchHandler(jc jape.Context) { + var ids []types.TransactionID + if jc.Decode(&ids) != nil { + return + } else if len(ids) > maxIDs { + jc.Error(ErrTooManyIDs, http.StatusBadRequest) + return + } + + txns, err := s.e.V2Transactions(ids) + if jc.Check("failed to get transactions", err) != nil { + return + } + jc.Encode(txns) +} + +func (s *server) addressessAddressUtxosSiacoinHandler(jc jape.Context) { var address types.Address if jc.DecodeParam("address", &address) != nil { return } - limit := uint64(100) + limit := defaultLimit offset := uint64(0) if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { return } + if limit > maxLimit { + limit = maxLimit + } - unspentSiacoinOutputs, err := s.e.UnspentSiacoinOutputs(address, limit, offset) + outputs, err := s.e.UnspentSiacoinOutputs(address, offset, limit) if jc.Check("failed to get unspent siacoin outputs", err) != nil { return } - unspentSiafundOutputs, err := s.e.UnspentSiafundOutputs(address, limit, offset) - if jc.Check("failed to get unspent siafund outputs", err) != nil { + jc.Encode(outputs) +} + +func (s *server) addressessAddressUtxosSiafundHandler(jc jape.Context) { + var address types.Address + if jc.DecodeParam("address", &address) != nil { return } - jc.Encode(AddressUTXOsResponse{ - UnspentSiacoinOutputs: unspentSiacoinOutputs, - UnspentSiafundOutputs: unspentSiafundOutputs, - }) + limit := defaultLimit + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + if limit > maxLimit { + limit = maxLimit + } + + outputs, err := s.e.UnspentSiafundOutputs(address, offset, limit) + if jc.Check("failed to get unspent siacoin outputs", err) != nil { + return + } + jc.Encode(outputs) } -func (s *server) explorerAddressessAddressBalanceHandler(jc jape.Context) { +func (s *server) addressessAddressBalanceHandler(jc jape.Context) { var address types.Address if jc.DecodeParam("address", &address) != nil { return @@ -241,9 +480,107 @@ func (s *server) explorerAddressessAddressBalanceHandler(jc jape.Context) { }) } -func (s *server) explorerContractIDHandler(jc jape.Context) { - errNotFound := errors.New("no contract found") +func (s *server) addressessAddressEventsHandler(jc jape.Context) { + var address types.Address + if jc.DecodeParam("address", &address) != nil { + return + } + + limit := defaultLimit + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + if limit > maxLimit { + limit = maxLimit + } + + events, err := s.e.AddressEvents(address, offset, limit) + if jc.Check("failed to get address events", err) != nil { + return + } + + jc.Encode(events) +} + +func (s *server) addressessAddressEventsUnconfirmedHandler(jc jape.Context) { + var address types.Address + if jc.DecodeParam("address", &address) != nil { + return + } + + events, err := s.e.AddressUnconfirmedEvents(address) + if jc.Check("failed to get unconfirmed address events", err) != nil { + return + } + + jc.Encode(events) +} + +func (s *server) eventsIDHandler(jc jape.Context) { + var id types.Hash256 + if jc.DecodeParam("id", &id) != nil { + return + } + + events, err := s.e.Events([]types.Hash256{id}) + if err != nil { + return + } else if len(events) > 0 { + jc.Encode(events[0]) + return + } + v1, v2 := s.cm.PoolTransactions(), s.cm.V2PoolTransactions() + events, err = s.e.UnconfirmedEvents(types.ChainIndex{}, types.CurrentTimestamp(), v1, v2) + if jc.Check("failed to annotate events", err) != nil { + return + } + for _, event := range events { + if event.ID == id { + jc.Encode(event) + return + } + } + + jc.Error(ErrEventNotFound, http.StatusNotFound) +} + +func (s *server) outputsSiacoinHandler(jc jape.Context) { + var id types.SiacoinOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + outputs, err := s.e.SiacoinElements([]types.SiacoinOutputID{id}) + if jc.Check("failed to get siacoin elements", err) != nil { + return + } else if len(outputs) == 0 { + jc.Error(ErrSiacoinOutputNotFound, http.StatusNotFound) + return + } + + jc.Encode(outputs[0]) +} + +func (s *server) outputsSiafundHandler(jc jape.Context) { + var id types.SiafundOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + outputs, err := s.e.SiafundElements([]types.SiafundOutputID{id}) + if jc.Check("failed to get siafund elements", err) != nil { + return + } else if len(outputs) == 0 { + jc.Error(ErrSiafundOutputNotFound, http.StatusNotFound) + return + } + + jc.Encode(outputs[0]) +} + +func (s *server) contractsIDHandler(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -252,18 +589,34 @@ func (s *server) explorerContractIDHandler(jc jape.Context) { if jc.Check("failed to get contract", err) != nil { return } else if len(fcs) == 0 { - jc.Error(errNotFound, http.StatusNotFound) + jc.Error(explorer.ErrContractNotFound, http.StatusNotFound) return } jc.Encode(fcs[0]) } -func (s *server) explorerContractsHandler(jc jape.Context) { +func (s *server) contractsIDRevisionsHandler(jc jape.Context) { + var id types.FileContractID + if jc.DecodeParam("id", &id) != nil { + return + } + + fcs, err := s.e.ContractRevisions(id) + if errors.Is(err, explorer.ErrContractNotFound) { + jc.Error(fmt.Errorf("%w: %v", err, id), http.StatusNotFound) + return + } else if jc.Check("failed to fetch contract revisions", err) != nil { + return + } + jc.Encode(fcs) +} + +func (s *server) contractsBatchHandler(jc jape.Context) { var ids []types.FileContractID if jc.Decode(&ids) != nil { return } else if len(ids) > maxIDs { - jc.Error(errTooManyIDs, http.StatusBadRequest) + jc.Error(ErrTooManyIDs, http.StatusBadRequest) return } @@ -274,14 +627,201 @@ func (s *server) explorerContractsHandler(jc jape.Context) { jc.Encode(fcs) } +func (s *server) v2ContractsIDHandler(jc jape.Context) { + var id types.FileContractID + if jc.DecodeParam("id", &id) != nil { + return + } + fcs, err := s.e.V2Contracts([]types.FileContractID{id}) + if jc.Check("failed to get contract", err) != nil { + return + } else if len(fcs) == 0 { + jc.Error(explorer.ErrContractNotFound, http.StatusNotFound) + return + } + jc.Encode(fcs[0]) +} + +func (s *server) v2ContractsBatchHandler(jc jape.Context) { + var ids []types.FileContractID + if jc.Decode(&ids) != nil { + return + } else if len(ids) > maxIDs { + jc.Error(ErrTooManyIDs, http.StatusBadRequest) + return + } + + fcs, err := s.e.V2Contracts(ids) + if jc.Check("failed to get contracts", err) != nil { + return + } + jc.Encode(fcs) +} + +func (s *server) v2ContractsIDRevisionsHandler(jc jape.Context) { + var id types.FileContractID + if jc.DecodeParam("id", &id) != nil { + return + } + + fcs, err := s.e.V2ContractRevisions(id) + if errors.Is(err, explorer.ErrContractNotFound) { + jc.Error(fmt.Errorf("%w: %v", err, id), http.StatusNotFound) + return + } else if jc.Check("failed to fetch contract revisions", err) != nil { + return + } + jc.Encode(fcs) +} + +func (s *server) v2PubkeyContractsHandler(jc jape.Context) { + var key types.PublicKey + if jc.DecodeParam("key", &key) != nil { + return + } + fcs, err := s.e.V2ContractsKey(key) + if jc.Check("failed to get contracts", err) != nil { + return + } else if len(fcs) == 0 { + jc.Error(explorer.ErrContractNotFound, http.StatusNotFound) + return + } + jc.Encode(fcs) +} + +func (s *server) pubkeyContractsHandler(jc jape.Context) { + var key types.PublicKey + if jc.DecodeParam("key", &key) != nil { + return + } + fcs, err := s.e.ContractsKey(key) + if jc.Check("failed to get contracts", err) != nil { + return + } else if len(fcs) == 0 { + jc.Error(explorer.ErrContractNotFound, http.StatusNotFound) + return + } + jc.Encode(fcs) +} + +func (s *server) pubkeyHostHandler(jc jape.Context) { + var key types.PublicKey + if jc.DecodeParam("key", &key) != nil { + return + } + hosts, err := s.e.Hosts([]types.PublicKey{key}) + if jc.Check("failed to get host", err) != nil { + return + } else if len(hosts) == 0 { + jc.Error(ErrHostNotFound, http.StatusNotFound) + return + } + jc.Encode(hosts[0]) +} + +func (s *server) pubkeyHostScanHandler(jc jape.Context) { + if !s.checkAuth(jc) { + return + } + + var key types.PublicKey + if jc.DecodeParam("key", &key) != nil { + return + } + + scans, err := s.e.ScanHosts(key) + if jc.Check("non host error when attempting to scan hosts", err) != nil { + return + } else if len(scans) == 0 { + jc.Error(ErrHostNotFound, http.StatusNotFound) + return + } + + jc.Encode(scans[0]) +} + +func (s *server) hostsHandler(jc jape.Context) { + var params explorer.HostQuery + if jc.Decode(¶ms) != nil { + return + } + + limit := defaultLimit + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + if limit > maxLimit { + limit = maxLimit + } + + dir := explorer.HostSortAsc + sortBy := explorer.HostSortDateCreated + if jc.DecodeForm("dir", &dir) != nil || jc.DecodeForm("sort", &sortBy) != nil { + return + } + + hosts, err := s.e.QueryHosts(params, sortBy, dir, offset, limit) + if errors.Is(err, explorer.ErrNoSortColumn) { + jc.Error(err, http.StatusBadRequest) + return + } else if jc.Check("failed to query hosts", err) != nil { + return + } + jc.Encode(hosts) +} + +func (s *server) searchIDHandler(jc jape.Context) { + var id string + if jc.DecodeParam("id", &id) != nil { + return + } + + result, err := s.e.Search(id) + if errors.Is(err, explorer.ErrNoSearchResults) { + jc.Error(err, http.StatusNotFound) + return + } else if errors.Is(err, explorer.ErrSearchParse) { + jc.Error(err, http.StatusBadRequest) + return + } else if jc.Check("failed to search ID", err) != nil { + return + } + jc.Encode(result) +} + +func (s *server) exchangeRateHandler(jc jape.Context) { + var currency string + if jc.DecodeParam("currency", ¤cy) != nil { + return + } + if currency == "" { + jc.Error(errors.New("provide a currency value such as USD or EUR"), http.StatusNotFound) + return + } + + currency = strings.ToUpper(currency) + price, err := s.ex.Last(currency) + if jc.Check("failed to get exchange rate", err) != nil { + return + } + jc.Encode(price) +} + // NewServer returns an HTTP handler that serves the explored API. -func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { +func NewServer(e Explorer, cm ChainManager, s Syncer, ex exchangerates.Source, apiPassword string) http.Handler { srv := server{ - cm: cm, - e: e, - s: s, + cm: cm, + e: e, + s: s, + ex: ex, + apiPassword: apiPassword, + startTime: time.Now().UTC(), } + return jape.Mux(map[string]jape.Handler{ + "GET /state": srv.stateHandler, + "GET /syncer/peers": srv.syncerPeersHandler, "POST /syncer/connect": srv.syncerConnectHandler, "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, @@ -289,14 +829,57 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - "GET /explorer/tip": srv.explorerTipHandler, - "GET /explorer/tip/:height": srv.explorerTipHeightHandler, - "GET /explorer/block/:id": srv.explorerBlockHandler, - "GET /explorer/transactions/:id": srv.explorerTransactionsIDHandler, - "POST /explorer/transactions": srv.explorerTransactionsHandler, - "GET /explorer/addresses/:address/utxos": srv.explorerAddressessAddressUtxosHandler, - "GET /explorer/addresses/:address/balance": srv.explorerAddressessAddressBalanceHandler, - "GET /explorer/contracts/:id": srv.explorerContractIDHandler, - "POST /explorer/contracts": srv.explorerContractsHandler, + "GET /consensus/network": srv.consensusNetworkHandler, + "GET /consensus/state": srv.consensusStateHandler, + "GET /consensus/tip": srv.consensusTipHandler, + "GET /consensus/tip/:height": srv.consensusTipHeightHandler, + + "GET /explorer/tip": srv.explorerTipHandler, + + "GET /blocks/:id": srv.blocksIDHandler, + + "GET /transactions/:id": srv.transactionsIDHandler, + "POST /transactions": srv.transactionsBatchHandler, + "GET /transactions/:id/indices": srv.transactionsIDIndicesHandler, + + "GET /v2/transactions/:id": srv.v2TransactionsIDHandler, + "POST /v2/transactions": srv.v2TransactionsBatchHandler, + "GET /v2/transactions/:id/indices": srv.v2TransactionsIDIndicesHandler, + + "GET /addresses/:address/utxos/siacoin": srv.addressessAddressUtxosSiacoinHandler, + "GET /addresses/:address/utxos/siafund": srv.addressessAddressUtxosSiafundHandler, + "GET /addresses/:address/events": srv.addressessAddressEventsHandler, + "GET /addresses/:address/events/unconfirmed": srv.addressessAddressEventsUnconfirmedHandler, + "GET /addresses/:address/balance": srv.addressessAddressBalanceHandler, + + "GET /events/:id": srv.eventsIDHandler, + + "GET /outputs/siacoin/:id": srv.outputsSiacoinHandler, + "GET /outputs/siafund/:id": srv.outputsSiafundHandler, + + "GET /contracts/:id": srv.contractsIDHandler, + "GET /contracts/:id/revisions": srv.contractsIDRevisionsHandler, + "POST /contracts": srv.contractsBatchHandler, + + "GET /v2/contracts/:id": srv.v2ContractsIDHandler, + "GET /v2/contracts/:id/revisions": srv.v2ContractsIDRevisionsHandler, + "POST /v2/contracts": srv.v2ContractsBatchHandler, + + "GET /v2/pubkey/:key/contracts": srv.v2PubkeyContractsHandler, + + "GET /pubkey/:key/contracts": srv.pubkeyContractsHandler, + + "GET /hosts/:key": srv.pubkeyHostHandler, + "POST /hosts/:key/scan": srv.pubkeyHostScanHandler, + + "GET /metrics/block": srv.blocksMetricsHandler, + "GET /metrics/block/:id": srv.blocksMetricsIDHandler, + "GET /metrics/host": srv.hostMetricsHandler, + + "POST /hosts": srv.hostsHandler, + + "GET /search/:id": srv.searchIDHandler, + + "GET /exchange-rate/siacoin/:currency": srv.exchangeRateHandler, }) } diff --git a/build/build.go b/build/build.go new file mode 100644 index 0000000..4cf0b34 --- /dev/null +++ b/build/build.go @@ -0,0 +1,21 @@ +// Package build contains build-time information. +package build + +//go:generate go run gen.go + +import "time" + +// Commit returns the commit hash of explored +func Commit() string { + return commit +} + +// Version returns the version of explored +func Version() string { + return version +} + +// Time returns the time at which the binary was built. +func Time() time.Time { + return time.Unix(buildTime, 0) +} diff --git a/build/gen.go b/build/gen.go new file mode 100644 index 0000000..28c6f56 --- /dev/null +++ b/build/gen.go @@ -0,0 +1,114 @@ +//go:build ignore + +// This script generates meta.go which contains version info for the explored binary. It can be run with `go generate`. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "strings" + "text/template" + "time" +) + +const logFormat = `{%n "commit": "%H",%n "shortCommit": "%h",%n "timestamp": "%cD",%n "tag": "%(describe:tags=true)"%n}` + +type ( + gitTime time.Time + + gitMeta struct { + Commit string `json:"commit"` + ShortCommit string `json:"shortCommit"` + Timestamp gitTime `json:"timestamp"` + Tag string `json:"tag"` + } +) + +var buildTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. +// This file was generated by go generate at {{ .RunTime }}. +package build + +const ( + commit = "{{ .Commit }}" + version = "{{ .Version }}" + buildTime = {{ .UnixTimestamp }} +) +`)) + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (t *gitTime) UnmarshalJSON(buf []byte) error { + timeFormats := []string{ + time.RFC1123Z, + "Mon, 2 Jan 2006 15:04:05 -0700", + "2006-01-02 15:04:05 -0700", + time.UnixDate, + time.ANSIC, + time.RFC3339, + time.RFC1123, + } + + for _, format := range timeFormats { + parsed, err := time.Parse(format, strings.Trim(string(buf), `"`)) + if err == nil { + *t = gitTime(parsed) + return nil + } + } + return errors.New("failed to parse time") +} + +func getGitMeta() (meta gitMeta, _ error) { + cmd := exec.Command("git", "log", "-1", "--pretty=format:"+logFormat+"") + buf, err := cmd.Output() + if err != nil { + if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 { + return gitMeta{}, fmt.Errorf("command failed: %w", errors.New(string(err.Stderr))) + } + return gitMeta{}, fmt.Errorf("failed to execute command: %w", err) + } else if err := json.Unmarshal(buf, &meta); err != nil { + return gitMeta{}, fmt.Errorf("failed to unmarshal json: %w", err) + } + return +} + +func main() { + meta, err := getGitMeta() + if err != nil { + log.Fatalln(err) + } + + commit := meta.ShortCommit + version := meta.Tag + if len(version) == 0 { + // no version, use commit and current time for development + version = commit + meta.Timestamp = gitTime(time.Now()) + } + + f, err := os.Create("meta.go") + if err != nil { + log.Fatalln(err) + } + defer f.Close() + + err = buildTemplate.Execute(f, struct { + Commit string + Version string + UnixTimestamp int64 + + RunTime string + }{ + Commit: commit, + Version: version, + UnixTimestamp: time.Time(meta.Timestamp).Unix(), + + RunTime: time.Now().Format(time.RFC3339), + }) + if err != nil { + log.Fatalln(err) + } +} diff --git a/build/meta.go b/build/meta.go new file mode 100644 index 0000000..05dd9fc --- /dev/null +++ b/build/meta.go @@ -0,0 +1,7 @@ +package build + +const ( + commit = "?" + version = "?" + buildTime = 0 +) diff --git a/cmd/explored/main.go b/cmd/explored/main.go index 271da84..339cffc 100644 --- a/cmd/explored/main.go +++ b/cmd/explored/main.go @@ -1,118 +1,416 @@ package main import ( + "context" + "errors" "flag" "fmt" - "log" "net" + "net/http" "os" "os/signal" - "runtime/debug" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/explored/api" + "go.sia.tech/explored/build" + "go.sia.tech/explored/config" + "go.sia.tech/explored/exchangerates" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/syncerutil" + "go.sia.tech/explored/persist/sqlite" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "golang.org/x/term" + "gopkg.in/yaml.v3" + "lukechampine.com/upnp" ) -var commit = "?" -var timestamp = "?" +var cfg = config.Config{ + Directory: ".", + HTTP: config.HTTP{ + Address: ":9980", + Password: os.Getenv("EXPLORED_API_PASSWORD"), + }, + Syncer: config.Syncer{ + Address: ":9981", + Bootstrap: true, + EnableUPNP: false, + }, + Scanner: config.Scanner{ + NumThreads: 100, + ScanTimeout: 1 * time.Minute, + ScanFrequency: 15 * time.Second, + ScanInterval: 1 * time.Hour, + MinLastAnnouncement: 365 * 24 * time.Hour, + }, + ExchangeRates: config.ExchangeRates{ + Refresh: 3 * time.Second, + }, + Consensus: config.Consensus{ + Network: "mainnet", + }, + Index: config.Index{ + BatchSize: 1000, + }, + Log: config.Log{ + Level: "info", + StdOut: config.StdOut{ + Enabled: true, + Format: "human", + EnableANSI: runtime.GOOS != "windows", + }, + File: config.LogFile{ + Enabled: true, + Format: "json", + }, + }, +} -func init() { - info, ok := debug.ReadBuildInfo() - if !ok { +// checkFatalError prints an error message to stderr and exits with a 1 exit code. If err is nil, this is a no-op. +func checkFatalError(context string, err error) { + if err == nil { return } - modified := false - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - commit = setting.Value[:8] - case "vcs.time": - timestamp = setting.Value - case "vcs.modified": - modified = setting.Value == "true" - } + os.Stderr.WriteString(fmt.Sprintf("%s: %s\n", context, err)) + os.Exit(1) +} + +// tryLoadConfig loads the config file specified by the EXPLORED_CONFIG_FILE. If +// the config file does not exist, it will not be loaded. +func tryLoadConfig() { + configPath := "explored.yml" + if str := os.Getenv("EXPLORED_CONFIG_FILE"); str != "" { + configPath = str } - if modified { - commit += " (modified)" + + // If the config file doesn't exist, don't try to load it. + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return } + + f, err := os.Open(configPath) + checkFatalError("failed to open config file", err) + defer f.Close() + + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + + checkFatalError("failed to decode config file", dec.Decode(&cfg)) +} + +// jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for +// parsing. +func jsonEncoder() zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.TimeKey = "timestamp" + return zapcore.NewJSONEncoder(cfg) } -func check(context string, err error, logger *zap.Logger) { +// humanEncoder returns a zapcore.Encoder that encodes logs as human-readable +// text. +func humanEncoder(showColors bool) zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.EncodeDuration = zapcore.StringDurationEncoder + + if showColors { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + } else { + cfg.EncodeLevel = zapcore.CapitalLevelEncoder + } + + cfg.StacktraceKey = "" + cfg.CallerKey = "" + return zapcore.NewConsoleEncoder(cfg) +} + +func parseLogLevel(level string) zap.AtomicLevel { + switch level { + case "debug": + return zap.NewAtomicLevelAt(zap.DebugLevel) + case "info": + return zap.NewAtomicLevelAt(zap.InfoLevel) + case "warn": + return zap.NewAtomicLevelAt(zap.WarnLevel) + case "error": + return zap.NewAtomicLevelAt(zap.ErrorLevel) + default: + fmt.Printf("invalid log level %q", level) + os.Exit(1) + } + panic("unreachable") +} + +func forwardUPNP(ctx context.Context, addr string, log *zap.Logger) string { + // wrapped so the context is appropriately canceled + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + d, err := upnp.Discover(ctx) if err != nil { - log.Fatalf("%v: %v", context, err) + log.Warn("WARN: couldn't discover UPnP device:", zap.Error(err)) + return "" + } + + _, portStr, _ := net.SplitHostPort(addr) + port, _ := strconv.Atoi(portStr) + if !d.IsForwarded(uint16(port), "TCP") { + if err := d.Forward(uint16(port), "TCP", "explored"); err != nil { + log.Warn("WARN: couldn't forward port:", zap.Error(err)) + } else { + log.Debug("p2p: Forwarded port", zap.Int("port", port)) + } } + + ip, err := d.ExternalIP() + if err != nil { + log.Warn("WARN: couldn't determine external IP:", zap.Error(err)) + return "" + } + log.Debug("p2p: External IP is", zap.String("ip", ip)) + return net.JoinHostPort(ip, portStr) } -func getAPIPassword(logger *zap.Logger) string { - apiPassword := os.Getenv("EXPLORED_API_PASSWORD") - if apiPassword != "" { - logger.Info("env: Using EXPLORED_API_PASSWORD environment variable") - } else { - fmt.Print("Enter API password: ") - pw, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - check("Could not read API password:", err, logger) - if err != nil { - log.Fatal(err) +func runRootCmd(ctx context.Context, log *zap.Logger) error { + var network *consensus.Network + var genesisBlock types.Block + + switch cfg.Consensus.Network { + case "mainnet": + network, genesisBlock = chain.Mainnet() + cfg.Syncer.Peers = append(cfg.Syncer.Peers, syncer.MainnetBootstrapPeers...) + case "zen": + network, genesisBlock = chain.TestnetZen() + cfg.Syncer.Peers = append(cfg.Syncer.Peers, syncer.ZenBootstrapPeers...) + case "anagami": + network, genesisBlock = chain.TestnetAnagami() + cfg.Syncer.Peers = append(cfg.Syncer.Peers, syncer.AnagamiBootstrapPeers...) + default: + log.Fatal("network must be 'mainnet', 'zen', or 'anagami'", zap.String("network", cfg.Consensus.Network)) + } + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) + if err != nil { + return fmt.Errorf("failed to open bolt database: %w", err) + } + defer bdb.Close() + + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) + if err != nil { + return fmt.Errorf("failed to create chain store: %w", err) + } + cm := chain.NewManager(dbstore, tipState) + + store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + return fmt.Errorf("failed to open sqlite database: %w", err) + } + defer store.Close() + + syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) + if err != nil { + return fmt.Errorf("failed to create listener: %w", err) + } + defer syncerListener.Close() + + httpListener, err := net.Listen("tcp", cfg.HTTP.Address) + if err != nil { + return fmt.Errorf("failed to create listener: %w", err) + } + defer httpListener.Close() + + syncerAddr := syncerListener.Addr().String() + if cfg.Syncer.EnableUPNP { + remoteIP := forwardUPNP(ctx, cfg.Syncer.Address, log) + if remoteIP != "" { + syncerAddr = remoteIP } - apiPassword = string(pw) } - return apiPassword + + // peers will reject us if our hostname is empty or unspecified, so use loopback + host, port, _ := net.SplitHostPort(syncerAddr) + if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { + syncerAddr = net.JoinHostPort("127.0.0.1", port) + } + + ps, err := syncerutil.NewJSONPeerStore(filepath.Join(cfg.Directory, "peers.json")) + if err != nil { + return fmt.Errorf("failed to open peer store: %w", err) + } + for _, peer := range cfg.Syncer.Peers { + ps.AddPeer(peer) + } + + header := gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerAddr, + } + s := syncer.New(syncerListener, cm, ps, header, syncer.WithLogger(log.Named("syncer")), syncer.WithMaxInboundPeers(256)) + defer s.Close() + go s.Run() + + e, err := explorer.NewExplorer(cm, store, cfg.Index, cfg.Scanner, log.Named("explorer")) + if err != nil { + return fmt.Errorf("failed to create explorer: %w", err) + } + timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 60*time.Second) + defer timeoutCancel() + defer e.Shutdown(timeoutCtx) + + var sources []exchangerates.Source + sources = append(sources, exchangerates.NewKraken(map[string]string{ + exchangerates.CurrencyUSD: exchangerates.KrakenPairSiacoinUSD, + exchangerates.CurrencyEUR: exchangerates.KrakenPairSiacoinEUR, + exchangerates.CurrencyBTC: exchangerates.KrakenPairSiacoinBTC, + }, cfg.ExchangeRates.Refresh)) + + coinGeckoPro, coinGeckoAPIKey := false, os.Getenv("COINGECKO_DEMO_API_KEY") + if coinGeckoAPIKey == "" { + coinGeckoPro, coinGeckoAPIKey = true, os.Getenv("COINGECKO_PRO_API_KEY") + } + if coinGeckoAPIKey != "" { + sources = append(sources, exchangerates.NewCoinGecko(coinGeckoPro, coinGeckoAPIKey, map[string]string{ + exchangerates.CurrencyUSD: exchangerates.CoinGeckoCurrencyUSD, + exchangerates.CurrencyEUR: exchangerates.CoinGeckoCurrencyEUR, + exchangerates.CurrencyCAD: exchangerates.CoinGeckoCurrencyCAD, + exchangerates.CurrencyAUD: exchangerates.CoinGeckoCurrencyAUD, + exchangerates.CurrencyGBP: exchangerates.CoinGeckoCurrencyGBP, + exchangerates.CurrencyJPY: exchangerates.CoinGeckoCurrencyJPY, + exchangerates.CurrencyCNY: exchangerates.CoinGeckoCurrencyCNY, + exchangerates.CurrencyETH: exchangerates.CoinGeckoCurrencyETH, + exchangerates.CurrencyBTC: exchangerates.CoinGeckoCurrencyBTC, + }, exchangerates.CoinGeckoTokenSiacoin, cfg.ExchangeRates.Refresh)) + } + + ex, err := exchangerates.NewAverager(true, sources...) + if err != nil { + return fmt.Errorf("failed to create exchange rate source: %w", err) + } + go ex.Start(ctx) + + api := api.NewServer(e, cm, s, ex, cfg.HTTP.Password) + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api") { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") + api.ServeHTTP(w, r) + return + } + http.NotFound(w, r) + }), + ReadTimeout: 15 * time.Second, + } + defer server.Close() + + go func() { + if err := server.Serve(httpListener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal("http server failed", zap.Error(err)) + } + }() + + log.Info("explored started", zap.String("network", cfg.Consensus.Network), zap.String("version", build.Version()), zap.String("http", cfg.HTTP.Address), zap.String("syncer", syncerAddr)) + + <-ctx.Done() + log.Info("shutting down") + time.AfterFunc(3*time.Minute, func() { + log.Fatal("failed to shut down within 3 minutes") + }) + + return nil } func main() { - // configure console logging note: this is configured before anything else - // to have consistent logging. File logging will be added after the cli - // flags and config is parsed - consoleCfg := zap.NewProductionEncoderConfig() - consoleCfg.TimeKey = "" // prevent duplicate timestamps - consoleCfg.EncodeTime = zapcore.RFC3339TimeEncoder - consoleCfg.EncodeDuration = zapcore.StringDurationEncoder - consoleCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder - consoleCfg.StacktraceKey = "" - consoleCfg.CallerKey = "" - consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg) - - // only log info messages to console unless stdout logging is enabled - consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.DebugLevel)) - log := zap.New(consoleCore, zap.AddCaller()) - defer log.Sync() - // redirect stdlib log to zap - zap.RedirectStdLog(log.Named("stdlib")) + tryLoadConfig() - gatewayAddr := flag.String("addr", ":9981", "p2p address to listen on") - apiAddr := flag.String("http", "localhost:9980", "address to serve API on") - dir := flag.String("dir", ".", "directory to store node state in") - network := flag.String("network", "mainnet", "network to connect to") - upnp := flag.Bool("upnp", true, "attempt to forward ports and discover IP with UPnP") + flag.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") + flag.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") + flag.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") + flag.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") + flag.BoolVar(&cfg.Syncer.EnableUPNP, "upnp", cfg.Syncer.EnableUPNP, "attempt to forward ports and discover IP with UPnP") flag.Parse() - log.Info("explored v0.0.0") if flag.Arg(0) == "version" { - log.Info("Commit Hash:", zap.String("hash", commit)) - log.Info("Commit Date:", zap.String("date", timestamp)) + fmt.Println("explored", build.Version()) + fmt.Println("Commit:", build.Commit()) + fmt.Println("Build Date:", build.Time()) return } - apiPassword := getAPIPassword(log) - l, err := net.Listen("tcp", *apiAddr) - if err != nil { - log.Fatal("Failed to create listener", zap.Error(err)) + checkFatalError("failed to open log file", os.MkdirAll(cfg.Directory, 0700)) + + var logCores []zapcore.Core + if cfg.Log.StdOut.Enabled { + // if no log level is set for stdout, use the global log level + if cfg.Log.StdOut.Level == "" { + cfg.Log.StdOut.Level = cfg.Log.Level + } + + var encoder zapcore.Encoder + switch cfg.Log.StdOut.Format { + case "json": + encoder = jsonEncoder() + default: // stdout defaults to human + encoder = humanEncoder(cfg.Log.StdOut.EnableANSI) + } + + // create the stdout logger + level := parseLogLevel(cfg.Log.StdOut.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), level)) } - n, err := newNode(*gatewayAddr, *dir, *network, *upnp, log) - if err != nil { - log.Fatal("Failed to create node", zap.Error(err)) - } - log.Info("p2p: Listening on", zap.String("addr", n.s.Addr())) - stop := n.Start() - log.Info("api: Listening on", zap.String("addr", l.Addr().String())) - go startWeb(l, n, apiPassword) - - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt) - <-signalCh - log.Info("Shutting down...") - stop() + if cfg.Log.File.Enabled { + // if no log level is set for file, use the global log level + if cfg.Log.File.Level == "" { + cfg.Log.File.Level = cfg.Log.Level + } + + // normalize log path + if cfg.Log.File.Path == "" { + cfg.Log.File.Path = filepath.Join(cfg.Directory, "explored.log") + } + + // configure file logging + var encoder zapcore.Encoder + switch cfg.Log.File.Format { + case "human": + encoder = humanEncoder(false) // disable colors in file log + default: // log file defaults to JSON + encoder = jsonEncoder() + } + + fileWriter, closeFn, err := zap.Open(cfg.Log.File.Path) + checkFatalError("failed to open log file", err) + defer closeFn() + + // create the file logger + level := parseLogLevel(cfg.Log.File.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(fileWriter), level)) + } + + var log *zap.Logger + if len(logCores) == 1 { + log = zap.New(logCores[0], zap.AddCaller()) + } else { + log = zap.New(zapcore.NewTee(logCores...), zap.AddCaller()) + } + defer log.Sync() + + // redirect stdlib log to zap + zap.RedirectStdLog(log.Named("stdlib")) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + checkFatalError("daemon startup failed", runRootCmd(ctx, log)) } diff --git a/cmd/explored/node.go b/cmd/explored/node.go deleted file mode 100644 index 190f06a..0000000 --- a/cmd/explored/node.go +++ /dev/null @@ -1,233 +0,0 @@ -package main - -import ( - "context" - "errors" - "net" - "path/filepath" - "strconv" - "time" - - bolt "go.etcd.io/bbolt" - "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/syncer" - "go.sia.tech/explored/explorer" - "go.sia.tech/explored/internal/syncerutil" - "go.sia.tech/explored/persist/sqlite" - "go.uber.org/zap" - "lukechampine.com/upnp" -) - -var mainnetBootstrap = []string{ - "108.227.62.195:9981", - "139.162.81.190:9991", - "144.217.7.188:9981", - "147.182.196.252:9981", - "15.235.85.30:9981", - "167.235.234.84:9981", - "173.235.144.230:9981", - "198.98.53.144:7791", - "199.27.255.169:9981", - "2.136.192.200:9981", - "213.159.50.43:9981", - "24.253.116.61:9981", - "46.249.226.103:9981", - "5.165.236.113:9981", - "5.252.226.131:9981", - "54.38.120.222:9981", - "62.210.136.25:9981", - "63.135.62.123:9981", - "65.21.93.245:9981", - "75.165.149.114:9981", - "77.51.200.125:9981", - "81.6.58.121:9981", - "83.194.193.156:9981", - "84.39.246.63:9981", - "87.99.166.34:9981", - "91.214.242.11:9981", - "93.105.88.181:9981", - "93.180.191.86:9981", - "94.130.220.162:9981", -} - -var zenBootstrap = []string{ - "147.135.16.182:9881", - "147.135.39.109:9881", - "51.81.208.10:9881", -} - -type boltDB struct { - tx *bolt.Tx - db *bolt.DB -} - -func (db *boltDB) newTx() (err error) { - if db.tx == nil { - db.tx, err = db.db.Begin(true) - } - return -} - -func (db *boltDB) Bucket(name []byte) chain.DBBucket { - if err := db.newTx(); err != nil { - panic(err) - } - - b := db.tx.Bucket(name) - if b == nil { - return nil - } - return b -} - -func (db *boltDB) CreateBucket(name []byte) (chain.DBBucket, error) { - if err := db.newTx(); err != nil { - return nil, err - } - - b, err := db.tx.CreateBucket(name) - if b == nil { - return nil, err - } - return b, nil -} - -func (db *boltDB) Flush() error { - if db.tx == nil { - return nil - } - - if err := db.tx.Commit(); err != nil { - return err - } - db.tx = nil - return nil -} - -func (db *boltDB) Cancel() { - if db.tx == nil { - return - } - - db.tx.Rollback() - db.tx = nil -} - -func (db *boltDB) Close() error { - db.Flush() - return db.db.Close() -} - -type node struct { - cm *chain.Manager - s *syncer.Syncer - e *explorer.Explorer - - Start func() (stop func()) -} - -func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Logger) (*node, error) { - var network *consensus.Network - var genesisBlock types.Block - var bootstrapPeers []string - switch chainNetwork { - case "mainnet": - network, genesisBlock = chain.Mainnet() - bootstrapPeers = mainnetBootstrap - case "zen": - network, genesisBlock = chain.TestnetZen() - bootstrapPeers = zenBootstrap - default: - return nil, errors.New("invalid network: must be one of 'mainnet' or 'zen'") - } - - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) - if err != nil { - return nil, err - } - db := &boltDB{db: bdb} - dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) - if err != nil { - return nil, err - } - cm := chain.NewManager(dbstore, tipState) - - store, err := sqlite.OpenDatabase(filepath.Join(dir, "./explore.db"), logger) - if err != nil { - return nil, err - } - - e, err := explorer.NewExplorer(cm, store, logger.Named("explorer")) - if err != nil { - return nil, err - } - - l, err := net.Listen("tcp", addr) - if err != nil { - return nil, err - } - syncerAddr := l.Addr().String() - if useUPNP { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if d, err := upnp.Discover(ctx); err != nil { - logger.Warn("WARN: couldn't discover UPnP device:", zap.Error(err)) - } else { - _, portStr, _ := net.SplitHostPort(addr) - port, _ := strconv.Atoi(portStr) - if !d.IsForwarded(uint16(port), "TCP") { - if err := d.Forward(uint16(port), "TCP", "explored"); err != nil { - logger.Warn("WARN: couldn't forward port:", zap.Error(err)) - } else { - logger.Info("p2p: Forwarded port", zap.Int("port", port)) - } - } - if ip, err := d.ExternalIP(); err != nil { - logger.Warn("WARN: couldn't determine external IP:", zap.Error(err)) - } else { - logger.Info("p2p: External IP is", zap.String("ip", ip)) - syncerAddr = net.JoinHostPort(ip, portStr) - } - } - } - // peers will reject us if our hostname is empty or unspecified, so use loopback - host, port, _ := net.SplitHostPort(syncerAddr) - if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { - syncerAddr = net.JoinHostPort("127.0.0.1", port) - } - - ps, err := syncerutil.NewJSONPeerStore(filepath.Join(dir, "peers.json")) - if err != nil { - return nil, err - } - for _, peer := range bootstrapPeers { - ps.AddPeer(peer) - } - header := gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerAddr, - } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(logger.Named("syncer"))) - - return &node{ - cm: cm, - s: s, - e: e, - Start: func() func() { - ch := make(chan struct{}) - go func() { - s.Run() - close(ch) - }() - return func() { - l.Close() - <-ch - db.Close() - } - }, - }, nil -} diff --git a/cmd/explored/web.go b/cmd/explored/web.go deleted file mode 100644 index c4bc646..0000000 --- a/cmd/explored/web.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "net" - "net/http" - "strings" - - "go.sia.tech/explored/api" - "go.sia.tech/jape" -) - -func startWeb(l net.Listener, node *node, password string) error { - renter := api.NewServer(node.e, node.cm, node.s) - api := jape.BasicAuth(password)(renter) - return http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api") { - r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") - api.ServeHTTP(w, r) - return - } - http.NotFound(w, r) - })) -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..852a78d --- /dev/null +++ b/config/config.go @@ -0,0 +1,96 @@ +package config + +import "time" + +type ( + // HTTP contains the configuration for the HTTP server. + HTTP struct { + Address string `yaml:"address,omitempty"` + Password string `yaml:"password,omitempty"` + } + + // Syncer contains the configuration for the syncer. + Syncer struct { + Address string `yaml:"address,omitempty"` + Bootstrap bool `yaml:"bootstrap,omitempty"` + EnableUPNP bool `yaml:"enableUPnP,omitempty"` + Peers []string `yaml:"peers,omitempty"` + } + + // Scanner contains the configuration for the host scanner. + Scanner struct { + // NumThreads represents the maximum number of hosts we will + // simultaneously scan. + NumThreads uint64 `yaml:"numThreads,omitempty"` + // ScanTimeout represents the maximum amount of time we will spend scanning + // a single host. + ScanTimeout time.Duration `yaml:"scanTimeout,omitempty"` + // ScanFrequency represents the amount of time we will wait before + // calling HostsForScanning again if the previous call returned zero + // hosts to scan. + ScanFrequency time.Duration `yaml:"scanFrequency,omitempty"` + // ScanInterval represents how frequently hosts will be scanned. If a + // scan is successful, the hosts next scan time will be set to + // the current time plus MaxLastScan. If it fails, the next scan time + // is set to the current time plus MaxLastScan * pow(2, # of + // consecutive failed scans). + ScanInterval time.Duration `yaml:"scanInterval,omitempty"` + // MinLastAnnouncement represents how far back we will search for + // announcements to find hosts to scan. + MinLastAnnouncement time.Duration `yaml:"minLastAnnouncement,omitempty"` + } + + // Consensus contains the configuration for the consensus set. + Consensus struct { + Network string `yaml:"network,omitempty"` + } + + // Index contains the configuration for the blockchain indexer + Index struct { + BatchSize int `yaml:"batchSize,omitempty"` + } + + // ExchangeRates contains the configuration for the exchange rate clients. + ExchangeRates struct { + // refresh exchange rates this often + Refresh time.Duration + } + + // LogFile configures the file output of the logger. + LogFile struct { + Enabled bool `yaml:"enabled,omitempty"` + Level string `yaml:"level,omitempty"` // override the file log level + Format string `yaml:"format,omitempty"` + // Path is the path of the log file. + Path string `yaml:"path,omitempty"` + } + + // StdOut configures the standard output of the logger. + StdOut struct { + Level string `yaml:"level,omitempty"` // override the stdout log level + Enabled bool `yaml:"enabled,omitempty"` + Format string `yaml:"format,omitempty"` + EnableANSI bool `yaml:"enableANSI,omitempty"` //nolint:tagliatelle + } + + // Log contains the configuration for the logger. + Log struct { + Level string `yaml:"level,omitempty"` // global log level + StdOut StdOut `yaml:"stdout,omitempty"` + File LogFile `yaml:"file,omitempty"` + } + + // Config contains the configuration for the host. + Config struct { + Directory string `yaml:"directory,omitempty"` + AutoOpenWebUI bool `yaml:"autoOpenWebUI,omitempty"` + + HTTP HTTP `yaml:"http,omitempty"` + Consensus Consensus `yaml:"consensus,omitempty"` + Syncer Syncer `yaml:"syncer,omitempty"` + Scanner Scanner `yaml:"scanner,omitempty"` + ExchangeRates ExchangeRates `yaml:"exchangeRates,omitempty"` + Log Log `yaml:"log,omitempty"` + Index Index `yaml:"index,omitempty"` + } +) diff --git a/exchangerates/coingecko.go b/exchangerates/coingecko.go new file mode 100644 index 0000000..3f14e84 --- /dev/null +++ b/exchangerates/coingecko.go @@ -0,0 +1,166 @@ +package exchangerates + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" +) + +const ( + // CoinGeckoTokenSiacoin is the token ID of Siacoin in CoinGecko + CoinGeckoTokenSiacoin = "siacoin" +) + +const ( + // CoinGeckoCurrencyUSD is the name of US dollars in CoinGecko. + CoinGeckoCurrencyUSD = "usd" + // CoinGeckoCurrencyEUR is the name of euros in CoinGecko. + CoinGeckoCurrencyEUR = "eur" + // CoinGeckoCurrencyCAD is the name of Canadian dollars in CoinGecko. + CoinGeckoCurrencyCAD = "cad" + // CoinGeckoCurrencyAUD is the name of Australian dollars in CoinGecko. + CoinGeckoCurrencyAUD = "aud" + // CoinGeckoCurrencyGBP is the name of British pounds in CoinGecko. + CoinGeckoCurrencyGBP = "gbp" + // CoinGeckoCurrencyJPY is the name of Japanese yen in CoinGecko. + CoinGeckoCurrencyJPY = "jpy" + // CoinGeckoCurrencyCNY is the name of Chinese yuan in CoinGecko. + CoinGeckoCurrencyCNY = "cny" + // CoinGeckoCurrencyBTC is the name of Bitcoin in CoinGecko. + CoinGeckoCurrencyBTC = "btc" + // CoinGeckoCurrencyETH is the name of Ethereum in CoinGecko. + CoinGeckoCurrencyETH = "eth" +) + +const ( + demoBaseURL = "https://api.coingecko.com" + proBaseURL = "https://pro-api.coingecko.com" +) + +type coinGeckoAPI struct { + pro bool + apiKey string + client http.Client +} + +func newCoinGeckoAPI(pro bool, apiKey string) *coinGeckoAPI { + return &coinGeckoAPI{pro: pro, apiKey: apiKey} +} + +type coinGeckoPriceResponse map[string]map[string]float64 + +func (c *coinGeckoAPI) baseURL() string { + if c.pro { + return proBaseURL + } + return demoBaseURL +} + +// See https://docs.coingecko.com/reference/simple-price +func (c *coinGeckoAPI) tickers(ctx context.Context, currencies []string, token string) (map[string]float64, error) { + vsCurrencies := strings.ToLower(strings.Join(currencies, ",")) + token = strings.ToLower(token) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf( + "%s/api/v3/simple/price?vs_currencies=%s&ids=%s", + c.baseURL(), vsCurrencies, token), nil) + if err != nil { + return nil, err + } + request.Header.Set("accept", "application/json") + if c.pro { + request.Header.Set("x-cg-pro-api-key", c.apiKey) + } else { + request.Header.Set("x-cg-demo-api-key", c.apiKey) + } + + response, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var parsed coinGeckoPriceResponse + if err := json.NewDecoder(response.Body).Decode(&parsed); err != nil { + return nil, err + } + + asset, ok := parsed[token] + if !ok { + return nil, fmt.Errorf("no asset %s", token) + } + + return asset, nil +} + +type coinGecko struct { + token string + pairMap map[string]string // User-specified currency -> CoinGecko currency + refresh time.Duration + client *coinGeckoAPI + + mu sync.Mutex + rates map[string]float64 // CoinGecko currency -> rate + err error +} + +// NewCoinGecko creates an Source with user-specified mappings +func NewCoinGecko(pro bool, apiKey string, pairMap map[string]string, token string, refresh time.Duration) Source { + return &coinGecko{ + token: token, + pairMap: pairMap, + refresh: refresh, + client: newCoinGeckoAPI(pro, apiKey), + rates: make(map[string]float64), + } +} + +// Start implements Source. +func (c *coinGecko) Start(ctx context.Context) { + ticker := time.NewTicker(c.refresh) + defer ticker.Stop() + + var currencies []string + for _, coinGeckoCurrency := range c.pairMap { + currencies = append(currencies, coinGeckoCurrency) + } + + c.mu.Lock() + c.rates, c.err = c.client.tickers(ctx, currencies, c.token) + c.mu.Unlock() + + for { + select { + case <-ticker.C: + c.mu.Lock() + c.rates, c.err = c.client.tickers(ctx, currencies, c.token) + c.mu.Unlock() + case <-ctx.Done(): + c.mu.Lock() + c.err = ctx.Err() + c.mu.Unlock() + return + } + } +} + +// Last implements Source. +func (c *coinGecko) Last(currency string) (float64, error) { + c.mu.Lock() + defer c.mu.Unlock() + + coinGeckoCurrency, exists := c.pairMap[currency] + if !exists { + return 0, fmt.Errorf("currency %s not mapped to a CoinGecko currency", currency) + } + + rate, ok := c.rates[coinGeckoCurrency] + if !ok { + return 0, fmt.Errorf("rate for currency %s not available", currency) + } + return rate, c.err +} diff --git a/exchangerates/exchangerates.go b/exchangerates/exchangerates.go new file mode 100644 index 0000000..cc34413 --- /dev/null +++ b/exchangerates/exchangerates.go @@ -0,0 +1,77 @@ +package exchangerates + +import ( + "context" + "errors" +) + +const ( + // CurrencyUSD represents US dollars. + CurrencyUSD = "USD" + // CurrencyEUR represents euros. + CurrencyEUR = "EUR" + // CurrencyCAD represents Canadian dollars. + CurrencyCAD = "CAD" + // CurrencyAUD represents Australian dollars. + CurrencyAUD = "AUD" + // CurrencyGBP represents British pounds. + CurrencyGBP = "GBP" + // CurrencyJPY represents Japanese yen. + CurrencyJPY = "JPY" + // CurrencyCNY represents Chinese yuan. + CurrencyCNY = "CNY" + // CurrencyBTC represents Bitcoin. + CurrencyBTC = "BTC" + // CurrencyETH represents Ethereum. + CurrencyETH = "ETH" +) + +// An Source returns the price of 1 unit of an asset in USD. +type Source interface { + Last(currency string) (float64, error) + Start(ctx context.Context) +} + +type averager struct { + ignoreOnError bool + sources []Source +} + +// NewAverager returns an exchange rate source that averages multiple exchange +// rates. +func NewAverager(ignoreOnError bool, sources ...Source) (Source, error) { + if len(sources) == 0 { + return nil, errors.New("no sources provided") + } + return &averager{ + ignoreOnError: ignoreOnError, + sources: sources, + }, nil +} + +// Start implements Source. +func (a *averager) Start(ctx context.Context) { + for i := range a.sources { + go a.sources[i].Start(ctx) + } +} + +// Last implements Source. +func (a *averager) Last(currency string) (float64, error) { + sum, count := 0.0, 0.0 + for i := range a.sources { + if v, err := a.sources[i].Last(currency); err == nil { + if v != 0 { + sum += v + count++ + } + } else if !a.ignoreOnError { + return 0, err + } + } + + if count == 0 { + return 0, errors.New("no sources working") + } + return sum / count, nil +} diff --git a/exchangerates/exchangerates_test.go b/exchangerates/exchangerates_test.go new file mode 100644 index 0000000..4b97661 --- /dev/null +++ b/exchangerates/exchangerates_test.go @@ -0,0 +1,144 @@ +package exchangerates + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type constantExchangeRateSource struct { + x float64 + + mu sync.Mutex + rate float64 +} + +func (c *constantExchangeRateSource) Start(ctx context.Context) { + c.mu.Lock() + c.rate = c.x + c.mu.Unlock() +} + +func (c *constantExchangeRateSource) Last(string) (rate float64, err error) { + c.mu.Lock() + rate, err = c.rate, nil + c.mu.Unlock() + return +} + +func newConstantExchangeRateSource(x float64) *constantExchangeRateSource { + return &constantExchangeRateSource{x: x} +} + +type errorExchangeRateSource struct{} + +func (c *errorExchangeRateSource) Start(ctx context.Context) {} + +func (c *errorExchangeRateSource) Last(string) (float64, error) { + return -1, errors.New("error") +} + +func TestAveragerLastBeforeStart(t *testing.T) { + averager, err := NewAverager(false, newConstantExchangeRateSource(1.0)) + if err != nil { + t.Fatal(err) + } + if _, err := averager.Last(CurrencyUSD); err == nil { + t.Fatal("should be error if we call Last before Start") + } +} + +func TestAverager(t *testing.T) { + const interval = time.Second + + const ( + p1 = 1.0 + p2 = 10.0 + p3 = 100.0 + ) + s1 := newConstantExchangeRateSource(p1) + s2 := newConstantExchangeRateSource(p2) + s3 := newConstantExchangeRateSource(p3) + errorSource := &errorExchangeRateSource{} + + tests := []struct { + name string + ignoreOnError bool + sources []Source + expectedPrice float64 + expectError bool + errorMessage string + }{ + { + name: "No sources provided", + ignoreOnError: true, + sources: nil, + expectError: true, + errorMessage: "Should have gotten error for averager with no sources", + }, + { + name: "All sources fail", + ignoreOnError: true, + sources: []Source{errorSource, errorSource, errorSource}, + expectError: true, + errorMessage: "Should have gotten error for averager with no working sources", + }, + { + name: "Valid sources without errors", + ignoreOnError: false, + sources: []Source{s1, s2, s3}, + expectedPrice: (p1 + p2 + p3) / 3, + expectError: false, + }, + { + name: "One error source without ignoreOnError", + ignoreOnError: false, + sources: []Source{s1, s2, s3, errorSource}, + expectError: true, + errorMessage: "Should have gotten error for averager with error source and ignoreOnError disabled", + }, + { + name: "One error source with ignoreOnError", + ignoreOnError: true, + sources: []Source{s1, s2, s3, errorSource}, + expectedPrice: (p1 + p2 + p3) / 3, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + averager, err := NewAverager(tt.ignoreOnError, tt.sources...) + if err != nil { + if !tt.expectError { + t.Fatal(err) + } + return + } + go averager.Start(ctx) + + time.Sleep(2 * interval) + + price, err := averager.Last(CurrencyUSD) + if tt.expectError { + if err == nil { + t.Fatal(tt.errorMessage) + } + return + } + + if err != nil { + t.Fatal(err) + } + + if price != tt.expectedPrice { + t.Fatalf("wrong price, got %v, expected %v", price, tt.expectedPrice) + } + }) + } +} diff --git a/exchangerates/kraken.go b/exchangerates/kraken.go new file mode 100644 index 0000000..e96a2d1 --- /dev/null +++ b/exchangerates/kraken.go @@ -0,0 +1,136 @@ +package exchangerates + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // KrakenPairSiacoinUSD is the ID of SC/USD pair in Kraken + KrakenPairSiacoinUSD = "SCUSD" + // KrakenPairSiacoinEUR is the ID of SC/EUR pair in Kraken + KrakenPairSiacoinEUR = "SCEUR" + // KrakenPairSiacoinBTC is the ID of SC/BTC pair in Kraken + KrakenPairSiacoinBTC = "SCXBT" +) + +type krakenAPI struct { + client http.Client +} + +type krakenPriceResponse struct { + Error []any `json:"error"` + Result map[string]struct { + B []string `json:"b"` + } `json:"result"` +} + +func newKrakenAPI() *krakenAPI { + return &krakenAPI{} +} + +func (k *krakenAPI) tickers(ctx context.Context, pairs []string) (map[string]float64, error) { + pairParam := strings.Join(pairs, ",") + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.kraken.com/0/public/Ticker?pair="+url.PathEscape(pairParam), nil) + if err != nil { + return nil, err + } + + response, err := k.client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var parsed krakenPriceResponse + if err := json.NewDecoder(response.Body).Decode(&parsed); err != nil { + return nil, err + } + + rates := make(map[string]float64) + for pair, data := range parsed.Result { + if len(data.B) == 0 { + continue + } + price, err := strconv.ParseFloat(data.B[0], 64) + if err != nil { + return nil, err + } + rates[pair] = price + } + + return rates, nil +} + +type kraken struct { + pairMap map[string]string // User-specified currency -> Kraken pair + refresh time.Duration + client *krakenAPI + + mu sync.Mutex + rates map[string]float64 // Kraken pair -> rate + err error +} + +// NewKraken returns an Source that gets data from Kraken. +func NewKraken(pairMap map[string]string, refresh time.Duration) Source { + return &kraken{ + pairMap: pairMap, + refresh: refresh, + client: newKrakenAPI(), + rates: make(map[string]float64), + } +} + +// Start implements Source. +func (k *kraken) Start(ctx context.Context) { + ticker := time.NewTicker(k.refresh) + defer ticker.Stop() + + var krakenPairs []string + for _, krakenPair := range k.pairMap { + krakenPairs = append(krakenPairs, krakenPair) + } + + k.mu.Lock() + k.rates, k.err = k.client.tickers(ctx, krakenPairs) + k.mu.Unlock() + + for { + select { + case <-ticker.C: + k.mu.Lock() + k.rates, k.err = k.client.tickers(ctx, krakenPairs) + k.mu.Unlock() + case <-ctx.Done(): + k.mu.Lock() + k.err = ctx.Err() + k.mu.Unlock() + return + } + } +} + +// Last implements Source. +func (k *kraken) Last(currency string) (float64, error) { + k.mu.Lock() + defer k.mu.Unlock() + + krakenPair, exists := k.pairMap[currency] + if !exists { + return 0, fmt.Errorf("currency %s not mapped to a Kraken pair", currency) + } + + rate, ok := k.rates[krakenPair] + if !ok { + return 0, fmt.Errorf("rate for pair %s not available", krakenPair) + } + return rate, k.err +} diff --git a/exchangerates/kraken_test.go b/exchangerates/kraken_test.go new file mode 100644 index 0000000..0d2f27e --- /dev/null +++ b/exchangerates/kraken_test.go @@ -0,0 +1,37 @@ +package exchangerates + +import ( + "context" + "testing" + "time" +) + +func TestKraken(t *testing.T) { + const interval = time.Second + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + kraken := NewKraken(map[string]string{ + CurrencyUSD: KrakenPairSiacoinUSD, + CurrencyEUR: KrakenPairSiacoinEUR, + }, interval) + go kraken.Start(ctx) + + time.Sleep(2 * interval) + if price, err := kraken.Last("USD"); err != nil { + t.Fatal(err) + } else if price <= 0.0 { + t.Fatalf("invalid price: %f", price) + } + + if price, err := kraken.Last("EUR"); err != nil { + t.Fatal(err) + } else if price <= 0.0 { + t.Fatalf("invalid price: %f", price) + } + + if _, err := kraken.Last("UNK"); err == nil { + t.Fatal("should fail for unmapped currency") + } +} diff --git a/explorer/events.go b/explorer/events.go index bbd5fc3..8d677d1 100644 --- a/explorer/events.go +++ b/explorer/events.go @@ -7,166 +7,399 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/wallet" ) -// event type constants -const ( - EventTypeTransaction = "transaction" - EventTypeMinerPayout = "miner payout" - EventTypeContractPayout = "contract payout" - EventTypeSiafundClaim = "siafund claim" - EventTypeFoundationSubsidy = "foundation subsidy" -) +type ( + // An EventPayout represents a miner payout, siafund claim, or foundation + // subsidy. + EventPayout struct { + SiacoinElement SiacoinOutput `json:"siacoinElement"` + } -type eventData interface { - EventType() string -} + // An EventV1Transaction pairs a v1 transaction with its spent siacoin and + // siafund elements. + EventV1Transaction struct { + Transaction Transaction `json:"transaction"` + } -// An Event is something interesting that happened on the Sia blockchain. -type Event struct { - ID types.Hash256 `json:"id"` - Index types.ChainIndex `json:"index"` - Timestamp time.Time `json:"timestamp"` - MaturityHeight uint64 `json:"maturityHeight"` - Relevant []types.Address `json:"relevant"` - Data eventData `json:"data"` -} + // An EventV1ContractResolution represents a file contract payout from a v1 + // contract. + EventV1ContractResolution struct { + Parent ExtendedFileContract `json:"parent"` + SiacoinElement SiacoinOutput `json:"siacoinElement"` + Missed bool `json:"missed"` + } -// EventType implements Event. -func (*EventTransaction) EventType() string { return EventTypeTransaction } - -// EventType implements Event. -func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } - -// EventType implements Event. -func (*EventFoundationSubsidy) EventType() string { return EventTypeFoundationSubsidy } - -// EventType implements Event. -func (*EventContractPayout) EventType() string { return EventTypeContractPayout } - -// MarshalJSON implements json.Marshaler. -func (e Event) MarshalJSON() ([]byte, error) { - val, _ := json.Marshal(e.Data) - return json.Marshal(struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Val json.RawMessage `json:"val"` - }{ - ID: e.ID, - Timestamp: e.Timestamp, - Index: e.Index, - Relevant: e.Relevant, - Type: e.Data.EventType(), - Val: val, - }) -} + // An EventV2ContractResolution represents a file contract payout from a v2 + // contract. + EventV2ContractResolution struct { + Resolution V2FileContractResolution `json:"resolution"` + SiacoinElement SiacoinOutput `json:"siacoinElement"` + Missed bool `json:"missed"` + } + + // EventV2Transaction is a transaction event that includes the transaction + EventV2Transaction V2Transaction + + // EventData contains the data associated with an event. + EventData interface { + isEvent() bool + } -// UnmarshalJSON implements json.Unarshaler. -func (e *Event) UnmarshalJSON(data []byte) error { - var s struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Val json.RawMessage `json:"val"` - } - if err := json.Unmarshal(data, &s); err != nil { + // An Event is a transaction or other event that affects the wallet including + // miner payouts, siafund claims, and file contract payouts. + Event struct { + ID types.Hash256 `json:"id"` + Index types.ChainIndex `json:"index"` + Confirmations uint64 `json:"confirmations"` + Type string `json:"type"` + Data EventData `json:"data"` + MaturityHeight uint64 `json:"maturityHeight"` + Timestamp time.Time `json:"timestamp"` + Relevant []types.Address `json:"relevant,omitempty"` + } +) + +func (EventPayout) isEvent() bool { return true } +func (EventV1Transaction) isEvent() bool { return true } +func (EventV1ContractResolution) isEvent() bool { return true } +func (EventV2Transaction) isEvent() bool { return true } +func (EventV2ContractResolution) isEvent() bool { return true } + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (e *Event) UnmarshalJSON(b []byte) error { + var je struct { + ID types.Hash256 `json:"id"` + Index types.ChainIndex `json:"index"` + Confirmations uint64 `json:"confirmations"` + Timestamp time.Time `json:"timestamp"` + MaturityHeight uint64 `json:"maturityHeight"` + Type string `json:"type"` + Data json.RawMessage `json:"data"` + Relevant []types.Address `json:"relevant,omitempty"` + } + if err := json.Unmarshal(b, &je); err != nil { return err } - e.ID = s.ID - e.Timestamp = s.Timestamp - e.Index = s.Index - e.Relevant = s.Relevant - switch s.Type { - case (*EventTransaction)(nil).EventType(): - e.Data = new(EventTransaction) - case (*EventMinerPayout)(nil).EventType(): - e.Data = new(EventMinerPayout) - case (*EventContractPayout)(nil).EventType(): - e.Data = new(EventContractPayout) - } - if e.Data == nil { - return fmt.Errorf("unknown event type %q", s.Type) - } - return json.Unmarshal(s.Val, e.Data) -} -// A HostAnnouncement represents a host announcement within an EventTransaction. -type HostAnnouncement struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` + e.ID = je.ID + e.Index = je.Index + e.Confirmations = je.Confirmations + e.Timestamp = je.Timestamp + e.MaturityHeight = je.MaturityHeight + e.Type = je.Type + e.Relevant = je.Relevant + + var err error + switch je.Type { + case wallet.EventTypeMinerPayout, wallet.EventTypeFoundationSubsidy, wallet.EventTypeSiafundClaim: + var data EventPayout + err = json.Unmarshal(je.Data, &data) + e.Data = data + case wallet.EventTypeV1ContractResolution: + var data EventV1ContractResolution + err = json.Unmarshal(je.Data, &data) + e.Data = data + case wallet.EventTypeV2ContractResolution: + var data EventV2ContractResolution + err = json.Unmarshal(je.Data, &data) + e.Data = data + case wallet.EventTypeV1Transaction: + var data EventV1Transaction + err = json.Unmarshal(je.Data, &data) + e.Data = data + case wallet.EventTypeV2Transaction: + var data EventV2Transaction + err = json.Unmarshal(je.Data, &data) + e.Data = data + default: + return fmt.Errorf("unknown event type: %v", je.Type) + } + return err } -// A SiafundInput represents a siafund input within an EventTransaction. -type EventSiafundInput struct { - SiafundElement types.SiafundElement `json:"siafundElement"` - ClaimElement types.SiacoinElement `json:"claimElement"` +// A ChainUpdate is a set of changes to the consensus state. +type ChainUpdate interface { + SiacoinElementDiffs() []consensus.SiacoinElementDiff + SiafundElementDiffs() []consensus.SiafundElementDiff + FileContractElementDiffs() []consensus.FileContractElementDiff + V2FileContractElementDiffs() []consensus.V2FileContractElementDiff } -// A FileContract represents a file contract within an EventTransaction. -type EventFileContract struct { - FileContract types.FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - ValidOutputs []types.SiacoinElement `json:"validOutputs,omitempty"` -} +// RelevantAddressesV1 returns all the relevant addresses to a V1 transaction. +func RelevantAddressesV1(txn types.Transaction) []types.Address { + addresses := make(map[types.Address]struct{}) + for _, sco := range txn.SiacoinOutputs { + addresses[sco.Address] = struct{}{} + } + for _, sci := range txn.SiacoinInputs { + addresses[sci.UnlockConditions.UnlockHash()] = struct{}{} + } + for _, sfo := range txn.SiafundOutputs { + addresses[sfo.Address] = struct{}{} + } + for _, sfi := range txn.SiafundInputs { + addresses[sfi.UnlockConditions.UnlockHash()] = struct{}{} + } + for _, fc := range txn.FileContracts { + for _, vpo := range fc.ValidProofOutputs { + addresses[vpo.Address] = struct{}{} + } + for _, mpo := range fc.MissedProofOutputs { + addresses[mpo.Address] = struct{}{} + } + } + for _, fcr := range txn.FileContractRevisions { + for _, vpo := range fcr.FileContract.ValidProofOutputs { + addresses[vpo.Address] = struct{}{} + } + for _, mpo := range fcr.FileContract.MissedProofOutputs { + addresses[mpo.Address] = struct{}{} + } + } -// A EventV2FileContract represents a v2 file contract within an EventTransaction. -type EventV2FileContract struct { - FileContract types.V2FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.V2FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - Resolution types.V2FileContractResolutionType `json:"resolution,omitempty"` - Outputs []types.SiacoinElement `json:"outputs,omitempty"` + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) + } + return relevant } -// An EventTransaction represents a transaction that affects the wallet. -type EventTransaction struct { - SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` - SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` - SiafundInputs []EventSiafundInput `json:"siafundInputs"` - SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` - FileContracts []EventFileContract `json:"fileContracts"` - V2FileContracts []EventV2FileContract `json:"v2FileContracts"` - HostAnnouncements []HostAnnouncement `json:"hostAnnouncements"` - Fee types.Currency `json:"fee"` -} +// RelevantAddressesV2 returns all the relevant addresses to a V2 transaction. +func RelevantAddressesV2(txn types.V2Transaction) []types.Address { + addresses := make(map[types.Address]struct{}) + for _, sco := range txn.SiacoinOutputs { + addresses[sco.Address] = struct{}{} + } + for _, sci := range txn.SiacoinInputs { + addresses[sci.Parent.SiacoinOutput.Address] = struct{}{} + } + for _, sfo := range txn.SiafundOutputs { + addresses[sfo.Address] = struct{}{} + } + for _, sfi := range txn.SiafundInputs { + addresses[sfi.Parent.SiafundOutput.Address] = struct{}{} + } + for _, fc := range txn.FileContracts { + addresses[fc.HostOutput.Address] = struct{}{} + addresses[fc.RenterOutput.Address] = struct{}{} + } + for _, fcr := range txn.FileContractRevisions { + addresses[fcr.Parent.V2FileContract.HostOutput.Address] = struct{}{} + addresses[fcr.Parent.V2FileContract.RenterOutput.Address] = struct{}{} + addresses[fcr.Revision.HostOutput.Address] = struct{}{} + addresses[fcr.Revision.RenterOutput.Address] = struct{}{} + } + for _, fcr := range txn.FileContractResolutions { + addresses[fcr.Parent.V2FileContract.HostOutput.Address] = struct{}{} + addresses[fcr.Parent.V2FileContract.RenterOutput.Address] = struct{}{} + if v, ok := fcr.Resolution.(*types.V2FileContractRenewal); ok { + addresses[v.NewContract.HostOutput.Address] = struct{}{} + addresses[v.NewContract.RenterOutput.Address] = struct{}{} + } + } -// An EventMinerPayout represents a miner payout from a block. -type EventMinerPayout struct { - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) + } + return relevant } -// EventFoundationSubsidy represents a foundation subsidy from a block. -type EventFoundationSubsidy struct { - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` -} +// CoreToExplorerV1Transaction converts a core/types.Transaction to an +// event.Transaction. Fields we do not have information are unfilled in the +// return value. +func CoreToExplorerV1Transaction(txn types.Transaction) (result Transaction) { + result.ID = txn.ID() + + coreToExplorerFC := func(fcID types.FileContractID, fc types.FileContract) ExtendedFileContract { + efc := ExtendedFileContract{ + ConfirmationTransactionID: result.ID, + ID: fcID, + Filesize: fc.Filesize, + FileMerkleRoot: fc.FileMerkleRoot, + WindowStart: fc.WindowStart, + WindowEnd: fc.WindowEnd, + Payout: fc.Payout, + UnlockHash: fc.UnlockHash, + RevisionNumber: fc.RevisionNumber, + } + for j, vpo := range fc.ValidProofOutputs { + efc.ValidProofOutputs = append(efc.ValidProofOutputs, ContractSiacoinOutput{ + ID: fcID.ValidOutputID(j), + SiacoinOutput: vpo, + }) + } + for j, mpo := range fc.MissedProofOutputs { + efc.MissedProofOutputs = append(efc.MissedProofOutputs, ContractSiacoinOutput{ + ID: fcID.MissedOutputID(j), + SiacoinOutput: mpo, + }) + } + return efc + } -// An EventContractPayout represents a file contract payout -type EventContractPayout struct { - FileContract types.FileContractElement `json:"fileContract"` - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` - Missed bool `json:"missed"` + for _, sci := range txn.SiacoinInputs { + result.SiacoinInputs = append(result.SiacoinInputs, SiacoinInput{ + SiacoinInput: sci, + }) + } + for i, sco := range txn.SiacoinOutputs { + sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(i), + SiacoinOutput: sco, + } + result.SiacoinOutputs = append(result.SiacoinOutputs, SiacoinOutput{ + SiacoinElement: sce, + }) + } + for _, sfi := range txn.SiafundInputs { + result.SiafundInputs = append(result.SiafundInputs, SiafundInput{ + SiafundInput: sfi, + }) + } + for i, sfo := range txn.SiafundOutputs { + sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(i), + SiafundOutput: sfo, + } + result.SiafundOutputs = append(result.SiafundOutputs, SiafundOutput{ + SiafundElement: sfe, + }) + } + for i, fc := range txn.FileContracts { + result.FileContracts = append(result.FileContracts, coreToExplorerFC(txn.FileContractID(i), fc)) + } + for _, fcr := range txn.FileContractRevisions { + result.FileContractRevisions = append(result.FileContractRevisions, FileContractRevision{ + ParentID: fcr.ParentID, + UnlockConditions: fcr.UnlockConditions, + ExtendedFileContract: coreToExplorerFC(fcr.ParentID, fcr.FileContract), + }) + } + for _, sp := range txn.StorageProofs { + result.StorageProofs = append(result.StorageProofs, sp) + } + for _, fee := range txn.MinerFees { + result.MinerFees = append(result.MinerFees, fee) + } + for _, arb := range txn.ArbitraryData { + result.ArbitraryData = append(result.ArbitraryData, arb) + } + for _, sig := range txn.Signatures { + result.Signatures = append(result.Signatures, sig) + } + + return } -// A ChainUpdate is a set of changes to the consensus state. -type ChainUpdate interface { - ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) - ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) +// CoreToExplorerV2Transaction converts a core/types.V2Transaction to an +// event.V2Transaction. Fields we do not have information are unfilled in the +// return value. +func CoreToExplorerV2Transaction(txn types.V2Transaction) (result V2Transaction) { + result.ID = txn.ID() + coreToExplorerFC := func(fcID types.FileContractID, fc types.V2FileContract) V2FileContract { + fce := types.V2FileContractElement{ + ID: fcID, + V2FileContract: fc, + } + + return V2FileContract{ + TransactionID: result.ID, + ConfirmationTransactionID: result.ID, + V2FileContractElement: fce, + } + } + + for _, sci := range txn.SiacoinInputs { + result.SiacoinInputs = append(result.SiacoinInputs, sci) + } + for i, sco := range txn.SiacoinOutputs { + sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(result.ID, i), + SiacoinOutput: sco, + } + result.SiacoinOutputs = append(result.SiacoinOutputs, SiacoinOutput{ + SiacoinElement: sce, + }) + } + for _, sfi := range txn.SiafundInputs { + result.SiafundInputs = append(result.SiafundInputs, sfi) + } + for i, sfo := range txn.SiafundOutputs { + sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(result.ID, i), + SiafundOutput: sfo, + } + result.SiafundOutputs = append(result.SiafundOutputs, SiafundOutput{ + SiafundElement: sfe, + }) + } + for i, fc := range txn.FileContracts { + result.FileContracts = append(result.FileContracts, coreToExplorerFC(txn.V2FileContractID(result.ID, i), fc)) + } + for _, fcr := range txn.FileContractRevisions { + parent := coreToExplorerFC(fcr.Parent.ID, fcr.Parent.V2FileContract) + parent.V2FileContractElement.StateElement = fcr.Parent.StateElement + result.FileContractRevisions = append(result.FileContractRevisions, V2FileContractRevision{ + Parent: parent, + Revision: coreToExplorerFC(fcr.Parent.ID, fcr.Revision), + }) + } + for _, fcr := range txn.FileContractResolutions { + parent := coreToExplorerFC(fcr.Parent.ID, fcr.Parent.V2FileContract) + parent.V2FileContractElement.StateElement = fcr.Parent.StateElement + + var res any + switch v := fcr.Resolution.(type) { + case *types.V2FileContractRenewal: + res = V2FileContractRenewal{ + FinalRenterOutput: v.FinalRenterOutput, + FinalHostOutput: v.FinalHostOutput, + RenterRollover: v.RenterRollover, + HostRollover: v.HostRollover, + NewContract: coreToExplorerFC(fcr.Parent.ID.V2RenewalID(), v.NewContract), + + RenterSignature: v.RenterSignature, + HostSignature: v.HostSignature, + } + case *types.V2StorageProof: + res = v + case *types.V2FileContractExpiration: + res = v + } + result.FileContractResolutions = append(result.FileContractResolutions, V2FileContractResolution{ + Parent: parent, + Type: V2ResolutionType(fcr.Resolution), + Resolution: res, + }) + } + + for _, attestation := range txn.Attestations { + result.Attestations = append(result.Attestations, attestation) + + var ha chain.V2HostAnnouncement + if ha.FromAttestation(attestation) == nil { + result.HostAnnouncements = append(result.HostAnnouncements, V2HostAnnouncement{ + V2HostAnnouncement: ha, + PublicKey: attestation.PublicKey, + }) + } + } + for _, arb := range txn.ArbitraryData { + result.ArbitraryData = append(result.ArbitraryData, arb) + } + result.NewFoundationAddress = txn.NewFoundationAddress + result.MinerFee = txn.MinerFee + + return } // AppliedEvents extracts a list of relevant events from a chain update. -func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { - var events []Event - addEvent := func(id types.Hash256, maturityHeight uint64, v eventData, relevant []types.Address) { +func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) (events []Event) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v EventData, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] @@ -183,296 +416,156 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f Index: cs.Index, MaturityHeight: maturityHeight, Relevant: unique, + Type: eventType, Data: v, }) } - anythingRelevant := func() (ok bool) { - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if ok || relevant(sce.SiacoinOutput.Address) { - ok = true - } - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if ok || relevant(sfe.SiafundOutput.Address) { - ok = true - } - }) - return - }() - if !anythingRelevant { - return nil - } - // collect all elements sces := make(map[types.SiacoinOutputID]types.SiacoinElement) sfes := make(map[types.SiafundOutputID]types.SiafundElement) - fces := make(map[types.FileContractID]types.FileContractElement) - v2fces := make(map[types.FileContractID]types.V2FileContractElement) - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - sce.MerkleProof = nil + for _, diff := range cu.SiacoinElementDiffs() { + sce := diff.SiacoinElement + sce.StateElement.MerkleProof = nil sces[types.SiacoinOutputID(sce.ID)] = sce - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - sfe.MerkleProof = nil - sfes[types.SiafundOutputID(sfe.ID)] = sfe - }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - fce.MerkleProof = nil - fces[types.FileContractID(fce.ID)] = fce - }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - fce.MerkleProof = nil - v2fces[types.FileContractID(fce.ID)] = fce - }) - - relevantTxn := func(txn types.Transaction) (addrs []types.Address) { - for _, sci := range txn.SiacoinInputs { - if sce := sces[sci.ParentID]; relevant(sce.SiacoinOutput.Address) { - addrs = append(addrs, sce.SiacoinOutput.Address) - } - } - for _, sco := range txn.SiacoinOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) - } - } - for _, sfi := range txn.SiafundInputs { - if sfe := sfes[sfi.ParentID]; relevant(sfe.SiafundOutput.Address) { - addrs = append(addrs, sfe.SiafundOutput.Address) - } - } - for _, sfo := range txn.SiafundOutputs { - if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) - } - } - return } - - relevantV2Txn := func(txn types.V2Transaction) (addrs []types.Address) { - for _, sci := range txn.SiacoinInputs { - if relevant(sci.Parent.SiacoinOutput.Address) { - addrs = append(addrs, sci.Parent.SiacoinOutput.Address) - } - } - for _, sco := range txn.SiacoinOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) - } - } - for _, sfi := range txn.SiafundInputs { - if relevant(sfi.Parent.SiafundOutput.Address) { - addrs = append(addrs, sfi.Parent.SiafundOutput.Address) - } - } - for _, sfo := range txn.SiafundOutputs { - if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) - } - } - return + for _, diff := range cu.SiafundElementDiffs() { + sfe := diff.SiafundElement + sfe.StateElement.MerkleProof = nil + sfes[types.SiafundOutputID(sfe.ID)] = sfe } // handle v1 transactions for _, txn := range b.Transactions { - relevant := relevantTxn(txn) - if len(relevant) == 0 { - continue - } - - e := &EventTransaction{ - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]EventSiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), - } - - for i := range txn.SiacoinInputs { - e.SiacoinInputs[i] = sces[txn.SiacoinInputs[i].ParentID] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(i)] - } - for i := range txn.SiafundInputs { - e.SiafundInputs[i] = EventSiafundInput{ - SiafundElement: sfes[txn.SiafundInputs[i].ParentID], - ClaimElement: sces[txn.SiafundClaimOutputID(i)], - } - } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(i)] - } - addContract := func(id types.FileContractID) *EventFileContract { - for i := range e.FileContracts { - if types.FileContractID(e.FileContracts[i].FileContract.ID) == id { - return &e.FileContracts[i] - } - } - e.FileContracts = append(e.FileContracts, EventFileContract{FileContract: fces[id]}) - return &e.FileContracts[len(e.FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.FileContractID(i)) - } - for i := range txn.FileContractRevisions { - fc := addContract(txn.FileContractRevisions[i].ParentID) - rev := txn.FileContractRevisions[i].FileContract - fc.Revision = &rev - } - for i := range txn.StorageProofs { - fc := addContract(txn.StorageProofs[i].ParentID) - fc.ValidOutputs = make([]types.SiacoinElement, len(fc.FileContract.FileContract.ValidProofOutputs)) - for i := range fc.ValidOutputs { - fc.ValidOutputs[i] = sces[types.FileContractID(fc.FileContract.ID).ValidOutputID(i)] - } - } - for _, arb := range txn.ArbitraryData { - var prefix types.Specifier - var uk types.UnlockKey - d := types.NewBufDecoder(arb) - prefix.DecodeFrom(d) - netAddress := d.ReadString() - uk.DecodeFrom(d) - if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") && - uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: *(*types.PublicKey)(uk.Key), - NetAddress: netAddress, - }) + for _, sfi := range txn.SiafundInputs { + sce, ok := sces[sfi.ParentID.ClaimOutputID()] + if ok { + addEvent(types.Hash256(sce.ID), sce.MaturityHeight, wallet.EventTypeSiafundClaim, EventPayout{ + SiacoinElement: SiacoinOutput{SiacoinElement: sce}, + }, []types.Address{sfi.ClaimAddress}) } } - for i := range txn.MinerFees { - e.Fee = e.Fee.Add(txn.MinerFees[i]) - } - addEvent(types.Hash256(txn.ID()), cs.Index.Height, e, relevant) // transaction maturity height is the current block height + relevant := RelevantAddressesV1(txn) + ev := EventV1Transaction{CoreToExplorerV1Transaction(txn)} + + addEvent(types.Hash256(txn.ID()), cs.Index.Height, wallet.EventTypeV1Transaction, ev, relevant) // transaction maturity height is the current block height } // handle v2 transactions for _, txn := range b.V2Transactions() { - relevant := relevantV2Txn(txn) - if len(relevant) == 0 { - continue - } - - txid := txn.ID() - e := &EventTransaction{ - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]EventSiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), - } - for i := range txn.SiacoinInputs { - // NOTE: here (and elsewhere), we fetch the element from our maps, - // rather than using the parent directly, because our copy has its - // Merkle proof nil'd out - e.SiacoinInputs[i] = sces[types.SiacoinOutputID(txn.SiacoinInputs[i].Parent.ID)] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(txid, i)] - } - for i := range txn.SiafundInputs { - sfoid := types.SiafundOutputID(txn.SiafundInputs[i].Parent.ID) - e.SiafundInputs[i] = EventSiafundInput{ - SiafundElement: sfes[sfoid], - ClaimElement: sces[sfoid.ClaimOutputID()], - } - } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(txid, i)] - } - addContract := func(id types.FileContractID) *EventV2FileContract { - for i := range e.V2FileContracts { - if types.FileContractID(e.V2FileContracts[i].FileContract.ID) == id { - return &e.V2FileContracts[i] - } - } - e.V2FileContracts = append(e.V2FileContracts, EventV2FileContract{FileContract: v2fces[id]}) - return &e.V2FileContracts[len(e.V2FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.V2FileContractID(txid, i)) - } - for _, fcr := range txn.FileContractRevisions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Revision = &fcr.Revision - } - for _, fcr := range txn.FileContractResolutions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Resolution = fcr.Resolution - fc.Outputs = []types.SiacoinElement{ - sces[types.FileContractID(fcr.Parent.ID).V2RenterOutputID()], - sces[types.FileContractID(fcr.Parent.ID).V2HostOutputID()], - } - } - for _, a := range txn.Attestations { - if a.Key == "HostAnnouncement" { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: a.PublicKey, - NetAddress: string(a.Value), - }) + for _, sfi := range txn.SiafundInputs { + sfe, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] + if ok { + addEvent(types.Hash256(sfe.ID), sfe.MaturityHeight, wallet.EventTypeSiafundClaim, EventPayout{ + SiacoinElement: SiacoinOutput{SiacoinElement: sfe}, + }, []types.Address{sfi.ClaimAddress}) } } - e.Fee = txn.MinerFee - addEvent(types.Hash256(txid), cs.Index.Height, e, relevant) // transaction maturity height is the current block height + relevant := RelevantAddressesV2(txn) + ev := EventV2Transaction(CoreToExplorerV2Transaction(txn)) + addEvent(types.Hash256(txn.ID()), cs.Index.Height, wallet.EventTypeV2Transaction, ev, relevant) // transaction maturity height is the current block height } - // handle missed contracts - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + // handle contracts + for _, diff := range cu.FileContractElementDiffs() { + fce, resolved, valid := diff.FileContractElement, diff.Resolved, diff.Valid if !resolved { return } + fce.StateElement.MerkleProof = nil + + var mpos, vpos []ContractSiacoinOutput + for _, mpo := range fce.FileContract.MissedProofOutputs { + mpos = append(mpos, ContractSiacoinOutput{SiacoinOutput: mpo}) + } + for _, vpo := range fce.FileContract.ValidProofOutputs { + vpos = append(vpos, ContractSiacoinOutput{SiacoinOutput: vpo}) + } + efc := ExtendedFileContract{ + ID: fce.ID, + Filesize: fce.FileContract.Filesize, + FileMerkleRoot: fce.FileContract.FileMerkleRoot, + WindowStart: fce.FileContract.WindowStart, + WindowEnd: fce.FileContract.WindowEnd, + Payout: fce.FileContract.Payout, + ValidProofOutputs: vpos, + MissedProofOutputs: mpos, + UnlockHash: fce.FileContract.UnlockHash, + RevisionNumber: fce.FileContract.RevisionNumber, + } + if valid { for i := range fce.FileContract.ValidProofOutputs { - if !relevant(fce.FileContract.ValidProofOutputs[i].Address) { - continue - } - - outputID := types.FileContractID(fce.ID).ValidOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ - FileContract: fce, - SiacoinOutput: sces[outputID], - Missed: false, - }, []types.Address{fce.FileContract.ValidProofOutputs[i].Address}) + address := fce.FileContract.ValidProofOutputs[i].Address + element := sces[types.FileContractID(fce.ID).ValidOutputID(i)] + + addEvent(types.Hash256(element.ID), element.MaturityHeight, wallet.EventTypeV1ContractResolution, EventV1ContractResolution{ + Parent: efc, + SiacoinElement: SiacoinOutput{SiacoinElement: element}, + Missed: false, + }, []types.Address{address}) } } else { for i := range fce.FileContract.MissedProofOutputs { - if !relevant(fce.FileContract.MissedProofOutputs[i].Address) { - continue - } - - outputID := types.FileContractID(fce.ID).MissedOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ - FileContract: fce, - SiacoinOutput: sces[outputID], - Missed: true, - }, []types.Address{fce.FileContract.MissedProofOutputs[i].Address}) + address := fce.FileContract.MissedProofOutputs[i].Address + element := sces[types.FileContractID(fce.ID).MissedOutputID(i)] + + addEvent(types.Hash256(element.ID), element.MaturityHeight, wallet.EventTypeV1ContractResolution, EventV1ContractResolution{ + Parent: efc, + SiacoinElement: SiacoinOutput{SiacoinElement: element}, + Missed: true, + }, []types.Address{address}) } } - }) + } + + for _, diff := range cu.V2FileContractElementDiffs() { + fce, res := diff.V2FileContractElement, diff.Resolution + if res == nil { + return + } + + fce.StateElement.MerkleProof = nil + + var missed bool + if _, ok := res.(*types.V2FileContractExpiration); ok { + missed = true + } + + resolutionType := V2ResolutionType(res) + addV2Resolution := func(element types.SiacoinElement) { + efc := V2FileContract{V2FileContractElement: fce} + addEvent(types.Hash256(element.ID), element.MaturityHeight, wallet.EventTypeV2ContractResolution, EventV2ContractResolution{ + Resolution: V2FileContractResolution{ + Parent: efc, + Type: resolutionType, + Resolution: res, + }, + SiacoinElement: SiacoinOutput{SiacoinElement: element}, + Missed: missed, + }, []types.Address{element.SiacoinOutput.Address}) + } + addV2Resolution(sces[types.FileContractID(fce.ID).V2RenterOutputID()]) + addV2Resolution(sces[types.FileContractID(fce.ID).V2HostOutputID()]) + } // handle block rewards for i := range b.MinerPayouts { - if relevant(b.MinerPayouts[i].Address) { - outputID := cs.Index.ID.MinerOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventMinerPayout{ - SiacoinOutput: sces[outputID], - }, []types.Address{b.MinerPayouts[i].Address}) - } + element := sces[cs.Index.ID.MinerOutputID(i)] + addEvent(types.Hash256(element.ID), element.MaturityHeight, wallet.EventTypeMinerPayout, EventPayout{ + SiacoinElement: SiacoinOutput{SiacoinElement: element}, + }, []types.Address{b.MinerPayouts[i].Address}) } // handle foundation subsidy - if relevant(cs.FoundationPrimaryAddress) { - outputID := cs.Index.ID.FoundationOutputID() - sce, ok := sces[outputID] - if ok { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventFoundationSubsidy{ - SiacoinOutput: sce, - }, []types.Address{cs.FoundationPrimaryAddress}) - } + element, ok := sces[cs.Index.ID.FoundationOutputID()] + if ok { + addEvent(types.Hash256(element.ID), element.MaturityHeight, wallet.EventTypeFoundationSubsidy, EventPayout{ + SiacoinElement: SiacoinOutput{SiacoinElement: element}, + }, []types.Address{element.SiacoinOutput.Address}) } return events diff --git a/explorer/events_test.go b/explorer/events_test.go new file mode 100644 index 0000000..4e722fb --- /dev/null +++ b/explorer/events_test.go @@ -0,0 +1,132 @@ +package explorer_test + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/wallet" + "go.sia.tech/explored/explorer" +) + +func TestEventMarshalling(t *testing.T) { + id := types.Hash256{31: 255} + index := types.ChainIndex{Height: 10} + timestamp := time.Now() + address := types.Address{31: 255} + + sco := explorer.SiacoinOutput{ + SiacoinElement: types.SiacoinElement{ + SiacoinOutput: types.SiacoinOutput{ + Address: address, + Value: types.Siacoins(1), + }, + }, + } + + events := []explorer.Event{ + { + ID: id, + Index: index, + Confirmations: 10, + Type: wallet.EventTypeMinerPayout, + Data: explorer.EventPayout{ + SiacoinElement: sco, + }, + MaturityHeight: 100, + Timestamp: timestamp, + Relevant: []types.Address{address}, + }, + { + ID: id, + Index: index, + Confirmations: 20, + Type: wallet.EventTypeV1Transaction, + Data: explorer.EventV1Transaction{ + Transaction: explorer.Transaction{ + SiacoinOutputs: []explorer.SiacoinOutput{sco}, + }, + }, + MaturityHeight: 200, + Timestamp: timestamp, + Relevant: []types.Address{address}, + }, + { + ID: id, + Index: index, + Confirmations: 30, + Type: wallet.EventTypeV1ContractResolution, + Data: explorer.EventV1ContractResolution{ + SiacoinElement: sco, + Missed: true, + }, + MaturityHeight: 300, + Timestamp: timestamp, + Relevant: []types.Address{address}, + }, + { + ID: id, + Index: index, + Confirmations: 40, + Type: wallet.EventTypeV2ContractResolution, + Data: explorer.EventV2ContractResolution{ + SiacoinElement: sco, + Missed: true, + }, + MaturityHeight: 400, + Timestamp: timestamp, + Relevant: []types.Address{address}, + }, + { + ID: id, + Index: index, + Confirmations: 50, + Type: wallet.EventTypeV2Transaction, + Data: explorer.EventV2Transaction(explorer.V2Transaction{ + SiacoinOutputs: []explorer.SiacoinOutput{sco}, + }), + MaturityHeight: 500, + Timestamp: timestamp, + Relevant: []types.Address{address}, + }, + } + + for _, event := range events { + data, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + + var unmarshalled explorer.Event + if err := json.Unmarshal(data, &unmarshalled); err != nil { + t.Fatal(err) + } + + if event.ID != unmarshalled.ID { + t.Errorf("ID: expected %v, got %v", event.ID, unmarshalled.ID) + } + if event.Index != unmarshalled.Index { + t.Errorf("Index: expected %v, got %v", event.Index, unmarshalled.Index) + } + if event.Confirmations != unmarshalled.Confirmations { + t.Errorf("Confirmations: expected %d, got %d", event.Confirmations, unmarshalled.Confirmations) + } + if event.Type != unmarshalled.Type { + t.Errorf("Type: expected %s, got %s", event.Type, unmarshalled.Type) + } + if event.MaturityHeight != unmarshalled.MaturityHeight { + t.Errorf("MaturityHeight: expected %d, got %d", event.MaturityHeight, unmarshalled.MaturityHeight) + } + if !event.Timestamp.Equal(unmarshalled.Timestamp) { + t.Errorf("Timestamp: expected %v, got %v", event.Timestamp, unmarshalled.Timestamp) + } + if !reflect.DeepEqual(event.Relevant, unmarshalled.Relevant) { + t.Errorf("Relevant: expected %v, got %v", event.Relevant, unmarshalled.Relevant) + } + if !reflect.DeepEqual(event.Data, unmarshalled.Data) { + t.Errorf("Data: expected %v, got %v", event.Data, unmarshalled.Data) + } + } +} diff --git a/explorer/explorer.go b/explorer/explorer.go index fbabeb8..1e15935 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -1,25 +1,55 @@ package explorer import ( + "context" "errors" "fmt" + "math" + "strings" "sync" + "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/config" + "go.sia.tech/explored/geoip" "go.uber.org/zap" ) var ( - // ErrNoTip is returned when Tip() is unable to find any blocks in the - // database and thus there is no tip. It does not mean there was an - // error in the underlying database. + // ErrNoTip is returned when we are unable to find the tip in the + // database or there is no tips at all. ErrNoTip = errors.New("no tip found") + + // ErrNoBlock is returned when we are unable to find the block in the + // database. + ErrNoBlock = errors.New("block not found") + + // ErrContractNotFound is returned when ContractRevisions is unable to find + // the specified contract ID. + ErrContractNotFound = errors.New("contract not found") + + // ErrSearchParse is returned when Search is unable to parse the specified + // ID. + ErrSearchParse = errors.New("error parsing ID") + + // ErrNoSearchResults is returned when Search is unable to find anything + // with the specified ID. + ErrNoSearchResults = errors.New("no search results") + + // ErrNoSortColumn is returned when a host query requests that we sort by a + // column that does not exist. + ErrNoSortColumn = errors.New("no such sort column") ) // A ChainManager manages the consensus state type ChainManager interface { + PoolTransactions() []types.Transaction + V2PoolTransactions() []types.V2Transaction + Tip() types.ChainIndex + TipState() consensus.State BestIndex(height uint64) (types.ChainIndex, bool) OnReorg(fn func(types.ChainIndex)) (cancel func()) @@ -29,81 +59,148 @@ type ChainManager interface { // A Store is a database that stores information about elements, contracts, // and blocks. type Store interface { + Close() error + + ResetChainState() error UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error + AddHostScans(scans ...HostScan) error Tip() (types.ChainIndex, error) Block(id types.BlockID) (Block, error) BestTip(height uint64) (types.ChainIndex, error) MerkleProof(leafIndex uint64) ([]types.Hash256, error) + Metrics(id types.BlockID) (Metrics, error) + HostMetrics() (HostMetrics, error) Transactions(ids []types.TransactionID) ([]Transaction, error) - AddressTransactions(addr types.Address, limit, offset uint64) (results []types.TransactionID, err error) - UnspentSiacoinOutputs(address types.Address, limit, offset uint64) ([]SiacoinOutput, error) - UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]SiafundOutput, error) + TransactionChainIndices(txid types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) + V2Transactions(ids []types.TransactionID) ([]V2Transaction, error) + V2TransactionChainIndices(txid types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) + UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]SiacoinOutput, error) + UnspentSiafundOutputs(address types.Address, offset, limit uint64) ([]SiafundOutput, error) + UnconfirmedEvents(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []Event, err error) + AddressEvents(address types.Address, offset, limit uint64) (events []Event, err error) + Events([]types.Hash256) ([]Event, error) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) - Contracts(ids []types.FileContractID) (result []FileContract, err error) - AddressEvents(address types.Address, offset, limit int) (events []Event, err error) + Contracts(ids []types.FileContractID) (result []ExtendedFileContract, err error) + ContractsKey(key types.PublicKey) (result []ExtendedFileContract, err error) + ContractRevisions(id types.FileContractID) (result []ExtendedFileContract, err error) + V2Contracts(ids []types.FileContractID) (result []V2FileContract, err error) + V2ContractsKey(key types.PublicKey) (result []V2FileContract, err error) + V2ContractRevisions(id types.FileContractID) (result []V2FileContract, err error) + SiacoinElements(ids []types.SiacoinOutputID) (result []SiacoinOutput, err error) + SiafundElements(ids []types.SiafundOutputID) (result []SiafundOutput, err error) + Search(id string) (SearchType, error) + + QueryHosts(params HostQuery, sortBy HostSortColumn, dir HostSortDir, offset, limit uint64) ([]Host, error) + HostsForScanning(minLastAnnouncement time.Time, limit uint64) ([]UnscannedHost, error) } // Explorer implements a Sia explorer. type Explorer struct { s Store - mu sync.Mutex + cm ChainManager + + scanCfg config.Scanner + locator geoip.Locator + + log *zap.Logger + + wg sync.WaitGroup + ctx context.Context + ctxCancel context.CancelFunc unsubscribe func() } -func syncStore(store Store, cm ChainManager, index types.ChainIndex) error { - for index != cm.Tip() { - crus, caus, err := cm.UpdatesSince(index, 1000) - if err != nil { - return fmt.Errorf("failed to subscribe to chain manager: %w", err) - } +func (e *Explorer) syncStore(index types.ChainIndex, batchSize int) error { + for index != e.cm.Tip() { + select { + case <-e.ctx.Done(): + return e.ctx.Err() + default: + crus, caus, err := e.cm.UpdatesSince(index, batchSize) + if err != nil { + return fmt.Errorf("failed to subscribe to chain manager: %w", err) + } - if err := store.UpdateChainState(crus, caus); err != nil { - return fmt.Errorf("failed to process updates: %w", err) - } - if len(crus) > 0 { - index = crus[len(crus)-1].State.Index - } - if len(caus) > 0 { - index = caus[len(caus)-1].State.Index + if err := e.s.UpdateChainState(crus, caus); err != nil { + return fmt.Errorf("failed to process updates: %w", err) + } + if len(crus) > 0 { + index = crus[len(crus)-1].State.Index + } + if len(caus) > 0 { + index = caus[len(caus)-1].State.Index + } } } return nil } // NewExplorer returns a Sia explorer. -func NewExplorer(cm ChainManager, store Store, log *zap.Logger) (*Explorer, error) { - e := &Explorer{s: store} - - tip, err := store.Tip() - if errors.Is(err, ErrNoTip) { - tip = types.ChainIndex{} - } else if err != nil { - return nil, fmt.Errorf("failed to get tip: %w", err) +func NewExplorer(cm ChainManager, store Store, indexCfg config.Index, scanCfg config.Scanner, log *zap.Logger) (*Explorer, error) { + ctx, ctxCancel := context.WithCancel(context.Background()) + e := &Explorer{ + s: store, + cm: cm, + scanCfg: scanCfg, + ctx: ctx, + ctxCancel: ctxCancel, + log: log, } - if err := syncStore(store, cm, tip); err != nil { - return nil, fmt.Errorf("failed to subscribe to chain manager: %w", err) + + locator, err := geoip.NewMaxMindLocator("") + if err != nil { + e.log.Info("failed to create geoip database:", zap.Error(err)) + return nil, err + } + e.locator = locator + + // add the genesis block if we do not have a tip + if _, err := e.s.Tip(); errors.Is(err, ErrNoTip) { + crus, caus, err := e.cm.UpdatesSince(types.ChainIndex{}, 1) + if err != nil { + return nil, fmt.Errorf("failed to get genesis block update: %w", err) + } + if err := e.s.UpdateChainState(crus, caus); err != nil { + return nil, fmt.Errorf("failed to process genesis block updates: %w", err) + } } reorgChan := make(chan types.ChainIndex, 1) + // get loop to start syncing immediately + reorgChan <- types.ChainIndex{} go func() { for range reorgChan { - e.mu.Lock() - lastTip, err := store.Tip() + lastTip, err := e.s.Tip() if errors.Is(err, ErrNoTip) { lastTip = types.ChainIndex{} } else if err != nil { - log.Error("failed to get tip", zap.Error(err)) + e.log.Error("failed to get tip", zap.Error(err)) } - if err := syncStore(store, cm, lastTip); err != nil { - log.Error("failed to sync store", zap.Error(err)) + if err := e.syncStore(lastTip, indexCfg.BatchSize); err != nil { + switch { + case errors.Is(err, context.Canceled): + return + case strings.Contains(err.Error(), "missing block at index"): + log.Warn("missing block at index, resetting chain state", zap.Stringer("id", lastTip.ID), zap.Uint64("height", lastTip.Height)) + if err := e.s.ResetChainState(); err != nil { + log.Panic("failed to reset explorer state", zap.Error(err)) + } + // trigger resync + select { + case reorgChan <- types.ChainIndex{}: + default: + } + default: + e.log.Panic("failed to sync store", zap.Error(err)) + } } - e.mu.Unlock() } }() + go e.scanLoop() - e.unsubscribe = cm.OnReorg(func(index types.ChainIndex) { + e.unsubscribe = e.cm.OnReorg(func(index types.ChainIndex) { select { case reorgChan <- index: default: @@ -112,6 +209,26 @@ func NewExplorer(cm ChainManager, store Store, log *zap.Logger) (*Explorer, erro return e, nil } +// Shutdown tries to close the scanning goroutines in the explorer. +func (e *Explorer) Shutdown(ctx context.Context) error { + e.ctxCancel() + e.locator.Close() + + done := make(chan struct{}) + go func() { + e.wg.Wait() + close(done) + }() + + // Wait for the WaitGroup to finish or the context to be cancelled + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // Tip returns the tip of the best known valid chain. func (e *Explorer) Tip() (types.ChainIndex, error) { return e.s.Tip() @@ -132,14 +249,38 @@ func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) { return e.s.MerkleProof(leafIndex) } +// Metrics returns various metrics about Sia. +func (e *Explorer) Metrics(id types.BlockID) (Metrics, error) { + return e.s.Metrics(id) +} + +// HostMetrics returns various metrics about currently available hosts. +func (e *Explorer) HostMetrics() (HostMetrics, error) { + return e.s.HostMetrics() +} + // Transactions returns the transactions with the specified IDs. func (e *Explorer) Transactions(ids []types.TransactionID) ([]Transaction, error) { return e.s.Transactions(ids) } -// AddressTransactions returns the transactions involving the address. -func (e *Explorer) AddressTransactions(addr types.Address, limit, offset uint64) (results []types.TransactionID, err error) { - return e.s.AddressTransactions(addr, limit, offset) +// TransactionChainIndices returns the chain indices of the blocks the transaction +// was included in. If the transaction has not been included in any blocks, the +// result will be nil,nil. +func (e *Explorer) TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) { + return e.s.TransactionChainIndices(id, offset, limit) +} + +// V2Transactions returns the v2 transactions with the specified IDs. +func (e *Explorer) V2Transactions(ids []types.TransactionID) ([]V2Transaction, error) { + return e.s.V2Transactions(ids) +} + +// V2TransactionChainIndices returns the chain indices of the blocks the +// transaction was included in. If the transaction has not been included in +// any blocks, the result will be nil,nil. +func (e *Explorer) V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) { + return e.s.V2TransactionChainIndices(id, offset, limit) } // UnspentSiacoinOutputs returns the unspent siacoin outputs owned by the @@ -154,17 +295,218 @@ func (e *Explorer) UnspentSiafundOutputs(address types.Address, offset, limit ui return e.s.UnspentSiafundOutputs(address, offset, limit) } +// AddressUnconfirmedEvents returns the unconfirmed events for a single address. +func (e *Explorer) AddressUnconfirmedEvents(address types.Address) ([]Event, error) { + relevantV1Txn := func(txn types.Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + return false + } + relevantV2Txn := func(txn types.V2Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.Parent.SiacoinOutput.Address == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.Parent.SiafundOutput.Address == address { + return true + } + } + return false + } + + index := e.cm.Tip() + index.Height++ + index.ID = types.BlockID{} + timestamp := time.Now() + + v1, v2 := e.cm.PoolTransactions(), e.cm.V2PoolTransactions() + + relevantV1 := v1[:0] + for _, txn := range v1 { + if !relevantV1Txn(txn) { + continue + } + relevantV1 = append(relevantV1, txn) + } + + relevantV2 := v2[:0] + for _, txn := range v2 { + if !relevantV2Txn(txn) { + continue + } + relevantV2 = append(relevantV2, txn) + } + + events, err := e.s.UnconfirmedEvents(index, timestamp, relevantV1, relevantV2) + if err != nil { + return nil, err + } + for i := range events { + events[i].Relevant = []types.Address{address} + } + return events, nil +} + +// UnconfirmedEvents annotates a list of unconfirmed transactions. +func (e *Explorer) UnconfirmedEvents(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) ([]Event, error) { + return e.s.UnconfirmedEvents(index, timestamp, v1, v2) +} + // AddressEvents returns the events of a single address. -func (e *Explorer) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) { +func (e *Explorer) AddressEvents(address types.Address, offset, limit uint64) (events []Event, err error) { return e.s.AddressEvents(address, offset, limit) } +// Events returns the events with the specified IDs. +func (e *Explorer) Events(ids []types.Hash256) ([]Event, error) { + return e.s.Events(ids) +} + // Balance returns the balance of an address. func (e *Explorer) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) { return e.s.Balance(address) } // Contracts returns the contracts with the specified IDs. -func (e *Explorer) Contracts(ids []types.FileContractID) (result []FileContract, err error) { +func (e *Explorer) Contracts(ids []types.FileContractID) (result []ExtendedFileContract, err error) { return e.s.Contracts(ids) } + +// ContractsKey returns the contracts for a particular ed25519 key. +func (e *Explorer) ContractsKey(key types.PublicKey) (result []ExtendedFileContract, err error) { + return e.s.ContractsKey(key) +} + +// ContractRevisions returns all the revisions of the contract with the +// specified ID. +func (e *Explorer) ContractRevisions(id types.FileContractID) (result []ExtendedFileContract, err error) { + return e.s.ContractRevisions(id) +} + +// V2Contracts returns the v2 contracts with the specified IDs. +func (e *Explorer) V2Contracts(ids []types.FileContractID) (result []V2FileContract, err error) { + return e.s.V2Contracts(ids) +} + +// V2ContractsKey returns the v2 contracts for a particular ed25519 key. +func (e *Explorer) V2ContractsKey(key types.PublicKey) (result []V2FileContract, err error) { + return e.s.V2ContractsKey(key) +} + +// V2ContractRevisions returns all the revisions of the v2 contract with the +// specified ID. +func (e *Explorer) V2ContractRevisions(id types.FileContractID) (result []V2FileContract, err error) { + return e.s.V2ContractRevisions(id) +} + +// SiacoinElements returns the siacoin elements with the specified IDs. +func (e *Explorer) SiacoinElements(ids []types.SiacoinOutputID) (result []SiacoinOutput, err error) { + return e.s.SiacoinElements(ids) +} + +// SiafundElements returns the siafund elements with the specified IDs. +func (e *Explorer) SiafundElements(ids []types.SiafundOutputID) (result []SiafundOutput, err error) { + return e.s.SiafundElements(ids) +} + +// Hosts returns the hosts with the specified public keys. +func (e *Explorer) Hosts(pks []types.PublicKey) ([]Host, error) { + return e.s.QueryHosts(HostQuery{PublicKeys: pks}, HostSortPublicKey, HostSortAsc, 0, math.MaxInt64) +} + +// QueryHosts returns the hosts matching the query parameters in the order +// specified by dir. +func (e *Explorer) QueryHosts(params HostQuery, sortBy HostSortColumn, dir HostSortDir, offset, limit uint64) ([]Host, error) { + return e.s.QueryHosts(params, sortBy, dir, offset, limit) +} + +// ScanHosts synchronously scans the provided host(s) and returns the resultant +// scan details. The errors encountered during scanner are contained in the +// HostScan.Error field. Errors retrieving hosts' net addresses from the +// database or writing the scans to the database will make the returned error +// value not equal to nil. +func (e *Explorer) ScanHosts(pks ...types.PublicKey) ([]HostScan, error) { + hosts, err := e.Hosts(pks) + if err != nil { + return nil, fmt.Errorf("failed to retrieve host: %w", err) + } else if len(hosts) == 0 { + return nil, fmt.Errorf("could not find any host with those pubkey(s)") + } + + scans := make([]HostScan, len(hosts)) + for i, host := range hosts { + unscannedHost := UnscannedHost{ + PublicKey: host.PublicKey, + V2: host.V2, + NetAddress: host.NetAddress, + V2NetAddresses: host.V2NetAddresses, + } + + if host.V2 { + scans[i], err = e.scanV2Host(unscannedHost) + } else { + scans[i], err = e.scanV1Host(unscannedHost) + } + + now := types.CurrentTimestamp() + if err != nil { + e.log.Debug("manual host scan failed", zap.Stringer("pk", host.PublicKey), zap.Error(err)) + scans[i] = HostScan{ + PublicKey: host.PublicKey, + Success: false, + Timestamp: now, + Error: func() *string { + str := err.Error() + return &str + }(), + } + } else { + e.log.Debug("manual host scan succeeded", zap.Stringer("pk", host.PublicKey)) + } + // We don't apply the exponential delay penalty to manually scanned hosts. + // Given that this would mostly be used by someone setting up or + // configuring their host, it seems wrong to use it here. + scans[i].NextScan = now.Add(e.scanCfg.ScanInterval) + } + + if err := e.s.AddHostScans(scans...); err != nil { + return nil, fmt.Errorf("failed to add host scans to DB: %w", err) + } + return scans, nil +} + +// Search returns the type of an element (siacoin element, siafund element, +// contract, v2 contract, transaction, v2 transaction, block, or host). +func (e *Explorer) Search(id string) (SearchType, error) { + return e.s.Search(id) +} diff --git a/explorer/explorer_test.go b/explorer/explorer_test.go new file mode 100644 index 0000000..2f81279 --- /dev/null +++ b/explorer/explorer_test.go @@ -0,0 +1,90 @@ +package explorer_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/explored/config" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" + "go.sia.tech/explored/persist/sqlite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +func TestChainMigration(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + + n, genesis := ctestutil.Network() + store, genesisState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis, chain.NewZapMigrationLogger(zap.NewNop())) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + b := testutil.MineBlock(cm.TipState(), nil, types.VoidAddress) + cs, au := consensus.ApplyBlock(cm.TipState(), b, consensus.V1BlockSupplement{}, time.Time{}) + + // add block to explorer store independent of chain manager + err = db.UpdateChainState(nil, []chain.ApplyUpdate{{ + ApplyUpdate: au, + State: cs, + Block: b, + }}) + if err != nil { + t.Fatal(err) + } + + explorerTip, err := db.Tip() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "explorer tip", cs.Index, explorerTip) + testutil.Equal(t, "cm tip", genesisState.Index, cm.Tip()) + + e, err := explorer.NewExplorer(cm, db, config.Index{BatchSize: 1000}, config.Scanner{}, log) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 1*time.Second) + defer timeoutCancel() + e.Shutdown(timeoutCtx) + }) + + time.Sleep(1 * time.Second) + + // the fact that the explorer has a block not contained in the chain + // manager's store should cause us to reset the state and reindex from + // scratch + + explorerTip, err = db.Tip() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "explorer tip", cm.Tip(), explorerTip) + testutil.Equal(t, "cm tip", genesisState.Index, cm.Tip()) + + // check that data is indexed in DB + for _, expected := range genesis.Transactions { + txns, err := db.Transactions([]types.TransactionID{expected.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 1, len(txns)) + testutil.CheckTransaction(t, expected, txns[0]) + } +} diff --git a/explorer/scan.go b/explorer/scan.go new file mode 100644 index 0000000..4622095 --- /dev/null +++ b/explorer/scan.go @@ -0,0 +1,270 @@ +package explorer + +import ( + "context" + "fmt" + "math" + "net" + "time" + + crhpv2 "go.sia.tech/core/rhp/v2" + crhpv3 "go.sia.tech/core/rhp/v3" + "go.sia.tech/core/types" + crhpv4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/coreutils/rhp/v4/siamux" + rhpv2 "go.sia.tech/explored/internal/rhp/v2" + rhpv3 "go.sia.tech/explored/internal/rhp/v3" + "go.uber.org/zap" +) + +func isSynced(b Block) bool { + return time.Since(b.Timestamp) <= 3*time.Hour +} + +func (e *Explorer) waitForSync() error { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + cs, err := e.Tip() + if err != nil { + e.log.Debug("Couldn't get tip, waiting", zap.Error(err)) + } else { + b, err := e.Block(cs.ID) + if err != nil { + return err + } else if isSynced(b) { + break + } + } + + select { + case <-e.ctx.Done(): + return e.ctx.Err() + case <-ticker.C: + continue + } + } + + return nil +} + +func rhpv2Settings(ctx context.Context, publicKey types.PublicKey, netAddress string) (crhpv2.HostSettings, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", netAddress) + if err != nil { + return crhpv2.HostSettings{}, fmt.Errorf("failed to connect to host: %w", err) + } + defer conn.Close() + + // default timeout if context doesn't have one + deadline := time.Now().Add(30 * time.Second) + if dl, ok := ctx.Deadline(); ok && !dl.IsZero() { + deadline = dl + } + if err := conn.SetDeadline(deadline); err != nil { + return crhpv2.HostSettings{}, fmt.Errorf("failed to set deadline: %w", err) + } + + t, err := crhpv2.NewRenterTransport(conn, publicKey) + if err != nil { + return crhpv2.HostSettings{}, fmt.Errorf("failed to establish rhpv2 transport: %w", err) + } + defer t.Close() + + settings, err := rhpv2.RPCSettings(ctx, t) + if err != nil { + return crhpv2.HostSettings{}, fmt.Errorf("failed to call settings RPC: %w", err) + } + return settings, nil +} + +func rhpv3PriceTable(ctx context.Context, publicKey types.PublicKey, netAddress string) (priceTable crhpv3.HostPriceTable, err error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", netAddress) + if err != nil { + return crhpv3.HostPriceTable{}, fmt.Errorf("failed to connect to siamux port: %w", err) + } + defer conn.Close() + + // default timeout if context doesn't have one + deadline := time.Now().Add(30 * time.Second) + if dl, ok := ctx.Deadline(); ok && !dl.IsZero() { + deadline = dl + } + if err := conn.SetDeadline(deadline); err != nil { + return crhpv3.HostPriceTable{}, fmt.Errorf("failed to set deadline: %w", err) + } + + v3Session, err := rhpv3.NewSession(ctx, conn, publicKey, nil, nil) + if err != nil { + return crhpv3.HostPriceTable{}, fmt.Errorf("failed to establish rhpv3 transport: %w", err) + } + defer v3Session.Close() + + table, err := v3Session.ScanPriceTable() + if err != nil { + return crhpv3.HostPriceTable{}, fmt.Errorf("failed to scan price table: %w", err) + } + return table, nil +} + +func (e *Explorer) scanV1Host(host UnscannedHost) (HostScan, error) { + ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.ScanTimeout) + defer cancel() + + settings, err := rhpv2Settings(ctx, host.PublicKey, host.NetAddress) + if err != nil { + return HostScan{}, fmt.Errorf("scanV1Host: failed to get host settings: %w", err) + } + + hostIP, _, err := net.SplitHostPort(settings.NetAddress) + if err != nil { + return HostScan{}, fmt.Errorf("scanV1Host: failed to parse net address: %w", err) + } + + table, err := rhpv3PriceTable(ctx, host.PublicKey, net.JoinHostPort(hostIP, settings.SiaMuxPort)) + if err != nil { + return HostScan{}, fmt.Errorf("scanV1Host: failed to get price table: %w", err) + } + + resolved, err := net.ResolveIPAddr("ip", hostIP) + if err != nil { + return HostScan{}, fmt.Errorf("scanV1Host: failed to resolve host address: %w", err) + } + + location, err := e.locator.Locate(resolved) + if err != nil { + e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) + } + + return HostScan{ + PublicKey: host.PublicKey, + Location: location, + Success: true, + Timestamp: types.CurrentTimestamp(), + + Settings: settings, + PriceTable: table, + }, nil +} + +func (e *Explorer) scanV2Host(host UnscannedHost) (HostScan, error) { + ctx, cancel := context.WithTimeout(e.ctx, e.scanCfg.ScanTimeout) + defer cancel() + + addr, ok := host.V2SiamuxAddr() + if !ok { + return HostScan{}, fmt.Errorf("host has no v2 siamux address") + } + + transport, err := siamux.Dial(ctx, addr, host.PublicKey) + if err != nil { + return HostScan{}, fmt.Errorf("failed to dial host: %w", err) + } + defer transport.Close() + + settings, err := crhpv4.RPCSettings(ctx, transport) + if err != nil { + return HostScan{}, fmt.Errorf("failed to get host settings: %w", err) + } + + hostIP, _, err := net.SplitHostPort(addr) + if err != nil { + return HostScan{}, fmt.Errorf("scanHost: failed to parse net address: %w", err) + } + + resolved, err := net.ResolveIPAddr("ip", hostIP) + if err != nil { + return HostScan{}, fmt.Errorf("scanHost: failed to resolve host address: %w", err) + } + + location, err := e.locator.Locate(resolved) + if err != nil { + e.log.Debug("Failed to resolve IP geolocation, not setting country code", zap.String("addr", host.NetAddress)) + } + + return HostScan{ + PublicKey: host.PublicKey, + Location: location, + Success: true, + Timestamp: types.CurrentTimestamp(), + + V2Settings: settings, + }, nil +} + +func (e *Explorer) isClosed() bool { + select { + case <-e.ctx.Done(): + return true + default: + return false + } +} + +func (e *Explorer) scanLoop() { + e.log.Info("Waiting for syncing to complete before scanning hosts") + // don't scan hosts till we're at least nearly done with syncing + if err := e.waitForSync(); err != nil { + e.log.Info("Interrupted before scanning started:", zap.Error(err)) + return + } + e.log.Info("Syncing complete, will begin scanning hosts") + + for !e.isClosed() { + now := types.CurrentTimestamp() + lastAnnouncementCutoff := now.Add(-e.scanCfg.MinLastAnnouncement) + + batch, err := e.s.HostsForScanning(lastAnnouncementCutoff, e.scanCfg.NumThreads) + if err != nil { + e.log.Info("failed to get hosts for scanning:", zap.Error(err)) + return + } else if len(batch) == 0 { + select { + case <-e.ctx.Done(): + e.log.Debug("shutdown:", zap.Error(e.ctx.Err())) + return + // wait until we call HostsForScanning again + case <-time.After(e.scanCfg.ScanFrequency): + continue // check again + } + } + + results := make([]HostScan, len(batch)) + for i, host := range batch { + e.wg.Add(1) + go func(i int, host UnscannedHost) { + defer e.wg.Done() + + var err error + if host.IsV2() { + results[i], err = e.scanV2Host(host) + } else { + results[i], err = e.scanV1Host(host) + } + now := types.CurrentTimestamp() + if err != nil { + e.log.Debug("host scan failed", zap.Stringer("pk", host.PublicKey), zap.Error(err)) + results[i] = HostScan{ + PublicKey: host.PublicKey, + Success: false, + Error: func() *string { + str := err.Error() + return &str + }(), + Timestamp: now, + NextScan: now.Add(e.scanCfg.ScanInterval * time.Duration(math.Pow(2, float64(host.FailedInteractionsStreak)+1))), + } + return + } else { + results[i].NextScan = now.Add(e.scanCfg.ScanInterval) + } + }(i, host) + } + e.wg.Wait() + + if err := e.s.AddHostScans(results...); err != nil { + e.log.Info("failed to add host scans to DB:", zap.Error(err)) + return + } + } +} diff --git a/explorer/types.go b/explorer/types.go index 1b52340..707df01 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -3,9 +3,17 @@ package explorer import ( "encoding/json" "errors" + "fmt" "time" + "go.sia.tech/core/consensus" + rhpv2 "go.sia.tech/core/rhp/v2" + rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/rhp/v4/siamux" + "go.sia.tech/explored/geoip" ) // A Source represents where a siacoin output came from. @@ -27,81 +35,556 @@ const ( ) // MarshalJSON implements json.Marshaler. -func (d Source) MarshalJSON() ([]byte, error) { - switch d { - case SourceInvalid: - return json.Marshal("invalid") - case SourceMinerPayout: - return json.Marshal("miner_payout") - case SourceTransaction: - return json.Marshal("transaction") +func (s Source) MarshalJSON() ([]byte, error) { + sourceToString := map[Source]string{ + SourceInvalid: "invalid", + SourceMinerPayout: "miner_payout", + SourceTransaction: "transaction", + SourceValidProofOutput: "valid_proof_output", + SourceMissedProofOutput: "missed_proof_output", + } + + str, ok := sourceToString[s] + if !ok { + str = "invalid" // "invalid" if source is unknown + } + return json.Marshal(str) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (s *Source) UnmarshalJSON(data []byte) error { + stringToSource := map[string]Source{ + "invalid": SourceInvalid, + "miner_payout": SourceMinerPayout, + "transaction": SourceTransaction, + "valid_proof_output": SourceValidProofOutput, + "missed_proof_output": SourceMissedProofOutput, + } + + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + + source, ok := stringToSource[str] + if !ok { + return errors.New("invalid source type") + } + + *s = source + return nil +} + +// A SearchType represents the type of element found during a search. +type SearchType string + +const ( + // SearchTypeInvalid means we were unable to find any element with the + // given identifier. + SearchTypeInvalid SearchType = "invalid" + // SearchTypeAddress means we found an address with the given ID. + SearchTypeAddress SearchType = "address" + // SearchTypeBlock means we found a block with the given ID. + SearchTypeBlock SearchType = "block" + // SearchTypeTransaction means we found a transaction with the given ID. + SearchTypeTransaction SearchType = "transaction" + // SearchTypeV2Transaction means we found a v2 transaction with the given ID. + SearchTypeV2Transaction SearchType = "v2Transaction" + // SearchTypeSiacoinElement means we found a siacoin element with the given + // ID. + SearchTypeSiacoinElement SearchType = "siacoinElement" + // SearchTypeSiafundElement means we found a siafund element with the given + // ID. + SearchTypeSiafundElement SearchType = "siafundElement" + // SearchTypeContract means we found a contract with the given ID. + SearchTypeContract SearchType = "contract" + // SearchTypeV2Contract means we found a V2 contract with the given ID. + SearchTypeV2Contract SearchType = "v2Contract" + // SearchTypeHost means we found a host with the given pubkey. + SearchTypeHost SearchType = "host" +) + +// A V2Resolution represents the type of a v2 file contract resolution. +type V2Resolution int + +const ( + // V2ResolutionInvalid represents an invalid resolution type. + V2ResolutionInvalid V2Resolution = iota + // V2ResolutionRenewal represents a renewal. + V2ResolutionRenewal + // V2ResolutionStorageProof represents a storage proof. + V2ResolutionStorageProof + // V2ResolutionExpiration represents contract expiry without renewal or a + // storage proof being submitted. + V2ResolutionExpiration +) + +// MarshalJSON implements json.Marshaler. +func (s V2Resolution) MarshalJSON() ([]byte, error) { + sourceToV2Resolution := map[V2Resolution]string{ + V2ResolutionInvalid: "invalid", + V2ResolutionRenewal: "renewal", + V2ResolutionStorageProof: "storage_proof", + V2ResolutionExpiration: "expiration", + } + + str, ok := sourceToV2Resolution[s] + if !ok { + str = "invalid" // "invalid" if source is unknown + } + return json.Marshal(str) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (s *V2Resolution) UnmarshalJSON(data []byte) error { + stringToV2Resolution := map[string]V2Resolution{ + "invalid": V2ResolutionInvalid, + "renewal": V2ResolutionRenewal, + "storage_proof": V2ResolutionStorageProof, + "expiration": V2ResolutionExpiration, + } + + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + + source, ok := stringToV2Resolution[str] + if !ok { + return errors.New("invalid resolution type") + } + + *s = source + return nil +} + +// V2ResolutionType determines the V2Resolution enum value from a v2 file +// contract resolution. +func V2ResolutionType(res types.V2FileContractResolutionType) (result V2Resolution) { + switch res.(type) { + case *types.V2FileContractRenewal: + result = V2ResolutionRenewal + case *types.V2StorageProof: + result = V2ResolutionStorageProof + case *types.V2FileContractExpiration: + result = V2ResolutionExpiration default: - return nil, errors.New("invalid Source value") + panic("unknown resolution type") } + return +} + +// A SiacoinInput is a types.SiacoinInput with information about the parent +// value. +type SiacoinInput struct { + Address types.Address `json:"address"` + Value types.Currency `json:"value"` + types.SiacoinInput +} + +// A SiafundInput is a types.SiafundInput with information about the parent +// value. +type SiafundInput struct { + Address types.Address `json:"address"` + Value uint64 `json:"value"` + types.SiafundInput } -// A SiacoinOutput is a types.SiacoinElement with an added field for the -// source. +// A SiacoinOutput is a types.SiacoinElement with added fields for the source +// and when it was spent. type SiacoinOutput struct { - Source Source `json:"source"` + Source Source `json:"source"` + SpentIndex *types.ChainIndex `json:"spentIndex"` types.SiacoinElement } -// A SiafundOutput is a types.SiafundElement. -type SiafundOutput types.SiafundElement +// A SiafundOutput is a types.SiafundElement with an added field for when it +// was spent. +type SiafundOutput struct { + SpentIndex *types.ChainIndex `json:"spentIndex"` + types.SiafundElement +} -// A FileContract is a types.FileContractElement that uses wrapped types -// internally. -type FileContract struct { - types.StateElement +// A ContractSiacoinOutput is a SiacoinOutput with an added field for its ID. +type ContractSiacoinOutput struct { + ID types.SiacoinOutputID `json:"id"` + types.SiacoinOutput +} +// A ExtendedFileContract is a FileContract with added fields for +// resolved/valid state, and when the transaction was confirmed and proved. +type ExtendedFileContract struct { Resolved bool `json:"resolved"` Valid bool `json:"valid"` - Filesize uint64 `json:"filesize"` - FileMerkleRoot types.Hash256 `json:"fileMerkleRoot"` - WindowStart uint64 `json:"windowStart"` - WindowEnd uint64 `json:"windowEnd"` - Payout types.Currency `json:"payout"` - ValidProofOutputs []types.SiacoinOutput `json:"validProofOutputs"` - MissedProofOutputs []types.SiacoinOutput `json:"missedProofOutputs"` - UnlockHash types.Hash256 `json:"unlockHash"` - RevisionNumber uint64 `json:"revisionNumber"` + TransactionID types.TransactionID `json:"transactionID"` + + ConfirmationIndex types.ChainIndex `json:"confirmationIndex"` + ConfirmationTransactionID types.TransactionID `json:"confirmationTransactionID"` + + ProofIndex *types.ChainIndex `json:"proofIndex"` + ProofTransactionID *types.TransactionID `json:"proofTransactionID"` + + ID types.FileContractID `json:"id"` + Filesize uint64 `json:"filesize"` + FileMerkleRoot types.Hash256 `json:"fileMerkleRoot"` + WindowStart uint64 `json:"windowStart"` + WindowEnd uint64 `json:"windowEnd"` + Payout types.Currency `json:"payout"` + ValidProofOutputs []ContractSiacoinOutput `json:"validProofOutputs"` + MissedProofOutputs []ContractSiacoinOutput `json:"missedProofOutputs"` + UnlockHash types.Address `json:"unlockHash"` + RevisionNumber uint64 `json:"revisionNumber"` } -// A FileContractRevision is a types.FileContractRevision that uses wrapped -// types internally. +// A FileContractRevision is a FileContract with extra fields for revision +// information. type FileContractRevision struct { ParentID types.FileContractID `json:"parentID"` UnlockConditions types.UnlockConditions `json:"unlockConditions"` - // NOTE: the Payout field of the contract is not "really" part of a - // revision. A revision cannot change the total payout, so the original siad - // code defines FileContractRevision as an entirely separate struct without - // a Payout field. Here, we instead reuse the FileContract type, which means - // we must treat its Payout field as invalid. To guard against developer - // error, we set it to a sentinel value when decoding it. - FileContract + + ExtendedFileContract } // A Transaction is a transaction that uses the wrapped types above. type Transaction struct { - SiacoinInputs []types.SiacoinInput `json:"siacoinInputs,omitempty"` - SiacoinOutputs []SiacoinOutput `json:"siacoinOutputs,omitempty"` - SiafundInputs []types.SiafundInput `json:"siafundInputs,omitempty"` - SiafundOutputs []SiafundOutput `json:"siafundOutputs,omitempty"` - FileContracts []FileContract `json:"fileContracts,omitempty"` - FileContractRevisions []FileContractRevision `json:"fileContractRevisions,omitempty"` - ArbitraryData [][]byte `json:"arbitraryData,omitempty"` + ID types.TransactionID `json:"id"` + + SiacoinInputs []SiacoinInput `json:"siacoinInputs,omitempty"` + SiacoinOutputs []SiacoinOutput `json:"siacoinOutputs,omitempty"` + SiafundInputs []SiafundInput `json:"siafundInputs,omitempty"` + SiafundOutputs []SiafundOutput `json:"siafundOutputs,omitempty"` + FileContracts []ExtendedFileContract `json:"fileContracts,omitempty"` + FileContractRevisions []FileContractRevision `json:"fileContractRevisions,omitempty"` + StorageProofs []types.StorageProof `json:"storageProofs,omitempty"` + MinerFees []types.Currency `json:"minerFees,omitempty"` + ArbitraryData [][]byte `json:"arbitraryData,omitempty"` + Signatures []types.TransactionSignature `json:"signatures,omitempty"` + + HostAnnouncements []chain.HostAnnouncement `json:"hostAnnouncements,omitempty"` +} + +// A V2FileContract is a v2 file contract. +type V2FileContract struct { + TransactionID types.TransactionID `json:"transactionID"` + + RenewedFrom *types.FileContractID `json:"renewedFrom"` + RenewedTo *types.FileContractID `json:"renewedTo"` + + ConfirmationIndex types.ChainIndex `json:"confirmationIndex"` + ConfirmationTransactionID types.TransactionID `json:"confirmationTransactionID"` + + ResolutionType *V2Resolution `json:"resolutionType"` + ResolutionIndex *types.ChainIndex `json:"resolutionIndex"` + ResolutionTransactionID *types.TransactionID `json:"resolutionTransactionID"` + + types.V2FileContractElement +} + +// A V2FileContractRevision is a V2 file contract revision with the +// explorer V2FileContract type. +type V2FileContractRevision struct { + Parent V2FileContract `json:"parent"` + Revision V2FileContract `json:"revision"` +} + +// A V2HostAnnouncement is a types.V2HostAnnouncement list of net addresses +// with the host public key attached. +type V2HostAnnouncement struct { + PublicKey types.PublicKey `json:"publicKey"` + chain.V2HostAnnouncement +} + +// A V2FileContractRenewal renews a file contract. +type V2FileContractRenewal struct { + FinalRenterOutput types.SiacoinOutput `json:"finalRenterOutput"` + FinalHostOutput types.SiacoinOutput `json:"finalHostOutput"` + RenterRollover types.Currency `json:"renterRollover"` + HostRollover types.Currency `json:"hostRollover"` + NewContract V2FileContract `json:"newContract"` + + // signatures cover above fields + RenterSignature types.Signature `json:"renterSignature"` + HostSignature types.Signature `json:"hostSignature"` +} + +// A V2FileContractResolution closes a v2 file contract's payment channel. +// There are four resolution types: renewwal, storage proof, finalization, +// and expiration. +type V2FileContractResolution struct { + Parent V2FileContract `json:"parent"` + Type V2Resolution `json:"type"` + Resolution any `json:"resolution"` +} + +// A V2Transaction is a V2 transaction that uses the wrapped types above. +type V2Transaction struct { + ID types.TransactionID `json:"id"` + + SiacoinInputs []types.V2SiacoinInput `json:"siacoinInputs,omitempty"` + SiacoinOutputs []SiacoinOutput `json:"siacoinOutputs,omitempty"` + SiafundInputs []types.V2SiafundInput `json:"siafundInputs,omitempty"` + SiafundOutputs []SiafundOutput `json:"siafundOutputs,omitempty"` + + FileContracts []V2FileContract `json:"fileContracts,omitempty"` + FileContractRevisions []V2FileContractRevision `json:"fileContractRevisions,omitempty"` + FileContractResolutions []V2FileContractResolution `json:"fileContractResolutions,omitempty"` + + Attestations []types.Attestation `json:"attestations,omitempty"` + ArbitraryData []byte `json:"arbitraryData,omitempty"` + + NewFoundationAddress *types.Address `json:"newFoundationAddress,omitempty"` + MinerFee types.Currency `json:"minerFee"` + + HostAnnouncements []V2HostAnnouncement `json:"hostAnnouncements,omitempty"` +} + +// V2BlockData is a struct containing the fields from types.V2BlockData and our +// modified explorer.V2Transaction type. +type V2BlockData struct { + Height uint64 `json:"height"` + Commitment types.Hash256 `json:"commitment"` + Transactions []V2Transaction `json:"transactions"` } // A Block is a block containing wrapped transactions and siacoin // outputs for the miner payouts. type Block struct { - Height uint64 - + Height uint64 `json:"height"` ParentID types.BlockID `json:"parentID"` Nonce uint64 `json:"nonce"` Timestamp time.Time `json:"timestamp"` + LeafIndex uint64 `json:"leafIndex"` MinerPayouts []SiacoinOutput `json:"minerPayouts"` Transactions []Transaction `json:"transactions"` + + V2 *V2BlockData `json:"v2,omitempty"` +} + +// Metrics contains various statistics relevant to the health of the Sia network. +type Metrics struct { + // Current chain index + Index types.ChainIndex `json:"index"` + // Current difficulty + Difficulty consensus.Work `json:"difficulty"` + // Siafund pool value + SiafundTaxRevenue types.Currency `json:"siafundTaxRevenue"` + // Total announced hosts + TotalHosts uint64 `json:"totalHosts"` + // Number of leaves in the accumulator + NumLeaves uint64 `json:"numLeaves"` + // Number of active contracts + ActiveContracts uint64 `json:"activeContracts"` + // Number of failed contracts + FailedContracts uint64 `json:"failedContracts"` + // Number of successful contracts + SuccessfulContracts uint64 `json:"successfulContracts"` + // Current storage utilization, in bytes + StorageUtilization uint64 `json:"storageUtilization"` + // Current circulating supply + CirculatingSupply types.Currency `json:"circulatingSupply"` + // Total contract revenue + ContractRevenue types.Currency `json:"contractRevenue"` +} + +// HostScan represents the results of a host scan. +type HostScan struct { + PublicKey types.PublicKey `json:"publicKey"` + Location geoip.Location `json:"location"` + Success bool `json:"success"` + Error *string `json:"error"` + Timestamp time.Time `json:"timestamp"` + NextScan time.Time `json:"nextScan"` + + Settings rhpv2.HostSettings `json:"settings"` + PriceTable rhpv3.HostPriceTable `json:"priceTable"` + + V2Settings rhpv4.HostSettings `json:"v2Settings"` +} + +// UnscannedHost represents the metadata needed to scan a host. +type UnscannedHost struct { + PublicKey types.PublicKey `json:"publicKey"` + V2 bool `json:"v2"` + NetAddress string `json:"netAddress"` + V2NetAddresses []chain.NetAddress `json:"v2NetAddresses,omitempty"` + FailedInteractionsStreak uint64 `json:"failedInteractionsStreak"` +} + +// V2SiamuxAddr returns the `Address` of the first TCP siamux `NetAddress` it +// finds in the host's list of net addresses. The protocol for this address is +// ProtocolTCPSiaMux. +func (h UnscannedHost) V2SiamuxAddr() (string, bool) { + for _, netAddr := range h.V2NetAddresses { + if netAddr.Protocol == siamux.Protocol { + return netAddr.Address, true + } + } + return "", false +} + +// IsV2 returns whether a host supports V2 or not. +func (h UnscannedHost) IsV2() bool { + return len(h.V2NetAddresses) > 0 +} + +// Host represents a host and the information gathered from scanning it. +type Host struct { + PublicKey types.PublicKey `json:"publicKey"` + V2 bool `json:"v2"` + NetAddress string `json:"netAddress"` + V2NetAddresses []chain.NetAddress `json:"v2NetAddresses,omitempty"` + + Location geoip.Location `json:"location"` + + KnownSince time.Time `json:"knownSince"` + LastScan time.Time `json:"lastScan"` + LastScanSuccessful bool `json:"lastScanSuccessful"` + LastScanError *string `json:"lastScanError"` + LastAnnouncement time.Time `json:"lastAnnouncement"` + NextScan time.Time `json:"nextScan"` + TotalScans uint64 `json:"totalScans"` + SuccessfulInteractions uint64 `json:"successfulInteractions"` + FailedInteractions uint64 `json:"failedInteractions"` + + Settings rhpv2.HostSettings `json:"settings"` + PriceTable rhpv3.HostPriceTable `json:"priceTable"` + + V2Settings rhpv4.HostSettings `json:"v2Settings"` +} + +// HostMetrics represents averages of scanned information from hosts. +type HostMetrics struct { + // Number of hosts that were up as of there last scan + ActiveHosts uint64 `json:"activeHosts"` + // Total storage of all active hosts, in bytes + TotalStorage uint64 `json:"totalStorage"` + // Remaining storage of all active hosts, in bytes (storage utilization is + // equal to TotalStorage - RemainingStorage) + RemainingStorage uint64 `json:"remainingStorage"` + + Settings rhpv2.HostSettings `json:"settings"` + PriceTable rhpv3.HostPriceTable `json:"priceTable"` + V2Settings rhpv4.HostSettings `json:"v2Settings"` +} + +// HostSortDir represents the sorting direction for host filtering. +type HostSortDir string + +const ( + // HostSortAsc means sorting in ascending order. + HostSortAsc HostSortDir = "asc" + // HostSortDesc means sorting in descending order. + HostSortDesc HostSortDir = "desc" +) + +// MarshalText implements encoding.TextMarshaler. +func (h HostSortDir) MarshalText() ([]byte, error) { + switch h { + case HostSortAsc, HostSortDesc: + return []byte(h), nil + default: + return nil, fmt.Errorf("invalid HostSortDir: %s", h) + } +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (h *HostSortDir) UnmarshalText(data []byte) error { + switch string(data) { + case string(HostSortAsc): + *h = HostSortAsc + case string(HostSortDesc): + *h = HostSortDesc + default: + return fmt.Errorf("invalid HostSortDir: %s", data) + } + return nil +} + +// HostSortColumn represents the sorting column for host filtering. +type HostSortColumn string + +const ( + // HostSortDateCreated sorts hosts in the order they were first announced. + HostSortDateCreated HostSortColumn = "date_created" + // HostSortNetAddress sorts hosts by their net address. + HostSortNetAddress HostSortColumn = "net_address" + // HostSortPublicKey sorts hosts by their public key + HostSortPublicKey HostSortColumn = "public_key" + // HostSortAcceptingContracts sorts hosts by whether they accept contracts. + HostSortAcceptingContracts HostSortColumn = "accepting_contracts" + // HostSortUptime sorts hosts by their uptime. + HostSortUptime HostSortColumn = "uptime" + // HostSortStoragePrice sorts hosts by their storage price. + HostSortStoragePrice HostSortColumn = "storage_price" + // HostSortContractPrice sorts hosts by their contract price. + HostSortContractPrice HostSortColumn = "contract_price" + // HostSortDownloadPrice sorts hosts by their download price. + HostSortDownloadPrice HostSortColumn = "download_price" + // HostSortUploadPrice sorts hosts by their upload price. + HostSortUploadPrice HostSortColumn = "upload_price" + // HostSortUsedStorage sorts hosts by their used storage. + HostSortUsedStorage HostSortColumn = "used_storage" + // HostSortTotalStorage sorts hosts by their total storage. + HostSortTotalStorage HostSortColumn = "total_storage" +) + +// MarshalText implements encoding.TextMarshaler. +func (h HostSortColumn) MarshalText() ([]byte, error) { + switch h { + case HostSortDateCreated, HostSortNetAddress, HostSortPublicKey, HostSortAcceptingContracts, + HostSortUptime, HostSortStoragePrice, HostSortContractPrice, HostSortDownloadPrice, + HostSortUploadPrice, HostSortUsedStorage, HostSortTotalStorage: + return []byte(h), nil + default: + return nil, fmt.Errorf("invalid HostSortColumn: %s", h) + } +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (h *HostSortColumn) UnmarshalText(data []byte) error { + switch string(data) { + case string(HostSortDateCreated): + *h = HostSortDateCreated + case string(HostSortNetAddress): + *h = HostSortNetAddress + case string(HostSortPublicKey): + *h = HostSortPublicKey + case string(HostSortAcceptingContracts): + *h = HostSortAcceptingContracts + case string(HostSortUptime): + *h = HostSortUptime + case string(HostSortStoragePrice): + *h = HostSortStoragePrice + case string(HostSortContractPrice): + *h = HostSortContractPrice + case string(HostSortDownloadPrice): + *h = HostSortDownloadPrice + case string(HostSortUploadPrice): + *h = HostSortUploadPrice + case string(HostSortUsedStorage): + *h = HostSortUsedStorage + case string(HostSortTotalStorage): + *h = HostSortTotalStorage + default: + return fmt.Errorf("invalid HostSortColumn: %s", data) + } + return nil +} + +// HostQuery defines the filter and sort parameters for querying hosts. +type HostQuery struct { + V2 *bool `json:"v2,omitempty"` + PublicKeys []types.PublicKey `json:"publicKeys,omitempty"` + NetAddresses []string `json:"netAddresses,omitempty"` + MinUptime *float64 `json:"minUptime,omitempty"` + MinDuration *uint64 `json:"minDuration,omitempty"` + MaxStoragePrice *types.Currency `json:"maxStoragePrice,omitempty"` + MaxContractPrice *types.Currency `json:"maxContractPrice,omitempty"` + MaxUploadPrice *types.Currency `json:"maxUploadPrice,omitempty"` + MaxDownloadPrice *types.Currency `json:"maxDownloadPrice,omitempty"` + MaxBaseRPCPrice *types.Currency `json:"maxBaseRPCPrice,omitempty"` + MaxSectorAccessPrice *types.Currency `json:"maxSectorAccessPrice,omitempty"` + AcceptContracts *bool `json:"acceptContracts,omitempty"` + Online *bool `json:"online,omitempty"` } diff --git a/explorer/update.go b/explorer/update.go index 48557dd..99b08bf 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -13,6 +13,20 @@ type ( FileContractElement types.FileContractElement Revision *types.FileContractElement Resolved, Valid bool + + ConfirmationTransactionID *types.TransactionID + ProofTransactionID *types.TransactionID + } + + // V2FileContractUpdate represents a v2 file contract from a consensus + // update. + V2FileContractUpdate struct { + FileContractElement types.V2FileContractElement + Revision *types.V2FileContractElement + Resolution types.V2FileContractResolutionType + + ConfirmationTransactionID *types.TransactionID + ResolutionTransactionID *types.TransactionID } // A DBFileContract represents a file contract element in the DB. @@ -31,26 +45,33 @@ type ( // An UpdateState contains information relevant to the block being applied // or reverted. UpdateState struct { - Block types.Block - Index types.ChainIndex + Block types.Block + ChainIndexElement types.ChainIndexElement Events []Event + Metrics Metrics TreeUpdates []TreeNodeUpdate - Sources map[types.SiacoinOutputID]Source - NewSiacoinElements []types.SiacoinElement - SpentSiacoinElements []types.SiacoinElement - EphemeralSiacoinElements []types.SiacoinElement + HostAnnouncements []chain.HostAnnouncement + V2HostAnnouncements []V2HostAnnouncement + + NewSiacoinElements []SiacoinOutput + SpentSiacoinElements []SiacoinOutput + EphemeralSiacoinElements []SiacoinOutput NewSiafundElements []types.SiafundElement SpentSiafundElements []types.SiafundElement EphemeralSiafundElements []types.SiafundElement - FileContractElements []FileContractUpdate + FileContractElements []FileContractUpdate + V2FileContractElements []V2FileContractUpdate } // An UpdateTx atomically updates the state of a store. UpdateTx interface { + Metrics(height uint64) (Metrics, error) + HostExists(pubkey types.PublicKey) (bool, error) + ApplyIndex(state UpdateState) error RevertIndex(state UpdateState) error } @@ -67,57 +88,82 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { for i := range txn.SiacoinOutputs { sources[txn.SiacoinOutputID(i)] = SourceTransaction } - - for i := range txn.FileContracts { - fcid := txn.FileContractID(i) - for j := range txn.FileContracts[i].ValidProofOutputs { - sources[fcid.ValidOutputID(j)] = SourceValidProofOutput - } - for j := range txn.FileContracts[i].MissedProofOutputs { - sources[fcid.MissedOutputID(j)] = SourceMissedProofOutput - } - } } - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cau.Block.Transactions { + for _, txn := range cau.Block.V2Transactions() { + txnID := txn.ID() for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + sources[txn.SiacoinOutputID(txnID, i)] = SourceTransaction } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true + } + + for _, diff := range cau.FileContractElementDiffs() { + if diff.Resolved { + fcID := diff.FileContractElement.ID + if diff.Valid { + for i := range diff.FileContractElement.FileContract.ValidProofOutputs { + sources[fcID.ValidOutputID(i)] = SourceValidProofOutput + } + } else { + for i := range diff.FileContractElement.FileContract.MissedProofOutputs { + sources[fcID.MissedOutputID(i)] = SourceMissedProofOutput + } + } } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + + for _, diff := range cau.V2FileContractElementDiffs() { + if diff.Resolution != nil { + fcID := diff.V2FileContractElement.ID + switch r := diff.Resolution.(type) { + case *types.V2FileContractRenewal: + sources[fcID.V2RenterOutputID()] = SourceValidProofOutput + sources[fcID.V2HostOutputID()] = SourceValidProofOutput + case *types.V2StorageProof: + sources[fcID.V2RenterOutputID()] = SourceValidProofOutput + sources[fcID.V2HostOutputID()] = SourceValidProofOutput + case *types.V2FileContractExpiration: + sources[fcID.V2RenterOutputID()] = SourceMissedProofOutput + sources[fcID.V2HostOutputID()] = SourceMissedProofOutput + default: + panic(fmt.Sprintf("unhandled resolution type %T", r)) + } } } // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement - cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) - return + var newSiacoinElements, spentSiacoinElements []SiacoinOutput + var ephemeralSiacoinElements []SiacoinOutput + for _, diff := range cau.SiacoinElementDiffs() { + created, spent, se := diff.Created, diff.Spent, diff.SiacoinElement + if created && spent { + ephemeralSiacoinElements = append(ephemeralSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[se.ID], + }) + continue } if spent { - spentSiacoinElements = append(spentSiacoinElements, se) + spentSiacoinElements = append(spentSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[se.ID], + }) } else { - newSiacoinElements = append(newSiacoinElements, se) + newSiacoinElements = append(newSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[se.ID], + }) } - }) + } var newSiafundElements, spentSiafundElements []types.SiafundElement var ephemeralSiafundElements []types.SiafundElement - cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { + for _, diff := range cau.SiafundElementDiffs() { + created, spent, se := diff.Created, diff.Spent, diff.SiafundElement + if created && spent { ephemeralSiafundElements = append(ephemeralSiafundElements, se) - return + continue } if spent { @@ -125,17 +171,86 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } else { newSiafundElements = append(newSiafundElements, se) } - }) + } + + fceMap := make(map[types.FileContractID]FileContractUpdate) + for _, diff := range cau.FileContractElementDiffs() { + var rev *types.FileContractElement + if revision, ok := diff.RevisionElement(); ok { + rev = &revision + } + fceMap[diff.FileContractElement.ID] = FileContractUpdate{ + FileContractElement: diff.FileContractElement, + Revision: rev, + Resolved: diff.Resolved, + Valid: diff.Valid, + } + } + + for _, txn := range cau.Block.Transactions { + txnID := txn.ID() + for i := range txn.FileContracts { + fcID := txn.FileContractID(i) + + v := fceMap[fcID] + v.ConfirmationTransactionID = &txnID + fceMap[fcID] = v + } + for _, sp := range txn.StorageProofs { + fcID := sp.ParentID + + v := fceMap[fcID] + v.ProofTransactionID = &txnID + fceMap[fcID] = v + } + } var fces []FileContractUpdate - cau.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - fces = append(fces, FileContractUpdate{ - FileContractElement: fce, + for _, fce := range fceMap { + fces = append(fces, fce) + } + + v2FceMap := make(map[types.FileContractID]V2FileContractUpdate) + for _, diff := range cau.V2FileContractElementDiffs() { + var rev *types.V2FileContractElement + if revision, ok := diff.V2RevisionElement(); ok { + rev = &revision + } + v2FceMap[types.FileContractID(diff.V2FileContractElement.ID)] = V2FileContractUpdate{ + FileContractElement: diff.V2FileContractElement, Revision: rev, - Resolved: resolved, - Valid: valid, - }) - }) + Resolution: diff.Resolution, + } + } + for _, txn := range cau.Block.V2Transactions() { + txnID := txn.ID() + for i := range txn.FileContracts { + fcID := txn.V2FileContractID(txnID, i) + + v := v2FceMap[fcID] + v.ConfirmationTransactionID = &txnID + v2FceMap[fcID] = v + } + for _, fcr := range txn.FileContractResolutions { + fcID := types.FileContractID(fcr.Parent.ID) + + v := v2FceMap[fcID] + v.ResolutionTransactionID = &txnID + v2FceMap[fcID] = v + + if _, ok := fcr.Resolution.(*types.V2FileContractRenewal); ok { + renewalID := fcID.V2RenewalID() + v := v2FceMap[renewalID] + v.ConfirmationTransactionID = &txnID + v2FceMap[renewalID] = v + } + } + } + + var v2Fces []V2FileContractUpdate + for _, fce := range v2FceMap { + v2Fces = append(v2Fces, fce) + } var treeUpdates []TreeNodeUpdate cau.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { @@ -146,17 +261,40 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { }) }) - relevant := func(types.Address) bool { return true } - events := AppliedEvents(cau.State, cau.Block, cau, relevant) + var hostAnnouncements []chain.HostAnnouncement + for _, txn := range cau.Block.Transactions { + for _, arb := range txn.ArbitraryData { + var ha chain.HostAnnouncement + if ha.FromArbitraryData(arb) { + hostAnnouncements = append(hostAnnouncements, ha) + } + } + } + var v2HostAnnouncements []V2HostAnnouncement + for _, txn := range cau.Block.V2Transactions() { + for _, a := range txn.Attestations { + var ha chain.V2HostAnnouncement + if ha.FromAttestation(a) == nil { + v2HostAnnouncements = append(v2HostAnnouncements, V2HostAnnouncement{ + PublicKey: a.PublicKey, + V2HostAnnouncement: ha, + }) + } + } + } + + events := AppliedEvents(cau.State, cau.Block, cau) state := UpdateState{ - Block: cau.Block, - Index: cau.State.Index, + Block: cau.Block, + ChainIndexElement: cau.ChainIndexElement(), Events: events, TreeUpdates: treeUpdates, - Sources: sources, + HostAnnouncements: hostAnnouncements, + V2HostAnnouncements: v2HostAnnouncements, + NewSiacoinElements: newSiacoinElements, SpentSiacoinElements: spentSiacoinElements, EphemeralSiacoinElements: ephemeralSiacoinElements, @@ -165,52 +303,62 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { SpentSiafundElements: spentSiafundElements, EphemeralSiafundElements: ephemeralSiafundElements, - FileContractElements: fces, + FileContractElements: fces, + V2FileContractElements: v2Fces, + } + + var err error + var prevMetrics Metrics + if cau.State.Index.Height > 0 { + prevMetrics, err = tx.Metrics(cau.State.Index.Height - 1) + if err != nil { + return err + } } + state.Metrics, err = updateMetrics(tx, state, prevMetrics) + if err != nil { + return err + } + state.Metrics.Index = cau.State.Index + state.Metrics.Difficulty = cau.State.Difficulty + state.Metrics.SiafundTaxRevenue = cau.State.SiafundTaxRevenue + state.Metrics.NumLeaves = cau.State.Elements.NumLeaves + return tx.ApplyIndex(state) } // revertChainUpdate atomically reverts a chain update from a store func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cru.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - } - // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement - cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) - return + var newSiacoinElements, spentSiacoinElements []SiacoinOutput + var ephemeralSiacoinElements []SiacoinOutput + for _, diff := range cru.SiacoinElementDiffs() { + created, spent, se := diff.Created, diff.Spent, diff.SiacoinElement + if created && spent { + ephemeralSiacoinElements = append(ephemeralSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) + continue } if spent { - newSiacoinElements = append(newSiacoinElements, se) + newSiacoinElements = append(newSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) } else { - spentSiacoinElements = append(spentSiacoinElements, se) + spentSiacoinElements = append(spentSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) } - }) + } var newSiafundElements, spentSiafundElements []types.SiafundElement var ephemeralSiafundElements []types.SiafundElement - cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { + for _, diff := range cru.SiafundElementDiffs() { + created, spent, se := diff.Created, diff.Spent, diff.SiafundElement + if created && spent { ephemeralSiafundElements = append(ephemeralSiafundElements, se) - return + continue } if spent { @@ -218,17 +366,78 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } else { spentSiafundElements = append(spentSiafundElements, se) } - }) + } + + fceMap := make(map[types.FileContractID]FileContractUpdate) + for _, diff := range cru.FileContractElementDiffs() { + var rev *types.FileContractElement + if revision, ok := diff.RevisionElement(); ok { + rev = &revision + } + fceMap[diff.FileContractElement.ID] = FileContractUpdate{ + FileContractElement: diff.FileContractElement, + Revision: rev, + Resolved: diff.Resolved, + Valid: diff.Valid, + } + } + for _, txn := range cru.Block.Transactions { + txnID := txn.ID() + for i := range txn.FileContracts { + fcID := txn.FileContractID(i) + + v := fceMap[fcID] + v.ConfirmationTransactionID = &txnID + fceMap[fcID] = v + } + for _, sp := range txn.StorageProofs { + fcID := sp.ParentID + + v := fceMap[fcID] + v.ProofTransactionID = &txnID + fceMap[fcID] = v + } + } var fces []FileContractUpdate - cru.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - fces = append(fces, FileContractUpdate{ - FileContractElement: fce, + for _, fce := range fceMap { + fces = append(fces, fce) + } + + v2FceMap := make(map[types.FileContractID]V2FileContractUpdate) + for _, diff := range cru.V2FileContractElementDiffs() { + var rev *types.V2FileContractElement + if revision, ok := diff.V2RevisionElement(); ok { + rev = &revision + } + v2FceMap[types.FileContractID(diff.V2FileContractElement.ID)] = V2FileContractUpdate{ + FileContractElement: diff.V2FileContractElement, Revision: rev, - Resolved: resolved, - Valid: valid, - }) - }) + Resolution: diff.Resolution, + } + } + for _, txn := range cru.Block.V2Transactions() { + txnID := txn.ID() + for i := range txn.FileContracts { + fcID := txn.V2FileContractID(txn.ID(), i) + + v := v2FceMap[fcID] + v.ConfirmationTransactionID = &txnID + v2FceMap[fcID] = v + } + for _, fcr := range txn.FileContractResolutions { + fcID := types.FileContractID(fcr.Parent.ID) + + v := v2FceMap[fcID] + v.ResolutionTransactionID = &txnID + v2FceMap[fcID] = v + } + } + + var v2Fces []V2FileContractUpdate + for _, fce := range v2FceMap { + v2Fces = append(v2Fces, fce) + } var treeUpdates []TreeNodeUpdate cru.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { @@ -240,8 +449,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. }) state := UpdateState{ - Block: cru.Block, - Index: revertedIndex, + Block: cru.Block, + TreeUpdates: treeUpdates, NewSiacoinElements: newSiacoinElements, @@ -252,11 +461,95 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. SpentSiafundElements: spentSiafundElements, EphemeralSiafundElements: ephemeralSiafundElements, - FileContractElements: fces, + FileContractElements: fces, + V2FileContractElements: v2Fces, } + state.Metrics.Index = revertedIndex + return tx.RevertIndex(state) } +func updateMetrics(tx UpdateTx, s UpdateState, metrics Metrics) (Metrics, error) { + seenHosts := make(map[types.PublicKey]struct{}) + for _, host := range s.HostAnnouncements { + if _, ok := seenHosts[host.PublicKey]; ok { + continue + } + + exists, err := tx.HostExists(host.PublicKey) + if err != nil { + return Metrics{}, err + } + if !exists { + // we haven't seen this host yet, increment count + metrics.TotalHosts++ + seenHosts[host.PublicKey] = struct{}{} + } + } + for _, host := range s.V2HostAnnouncements { + if _, ok := seenHosts[host.PublicKey]; ok { + continue + } + + exists, err := tx.HostExists(host.PublicKey) + if err != nil { + return Metrics{}, err + } + if !exists { + // we haven't seen this host yet, increment count + metrics.TotalHosts++ + seenHosts[host.PublicKey] = struct{}{} + } + } + + for _, fce := range s.FileContractElements { + fc := fce.FileContractElement.FileContract + if fce.Revision != nil { + fc = fce.Revision.FileContract + } + + if fce.Resolved { + metrics.ActiveContracts-- + metrics.StorageUtilization -= fc.Filesize + } else if fce.Revision == nil { + // don't count revision as a new contract + metrics.ActiveContracts++ + metrics.StorageUtilization += fc.Filesize + } else { + // filesize changed + metrics.StorageUtilization += (fc.Filesize - fce.FileContractElement.FileContract.Filesize) + } + + if fce.Resolved { + if !fce.Valid { + metrics.FailedContracts++ + } else { + metrics.SuccessfulContracts++ + for _, vpo := range fc.ValidProofOutputs { + metrics.ContractRevenue = metrics.ContractRevenue.Add(vpo.Value) + } + } + } + } + + for _, sce := range s.NewSiacoinElements { + sco := sce.SiacoinOutput + if sco.Address == types.VoidAddress { + continue + } + metrics.CirculatingSupply = metrics.CirculatingSupply.Add(sco.Value) + } + for _, sce := range s.SpentSiacoinElements { + sco := sce.SiacoinOutput + if sco.Address == types.VoidAddress { + continue + } + metrics.CirculatingSupply = metrics.CirculatingSupply.Sub(sco.Value) + } + + return metrics, nil +} + // UpdateChainState applies the reverts and updates. func UpdateChainState(tx UpdateTx, crus []chain.RevertUpdate, caus []chain.ApplyUpdate) error { for _, cru := range crus { diff --git a/geoip/GeoLite2-City.mmdb b/geoip/GeoLite2-City.mmdb new file mode 100644 index 0000000..5f2af95 Binary files /dev/null and b/geoip/GeoLite2-City.mmdb differ diff --git a/geoip/geoip.go b/geoip/geoip.go new file mode 100644 index 0000000..13fd413 --- /dev/null +++ b/geoip/geoip.go @@ -0,0 +1,81 @@ +package geoip + +import ( + _ "embed" // needed for geolocation database + "errors" + "net" + "sync" + + "github.com/oschwald/geoip2-golang" +) + +//go:embed GeoLite2-City.mmdb +var maxMindCityDB []byte + +// A Location represents an ISO 3166-1 A-2 country codes and an approximate +// latitude/longitude. +type Location struct { + CountryCode string `json:"countryCode"` + + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} + +// A Locator maps IP addresses to their location. +// It is assumed that it implementations are thread-safe. +type Locator interface { + // Close closes the Locator. + Close() error + // Locate maps IP addresses to a Location. + Locate(ip *net.IPAddr) (Location, error) +} + +type maxMindLocator struct { + mu sync.Mutex + + db *geoip2.Reader +} + +// Locate implements Locator. +func (m *maxMindLocator) Locate(addr *net.IPAddr) (Location, error) { + if addr == nil { + return Location{}, errors.New("nil IP") + } + m.mu.Lock() + defer m.mu.Unlock() + + record, err := m.db.City(addr.IP) + if err != nil { + return Location{}, err + } + return Location{ + CountryCode: record.Country.IsoCode, + Latitude: record.Location.Latitude, + Longitude: record.Location.Longitude, + }, nil +} + +// Close implements Locator. +func (m *maxMindLocator) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + return m.db.Close() +} + +// NewMaxMindLocator returns a Locator that uses an underlying MaxMind +// database. If no path is provided, a default embedded GeoLite2-City database +// is used. +func NewMaxMindLocator(path string) (Locator, error) { + var db *geoip2.Reader + var err error + if path == "" { + db, err = geoip2.FromBytes(maxMindCityDB) + } else { + db, err = geoip2.Open(path) + } + if err != nil { + return nil, err + } + + return &maxMindLocator{db: db}, nil +} diff --git a/go.mod b/go.mod index b702f8f..51d2a66 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,40 @@ module go.sia.tech/explored -go 1.21.6 +go 1.23.2 + +toolchain go1.24.2 require ( - github.com/mattn/go-sqlite3 v1.14.22 - go.etcd.io/bbolt v1.3.9 - go.sia.tech/core v0.2.2 - go.sia.tech/coreutils v0.0.4-0.20240327130436-3fc21abba2db - go.sia.tech/jape v0.11.1 + github.com/google/go-cmp v0.7.0 + github.com/mattn/go-sqlite3 v1.14.28 + github.com/oschwald/geoip2-golang v1.11.0 + go.sia.tech/core v0.12.0 + go.sia.tech/coreutils v0.13.2 + go.sia.tech/jape v0.13.1 go.uber.org/zap v1.27.0 - golang.org/x/term v0.19.0 - lukechampine.com/frand v1.4.2 + gopkg.in/yaml.v3 v3.0.1 + lukechampine.com/frand v1.5.1 lukechampine.com/upnp v0.3.0 ) require ( - github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - go.sia.tech/mux v1.2.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/tools v0.7.0 // indirect + github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.51.0 // indirect + github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + go.etcd.io/bbolt v1.4.0 // indirect + go.sia.tech/mux v1.4.0 // indirect + go.uber.org/mock v0.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + golang.org/x/tools v0.32.0 // indirect ) diff --git a/go.sum b/go.sum index d7f9bbd..47a004e 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,91 @@ -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.sia.tech/core v0.2.2 h1:33RJrt08o7KyUOY4tITH6ECmRq1lhtapqc/SncIF/2A= -go.sia.tech/core v0.2.2/go.mod h1:Zk7HaybEPgkPC1p6e6tTQr8PIeZClTgNcLNGYDLQJeE= -go.sia.tech/coreutils v0.0.4-0.20240327130436-3fc21abba2db h1:nfhcgN3zfwd+GdDUCrNmV4Ajf8VZSQcoXjGGmfs7V9E= -go.sia.tech/coreutils v0.0.4-0.20240327130436-3fc21abba2db/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= -go.sia.tech/jape v0.11.1 h1:M7IP+byXL7xOqzxcHUQuXW+q3sYMkYzmMlMw+q8ZZw0= -go.sia.tech/jape v0.11.1/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= -go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= -go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.51.0 h1:K8exxe9zXxeRKxaXxi/GpUqYiTrtdiWP8bo1KFya6Wc= +github.com/quic-go/quic-go v0.51.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.sia.tech/core v0.12.0 h1:nuHjUE3MnYPQ+BKo44DD64332jBGVZlH657c5RIomXw= +go.sia.tech/core v0.12.0/go.mod h1:ycpNTb9Y7Vtnq6HQ3iqOxeywLnjDs0udD3ZVq6KE13Y= +go.sia.tech/coreutils v0.13.2 h1:SlxQ6onhdjGrTB8I/TxFwIbFTM0mzH5amDoAQMkjmOI= +go.sia.tech/coreutils v0.13.2/go.mod h1:RELgLyNq1oCRBAXnMvGUefASRJWjju6jgS6ioYbZGFE= +go.sia.tech/jape v0.13.1 h1:gBTQCIXVFzUTs6mFA3NcKo/55SzlFYCzb4WbvGeaAF4= +go.sia.tech/jape v0.13.1/go.mod h1:ZZe2SKMWfpV0mpZWld/Rv4GGXpr51liyfUkes2ikw54= +go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= +go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= -lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= +lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w= +lukechampine.com/frand v1.5.1/go.mod h1:4VstaWc2plN4Mjr10chUD46RAVGWhpkZ5Nja8+Azp0Q= lukechampine.com/upnp v0.3.0 h1:UVCD6eD6fmJmwak6DVE3vGN+L46Fk8edTcC6XYCb6C4= lukechampine.com/upnp v0.3.0/go.mod h1:sOuF+fGSDKjpUm6QI0mfb82ScRrhj8bsqsD78O5nK1k= diff --git a/internal/rhp/v2/rhp.go b/internal/rhp/v2/rhp.go new file mode 100644 index 0000000..a2f260c --- /dev/null +++ b/internal/rhp/v2/rhp.go @@ -0,0 +1,851 @@ +package rhp + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net" + "sort" + "time" + + rhp2 "go.sia.tech/core/rhp/v2" + "go.sia.tech/core/types" +) + +const ( + // minMessageSize is the minimum size of an RPC message + minMessageSize = 4096 +) + +var ( + errContractLocked = errors.New("contract is locked by another party") + errContractFinalized = errors.New("contract cannot be revised further") + errInsufficientCollateral = errors.New("insufficient collateral") + errInsufficientFunds = errors.New("insufficient funds") + errInvalidMerkleProof = errors.New("host supplied invalid Merkle proof") +) + +// RPCSettings calls the Settings RPC, returning the host's reported settings. +func RPCSettings(ctx context.Context, t *rhp2.Transport) (settings rhp2.HostSettings, err error) { + var resp rhp2.RPCSettingsResponse + if err := t.Call(rhp2.RPCSettingsID, nil, &resp); err != nil { + return rhp2.HostSettings{}, err + } else if err := json.Unmarshal(resp.Settings, &settings); err != nil { + return rhp2.HostSettings{}, fmt.Errorf("couldn't unmarshal json: %w", err) + } + + return settings, nil +} + +// RPCFormContract forms a contract with a host. +func RPCFormContract(ctx context.Context, t *rhp2.Transport, renterKey types.PrivateKey, txnSet []types.Transaction) (_ rhp2.ContractRevision, _ []types.Transaction, err error) { + // strip our signatures before sending + parents, txn := txnSet[:len(txnSet)-1], txnSet[len(txnSet)-1] + renterContractSignatures := txn.Signatures + txnSet[len(txnSet)-1].Signatures = nil + + // create request + renterPubkey := renterKey.PublicKey() + req := &rhp2.RPCFormContractRequest{ + Transactions: txnSet, + RenterKey: renterPubkey.UnlockKey(), + } + if err := t.WriteRequest(rhp2.RPCFormContractID, req); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + // execute form contract RPC + var resp rhp2.RPCFormContractAdditions + if err := t.ReadResponse(&resp, 65536); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + // merge host additions with txn + txn.SiacoinInputs = append(txn.SiacoinInputs, resp.Inputs...) + txn.SiacoinOutputs = append(txn.SiacoinOutputs, resp.Outputs...) + + // create initial (no-op) revision, transaction, and signature + fc := txn.FileContracts[0] + initRevision := types.FileContractRevision{ + ParentID: txn.FileContractID(0), + UnlockConditions: types.UnlockConditions{ + PublicKeys: []types.UnlockKey{ + renterPubkey.UnlockKey(), + t.HostKey().UnlockKey(), + }, + SignaturesRequired: 2, + }, + FileContract: types.FileContract{ + RevisionNumber: 1, + Filesize: fc.Filesize, + FileMerkleRoot: fc.FileMerkleRoot, + WindowStart: fc.WindowStart, + WindowEnd: fc.WindowEnd, + ValidProofOutputs: fc.ValidProofOutputs, + MissedProofOutputs: fc.MissedProofOutputs, + UnlockHash: fc.UnlockHash, + }, + } + revSig := renterKey.SignHash(hashRevision(initRevision)) + renterRevisionSig := types.TransactionSignature{ + ParentID: types.Hash256(initRevision.ParentID), + CoveredFields: types.CoveredFields{FileContractRevisions: []uint64{0}}, + PublicKeyIndex: 0, + Signature: revSig[:], + } + + // write our signatures + renterSigs := &rhp2.RPCFormContractSignatures{ + ContractSignatures: renterContractSignatures, + RevisionSignature: renterRevisionSig, + } + if err := t.WriteResponse(renterSigs); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + // read the host's signatures and merge them with our own + var hostSigs rhp2.RPCFormContractSignatures + if err := t.ReadResponse(&hostSigs, minMessageSize); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + txn.Signatures = append(renterContractSignatures, hostSigs.ContractSignatures...) + signedTxnSet := append(resp.Parents, append(parents, txn)...) + return rhp2.ContractRevision{ + Revision: initRevision, + Signatures: [2]types.TransactionSignature{ + renterRevisionSig, + hostSigs.RevisionSignature, + }, + }, signedTxnSet, nil +} + +// RHP2Session represents a session with a host +type RHP2Session struct { + transport *rhp2.Transport + revision rhp2.ContractRevision + key types.PrivateKey + appendRoots []types.Hash256 + settings rhp2.HostSettings + lastSeen time.Time +} + +// NewRHP2Session returns a new rhp2 session +func NewRHP2Session(t *rhp2.Transport, key types.PrivateKey, rev rhp2.ContractRevision, settings rhp2.HostSettings) *RHP2Session { + return &RHP2Session{ + transport: t, + key: key, + revision: rev, + settings: settings, + } +} + +// Append appends the sector to the contract +func (s *RHP2Session) Append(ctx context.Context, sector *[rhp2.SectorSize]byte, price, collateral types.Currency) (types.Hash256, error) { + err := s.Write(ctx, []rhp2.RPCWriteAction{{ + Type: rhp2.RPCWriteActionAppend, + Data: sector[:], + }}, price, collateral) + if err != nil { + return types.Hash256{}, err + } + return s.appendRoots[0], nil +} + +// Close closes the underlying transport +func (s *RHP2Session) Close() (err error) { + return s.closeTransport() +} + +// Delete deletes the sectors at the given indices from the contract +func (s *RHP2Session) Delete(ctx context.Context, sectorIndices []uint64, price types.Currency) error { + if len(sectorIndices) == 0 { + return nil + } + + // sort in descending order so that we can use 'range' + sort.Slice(sectorIndices, func(i, j int) bool { + return sectorIndices[i] > sectorIndices[j] + }) + + // iterate backwards from the end of the contract, swapping each "good" + // sector with one of the "bad" sectors. + var actions []rhp2.RPCWriteAction + cIndex := s.revision.NumSectors() - 1 + for _, rIndex := range sectorIndices { + if cIndex != rIndex { + // swap a "good" sector for a "bad" sector + actions = append(actions, rhp2.RPCWriteAction{ + Type: rhp2.RPCWriteActionSwap, + A: uint64(cIndex), + B: uint64(rIndex), + }) + } + cIndex-- + } + // trim all "bad" sectors + actions = append(actions, rhp2.RPCWriteAction{ + Type: rhp2.RPCWriteActionTrim, + A: uint64(len(sectorIndices)), + }) + + // request the swap+delete operation + // + // NOTE: siad hosts will accept up to 20 MiB of data in the request, + // which should be sufficient to delete up to 2.5 TiB of sector data + // at a time. + return s.Write(ctx, actions, price, types.ZeroCurrency) +} + +// HostKey returns the host's public key +func (s *RHP2Session) HostKey() types.PublicKey { return s.revision.HostKey() } + +// Read reads the given sections +func (s *RHP2Session) Read(ctx context.Context, w io.Writer, sections []rhp2.RPCReadRequestSection, price types.Currency) (err error) { + empty := true + for _, s := range sections { + empty = empty && s.Length == 0 + } + if empty || len(sections) == 0 { + return nil + } + + if !s.isRevisable() { + return errContractFinalized + } else if !s.sufficientFunds(price) { + return errInsufficientFunds + } + + // construct new revision + rev := s.revision.Revision + rev.RevisionNumber++ + newValid, newMissed := updateRevisionOutputs(&rev, price, types.ZeroCurrency) + revisionHash := hashRevision(rev) + renterSig := s.key.SignHash(revisionHash) + + // construct the request + req := &rhp2.RPCReadRequest{ + Sections: sections, + MerkleProof: true, + + RevisionNumber: rev.RevisionNumber, + ValidProofValues: newValid, + MissedProofValues: newMissed, + Signature: renterSig, + } + + var hostSig *types.Signature + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.WriteRequest(rhp2.RPCReadID, req); err != nil { + return err + } + + // ensure we send RPCLoopReadStop before returning + defer transport.WriteResponse(&rhp2.RPCReadStop) + + // read all sections + for _, sec := range sections { + hostSig, err = s.readSection(w, transport, sec) + if err != nil { + return err + } + if hostSig != nil { + break // exit the loop; they won't be sending any more data + } + } + + // the host is required to send a signature; if they haven't sent one + // yet, they should send an empty ReadResponse containing just the + // signature. + if hostSig == nil { + var resp rhp2.RPCReadResponse + if err := transport.ReadResponse(&resp, 4096); err != nil { + return wrapResponseErr(err, "couldn't read signature", "host rejected Read request") + } + hostSig = &resp.Signature + } + return nil + }); err != nil { + return err + } + + // verify the host signature + if !s.HostKey().VerifyHash(revisionHash, *hostSig) { + return errors.New("host's signature is invalid") + } + s.revision.Revision = rev + s.revision.Signatures[0].Signature = renterSig[:] + s.revision.Signatures[1].Signature = hostSig[:] + + return nil +} + +// Reconnect reconnects to the host +func (s *RHP2Session) Reconnect(ctx context.Context, hostIP string, hostKey types.PublicKey, renterKey types.PrivateKey, contractID types.FileContractID) (err error) { + s.closeTransport() + + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", hostIP) + if err != nil { + return err + } + s.transport, err = rhp2.NewRenterTransport(conn, hostKey) + if err != nil { + return err + } + + s.key = renterKey + if err = s.lock(ctx, contractID, renterKey, 10*time.Second); err != nil { + s.closeTransport() + return err + } + + if err := s.updateSettings(ctx); err != nil { + s.closeTransport() + return err + } + + s.lastSeen = time.Now() + return nil +} + +// Refresh refreshes the session +func (s *RHP2Session) Refresh(ctx context.Context, sessionTTL time.Duration, renterKey types.PrivateKey, contractID types.FileContractID) error { + if s.transport == nil { + return errors.New("no transport") + } + + if time.Since(s.lastSeen) >= sessionTTL { + // use RPCSettings as a generic "ping" + if err := s.updateSettings(ctx); err != nil { + return err + } + } + + if s.revision.ID() != contractID { + // connected, but not locking the correct contract + if s.revision.ID() != (types.FileContractID{}) { + if err := s.unlock(ctx); err != nil { + return err + } + } + if err := s.lock(ctx, contractID, renterKey, 10*time.Second); err != nil { + return err + } + + s.key = renterKey + if err := s.updateSettings(ctx); err != nil { + return err + } + } + s.lastSeen = time.Now() + return nil +} + +// RenewContract renews the contract +func (s *RHP2Session) RenewContract(ctx context.Context, txnSet []types.Transaction, finalPayment types.Currency) (_ rhp2.ContractRevision, _ []types.Transaction, err error) { + // strip our signatures before sending + parents, txn := txnSet[:len(txnSet)-1], txnSet[len(txnSet)-1] + renterContractSignatures := txn.Signatures + txnSet[len(txnSet)-1].Signatures = nil + + // construct the final revision of the old contract + finalOldRevision := s.revision.Revision + newValid, _ := updateRevisionOutputs(&finalOldRevision, finalPayment, types.ZeroCurrency) + finalOldRevision.MissedProofOutputs = finalOldRevision.ValidProofOutputs + finalOldRevision.Filesize = 0 + finalOldRevision.FileMerkleRoot = types.Hash256{} + finalOldRevision.RevisionNumber = math.MaxUint64 + + // construct the renew request + req := &rhp2.RPCRenewAndClearContractRequest{ + Transactions: txnSet, + RenterKey: s.revision.Revision.UnlockConditions.PublicKeys[0], + FinalValidProofValues: newValid, + FinalMissedProofValues: newValid, + } + + // send the request + var resp rhp2.RPCFormContractAdditions + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.WriteRequest(rhp2.RPCRenewClearContractID, req); err != nil { + return err + } + return transport.ReadResponse(&resp, 65536) + }); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + // merge host additions with txn + txn.SiacoinInputs = append(txn.SiacoinInputs, resp.Inputs...) + txn.SiacoinOutputs = append(txn.SiacoinOutputs, resp.Outputs...) + + // create initial (no-op) revision, transaction, and signature + fc := txn.FileContracts[0] + initRevision := types.FileContractRevision{ + ParentID: txn.FileContractID(0), + UnlockConditions: s.revision.Revision.UnlockConditions, + FileContract: types.FileContract{ + RevisionNumber: 1, + Filesize: fc.Filesize, + FileMerkleRoot: fc.FileMerkleRoot, + WindowStart: fc.WindowStart, + WindowEnd: fc.WindowEnd, + ValidProofOutputs: fc.ValidProofOutputs, + MissedProofOutputs: fc.MissedProofOutputs, + UnlockHash: fc.UnlockHash, + }, + } + revSig := s.key.SignHash(hashRevision(initRevision)) + renterRevisionSig := types.TransactionSignature{ + ParentID: types.Hash256(initRevision.ParentID), + CoveredFields: types.CoveredFields{FileContractRevisions: []uint64{0}}, + PublicKeyIndex: 0, + Signature: revSig[:], + } + + // create signatures + finalRevSig := s.key.SignHash(hashRevision(finalOldRevision)) + renterSigs := &rhp2.RPCRenewAndClearContractSignatures{ + ContractSignatures: renterContractSignatures, + RevisionSignature: renterRevisionSig, + FinalRevisionSignature: finalRevSig, + } + + // send the signatures and read the host's signatures + var hostSigs rhp2.RPCRenewAndClearContractSignatures + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.WriteResponse(renterSigs); err != nil { + return err + } + return transport.ReadResponse(&hostSigs, 4096) + }); err != nil { + return rhp2.ContractRevision{}, nil, err + } + + // merge host signatures with our own + txn.Signatures = append(renterContractSignatures, hostSigs.ContractSignatures...) + signedTxnSet := append(resp.Parents, append(parents, txn)...) + return rhp2.ContractRevision{ + Revision: initRevision, + Signatures: [2]types.TransactionSignature{renterRevisionSig, hostSigs.RevisionSignature}, + }, signedTxnSet, nil +} + +// Revision returns the contract revision +func (s *RHP2Session) Revision() (rev rhp2.ContractRevision) { + b, _ := json.Marshal(s.revision) // deep copy + if err := json.Unmarshal(b, &rev); err != nil { + panic(err) // should never happen + } + return rev +} + +// RPCAppendCost returns the cost of a single append +func (s *RHP2Session) RPCAppendCost(remainingDuration uint64) (price types.Currency, collateral types.Currency, err error) { + var sector [rhp2.SectorSize]byte + actions := []rhp2.RPCWriteAction{{Type: rhp2.RPCWriteActionAppend, Data: sector[:]}} + cost, err := s.settings.RPCWriteCost(actions, s.revision.Revision.Filesize/rhp2.SectorSize, remainingDuration, true) + if err != nil { + return types.ZeroCurrency, types.ZeroCurrency, err + } + price, collateral = cost.Total() + return +} + +// SectorRoots returns n roots at offset. +func (s *RHP2Session) SectorRoots(ctx context.Context, offset, n uint64, price types.Currency) (roots []types.Hash256, err error) { + if !s.isRevisable() { + return nil, errContractFinalized + } else if offset+n > s.revision.NumSectors() { + return nil, errors.New("requested range is out-of-bounds") + } else if n == 0 { + return nil, nil + } else if !s.sufficientFunds(price) { + return nil, errInsufficientFunds + } + + // construct new revision + rev := s.revision.Revision + rev.RevisionNumber++ + newValid, newMissed := updateRevisionOutputs(&rev, price, types.ZeroCurrency) + revisionHash := hashRevision(rev) + + req := &rhp2.RPCSectorRootsRequest{ + RootOffset: uint64(offset), + NumRoots: uint64(n), + + RevisionNumber: rev.RevisionNumber, + ValidProofValues: newValid, + MissedProofValues: newMissed, + Signature: s.key.SignHash(revisionHash), + } + + // execute the sector roots RPC + var resp rhp2.RPCSectorRootsResponse + err = s.withTransport(ctx, func(t *rhp2.Transport) error { + if err := t.WriteRequest(rhp2.RPCSectorRootsID, req); err != nil { + return err + } else if err := t.ReadResponse(&resp, uint64(4096+32*n)); err != nil { + readCtx := fmt.Sprintf("couldn't read %v response", rhp2.RPCSectorRootsID) + rejectCtx := fmt.Sprintf("host rejected %v request", rhp2.RPCSectorRootsID) + return wrapResponseErr(err, readCtx, rejectCtx) + } else { + return nil + } + }) + if err != nil { + return nil, err + } + + // verify the host signature + if !s.HostKey().VerifyHash(revisionHash, resp.Signature) { + return nil, errors.New("host's signature is invalid") + } + s.revision.Revision = rev + s.revision.Signatures[0].Signature = req.Signature[:] + s.revision.Signatures[1].Signature = resp.Signature[:] + + // verify the proof + if !rhp2.VerifySectorRangeProof(resp.MerkleProof, resp.SectorRoots, offset, offset+n, s.revision.NumSectors(), rev.FileMerkleRoot) { + return nil, errInvalidMerkleProof + } + return resp.SectorRoots, nil +} + +// Settings returns the host settings +func (s *RHP2Session) Settings() *rhp2.HostSettings { return &s.settings } + +// Write performs the given write actions +func (s *RHP2Session) Write(ctx context.Context, actions []rhp2.RPCWriteAction, price, collateral types.Currency) (err error) { + if !s.isRevisable() { + return errContractFinalized + } else if len(actions) == 0 { + return nil + } else if !s.sufficientFunds(price) { + return errInsufficientFunds + } else if !s.sufficientCollateral(collateral) { + return errInsufficientCollateral + } + + rev := s.revision.Revision + newFilesize := rev.Filesize + for _, action := range actions { + switch action.Type { + case rhp2.RPCWriteActionAppend: + newFilesize += rhp2.SectorSize + case rhp2.RPCWriteActionTrim: + newFilesize -= rhp2.SectorSize * action.A + } + } + + // calculate new revision outputs + newValid, newMissed := updateRevisionOutputs(&rev, price, collateral) + + // compute appended roots in parallel with I/O + precompChan := make(chan struct{}) + go func() { + s.appendRoots = s.appendRoots[:0] + for _, action := range actions { + if action.Type == rhp2.RPCWriteActionAppend { + s.appendRoots = append(s.appendRoots, rhp2.SectorRoot((*[rhp2.SectorSize]byte)(action.Data))) + } + } + close(precompChan) + }() + // ensure that the goroutine has exited before we return + defer func() { <-precompChan }() + + // create request + req := &rhp2.RPCWriteRequest{ + Actions: actions, + MerkleProof: true, + + RevisionNumber: rev.RevisionNumber + 1, + ValidProofValues: newValid, + MissedProofValues: newMissed, + } + + // send request and read merkle proof + var merkleResp rhp2.RPCWriteMerkleProof + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.WriteRequest(rhp2.RPCWriteID, req); err != nil { + return err + } else if err := transport.ReadResponse(&merkleResp, 4096); err != nil { + return wrapResponseErr(err, "couldn't read Merkle proof response", "host rejected Write request") + } else { + return nil + } + }); err != nil { + return err + } + + // verify proof + proofHashes := merkleResp.OldSubtreeHashes + leafHashes := merkleResp.OldLeafHashes + oldRoot, newRoot := types.Hash256(rev.FileMerkleRoot), merkleResp.NewMerkleRoot + <-precompChan + if newFilesize > 0 && !rhp2.VerifyDiffProof(actions, s.revision.NumSectors(), proofHashes, leafHashes, oldRoot, newRoot, s.appendRoots) { + err := errInvalidMerkleProof + s.withTransport(ctx, func(transport *rhp2.Transport) error { return transport.WriteResponseErr(err) }) + return err + } + + // update revision + rev.RevisionNumber++ + rev.Filesize = newFilesize + copy(rev.FileMerkleRoot[:], newRoot[:]) + revisionHash := hashRevision(rev) + renterSig := &rhp2.RPCWriteResponse{ + Signature: s.key.SignHash(revisionHash), + } + + // exchange signatures + var hostSig rhp2.RPCWriteResponse + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.WriteResponse(renterSig); err != nil { + return fmt.Errorf("couldn't write signature response: %w", err) + } else if err := transport.ReadResponse(&hostSig, 4096); err != nil { + return wrapResponseErr(err, "couldn't read signature response", "host rejected Write signature") + } else { + return nil + } + }); err != nil { + return err + } + + // verify the host signature + if !s.HostKey().VerifyHash(revisionHash, hostSig.Signature) { + return errors.New("host's signature is invalid") + } + s.revision.Revision = rev + s.revision.Signatures[0].Signature = renterSig.Signature[:] + s.revision.Signatures[1].Signature = hostSig.Signature[:] + return nil +} + +func (s *RHP2Session) closeTransport() error { + if s.transport != nil { + return s.transport.Close() + } + return nil +} + +func (s *RHP2Session) isRevisable() bool { + return s.revision.Revision.RevisionNumber < math.MaxUint64 +} + +func (s *RHP2Session) lock(ctx context.Context, id types.FileContractID, key types.PrivateKey, timeout time.Duration) (err error) { + req := &rhp2.RPCLockRequest{ + ContractID: id, + Signature: s.transport.SignChallenge(key), + Timeout: uint64(timeout.Milliseconds()), + } + + // execute lock RPC + var resp rhp2.RPCLockResponse + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + if err := transport.Call(rhp2.RPCLockID, req, &resp); err != nil { + return err + } + transport.SetChallenge(resp.NewChallenge) + return nil + }); err != nil { + return err + } + + // verify claimed revision + if len(resp.Signatures) != 2 { + return fmt.Errorf("host returned wrong number of signatures (expected 2, got %v)", len(resp.Signatures)) + } else if len(resp.Signatures[0].Signature) != 64 || len(resp.Signatures[1].Signature) != 64 { + return errors.New("signatures on claimed revision have wrong length") + } + revHash := hashRevision(resp.Revision) + if !key.PublicKey().VerifyHash(revHash, *(*types.Signature)(resp.Signatures[0].Signature)) { + return errors.New("renter's signature on claimed revision is invalid") + } else if !s.transport.HostKey().VerifyHash(revHash, *(*types.Signature)(resp.Signatures[1].Signature)) { + return errors.New("host's signature on claimed revision is invalid") + } else if !resp.Acquired { + return errContractLocked + } else if resp.Revision.RevisionNumber == math.MaxUint64 { + return errContractFinalized + } + s.revision = rhp2.ContractRevision{ + Revision: resp.Revision, + Signatures: [2]types.TransactionSignature{resp.Signatures[0], resp.Signatures[1]}, + } + return nil +} + +func (s *RHP2Session) readSection(w io.Writer, t *rhp2.Transport, sec rhp2.RPCReadRequestSection) (hostSig *types.Signature, _ error) { + // NOTE: normally, we would call ReadResponse here to read an AEAD RPC + // message, verify the tag and decrypt, and then pass the data to + // VerifyProof. As an optimization, we instead stream the message + // through a Merkle proof verifier before verifying the AEAD tag. + // Security therefore depends on the caller of Read discarding any data + // written to w in the event that verification fails. + msgReader, err := t.RawResponse(4096 + uint64(sec.Length)) + if err != nil { + return nil, wrapResponseErr(err, "couldn't read sector data", "host rejected Read request") + } + // Read the signature, which may or may not be present. + lenbuf := make([]byte, 8) + if _, err := io.ReadFull(msgReader, lenbuf); err != nil { + return nil, fmt.Errorf("couldn't read signature len: %w", err) + } + if n := binary.LittleEndian.Uint64(lenbuf); n > 0 { + hostSig = new(types.Signature) + if _, err := io.ReadFull(msgReader, hostSig[:]); err != nil { + return nil, fmt.Errorf("couldn't read signature: %w", err) + } + } + // stream the sector data into w and the proof verifier + if _, err := io.ReadFull(msgReader, lenbuf); err != nil { + return nil, fmt.Errorf("couldn't read data len: %w", err) + } else if binary.LittleEndian.Uint64(lenbuf) != uint64(sec.Length) { + return nil, errors.New("host sent wrong amount of sector data") + } + proofStart := sec.Offset / rhp2.LeafSize + proofEnd := proofStart + sec.Length/rhp2.LeafSize + rpv := rhp2.NewRangeProofVerifier(proofStart, proofEnd) + tee := io.TeeReader(io.LimitReader(msgReader, int64(sec.Length)), &segWriter{w: w}) + // the proof verifier Reads one segment at a time, so bufio is crucial + // for performance here + if _, err := rpv.ReadFrom(bufio.NewReaderSize(tee, 1<<16)); err != nil { + return nil, fmt.Errorf("couldn't stream sector data: %w", err) + } + // read the Merkle proof + if _, err := io.ReadFull(msgReader, lenbuf); err != nil { + return nil, fmt.Errorf("couldn't read proof len: %w", err) + } + if binary.LittleEndian.Uint64(lenbuf) != uint64(rhp2.RangeProofSize(rhp2.LeavesPerSector, proofStart, proofEnd)) { + return nil, errors.New("invalid proof size") + } + proof := make([]types.Hash256, binary.LittleEndian.Uint64(lenbuf)) + for i := range proof { + if _, err := io.ReadFull(msgReader, proof[i][:]); err != nil { + return nil, fmt.Errorf("couldn't read Merkle proof: %w", err) + } + } + // verify the message tag and the Merkle proof + if err := msgReader.VerifyTag(); err != nil { + return nil, err + } + if !rpv.Verify(proof, sec.MerkleRoot) { + return nil, errInvalidMerkleProof + } + return +} + +func (s *RHP2Session) sufficientFunds(price types.Currency) bool { + return s.revision.RenterFunds().Cmp(price) >= 0 +} + +func (s *RHP2Session) sufficientCollateral(collateral types.Currency) bool { + return s.revision.Revision.MissedProofOutputs[1].Value.Cmp(collateral) >= 0 +} + +func (s *RHP2Session) updateSettings(ctx context.Context) (err error) { + var resp rhp2.RPCSettingsResponse + if err := s.withTransport(ctx, func(transport *rhp2.Transport) error { + return transport.Call(rhp2.RPCSettingsID, nil, &resp) + }); err != nil { + return err + } + + if err := json.Unmarshal(resp.Settings, &s.settings); err != nil { + return fmt.Errorf("couldn't unmarshal json: %w", err) + } + return +} + +func (s *RHP2Session) unlock(ctx context.Context) (err error) { + s.revision = rhp2.ContractRevision{} + s.key = nil + + return s.withTransport(ctx, func(transport *rhp2.Transport) error { + return transport.WriteRequest(rhp2.RPCUnlockID, nil) + }) +} + +func (s *RHP2Session) withTransport(ctx context.Context, fn func(t *rhp2.Transport) error) (err error) { + errChan := make(chan error) + go func() { + defer close(errChan) + errChan <- fn(s.transport) + }() + + select { + case err = <-errChan: + return + case <-ctx.Done(): + _ = s.transport.ForceClose() // ignore error + if err = <-errChan; err == nil { + err = ctx.Err() + } + s.transport = nil + } + return +} + +func hashRevision(rev types.FileContractRevision) types.Hash256 { + h := types.NewHasher() + rev.EncodeTo(h.E) + return h.Sum() +} + +func updateRevisionOutputs(rev *types.FileContractRevision, cost, collateral types.Currency) (valid, missed []types.Currency) { + // allocate new slices; don't want to risk accidentally sharing memory + rev.ValidProofOutputs = append([]types.SiacoinOutput(nil), rev.ValidProofOutputs...) + rev.MissedProofOutputs = append([]types.SiacoinOutput(nil), rev.MissedProofOutputs...) + + // move valid payout from renter to host + rev.ValidProofOutputs[0].Value = rev.ValidProofOutputs[0].Value.Sub(cost) + rev.ValidProofOutputs[1].Value = rev.ValidProofOutputs[1].Value.Add(cost) + + // move missed payout from renter to void + rev.MissedProofOutputs[0].Value = rev.MissedProofOutputs[0].Value.Sub(cost) + rev.MissedProofOutputs[2].Value = rev.MissedProofOutputs[2].Value.Add(cost) + + // move collateral from host to void + rev.MissedProofOutputs[1].Value = rev.MissedProofOutputs[1].Value.Sub(collateral) + rev.MissedProofOutputs[2].Value = rev.MissedProofOutputs[2].Value.Add(collateral) + + return []types.Currency{rev.ValidProofOutputs[0].Value, rev.ValidProofOutputs[1].Value}, + []types.Currency{rev.MissedProofOutputs[0].Value, rev.MissedProofOutputs[1].Value, rev.MissedProofOutputs[2].Value} +} + +func wrapResponseErr(err error, readCtx, rejectCtx string) error { + if errors.As(err, new(*rhp2.RPCError)) { + return fmt.Errorf("%s: %w", rejectCtx, err) + } + if err != nil { + return fmt.Errorf("%s: %w", readCtx, err) + } + return nil +} + +type segWriter struct { + w io.Writer + buf [rhp2.LeafSize * 64]byte + len int +} + +func (sw *segWriter) Write(p []byte) (int, error) { + lenp := len(p) + for len(p) > 0 { + n := copy(sw.buf[sw.len:], p) + sw.len += n + p = p[n:] + segs := sw.buf[:sw.len-(sw.len%rhp2.LeafSize)] + if _, err := sw.w.Write(segs); err != nil { + return 0, err + } + sw.len = copy(sw.buf[:], sw.buf[len(segs):sw.len]) + } + return lenp, nil +} diff --git a/internal/rhp/v3/rhp.go b/internal/rhp/v3/rhp.go new file mode 100644 index 0000000..a745a77 --- /dev/null +++ b/internal/rhp/v3/rhp.go @@ -0,0 +1,815 @@ +package rhp + +import ( + "context" + "crypto/ed25519" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "math/bits" + "net" + + "go.sia.tech/core/consensus" + rhp2 "go.sia.tech/core/rhp/v2" + rhp3 "go.sia.tech/core/rhp/v3" + "go.sia.tech/core/types" +) + +type ( + // An accountPayment pays for usage using an ephemeral account + accountPayment struct { + Account rhp3.Account + PrivateKey types.PrivateKey + } + + // A contractPayment pays for usage using a contract + contractPayment struct { + Revision *rhp2.ContractRevision + RefundAccount rhp3.Account + RenterKey types.PrivateKey + } + + // A PaymentMethod facilitates payments to the host using either a contract + // or an ephemeral account + PaymentMethod interface { + Pay(amount types.Currency, height uint64) (rhp3.PaymentMethod, bool) + } + + // A Wallet funds and signs transactions + Wallet interface { + Address() types.Address + FundTransaction(txn *types.Transaction, amount types.Currency) ([]types.Hash256, func(), error) + SignTransaction(cs consensus.State, txn *types.Transaction, toSign []types.Hash256, cf types.CoveredFields) error + } + + // A ChainManager is used to get the current consensus state + ChainManager interface { + TipState() consensus.State + } +) + +type ( + // A Session is an RHP3 session with the host + Session struct { + hostKey types.PublicKey + cm ChainManager + w Wallet + t *rhp3.Transport + + pt rhp3.HostPriceTable + } +) + +// Pay implements PaymentMethod +func (cp *contractPayment) Pay(amount types.Currency, height uint64) (rhp3.PaymentMethod, bool) { + req, ok := rhp3.PayByContract(&cp.Revision.Revision, amount, cp.RefundAccount, cp.RenterKey) + return &req, ok +} + +// Pay implements PaymentMethod +func (ap *accountPayment) Pay(amount types.Currency, height uint64) (rhp3.PaymentMethod, bool) { + expirationHeight := height + 6 + req := rhp3.PayByEphemeralAccount(ap.Account, amount, expirationHeight, ap.PrivateKey) + return &req, true +} + +// RegisterPriceTable registers the price table with the host +func (s *Session) RegisterPriceTable(payment PaymentMethod) (rhp3.HostPriceTable, error) { + stream := s.t.DialStream() + defer stream.Close() + + if err := stream.WriteRequest(rhp3.RPCUpdatePriceTableID, nil); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to write request: %w", err) + } + var resp rhp3.RPCUpdatePriceTableResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to read response: %w", err) + } + + var pt rhp3.HostPriceTable + if err := json.Unmarshal(resp.PriceTableJSON, &pt); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to unmarshal price table: %w", err) + } else if err := s.processPayment(stream, payment, pt.UpdatePriceTableCost); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to pay: %w", err) + } + var confirmResp rhp3.RPCPriceTableResponse + if err := stream.ReadResponse(&confirmResp, 4096); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to read response: %w", err) + } + s.pt = pt + return pt, nil +} + +// FundAccount funds the account with the given amount +func (s *Session) FundAccount(account rhp3.Account, payment PaymentMethod, amount types.Currency) (types.Currency, error) { + stream := s.t.DialStream() + defer stream.Close() + + if err := stream.WriteRequest(rhp3.RPCFundAccountID, &s.pt.UID); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write request: %w", err) + } + + req := &rhp3.RPCFundAccountRequest{ + Account: account, + } + if err := stream.WriteResponse(req); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.FundAccountCost.Add(amount)); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to pay: %w", err) + } + + var resp rhp3.RPCFundAccountResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + return resp.Balance, nil +} + +// AccountBalance retrieves the balance of the given account +func (s *Session) AccountBalance(account rhp3.Account, payment PaymentMethod) (types.Currency, error) { + stream := s.t.DialStream() + defer stream.Close() + + if err := stream.WriteRequest(rhp3.RPCAccountBalanceID, &s.pt.UID); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write request: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.AccountBalanceCost); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to pay: %w", err) + } + + req := rhp3.RPCAccountBalanceRequest{ + Account: account, + } + if err := stream.WriteResponse(&req); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } + + var resp rhp3.RPCAccountBalanceResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + return resp.Balance, nil +} + +// Revision retrieves the latest revision of the contract +func (s *Session) Revision(contractID types.FileContractID) (types.FileContractRevision, error) { + stream := s.t.DialStream() + defer stream.Close() + + req := rhp3.RPCLatestRevisionRequest{ + ContractID: contractID, + } + if err := stream.WriteRequest(rhp3.RPCLatestRevisionID, &req); err != nil { + return types.FileContractRevision{}, fmt.Errorf("failed to write request: %w", err) + } + var resp rhp3.RPCLatestRevisionResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return types.FileContractRevision{}, fmt.Errorf("failed to read response: %w", err) + } else if err := stream.WriteResponse(&s.pt.UID); err != nil { + return types.FileContractRevision{}, fmt.Errorf("failed to write price table uid: %w", err) + } + return resp.Revision, nil +} + +// StoreSector stores the given sector for the given duration +func (s *Session) StoreSector(sector *[rhp2.SectorSize]byte, duration uint64, payment PaymentMethod, budget types.Currency) error { + stream := s.t.DialStream() + defer stream.Close() + + req := rhp3.RPCExecuteProgramRequest{ + Program: []rhp3.Instruction{ + &rhp3.InstrStoreSector{ + DataOffset: 0, + Duration: duration, + }, + }, + ProgramData: sector[:], + } + + if err := stream.WriteRequest(rhp3.RPCExecuteProgramID, &s.pt.UID); err != nil { + return fmt.Errorf("failed to write request: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.InitBaseCost.Add(budget)); err != nil { + return fmt.Errorf("failed to pay: %w", err) + } else if err := stream.WriteResponse(&req); err != nil { + return fmt.Errorf("failed to write response: %w", err) + } + var cancelToken types.Specifier // unused + if err := stream.ReadResponse(&cancelToken, 4096); err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + + var resp rhp3.RPCExecuteProgramResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return fmt.Errorf("failed to read response: %w", err) + } else if resp.Error != nil { + return fmt.Errorf("failed to append sector: %w", resp.Error) + } + return nil +} + +// AppendSector appends a sector to the contract +func (s *Session) AppendSector(sector *[rhp2.SectorSize]byte, revision *rhp2.ContractRevision, renterKey types.PrivateKey, payment PaymentMethod, budget types.Currency) (types.Currency, error) { + stream := s.t.DialStream() + defer stream.Close() + + req := rhp3.RPCExecuteProgramRequest{ + FileContractID: revision.ID(), + Program: []rhp3.Instruction{ + &rhp3.InstrAppendSector{ + SectorDataOffset: 0, + ProofRequired: true, + }, + }, + ProgramData: sector[:], + } + + if err := stream.WriteRequest(rhp3.RPCExecuteProgramID, &s.pt.UID); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write request: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.InitBaseCost.Add(budget)); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to pay: %w", err) + } else if err := stream.WriteResponse(&req); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } + var cancelToken types.Specifier // unused + if err := stream.ReadResponse(&cancelToken, 4096); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + + var resp rhp3.RPCExecuteProgramResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } else if resp.Error != nil { + return types.ZeroCurrency, fmt.Errorf("failed to append sector: %w", resp.Error) + } else if resp.NewSize != revision.Revision.Filesize+rhp2.SectorSize { + return types.ZeroCurrency, fmt.Errorf("unexpected filesize: %v != %v", resp.NewSize, revision.Revision.Filesize+rhp2.SectorSize) + } + //TODO: validate proof + // revise the contract + revised := revision.Revision + revised.RevisionNumber++ + revised.Filesize = resp.NewSize + revised.FileMerkleRoot = resp.NewMerkleRoot + revised.ValidProofOutputs = make([]types.SiacoinOutput, len(revision.Revision.ValidProofOutputs)) + revised.MissedProofOutputs = make([]types.SiacoinOutput, len(revision.Revision.MissedProofOutputs)) + for i := range revision.Revision.ValidProofOutputs { + revised.ValidProofOutputs[i].Address = revision.Revision.ValidProofOutputs[i].Address + revised.ValidProofOutputs[i].Value = revision.Revision.ValidProofOutputs[i].Value + } + for i := range revision.Revision.MissedProofOutputs { + revised.MissedProofOutputs[i].Address = revision.Revision.MissedProofOutputs[i].Address + revised.MissedProofOutputs[i].Value = revision.Revision.MissedProofOutputs[i].Value + } + // subtract the storage revenue and collateral from the host's missed proof + // output and add it to the void + transfer := resp.AdditionalCollateral.Add(resp.FailureRefund) + revised.MissedProofOutputs[1].Value = revised.MissedProofOutputs[1].Value.Sub(transfer) + revised.MissedProofOutputs[2].Value = revised.MissedProofOutputs[2].Value.Add(transfer) + validProofValues := make([]types.Currency, len(revised.ValidProofOutputs)) + for i := range validProofValues { + validProofValues[i] = revised.ValidProofOutputs[i].Value + } + missedProofValues := make([]types.Currency, len(revised.MissedProofOutputs)) + for i := range missedProofValues { + missedProofValues[i] = revised.MissedProofOutputs[i].Value + } + + sigHash := hashRevision(revised) + finalizeReq := rhp3.RPCFinalizeProgramRequest{ + Signature: renterKey.SignHash(sigHash), + RevisionNumber: revised.RevisionNumber, + ValidProofValues: validProofValues, + MissedProofValues: missedProofValues, + } + if err := stream.WriteResponse(&finalizeReq); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } + var finalizeResp rhp3.RPCFinalizeProgramResponse + if err := stream.ReadResponse(&finalizeResp, 4096); err != nil { + return types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + revision.Revision = revised + revision.Signatures = [2]types.TransactionSignature{ + { + ParentID: types.Hash256(revised.ParentID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{ + FileContractRevisions: []uint64{0}, + }, + Signature: finalizeReq.Signature[:], + }, + { + ParentID: types.Hash256(revised.ParentID), + PublicKeyIndex: 1, + CoveredFields: types.CoveredFields{ + FileContractRevisions: []uint64{0}, + }, + Signature: finalizeResp.Signature[:], + }, + } + return resp.TotalCost, nil +} + +// ReadSector downloads a sector from the host. +func (s *Session) ReadSector(root types.Hash256, offset, length uint64, payment PaymentMethod, budget types.Currency) ([]byte, types.Currency, error) { + stream := s.t.DialStream() + defer stream.Close() + + programData := make([]byte, 48) + binary.LittleEndian.PutUint64(programData[0:8], length) + binary.LittleEndian.PutUint64(programData[8:16], offset) + copy(programData[16:], root[:]) + + req := rhp3.RPCExecuteProgramRequest{ + Program: []rhp3.Instruction{ + &rhp3.InstrReadSector{ + LengthOffset: 0, + OffsetOffset: 8, + MerkleRootOffset: 16, + ProofRequired: true, + }, + }, + ProgramData: programData, + } + + if err := stream.WriteRequest(rhp3.RPCExecuteProgramID, &s.pt.UID); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to write request: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.InitBaseCost.Add(budget)); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to pay: %w", err) + } else if err := stream.WriteResponse(&req); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } + var cancelToken types.Specifier // unused + if err := stream.ReadResponse(&cancelToken, 4096); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + + var resp rhp3.RPCExecuteProgramResponse + if err := stream.ReadResponse(&resp, 4096+length); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } else if resp.Error != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to append sector: %w", resp.Error) + } else if len(resp.Output) != int(length) { + return nil, types.ZeroCurrency, fmt.Errorf("unexpected output length: %v != %v", len(resp.Output), length) + } + return resp.Output, resp.TotalCost, nil +} + +// ReadOffset reads a sector from a contract at a given offset. +func (s *Session) ReadOffset(offset, length uint64, contractID types.FileContractID, payment PaymentMethod, budget types.Currency) ([]byte, types.Currency, error) { + stream := s.t.DialStream() + defer stream.Close() + + programData := make([]byte, 16) + binary.LittleEndian.PutUint64(programData[0:8], length) + binary.LittleEndian.PutUint64(programData[8:16], offset) + + req := rhp3.RPCExecuteProgramRequest{ + FileContractID: contractID, + Program: []rhp3.Instruction{ + &rhp3.InstrReadOffset{ + LengthOffset: 0, + OffsetOffset: 8, + ProofRequired: true, + }, + }, + ProgramData: programData, + } + + if err := stream.WriteRequest(rhp3.RPCExecuteProgramID, &s.pt.UID); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to write request: %w", err) + } else if err := s.processPayment(stream, payment, s.pt.InitBaseCost.Add(budget)); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to pay: %w", err) + } else if err := stream.WriteResponse(&req); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to write response: %w", err) + } + var cancelToken types.Specifier // unused + if err := stream.ReadResponse(&cancelToken, 4096); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } + + var resp rhp3.RPCExecuteProgramResponse + if err := stream.ReadResponse(&resp, 4096+length); err != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to read response: %w", err) + } else if resp.Error != nil { + return nil, types.ZeroCurrency, fmt.Errorf("failed to append sector: %w", resp.Error) + } else if len(resp.Output) != int(length) { + return nil, types.ZeroCurrency, fmt.Errorf("unexpected output length: %v != %v", len(resp.Output), length) + } + return resp.Output, resp.TotalCost, nil +} + +// ScanPriceTable retrieves the host's current price table +func (s *Session) ScanPriceTable() (rhp3.HostPriceTable, error) { + stream := s.t.DialStream() + defer stream.Close() + + if err := stream.WriteRequest(rhp3.RPCUpdatePriceTableID, nil); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to write request: %w", err) + } + var resp rhp3.RPCUpdatePriceTableResponse + if err := stream.ReadResponse(&resp, 4096); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to read response: %w", err) + } + + var pt rhp3.HostPriceTable + if err := json.Unmarshal(resp.PriceTableJSON, &pt); err != nil { + return rhp3.HostPriceTable{}, fmt.Errorf("failed to unmarshal price table: %w", err) + } + return pt, nil +} + +// RenewContract renews an existing contract with the host +func (s *Session) RenewContract(revision *rhp2.ContractRevision, hostAddr types.Address, renterKey types.PrivateKey, renterPayout, newCollateral types.Currency, endHeight uint64) (rhp2.ContractRevision, []types.Transaction, error) { + stream := s.t.DialStream() + defer stream.Close() + + state := s.cm.TipState() + + pt := s.pt + if err := stream.WriteRequest(rhp3.RPCRenewContractID, &pt.UID); err != nil { + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to write request: %w", err) + } else if pt.UID == (rhp3.SettingsID{}) { + // if the price table UID is the zero value, the host sends + // a temporary price table + var priceTableResp rhp3.RPCUpdatePriceTableResponse + if err := stream.ReadResponse(&priceTableResp, 4096); err != nil { + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to read response: %w", err) + } + if err := json.Unmarshal(priceTableResp.PriceTableJSON, &pt); err != nil { + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to unmarshal price table: %w", err) + } + } + + clearingValues := make([]types.Currency, len(revision.Revision.ValidProofOutputs)) + for i := range revision.Revision.ValidProofOutputs { + clearingValues[i] = revision.Revision.ValidProofOutputs[i].Value + } + + clearingRevision, err := clearingRevision(revision.Revision, clearingValues) + if err != nil { + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to create clearing revision: %w", err) + } + + txnFee := types.Siacoins(1) + renewal, baseCost := prepareContractRenewal(revision.Revision, s.w.Address(), renterKey, renterPayout, newCollateral, s.hostKey, hostAddr, pt, endHeight) + renewTxn := types.Transaction{ + MinerFees: []types.Currency{txnFee}, + FileContractRevisions: []types.FileContractRevision{clearingRevision}, + FileContracts: []types.FileContract{renewal}, + } + renterCost := rhp2.ContractRenewalCost(state, renewal, pt.ContractPrice, txnFee, baseCost) + toSign, release, err := s.w.FundTransaction(&renewTxn, renterCost) + if err != nil { + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to fund transaction: %w", err) + } + + clearingSigHash := hashFinalRevision(clearingRevision, renewal) + renewReq := &rhp3.RPCRenewContractRequest{ + TransactionSet: []types.Transaction{renewTxn}, + RenterKey: renterKey.PublicKey().UnlockKey(), + FinalRevisionSignature: renterKey.SignHash(clearingSigHash), + } + if err := stream.WriteResponse(renewReq); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to write renew request: %w", err) + } + + var hostAdditions rhp3.RPCRenewContractHostAdditions + if err := stream.ReadResponse(&hostAdditions, 4096); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to read host additions response: %w", err) + } else if !s.hostKey.VerifyHash(clearingSigHash, hostAdditions.FinalRevisionSignature) { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("host final revision signature invalid") + } + // add the host's additions to the transaction set + renewalParents := hostAdditions.Parents + renewTxn.SiacoinInputs = append(renewTxn.SiacoinInputs, hostAdditions.SiacoinInputs...) + renewTxn.SiacoinOutputs = append(renewTxn.SiacoinOutputs, hostAdditions.SiacoinOutputs...) + + // sign the transaction + if err := s.w.SignTransaction(state, &renewTxn, toSign, types.CoveredFields{WholeTransaction: true}); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to sign transaction: %w", err) + } + + renewRevision := initialRevision(&renewTxn, s.hostKey.UnlockKey(), renterKey.PublicKey().UnlockKey()) + renewSigHash := hashRevision(renewRevision) + renterSig := renterKey.SignHash(renewSigHash) + renterSigsResp := &rhp3.RPCRenewSignatures{ + TransactionSignatures: renewTxn.Signatures, + RevisionSignature: types.TransactionSignature{ + ParentID: types.Hash256(renewRevision.ParentID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{ + FileContractRevisions: []uint64{0}, + }, + Signature: renterSig[:], + }, + } + if err := stream.WriteResponse(renterSigsResp); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to write renter signatures: %w", err) + } + + var hostSigsResp rhp3.RPCRenewSignatures + if err := stream.ReadResponse(&hostSigsResp, 4096); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("failed to read host signatures: %w", err) + } else if err := validateHostRevisionSignature(hostSigsResp.RevisionSignature, renewRevision.ParentID, renewSigHash, s.hostKey); err != nil { + release() + return rhp2.ContractRevision{}, nil, fmt.Errorf("invalid host revision signature: %w", err) + } + return rhp2.ContractRevision{ + Revision: renewRevision, + Signatures: [2]types.TransactionSignature{ + renterSigsResp.RevisionSignature, + hostSigsResp.RevisionSignature, + }, + }, append(renewalParents, renewTxn), nil +} + +// Close closes the underlying transport +func (s *Session) Close() error { + return s.t.Close() +} + +// processPayment processes a payment using the given payment method +func (s *Session) processPayment(stream *rhp3.Stream, method PaymentMethod, amount types.Currency) error { + pm, ok := method.Pay(amount, s.cm.TipState().Index.Height) + if !ok { + return fmt.Errorf("payment method cannot pay %v", amount) + } + switch pm := pm.(type) { + case *rhp3.PayByEphemeralAccountRequest: + if err := stream.WriteResponse(&rhp3.PaymentTypeEphemeralAccount); err != nil { + return fmt.Errorf("failed to write payment request type: %w", err) + } else if err := stream.WriteResponse(pm); err != nil { + return fmt.Errorf("failed to write request: %w", err) + } + case *rhp3.PayByContractRequest: + if err := stream.WriteResponse(&rhp3.PaymentTypeContract); err != nil { + return fmt.Errorf("failed to write payment request type: %w", err) + } else if err := stream.WriteResponse(pm); err != nil { + return fmt.Errorf("failed to write request: %w", err) + } + var hostSigResp rhp3.PaymentResponse + if err := stream.ReadResponse(&hostSigResp, 4096); err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + } + return nil +} + +// clearingRevision returns a revision that locks a contract and sets the missed +// proof outputs to the valid proof outputs. +func clearingRevision(revision types.FileContractRevision, outputValues []types.Currency) (types.FileContractRevision, error) { + if revision.RevisionNumber == math.MaxUint64 { + return types.FileContractRevision{}, errors.New("contract is locked") + } else if len(outputValues) != len(revision.ValidProofOutputs) { + return types.FileContractRevision{}, errors.New("incorrect number of outputs") + } + + oldValid := revision.ValidProofOutputs + revision.ValidProofOutputs = make([]types.SiacoinOutput, len(outputValues)) + for i := range outputValues { + revision.ValidProofOutputs[i].Address = oldValid[i].Address + revision.ValidProofOutputs[i].Value = outputValues[i] + } + revision.MissedProofOutputs = revision.ValidProofOutputs + revision.RevisionNumber = math.MaxUint64 + revision.Filesize = 0 + revision.FileMerkleRoot = types.Hash256{} + return revision, nil +} + +func contractUnlockConditions(hostKey, renterKey types.UnlockKey) types.UnlockConditions { + return types.UnlockConditions{ + PublicKeys: []types.UnlockKey{renterKey, hostKey}, + SignaturesRequired: 2, + } +} + +// hashFinalRevision returns the hash of the final revision during contract renewal +func hashFinalRevision(clearing types.FileContractRevision, renewal types.FileContract) types.Hash256 { + h := types.NewHasher() + renewal.EncodeTo(h.E) + clearing.EncodeTo(h.E) + return h.Sum() +} + +// HashRevision returns the hash of rev. +func hashRevision(rev types.FileContractRevision) types.Hash256 { + h := types.NewHasher() + rev.EncodeTo(h.E) + return h.Sum() +} + +func validateHostRevisionSignature(sig types.TransactionSignature, fcID types.FileContractID, sigHash types.Hash256, hostKey types.PublicKey) error { + switch { + case sig.ParentID != types.Hash256(fcID): + return errors.New("revision signature has invalid parent ID") + case sig.PublicKeyIndex != 1: + return errors.New("revision signature has invalid public key index") + case len(sig.Signature) != ed25519.SignatureSize: + return errors.New("revision signature has invalid length") + case len(sig.CoveredFields.SiacoinInputs) != 0: + return errors.New("signature should not cover siacoin inputs") + case len(sig.CoveredFields.SiacoinOutputs) != 0: + return errors.New("signature should not cover siacoin outputs") + case len(sig.CoveredFields.FileContracts) != 0: + return errors.New("signature should not cover file contract") + case len(sig.CoveredFields.StorageProofs) != 0: + return errors.New("signature should not cover storage proofs") + case len(sig.CoveredFields.SiafundInputs) != 0: + return errors.New("signature should not cover siafund inputs") + case len(sig.CoveredFields.SiafundOutputs) != 0: + return errors.New("signature should not cover siafund outputs") + case len(sig.CoveredFields.MinerFees) != 0: + return errors.New("signature should not cover miner fees") + case len(sig.CoveredFields.ArbitraryData) != 0: + return errors.New("signature should not cover arbitrary data") + case len(sig.CoveredFields.Signatures) != 0: + return errors.New("signature should not cover signatures") + case len(sig.CoveredFields.FileContractRevisions) != 1: + return errors.New("signature should cover one file contract revision") + case sig.CoveredFields.FileContractRevisions[0] != 0: + return errors.New("signature should cover the first file contract revision") + case !hostKey.VerifyHash(sigHash, *(*types.Signature)(sig.Signature)): + return errors.New("revision signature is invalid") + } + return nil +} + +// InitialRevision returns the first revision of a file contract formation +// transaction. +func initialRevision(formationTxn *types.Transaction, hostPubKey, renterPubKey types.UnlockKey) types.FileContractRevision { + fc := formationTxn.FileContracts[0] + return types.FileContractRevision{ + ParentID: formationTxn.FileContractID(0), + UnlockConditions: contractUnlockConditions(hostPubKey, renterPubKey), + FileContract: types.FileContract{ + Filesize: fc.Filesize, + FileMerkleRoot: fc.FileMerkleRoot, + WindowStart: fc.WindowStart, + WindowEnd: fc.WindowEnd, + ValidProofOutputs: fc.ValidProofOutputs, + MissedProofOutputs: fc.MissedProofOutputs, + UnlockHash: fc.UnlockHash, + RevisionNumber: 1, + }, + } +} + +// calculateRenewalPayouts calculates the contract payouts for the host. +func calculateRenewalPayouts(fc types.FileContract, newCollateral types.Currency, pt rhp3.HostPriceTable, endHeight uint64) (hostValidPayout, hostMissedPayout, voidMissedPayout, basePrice types.Currency) { + // The host gets their contract fee, plus the cost of the data already in the + // contract, plus their collateral. In the event of a missed payout, the cost + // and collateral of the data already in the contract is subtracted from the + // host, and sent to the void instead. + // + // However, it is possible for this subtraction to underflow: this can happen if + // baseCollateral is large and MaxCollateral is small. We cannot simply replace + // the underflow with a zero, because the host performs the same subtraction and + // returns an error on underflow. Nor can we increase the valid payout, because + // the host calculates its collateral contribution by subtracting the contract + // price and base price from this payout, and we're already at MaxCollateral. + // Thus the host has conflicting requirements, and renewing the contract is + // impossible until they change their settings. + + // calculate base price and collateral + // if the contract height did not increase both prices are zero + basePrice = pt.RenewContractCost + var baseCollateral types.Currency + if contractEnd := uint64(endHeight + pt.WindowSize); contractEnd > fc.WindowEnd { + timeExtension := uint64(contractEnd - fc.WindowEnd) + basePrice = basePrice.Add(pt.WriteStoreCost.Mul64(fc.Filesize).Mul64(timeExtension)) + baseCollateral = pt.CollateralCost.Mul64(fc.Filesize).Mul64(timeExtension) + } + + // calculate payouts + hostValidPayout = pt.ContractPrice.Add(basePrice).Add(baseCollateral).Add(newCollateral) + voidMissedPayout = basePrice.Add(baseCollateral) + if hostValidPayout.Cmp(voidMissedPayout) < 0 { + // TODO: detect this elsewhere + panic("host's settings are unsatisfiable") + } + hostMissedPayout = hostValidPayout.Sub(voidMissedPayout) + return hostValidPayout, hostMissedPayout, voidMissedPayout, basePrice +} + +// NOTE: due to a bug in the transaction validation code, calculating payouts +// is way harder than it needs to be. Tax is calculated on the post-tax +// contract payout (instead of the sum of the renter and host payouts). So the +// equation for the payout is: +// +// payout = renterPayout + hostPayout + payout*tax +// ∴ payout = (renterPayout + hostPayout) / (1 - tax) +// +// This would work if 'tax' were a simple fraction, but because the tax must +// be evenly distributed among siafund holders, 'tax' is actually a function +// that multiplies by a fraction and then rounds down to the nearest multiple +// of the siafund count. Thus, when inverting the function, we have to make an +// initial guess and then fix the rounding error. +func taxAdjustedPayout(target types.Currency) types.Currency { + // compute initial guess as target * (1 / 1-tax); since this does not take + // the siafund rounding into account, the guess will be up to + // types.SiafundCount greater than the actual payout value. + guess := target.Mul64(1000).Div64(961) + + // now, adjust the guess to remove the rounding error. We know that: + // + // (target % types.SiafundCount) == (payout % types.SiafundCount) + // + // therefore, we can simply adjust the guess to have this remainder as + // well. The only wrinkle is that, since we know guess >= payout, if the + // guess remainder is smaller than the target remainder, we must subtract + // an extra types.SiafundCount. + // + // for example, if target = 87654321 and types.SiafundCount = 10000, then: + // + // initial_guess = 87654321 * (1 / (1 - tax)) + // = 91211572 + // target % 10000 = 4321 + // adjusted_guess = 91204321 + + mod64 := func(c types.Currency, v uint64) types.Currency { + var r uint64 + if c.Hi < v { + _, r = bits.Div64(c.Hi, c.Lo, v) + } else { + _, r = bits.Div64(0, c.Hi, v) + _, r = bits.Div64(r, c.Lo, v) + } + return types.NewCurrency64(r) + } + sfc := (consensus.State{}).SiafundCount() + tm := mod64(target, sfc) + gm := mod64(guess, sfc) + if gm.Cmp(tm) < 0 { + guess = guess.Sub(types.NewCurrency64(sfc)) + } + return guess.Add(tm).Sub(gm) +} + +func prepareContractRenewal(currentRevision types.FileContractRevision, renterAddress types.Address, renterKey types.PrivateKey, renterPayout, newCollateral types.Currency, hostKey types.PublicKey, hostAddr types.Address, host rhp3.HostPriceTable, endHeight uint64) (types.FileContract, types.Currency) { + hostValidPayout, hostMissedPayout, voidMissedPayout, basePrice := calculateRenewalPayouts(currentRevision.FileContract, newCollateral, host, endHeight) + return types.FileContract{ + Filesize: currentRevision.Filesize, + FileMerkleRoot: currentRevision.FileMerkleRoot, + WindowStart: uint64(endHeight), + WindowEnd: uint64(endHeight + host.WindowSize), + Payout: taxAdjustedPayout(renterPayout.Add(hostValidPayout)), + UnlockHash: types.UnlockConditions{ + PublicKeys: []types.UnlockKey{ + renterKey.PublicKey().UnlockKey(), + hostKey.UnlockKey(), + }, + SignaturesRequired: 2, + }.UnlockHash(), + RevisionNumber: 0, + ValidProofOutputs: []types.SiacoinOutput{ + {Value: renterPayout, Address: renterAddress}, + {Value: hostValidPayout, Address: hostAddr}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Value: renterPayout, Address: renterAddress}, + {Value: hostMissedPayout, Address: hostAddr}, + {Value: voidMissedPayout, Address: types.Address{}}, + }, + }, basePrice +} + +// ContractPayment creates a new payment method for a contract +func ContractPayment(revision *rhp2.ContractRevision, renterKey types.PrivateKey, refundAccount rhp3.Account) PaymentMethod { + return &contractPayment{ + Revision: revision, + RenterKey: renterKey, + RefundAccount: refundAccount, + } +} + +// AccountPayment creates a new payment method for an account +func AccountPayment(account rhp3.Account, privateKey types.PrivateKey) PaymentMethod { + return &accountPayment{ + Account: account, + PrivateKey: privateKey, + } +} + +// NewSession creates a new session with a host +func NewSession(ctx context.Context, conn net.Conn, hostKey types.PublicKey, cm ChainManager, w Wallet) (*Session, error) { + t, err := rhp3.NewRenterTransport(conn, hostKey) + if err != nil { + conn.Close() + return nil, fmt.Errorf("failed to create transport: %w", err) + } + + return &Session{ + hostKey: hostKey, + t: t, + w: w, + cm: cm, + }, nil +} diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go index 22a37f6..e2e6508 100644 --- a/internal/syncerutil/store.go +++ b/internal/syncerutil/store.go @@ -23,7 +23,7 @@ type EphemeralPeerStore struct { mu sync.Mutex } -func (eps *EphemeralPeerStore) banned(peer string) bool { +func (eps *EphemeralPeerStore) isBanned(peer string) bool { host, _, err := net.SplitHostPort(peer) if err != nil { return false // shouldn't happen @@ -62,7 +62,7 @@ func (eps *EphemeralPeerStore) Peers() ([]syncer.PeerInfo, error) { defer eps.mu.Unlock() var peers []syncer.PeerInfo for addr, p := range eps.peers { - if !eps.banned(addr) { + if !eps.isBanned(addr) { peers = append(peers, p) } } @@ -109,7 +109,7 @@ func (eps *EphemeralPeerStore) Ban(peer string, duration time.Duration, reason s func (eps *EphemeralPeerStore) Banned(peer string) (bool, error) { eps.mu.Lock() defer eps.mu.Unlock() - return eps.banned(peer), nil + return eps.isBanned(peer), nil } // NewEphemeralPeerStore initializes an EphemeralPeerStore. @@ -171,9 +171,9 @@ func (jps *JSONPeerStore) save() error { defer f.Close() if _, err = f.Write(js); err != nil { return err - } else if f.Sync(); err != nil { + } else if err := f.Sync(); err != nil { return err - } else if f.Close(); err != nil { + } else if err := f.Close(); err != nil { return err } else if err := os.Rename(jps.path+"_tmp", jps.path); err != nil { return err @@ -205,5 +205,8 @@ func NewJSONPeerStore(path string) (*JSONPeerStore, error) { EphemeralPeerStore: NewEphemeralPeerStore(), path: path, } - return jps, jps.load() + if err := jps.load(); err != nil { + return nil, err + } + return jps, nil } diff --git a/internal/testutil/chain.go b/internal/testutil/chain.go new file mode 100644 index 0000000..e89806e --- /dev/null +++ b/internal/testutil/chain.go @@ -0,0 +1,192 @@ +package testutil + +import ( + "math/bits" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" +) + +// ContractFilesize is the default file size of contracts formed with PrepareContractFormation. +const ContractFilesize = 10 + +// PrepareContractFormation creates a file contract using the specified +// renter/host keys, payouts, and start/end window. It is an easier to +// use version of rhp2.PrepareContractFormation because it doesn't require +// a host settings struct and sets a default file size. +func PrepareContractFormation(renterPubKey types.PublicKey, hostKey types.PublicKey, renterPayout, hostCollateral types.Currency, startHeight uint64, endHeight uint64, refundAddr types.Address) types.FileContract { + taxAdjustedPayout := func(target types.Currency) types.Currency { + guess := target.Mul64(1000).Div64(961) + mod64 := func(c types.Currency, v uint64) types.Currency { + var r uint64 + if c.Hi < v { + _, r = bits.Div64(c.Hi, c.Lo, v) + } else { + _, r = bits.Div64(0, c.Hi, v) + _, r = bits.Div64(r, c.Lo, v) + } + return types.NewCurrency64(r) + } + sfc := (consensus.State{}).SiafundCount() + tm := mod64(target, sfc) + gm := mod64(guess, sfc) + if gm.Cmp(tm) < 0 { + guess = guess.Sub(types.NewCurrency64(sfc)) + } + return guess.Add(tm).Sub(gm) + } + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{ + renterPubKey.UnlockKey(), + hostKey.UnlockKey(), + }, + SignaturesRequired: 2, + } + hostPayout := hostCollateral + payout := taxAdjustedPayout(renterPayout.Add(hostPayout)) + return types.FileContract{ + Filesize: ContractFilesize, + FileMerkleRoot: types.Hash256{}, + WindowStart: startHeight, + WindowEnd: endHeight, + Payout: payout, + UnlockHash: uc.UnlockHash(), + RevisionNumber: 0, + ValidProofOutputs: []types.SiacoinOutput{ + {Value: renterPayout, Address: refundAddr}, + {Value: hostPayout, Address: types.VoidAddress}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Value: renterPayout, Address: refundAddr}, + {Value: hostPayout, Address: types.VoidAddress}, + {Value: types.ZeroCurrency, Address: types.VoidAddress}, + }, + } +} + +// CreateAnnouncement creates a host announcement. +func CreateAnnouncement(priv types.PrivateKey, netAddress string) []byte { + return chain.HostAnnouncement{ + PublicKey: priv.PublicKey(), + NetAddress: netAddress, + }.ToArbitraryData(priv) +} + +// MineBlock mines sets the metadata fields of the block along with the +// transactions and then generates a valid nonce for the block. +func MineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + reward := state.BlockReward() + for _, txn := range txns { + for _, fee := range txn.MinerFees { + reward = reward.Add(fee) + } + } + + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: reward}}, + } + if !coreutils.FindBlockNonce(state, &b, time.Minute) { + panic("failed to mine test block quickly enough") + } + return b +} + +// MineV2Block mines sets the metadata fields of the block along with the +// transactions and then generates a valid nonce for the block. +func MineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { + reward := state.BlockReward() + for _, txn := range txns { + reward = reward.Add(txn.MinerFee) + } + + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: reward}}, + + V2: &types.V2BlockData{ + Transactions: txns, + Height: state.Index.Height + 1, + }, + } + b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +// SignTransactionWithContracts signs a transaction using the specified private +// keys, including contract revisions. +func SignTransactionWithContracts(cs consensus.State, pk, renterPK, hostPK types.PrivateKey, txn *types.Transaction) { + appendSig := func(key types.PrivateKey, pubkeyIndex uint64, parentID types.Hash256) { + sig := key.SignHash(cs.WholeSigHash(*txn, parentID, pubkeyIndex, 0, nil)) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: parentID, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + PublicKeyIndex: pubkeyIndex, + Signature: sig[:], + }) + } + for i := range txn.SiacoinInputs { + appendSig(pk, 0, types.Hash256(txn.SiacoinInputs[i].ParentID)) + } + for i := range txn.SiafundInputs { + appendSig(pk, 0, types.Hash256(txn.SiafundInputs[i].ParentID)) + } + for i := range txn.FileContractRevisions { + appendSig(renterPK, 0, types.Hash256(txn.FileContractRevisions[i].ParentID)) + appendSig(hostPK, 1, types.Hash256(txn.FileContractRevisions[i].ParentID)) + } +} + +// SignTransaction signs a transaction that does not have any revisions with +// the specified private key. +func SignTransaction(cs consensus.State, pk types.PrivateKey, txn *types.Transaction) { + if len(txn.FileContractRevisions) > 0 { + panic("use SignTransactionWithContracts instead") + } + SignTransactionWithContracts(cs, pk, types.PrivateKey{}, types.PrivateKey{}, txn) +} + +// SignV2TransactionWithContracts signs a transaction using the specified +// private keys, including contracts and revisions. +func SignV2TransactionWithContracts(cs consensus.State, pk, renterPK, hostPK types.PrivateKey, txn *types.V2Transaction) { + for i := range txn.SiacoinInputs { + txn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cs.InputSigHash(*txn))} + } + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cs.InputSigHash(*txn))} + } + for i := range txn.FileContracts { + txn.FileContracts[i].RenterSignature = renterPK.SignHash(cs.ContractSigHash(txn.FileContracts[i])) + txn.FileContracts[i].HostSignature = hostPK.SignHash(cs.ContractSigHash(txn.FileContracts[i])) + } + for i := range txn.FileContractRevisions { + txn.FileContractRevisions[i].Revision.RenterSignature = renterPK.SignHash(cs.ContractSigHash(txn.FileContractRevisions[i].Revision)) + txn.FileContractRevisions[i].Revision.HostSignature = hostPK.SignHash(cs.ContractSigHash(txn.FileContractRevisions[i].Revision)) + } + for i := range txn.FileContractResolutions { + if r, ok := txn.FileContractResolutions[i].Resolution.(*types.V2FileContractRenewal); ok { + r.RenterSignature = renterPK.SignHash(cs.RenewalSigHash(*r)) + r.HostSignature = hostPK.SignHash(cs.RenewalSigHash(*r)) + r.NewContract.RenterSignature = renterPK.SignHash(cs.ContractSigHash(r.NewContract)) + r.NewContract.HostSignature = hostPK.SignHash(cs.ContractSigHash(r.NewContract)) + } + } +} + +// SignV2Transaction signs a transaction that does not have any contracts with +// the specified private key. +func SignV2Transaction(cs consensus.State, pk types.PrivateKey, txn *types.V2Transaction) { + if len(txn.FileContracts) > 0 || len(txn.FileContractRevisions) > 0 { + panic("use SignV2TransactionWithContracts instead") + } + SignV2TransactionWithContracts(cs, pk, types.PrivateKey{}, types.PrivateKey{}, txn) +} diff --git a/internal/testutil/check.go b/internal/testutil/check.go new file mode 100644 index 0000000..408d917 --- /dev/null +++ b/internal/testutil/check.go @@ -0,0 +1,360 @@ +package testutil + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/explorer" +) + +// Equal checks if two values of the same type are equal and fails otherwise. +func Equal[T any](t *testing.T, desc string, expect, got T) { + t.Helper() + + if !cmp.Equal(expect, got, cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(consensus.Work{}), cmpopts.IgnoreUnexported(types.StateElement{}), cmpopts.IgnoreFields(types.StateElement{}, "MerkleProof")) { + t.Fatalf("%s expected != got, diff: %s", desc, cmp.Diff(expect, got)) + } +} + +// CheckBalance checks that an address has the balances we expect. +func CheckBalance(t *testing.T, db explorer.Store, addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { + t.Helper() + + sc, immatureSC, sf, err := db.Balance(addr) + if err != nil { + t.Fatal(err) + } + Equal(t, "siacoins", expectSC, sc) + Equal(t, "immature siacoins", expectImmatureSC, immatureSC) + Equal(t, "siafunds", expectSF, sf) +} + +// CheckTransaction checks the inputs and outputs of the retrieved transaction +// with the source transaction. +func CheckTransaction(t *testing.T, expectTxn types.Transaction, gotTxn explorer.Transaction) { + t.Helper() + + Equal(t, "siacoin inputs", len(expectTxn.SiacoinInputs), len(gotTxn.SiacoinInputs)) + for i := range expectTxn.SiacoinInputs { + expected := expectTxn.SiacoinInputs[i] + got := gotTxn.SiacoinInputs[i] + + if got.Value == types.ZeroCurrency { + t.Fatal("invalid value") + } + Equal(t, "parent ID", expected.ParentID, got.ParentID) + Equal(t, "unlock conditions", expected.UnlockConditions, got.UnlockConditions) + Equal(t, "address", expected.UnlockConditions.UnlockHash(), got.Address) + } + + Equal(t, "siacoin outputs", len(expectTxn.SiacoinOutputs), len(gotTxn.SiacoinOutputs)) + for i := range expectTxn.SiacoinOutputs { + expected := expectTxn.SiacoinOutputs[i] + got := gotTxn.SiacoinOutputs[i].SiacoinOutput + + Equal(t, "address", expected.Address, got.Address) + Equal(t, "value", expected.Value, got.Value) + Equal(t, "source", explorer.SourceTransaction, gotTxn.SiacoinOutputs[i].Source) + } + + Equal(t, "siafund inputs", len(expectTxn.SiafundInputs), len(gotTxn.SiafundInputs)) + for i := range expectTxn.SiafundInputs { + expected := expectTxn.SiafundInputs[i] + got := gotTxn.SiafundInputs[i] + + if got.Value == 0 { + t.Fatal("invalid value") + } + Equal(t, "parent ID", expected.ParentID, got.ParentID) + Equal(t, "claim address", expected.ClaimAddress, got.ClaimAddress) + Equal(t, "unlock conditions", expected.UnlockConditions, got.UnlockConditions) + Equal(t, "address", expected.UnlockConditions.UnlockHash(), got.Address) + } + + Equal(t, "siafund outputs", len(expectTxn.SiafundOutputs), len(gotTxn.SiafundOutputs)) + for i := range expectTxn.SiafundOutputs { + expected := expectTxn.SiafundOutputs[i] + got := gotTxn.SiafundOutputs[i].SiafundOutput + + Equal(t, "address", expected.Address, got.Address) + Equal(t, "value", expected.Value, got.Value) + } + + Equal(t, "arbitrary data", len(expectTxn.ArbitraryData), len(gotTxn.ArbitraryData)) + for i := range expectTxn.ArbitraryData { + Equal(t, "arbitrary data", expectTxn.ArbitraryData[i], gotTxn.ArbitraryData[i]) + } + + Equal(t, "miner fees", len(expectTxn.MinerFees), len(gotTxn.MinerFees)) + for i := range expectTxn.MinerFees { + Equal(t, "miner fee", expectTxn.MinerFees[i], gotTxn.MinerFees[i]) + } + + Equal(t, "signatures", len(expectTxn.Signatures), len(gotTxn.Signatures)) + for i := range expectTxn.Signatures { + expected := expectTxn.Signatures[i] + got := gotTxn.Signatures[i] + + Equal(t, "parent ID", expected.ParentID, got.ParentID) + Equal(t, "public key index", expected.PublicKeyIndex, got.PublicKeyIndex) + Equal(t, "timelock", expected.Timelock, got.Timelock) + Equal(t, "signature", expected.Signature, got.Signature) + + // reflect.DeepCheck treats empty slices as different from nil + // slices so these will differ because the decoder is doing + // cf.X = make([]uint64, d.ReadPrefix()) and the prefix is 0 + // testutil.Equal(t, "covered fields", expected.CoveredFields, got.CoveredFields) + } + + var hostAnnouncements []chain.HostAnnouncement + for _, arb := range expectTxn.ArbitraryData { + var ha chain.HostAnnouncement + if ha.FromArbitraryData(arb) { + hostAnnouncements = append(hostAnnouncements, ha) + } + } + Equal(t, "host announcements", len(hostAnnouncements), len(gotTxn.HostAnnouncements)) + for i := range hostAnnouncements { + expected := hostAnnouncements[i] + got := gotTxn.HostAnnouncements[i] + + Equal(t, "public key", expected.PublicKey, got.PublicKey) + Equal(t, "net address", expected.NetAddress, got.NetAddress) + } +} + +// CheckV2Transaction checks the inputs and outputs of the retrieved transaction +// with the source transaction. +func CheckV2Transaction(t *testing.T, expectTxn types.V2Transaction, gotTxn explorer.V2Transaction) { + t.Helper() + + Equal(t, "new foundation address", expectTxn.NewFoundationAddress, gotTxn.NewFoundationAddress) + Equal(t, "miner fee", expectTxn.MinerFee, gotTxn.MinerFee) + + Equal(t, "siacoin inputs", len(expectTxn.SiacoinInputs), len(gotTxn.SiacoinInputs)) + for i := range expectTxn.SiacoinInputs { + expected := expectTxn.SiacoinInputs[i] + got := gotTxn.SiacoinInputs[i] + + Equal(t, "address", expected.Parent.SiacoinOutput.Address, got.Parent.SiacoinOutput.Address) + Equal(t, "value", expected.Parent.SiacoinOutput.Value, got.Parent.SiacoinOutput.Value) + Equal(t, "maturity height", expected.Parent.MaturityHeight, got.Parent.MaturityHeight) + Equal(t, "id", expected.Parent.ID, got.Parent.ID) + Equal(t, "leaf index", expected.Parent.StateElement.LeafIndex, got.Parent.StateElement.LeafIndex) + if len(got.SatisfiedPolicy.Preimages) == 0 { + got.SatisfiedPolicy.Preimages = nil + } + Equal(t, "satisfied policy", expected.SatisfiedPolicy, got.SatisfiedPolicy) + } + + Equal(t, "siacoin outputs", len(expectTxn.SiacoinOutputs), len(gotTxn.SiacoinOutputs)) + for i := range expectTxn.SiacoinOutputs { + expected := expectTxn.SiacoinOutputs[i] + got := gotTxn.SiacoinOutputs[i].SiacoinOutput + + Equal(t, "address", expected.Address, got.Address) + Equal(t, "value", expected.Value, got.Value) + Equal(t, "source", explorer.SourceTransaction, gotTxn.SiacoinOutputs[i].Source) + } + + Equal(t, "siafund inputs", len(expectTxn.SiafundInputs), len(gotTxn.SiafundInputs)) + for i := range expectTxn.SiafundInputs { + expected := expectTxn.SiafundInputs[i] + got := gotTxn.SiafundInputs[i] + + Equal(t, "address", expected.Parent.SiafundOutput.Address, got.Parent.SiafundOutput.Address) + Equal(t, "value", expected.Parent.SiafundOutput.Value, got.Parent.SiafundOutput.Value) + Equal(t, "claim address", expected.ClaimAddress, got.ClaimAddress) + Equal(t, "id", expected.Parent.ID, got.Parent.ID) + Equal(t, "leaf index", expected.Parent.StateElement.LeafIndex, got.Parent.StateElement.LeafIndex) + if len(got.SatisfiedPolicy.Preimages) == 0 { + got.SatisfiedPolicy.Preimages = nil + } + Equal(t, "satisfied policy", expected.SatisfiedPolicy, got.SatisfiedPolicy) + } + + Equal(t, "siafund outputs", len(expectTxn.SiafundOutputs), len(gotTxn.SiafundOutputs)) + for i := range expectTxn.SiafundOutputs { + expected := expectTxn.SiafundOutputs[i] + got := gotTxn.SiafundOutputs[i].SiafundOutput + + Equal(t, "address", expected.Address, got.Address) + Equal(t, "value", expected.Value, got.Value) + } + + Equal(t, "file contracts", len(expectTxn.FileContracts), len(gotTxn.FileContracts)) + for i := range expectTxn.FileContracts { + expected := expectTxn.FileContracts[i] + got := gotTxn.FileContracts[i] + + Equal(t, "id", expectTxn.V2FileContractID(expectTxn.ID(), i), types.FileContractID(got.ID)) + CheckV2FC(t, expected, got) + } + + Equal(t, "file contract revision", len(expectTxn.FileContractRevisions), len(gotTxn.FileContractRevisions)) + for i := range expectTxn.FileContractRevisions { + expected := expectTxn.FileContractRevisions[i] + got := gotTxn.FileContractRevisions[i] + + Equal(t, "parent ID", expected.Parent.ID, got.Parent.ID) + Equal(t, "revision ID", expected.Parent.ID, got.Revision.ID) + CheckV2FC(t, expected.Parent.V2FileContract, got.Parent) + CheckV2FC(t, expected.Revision, got.Revision) + } + + Equal(t, "file contract resolutions", len(expectTxn.FileContractResolutions), len(gotTxn.FileContractResolutions)) + for i := range expectTxn.FileContractResolutions { + expected := expectTxn.FileContractResolutions[i] + got := gotTxn.FileContractResolutions[i] + + CheckV2FC(t, expected.Parent.V2FileContract, got.Parent) + + switch v := expected.Resolution.(type) { + case *types.V2FileContractRenewal: + if gotV, ok := got.Resolution.(*explorer.V2FileContractRenewal); !ok { + t.Fatalf("expected V2FileContractRenewal, got %v", reflect.TypeOf(got.Resolution)) + } else { + CheckV2FC(t, v.NewContract, gotV.NewContract) + + Equal(t, "type", explorer.V2ResolutionRenewal, got.Type) + Equal(t, "final renter output address", v.FinalRenterOutput.Address, gotV.FinalRenterOutput.Address) + Equal(t, "final renter output value", v.FinalRenterOutput.Value, gotV.FinalRenterOutput.Value) + Equal(t, "final host output address", v.FinalHostOutput.Address, gotV.FinalHostOutput.Address) + Equal(t, "final host output value", v.FinalHostOutput.Value, gotV.FinalHostOutput.Value) + Equal(t, "renter rollover", v.RenterRollover, gotV.RenterRollover) + Equal(t, "host rollover", v.HostRollover, gotV.HostRollover) + Equal(t, "renter signature", v.RenterSignature, gotV.RenterSignature) + Equal(t, "host signature", v.HostSignature, gotV.HostSignature) + } + case *types.V2StorageProof: + if gotV, ok := got.Resolution.(*types.V2StorageProof); !ok { + t.Fatalf("expected V2StorageProof, got %v", reflect.TypeOf(got.Resolution)) + } else { + Equal(t, "type", explorer.V2ResolutionStorageProof, got.Type) + Equal(t, "proof index", v.ProofIndex, gotV.ProofIndex) + Equal(t, "leaf", v.Leaf, gotV.Leaf) + Equal(t, "proof", v.Proof, gotV.Proof) + } + case *types.V2FileContractExpiration: + Equal(t, "type", explorer.V2ResolutionExpiration, got.Type) + if _, ok := got.Resolution.(*types.V2FileContractExpiration); !ok { + t.Fatalf("expected V2FileContractExpiration, got %v", reflect.TypeOf(got.Resolution)) + } + default: + t.Fatalf("invalid resolution type: %v", reflect.TypeOf(got.Resolution)) + } + } + + Equal(t, "attestations", len(expectTxn.Attestations), len(gotTxn.Attestations)) + for i := range expectTxn.Attestations { + expected := expectTxn.Attestations[i] + got := gotTxn.Attestations[i] + + Equal(t, "public key", expected.PublicKey, got.PublicKey) + Equal(t, "key", expected.Key, got.Key) + Equal(t, "value", expected.Value, got.Value) + Equal(t, "signature", expected.Signature, got.Signature) + } + + var hostAnnouncements []explorer.V2HostAnnouncement + for _, attestation := range expectTxn.Attestations { + var ha chain.V2HostAnnouncement + if ha.FromAttestation(attestation) == nil { + hostAnnouncements = append(hostAnnouncements, explorer.V2HostAnnouncement{ + V2HostAnnouncement: ha, + PublicKey: attestation.PublicKey, + }) + } + } + Equal(t, "host announcements", len(hostAnnouncements), len(gotTxn.HostAnnouncements)) + for i := range hostAnnouncements { + expected := []chain.NetAddress(hostAnnouncements[i].V2HostAnnouncement) + got := []chain.NetAddress(gotTxn.HostAnnouncements[i].V2HostAnnouncement) + + Equal(t, "public key", hostAnnouncements[i].PublicKey, gotTxn.HostAnnouncements[i].PublicKey) + Equal(t, "net addresses", len(expected), len(got)) + for j := range expected { + Equal(t, "protocol", expected[j].Protocol, got[j].Protocol) + Equal(t, "address", expected[j].Address, got[j].Address) + } + } + + Equal(t, "arbitrary data", len(expectTxn.ArbitraryData), len(gotTxn.ArbitraryData)) + for i := range expectTxn.ArbitraryData { + Equal(t, "arbitrary data value", expectTxn.ArbitraryData[i], gotTxn.ArbitraryData[i]) + } +} + +// CheckV2ChainIndices checks that the chain indices that a v2 transaction was +// in from the explorer match the expected chain indices. +func CheckV2ChainIndices(t *testing.T, db explorer.Store, txnID types.TransactionID, expected []types.ChainIndex) { + t.Helper() + + indices, err := db.V2TransactionChainIndices(txnID, 0, 100) + switch { + case err != nil: + t.Fatal(err) + case len(indices) != len(expected): + t.Fatalf("expected %d indices, got %d", len(expected), len(indices)) + } + for i := range indices { + Equal(t, "index", expected[i], indices[i]) + } +} + +// CheckFC checks the retrieved file contract with the source file contract in +// addition to checking the resolved and valid fields. +func CheckFC(t *testing.T, revision, resolved, valid bool, expected types.FileContract, got explorer.ExtendedFileContract) { + t.Helper() + + Equal(t, "resolved state", resolved, got.Resolved) + Equal(t, "valid state", valid, got.Valid) + + Equal(t, "filesize", expected.Filesize, got.Filesize) + Equal(t, "file merkle root", expected.FileMerkleRoot, got.FileMerkleRoot) + Equal(t, "window start", expected.WindowStart, got.WindowStart) + Equal(t, "window end", expected.WindowEnd, got.WindowEnd) + if !revision { + Equal(t, "payout", expected.Payout, got.Payout) + } + Equal(t, "unlock hash", expected.UnlockHash, got.UnlockHash) + Equal(t, "revision number", expected.RevisionNumber, got.RevisionNumber) + Equal(t, "valid proof outputs", len(expected.ValidProofOutputs), len(got.ValidProofOutputs)) + for i := range expected.ValidProofOutputs { + Equal(t, "valid proof output address", expected.ValidProofOutputs[i].Address, got.ValidProofOutputs[i].Address) + Equal(t, "valid proof output value", expected.ValidProofOutputs[i].Value, got.ValidProofOutputs[i].Value) + } + Equal(t, "missed proof outputs", len(expected.MissedProofOutputs), len(got.MissedProofOutputs)) + for i := range expected.MissedProofOutputs { + Equal(t, "missed proof output address", expected.MissedProofOutputs[i].Address, got.MissedProofOutputs[i].Address) + Equal(t, "missed proof output value", expected.MissedProofOutputs[i].Value, got.MissedProofOutputs[i].Value) + } +} + +// CheckV2FC checks the retrieved file contract with the source file contract +// in addition to checking the resolved and valid fields. +func CheckV2FC(t *testing.T, expected types.V2FileContract, got explorer.V2FileContract) { + t.Helper() + + gotFC := got.V2FileContractElement.V2FileContract + Equal(t, "capacity", expected.Capacity, gotFC.Capacity) + Equal(t, "filesize", expected.Filesize, gotFC.Filesize) + Equal(t, "proof height", expected.ProofHeight, gotFC.ProofHeight) + Equal(t, "expiration height", expected.ExpirationHeight, gotFC.ExpirationHeight) + Equal(t, "renter output address", expected.RenterOutput.Address, gotFC.RenterOutput.Address) + Equal(t, "renter output value", expected.RenterOutput.Address, gotFC.RenterOutput.Address) + Equal(t, "host output address", expected.HostOutput.Address, gotFC.HostOutput.Address) + Equal(t, "host output value", expected.HostOutput.Address, gotFC.HostOutput.Address) + Equal(t, "missed host value", expected.MissedHostValue, gotFC.MissedHostValue) + Equal(t, "total collateral", expected.TotalCollateral, gotFC.TotalCollateral) + Equal(t, "renter public key", expected.RenterPublicKey, gotFC.RenterPublicKey) + Equal(t, "host public key", expected.HostPublicKey, gotFC.HostPublicKey) + Equal(t, "revision number", expected.RevisionNumber, gotFC.RevisionNumber) + Equal(t, "renter signature", expected.RenterSignature, gotFC.RenterSignature) + Equal(t, "host signature", expected.HostSignature, gotFC.HostSignature) +} diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index dff6030..af8a40a 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -2,95 +2,128 @@ package sqlite import ( "database/sql" - "encoding/json" + "errors" "fmt" "go.sia.tech/core/types" "go.sia.tech/explored/explorer" ) -func scanEvent(s scanner) (ev explorer.Event, eventID int64, err error) { - var eventType string - var eventBuf []byte +func getAddressEvents(tx *txn, address types.Address, offset, limit uint64) (eventIDs []int64, err error) { + const query = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN address_balance sa ON ea.address_id = sa.id +WHERE sa.address = $1 +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT $2 OFFSET $3;` - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &eventType, &eventBuf) + rows, err := tx.Query(query, encode(address), limit, offset) if err != nil { - return + return nil, err } + defer rows.Close() - switch eventType { - case explorer.EventTypeTransaction: - var tx explorer.EventTransaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return explorer.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - ev.Data = &tx - case explorer.EventTypeContractPayout: - var m explorer.EventContractPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return explorer.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - ev.Data = &m - case explorer.EventTypeMinerPayout: - var m explorer.EventMinerPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return explorer.Event{}, 0, fmt.Errorf("failed to unmarshal payout event: %w", err) - } - ev.Data = &m - case explorer.EventTypeFoundationSubsidy: - var m explorer.EventFoundationSubsidy - if err = json.Unmarshal(eventBuf, &m); err != nil { - return explorer.Event{}, 0, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) - } - ev.Data = &m - default: - return explorer.Event{}, 0, fmt.Errorf("unknown event type: %s", eventType) + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + eventIDs = append(eventIDs, id) + } + return eventIDs, rows.Err() +} + +func getEventsByID(tx *txn, eventIDs []int64) (events []explorer.Event, err error) { + var scanHeight uint64 + err = tx.QueryRow(`SELECT COALESCE(MAX(height), 0) FROM blocks`).Scan(&scanHeight) + if err != nil { + return nil, fmt.Errorf("failed to get last indexed height: %w", err) + } + + stmt, err := tx.Prepare(`SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + b.height, + b.id, + CASE + WHEN $1 < b.height THEN 0 + ELSE $1 - b.height + END AS confirmations, + ev.event_type +FROM events ev +INNER JOIN blocks b ON (ev.block_id = b.id) +WHERE ev.id=$2`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]explorer.Event, 0, len(eventIDs)) + for i, id := range eventIDs { + event, _, err := scanEvent(tx, stmt.QueryRow(scanHeight, id)) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return nil, fmt.Errorf("failed to query event %d: %w", i, err) + } + events = append(events, event) } return } // AddressEvents returns the events of a single address. -func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []explorer.Event, err error) { +func (s *Store) AddressEvents(address types.Address, offset, limit uint64) (events []explorer.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ev.height, ev.block_id, ev.event_type, ev.event_data - FROM events ev - INNER JOIN event_addresses ea ON (ev.id = ea.event_id) - INNER JOIN address_balance sa ON (ea.address_id = sa.id) - WHERE sa.address = $1 - ORDER BY ev.maturity_height DESC, ev.id DESC - LIMIT $2 OFFSET $3` - - rows, err := tx.Query(query, encode(address), limit, offset) + dbIDs, err := getAddressEvents(tx, address, offset, limit) if err != nil { return err } - defer rows.Close() - for rows.Next() { - event, _, err := scanEvent(rows) - if err != nil { - return fmt.Errorf("failed to scan event: %w", err) - } + events, err = getEventsByID(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } - events = append(events, event) + for i := range events { + events[i].Relevant = []types.Address{address} } - return rows.Err() + return nil }) return } +func scanSiacoinOutput(s scanner) (sco explorer.SiacoinOutput, err error) { + var spentIndex types.ChainIndex + err = s.Scan(decode(&sco.ID), decode(&sco.StateElement.LeafIndex), &sco.Source, decodeNull(&spentIndex), &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)) + if spentIndex != (types.ChainIndex{}) { + sco.SpentIndex = &spentIndex + } + return +} + +func scanSiafundOutput(s scanner) (sfo explorer.SiafundOutput, err error) { + var spentIndex types.ChainIndex + err = s.Scan(decode(&sfo.ID), decode(&sfo.StateElement.LeafIndex), decodeNull(&spentIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)) + if spentIndex != (types.ChainIndex{}) { + sfo.SpentIndex = &spentIndex + } + return +} + // UnspentSiacoinOutputs implements explorer.Store. func (s *Store) UnspentSiacoinOutputs(address types.Address, offset, limit uint64) (result []explorer.SiacoinOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, source, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), offset, limit) + rows, err := tx.Query(`SELECT output_id, leaf_index, source, spent_index, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent_index IS NULL LIMIT ? OFFSET ?`, encode(address), limit, offset) if err != nil { return fmt.Errorf("failed to query siacoin outputs: %w", err) } defer rows.Close() for rows.Next() { - var sco explorer.SiacoinOutput - if err := rows.Scan(decode(&sco.StateElement.ID), decode(&sco.StateElement.LeafIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { + sco, err := scanSiacoinOutput(rows) + if err != nil { return fmt.Errorf("failed to scan siacoin output: %w", err) } result = append(result, sco) @@ -103,17 +136,78 @@ func (s *Store) UnspentSiacoinOutputs(address types.Address, offset, limit uint6 // UnspentSiafundOutputs implements explorer.Store. func (s *Store) UnspentSiafundOutputs(address types.Address, offset, limit uint64) (result []explorer.SiafundOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), offset, limit) + rows, err := tx.Query(`SELECT output_id, leaf_index, spent_index, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent_index IS NULL LIMIT ? OFFSET ?`, encode(address), limit, offset) + if err != nil { + return fmt.Errorf("failed to query siafund outputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + sfo, err := scanSiafundOutput(rows) + if err != nil { + return fmt.Errorf("failed to scan siafund output: %w", err) + } + result = append(result, sfo) + } + return nil + }) + return +} + +// SiacoinElements implements explorer.Store. +func (s *Store) SiacoinElements(ids []types.SiacoinOutputID) (result []explorer.SiacoinOutput, err error) { + err = s.transaction(func(tx *txn) error { + var encoded []any + for _, id := range ids { + encoded = append(encoded, encode(id)) + } + + rows, err := tx.Query(`SELECT output_id, leaf_index, source, spent_index, maturity_height, address, value FROM siacoin_elements WHERE output_id IN (`+queryPlaceHolders(len(encoded))+`)`, encoded...) + if err != nil { + return fmt.Errorf("failed to query siacoin outputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + sco, err := scanSiacoinOutput(rows) + if err != nil { + return fmt.Errorf("failed to scan siacoin output: %w", err) + } + sco.StateElement.MerkleProof, err = s.MerkleProof(sco.StateElement.LeafIndex) + if err != nil { + return fmt.Errorf("failed to get output merkle proof: %w", err) + } + + result = append(result, sco) + } + return nil + }) + return +} + +// SiafundElements implements explorer.Store. +func (s *Store) SiafundElements(ids []types.SiafundOutputID) (result []explorer.SiafundOutput, err error) { + err = s.transaction(func(tx *txn) error { + var encoded []any + for _, id := range ids { + encoded = append(encoded, encode(id)) + } + + rows, err := tx.Query(`SELECT output_id, leaf_index, spent_index, claim_start, address, value FROM siafund_elements WHERE output_id IN (`+queryPlaceHolders(len(encoded))+`)`, encoded...) if err != nil { return fmt.Errorf("failed to query siafund outputs: %w", err) } defer rows.Close() for rows.Next() { - var sfo explorer.SiafundOutput - if err := rows.Scan(decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { + sfo, err := scanSiafundOutput(rows) + if err != nil { return fmt.Errorf("failed to scan siafund output: %w", err) } + sfo.StateElement.MerkleProof, err = s.MerkleProof(sfo.StateElement.LeafIndex) + if err != nil { + return fmt.Errorf("failed to get output merkle proof: %w", err) + } result = append(result, sfo) } return nil diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index fad9345..24d62ad 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -1,6 +1,8 @@ package sqlite import ( + "database/sql" + "errors" "fmt" "go.sia.tech/core/types" @@ -10,23 +12,43 @@ import ( // Block implements explorer.Store. func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height) - if err != nil { - return err + var v2Height uint64 + var v2Commitment types.Hash256 + err := tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, leaf_index, v2_height, v2_commitment FROM blocks WHERE id = ?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decode(&result.LeafIndex), decodeNull(&v2Height), decodeNull(&v2Commitment)) + if errors.Is(err, sql.ErrNoRows) { + return explorer.ErrNoBlock + } else if err != nil { + return fmt.Errorf("failed to get block: %w", err) } - result.MinerPayouts, err = blockMinerPayouts(tx, id) if err != nil { return fmt.Errorf("failed to get miner payouts: %w", err) } + if (v2Height != 0 && v2Commitment != types.Hash256{}) { + result.V2 = new(explorer.V2BlockData) + result.V2.Height = v2Height + result.V2.Commitment = v2Commitment + + // get block transaction IDs + transactionIDs, err := blockV2TransactionIDs(tx, id) + if err != nil { + return fmt.Errorf("failed to get block transaction IDs: %w", err) + } + + result.V2.Transactions, err = getV2Transactions(tx, transactionIDs) + if err != nil { + return fmt.Errorf("failed to get transactions: %w", err) + } + } + // get block transaction IDs transactionIDs, err := blockTransactionIDs(tx, id) if err != nil { return fmt.Errorf("failed to get block transaction IDs: %w", err) } - result.Transactions, err = s.getTransactions(tx, transactionIDs) + result.Transactions, err = getTransactions(tx, transactionIDs) if err != nil { return fmt.Errorf("failed to get transactions: %w", err) } @@ -36,11 +58,28 @@ func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { return } +// Tip implements explorer.Store. +func (s *Store) Tip() (result types.ChainIndex, err error) { + const query = `SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1` + err = s.transaction(func(dbTxn *txn) error { + err := dbTxn.QueryRow(query).Scan(decode(&result.ID), &result.Height) + if errors.Is(err, sql.ErrNoRows) { + return explorer.ErrNoTip + } else if err != nil { + return err + } + return nil + }) + return +} + // BestTip implements explorer.Store. func (s *Store) BestTip(height uint64) (result types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { err = tx.QueryRow(`SELECT id, height FROM blocks WHERE height=?`, height).Scan(decode(&result.ID), decode(&result.Height)) - if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return explorer.ErrNoTip + } else if err != nil { return err } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index fa56615..7c1db28 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -6,6 +6,8 @@ import ( "encoding/json" "errors" "fmt" + "reflect" + "time" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" @@ -13,18 +15,23 @@ import ( ) type updateTx struct { - tx *txn - relevantAddresses map[types.Address]bool + tx *txn } -func (ut *updateTx) addBlock(b types.Block, height uint64) error { +func addBlock(tx *txn, b types.Block, cie types.ChainIndexElement, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) + var v2Height any + var v2Commitment any + if b.V2 != nil { + v2Height = encode(b.V2.Height) + v2Commitment = encode(b.V2.Commitment) + } + _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp, leaf_index, v2_height, v2_commitment) VALUES (?, ?, ?, ?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp), encode(cie.StateElement.LeafIndex), v2Height, v2Commitment) return err } -func (ut *updateTx) addMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) +func addMinerPayouts(tx *txn, bid types.BlockID, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("addMinerPayouts: failed to prepare statement: %w", err) } @@ -43,55 +50,23 @@ func (ut *updateTx) addMinerPayouts(bid types.BlockID, height uint64, scos []typ return nil } -func (s *Store) addTransactionAddresses(dbTxn txn, id int64, txn types.Transaction) error { - m := make(map[types.Address]struct{}) - for _, sci := range txn.SiacoinInputs { - m[sci.UnlockConditions.UnlockHash()] = struct{}{} - } - for _, sco := range txn.SiacoinOutputs { - m[sco.Address] = struct{}{} - } - for _, sfi := range txn.SiafundInputs { - m[sfi.UnlockConditions.UnlockHash()] = struct{}{} - } - for _, sfo := range txn.SiafundOutputs { - m[sfo.Address] = struct{}{} - } - for _, fc := range txn.FileContracts { - for _, vpo := range fc.ValidProofOutputs { - m[vpo.Address] = struct{}{} - } - for _, mpo := range fc.MissedProofOutputs { - m[mpo.Address] = struct{}{} - } - m[types.Address(fc.UnlockHash)] = struct{}{} - } - for _, fcr := range txn.FileContractRevisions { - for _, vpo := range fcr.FileContract.ValidProofOutputs { - m[vpo.Address] = struct{}{} - } - for _, mpo := range fcr.FileContract.MissedProofOutputs { - m[mpo.Address] = struct{}{} - } - m[fcr.UnlockConditions.UnlockHash()] = struct{}{} - } - - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_addresses(transaction_id, address) VALUES (?, ?);`) +func addMinerFees(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_miner_fees(transaction_id, transaction_order, fee) VALUES (?, ?, ?)`) if err != nil { - return fmt.Errorf("addTransactionAddresses: failed to prepare statement: %w", err) + return fmt.Errorf("addMinerFees: failed to prepare statement: %w", err) } defer stmt.Close() - for addr := range m { - if _, err := stmt.Exec(id, dbEncode(addr)); err != nil { - return fmt.Errorf("addTransactionAddresses: failed to execute statement: %w", err) + for i, fee := range txn.MinerFees { + if _, err := stmt.Exec(id, i, encode(fee)); err != nil { + return fmt.Errorf("addMinerFees: failed to execute statement: %w", err) } } return nil } -func (s *Store) addArbitraryData(dbTxn txn, id int64, txn types.Transaction) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) +func addArbitraryData(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addArbitraryData: failed to prepare statement: %w", err) } @@ -105,23 +80,43 @@ func (s *Store) addArbitraryData(dbTxn txn, id int64, txn types.Transaction) err return nil } -func (ut *updateTx) addSiacoinInputs(id int64, txn types.Transaction) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) +func addSignatures(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_signatures(transaction_id, transaction_order, parent_id, public_key_index, timelock, covered_fields, signature) VALUES (?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addMinerFees: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, sig := range txn.Signatures { + if _, err := stmt.Exec(id, i, encode(sig.ParentID), sig.PublicKeyIndex, encode(sig.Timelock), encode(sig.CoveredFields), sig.Signature); err != nil { + return fmt.Errorf("addMinerFees: failed to execute statement: %w", err) + } + } + return nil +} + +func addSiacoinInputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinInputs: failed to prepare statement: %w", err) } defer stmt.Close() for i, sci := range txn.SiacoinInputs { - if _, err := stmt.Exec(id, i, encode(sci.ParentID), encode(sci.UnlockConditions)); err != nil { + dbID, ok := dbIDs[sci.ParentID] + if !ok { + return errors.New("addSiacoinOutputs: dbID not in map") + } + + if _, err := stmt.Exec(id, i, dbID, encode(sci.UnlockConditions)); err != nil { return fmt.Errorf("addSiacoinInputs: failed to execute statement: %w", err) } } return nil } -func (ut *updateTx) addSiacoinOutputs(id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func addSiacoinOutputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinOutputs: failed to prepare statement: %w", err) } @@ -140,23 +135,29 @@ func (ut *updateTx) addSiacoinOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) addSiafundInputs(id int64, txn types.Transaction) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) +func addSiafundInputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundInputs: failed to prepare statement: %w", err) } defer stmt.Close() - for i, sci := range txn.SiafundInputs { - if _, err := stmt.Exec(id, i, encode(sci.ParentID), encode(sci.UnlockConditions), encode(sci.ClaimAddress)); err != nil { + for i, sfi := range txn.SiafundInputs { + dbID, ok := dbIDs[sfi.ParentID] + if !ok { + return errors.New("addSiafundOutputs: dbID not in map") + } + + if _, err := stmt.Exec(id, i, dbID, encode(sfi.UnlockConditions), encode(sfi.ClaimAddress)); err != nil { return fmt.Errorf("addSiafundInputs: failed to execute statement: %w", err) } } + return nil } -func (ut *updateTx) addSiafundOutputs(id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func addSiafundOutputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundOutputs: failed to prepare statement: %w", err) } @@ -175,25 +176,13 @@ func (ut *updateTx) addSiafundOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) addFileContracts(id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) +func addFileContracts(tx *txn, id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) - if err != nil { - return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) - } - defer validOutputsStmt.Close() - - missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) - if err != nil { - return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) - } - defer missedOutputsStmt.Close() - for i := range txn.FileContracts { dbID, ok := fcDBIds[explorer.DBFileContract{ID: txn.FileContractID(i), RevisionNumber: 0}] if !ok { @@ -203,41 +192,17 @@ func (ut *updateTx) addFileContracts(id int64, txn types.Transaction, fcDBIds ma if _, err := stmt.Exec(id, i, dbID); err != nil { return fmt.Errorf("addFileContracts: failed to execute transaction_file_contracts statement: %w", err) } - - for j, sco := range txn.FileContracts[i].ValidProofOutputs { - if _, err := validOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { - return fmt.Errorf("addFileContracts: failed to execute valid proof outputs statement: %w", err) - } - } - - for j, sco := range txn.FileContracts[i].MissedProofOutputs { - if _, err := missedOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { - return fmt.Errorf("addFileContracts: failed to execute missed proof outputs statement: %w", err) - } - } } return nil } -func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) +func addFileContractRevisions(tx *txn, id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) - if err != nil { - return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) - } - defer validOutputsStmt.Close() - - missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) - if err != nil { - return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) - } - defer missedOutputsStmt.Close() - for i := range txn.FileContractRevisions { fcr := &txn.FileContractRevisions[i] dbID, ok := dbIDs[explorer.DBFileContract{ID: fcr.ParentID, RevisionNumber: fcr.FileContract.RevisionNumber}] @@ -248,75 +213,160 @@ func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, db if _, err := stmt.Exec(id, i, dbID, encode(fcr.ParentID), encode(fcr.UnlockConditions)); err != nil { return fmt.Errorf("addFileContractRevisions: failed to execute statement: %w", err) } + } - for j, sco := range txn.FileContractRevisions[i].ValidProofOutputs { - if _, err := validOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { - return fmt.Errorf("addFileContractRevisions: failed to execute valid proof outputs statement: %w", err) - } - } + return nil +} - for j, sco := range txn.FileContractRevisions[i].MissedProofOutputs { - if _, err := missedOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { - return fmt.Errorf("addFileContractRevisions: failed to execute missed proof outputs statement: %w", err) - } - } +func addStorageProofs(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_storage_proofs(transaction_id, transaction_order, parent_id, leaf, proof) VALUES (?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addStorageProofs: failed to prepare statement: %w", err) } + defer stmt.Close() + for i, proof := range txn.StorageProofs { + if _, err := stmt.Exec(id, i, encode(proof.ParentID), proof.Leaf[:], encode(proof.Proof)); err != nil { + return fmt.Errorf("addStorageProofs: failed to execute statement: %w", err) + } + } return nil } -func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { - insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) - ON CONFLICT (transaction_id) DO UPDATE SET transaction_id=EXCLUDED.transaction_id -- technically a no-op, but necessary for the RETURNING clause - RETURNING id;`) +type txnDBId struct { + id int64 + exist bool +} + +func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[types.TransactionID]txnDBId, error) { + checkTransactionStmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare check transaction statement: %v", err) + } + defer checkTransactionStmt.Close() + + insertTransactionStmt, err := tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?)`) if err != nil { - return fmt.Errorf("failed to prepare insert transaction statement: %v", err) + return nil, fmt.Errorf("failed to prepare insert transaction statement: %v", err) } defer insertTransactionStmt.Close() - blockTransactionsStmt, err := ut.tx.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) + blockTransactionsStmt, err := tx.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) if err != nil { - return fmt.Errorf("failed to prepare block_transactions statement: %w", err) + return nil, fmt.Errorf("failed to prepare block_transactions statement: %w", err) } defer blockTransactionsStmt.Close() + txnDBIds := make(map[types.TransactionID]txnDBId) for i, txn := range txns { - var txnID int64 - err := insertTransactionStmt.QueryRow(encode(txn.ID())).Scan(&txnID) - if err != nil { - return fmt.Errorf("failed to insert into transactions: %w", err) + var exist bool + var dbID int64 + txnID := txn.ID() + if err := checkTransactionStmt.QueryRow(encode(txnID)).Scan(&dbID); err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to check if transaction exists: %w", err) + } else if err == nil { + exist = true + } + + if !exist { + result, err := insertTransactionStmt.Exec(encode(txnID)) + if err != nil { + return nil, fmt.Errorf("failed to insert into transactions: %w", err) + } + dbID, err = result.LastInsertId() + if err != nil { + return nil, fmt.Errorf("failed to get transaction ID: %w", err) + } + } + + // If we have the same transaction multiple times in one block, exist + // will be true after the above query after the first time the + // transaction is encountered by this loop. So we only set the value in + // the map for each transaction once. + if _, ok := txnDBIds[txnID]; !ok { + txnDBIds[txnID] = txnDBId{id: dbID, exist: exist} + } + + if _, err := blockTransactionsStmt.Exec(encode(bid), dbID, i); err != nil { + return nil, fmt.Errorf("failed to insert into block_transactions: %w", err) } + } + + return txnDBIds, nil +} - if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { - return fmt.Errorf("failed to insert into block_transactions: %w", err) - } else if err := s.addTransactionAddresses(dbTxn, txnID, txn); err != nil { - return fmt.Errorf("failed to add transaction addresses: %w", err) - } else if err := s.addArbitraryData(dbTxn, txnID, txn); err != nil { +func addTransactionFields(tx *txn, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId) error { + for _, txn := range txns { + txnID := txn.ID() + dbID, ok := txnDBIds[txnID] + if !ok { + panic(fmt.Errorf("txn %v should be in txnDBIds", txn.ID())) + } + + // transaction already exists, don't reinsert its fields + if dbID.exist { + continue + } + // set exist = true so we don't re-insert fields in case we have + // multiple of the same transaction in a block + txnDBIds[txnID] = txnDBId{id: dbID.id, exist: true} + + if err := addMinerFees(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add miner fees: %w", err) + } else if err := addArbitraryData(tx, dbID.id, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) - } else if err := ut.addSiacoinInputs(txnID, txn); err != nil { + } else if err := addSignatures(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add signatures: %w", err) + } else if err := addSiacoinInputs(tx, dbID.id, txn, scDBIds); err != nil { return fmt.Errorf("failed to add siacoin inputs: %w", err) - } else if err := ut.addSiacoinOutputs(txnID, txn, scDBIds); err != nil { + } else if err := addSiacoinOutputs(tx, dbID.id, txn, scDBIds); err != nil { return fmt.Errorf("failed to add siacoin outputs: %w", err) - } else if err := ut.addSiafundInputs(txnID, txn); err != nil { + } else if err := addSiafundInputs(tx, dbID.id, txn, sfDBIds); err != nil { return fmt.Errorf("failed to add siafund inputs: %w", err) - } else if err := ut.addSiafundOutputs(txnID, txn, sfDBIds); err != nil { + } else if err := addSiafundOutputs(tx, dbID.id, txn, sfDBIds); err != nil { return fmt.Errorf("failed to add siafund outputs: %w", err) - } else if err := ut.addFileContracts(txnID, txn, fcDBIds); err != nil { + } else if err := addFileContracts(tx, dbID.id, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract: %w", err) - } else if err := ut.addFileContractRevisions(txnID, txn, fcDBIds); err != nil { + } else if err := addFileContractRevisions(tx, dbID.id, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract revisions: %w", err) + } else if err := addStorageProofs(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add storage proofs: %w", err) } } + return nil } +func addHostAnnouncements(tx *txn, timestamp time.Time, hostAnnouncements []chain.HostAnnouncement, v2HostAnnouncements []explorer.V2HostAnnouncement) error { + hosts := make([]explorer.Host, 0, len(hostAnnouncements)+len(v2HostAnnouncements)) + for _, announcement := range hostAnnouncements { + hosts = append(hosts, explorer.Host{ + PublicKey: announcement.PublicKey, + NetAddress: announcement.NetAddress, + + KnownSince: timestamp, + LastAnnouncement: timestamp, + }) + } + for _, announcement := range v2HostAnnouncements { + hosts = append(hosts, explorer.Host{ + PublicKey: announcement.PublicKey, + V2NetAddresses: []chain.NetAddress(announcement.V2HostAnnouncement), + + KnownSince: timestamp, + LastAnnouncement: timestamp, + }) + } + return addHosts(tx, hosts) +} + type balance struct { sc types.Currency immatureSC types.Currency sf uint64 } -func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { +func updateBalances(tx *txn, height uint64, spentSiacoinElements, newSiacoinElements []explorer.SiacoinOutput, spentSiafundElements, newSiafundElements []types.SiafundElement) error { addresses := make(map[types.Address]balance) for _, sce := range spentSiacoinElements { addresses[sce.SiacoinOutput.Address] = balance{} @@ -331,26 +381,20 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco addresses[sfe.SiafundOutput.Address] = balance{} } - var addressList []any - for address := range addresses { - addressList = append(addressList, encode(address)) - } - - rows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance - FROM address_balance - WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) + balanceRowsStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance + FROM address_balance + WHERE address = ?`) if err != nil { - return fmt.Errorf("updateBalances: failed to query address_balance: %w", err) + return fmt.Errorf("updateBalances: failed to prepare address_balance statement: %w", err) } - defer rows.Close() + defer balanceRowsStmt.Close() - for rows.Next() { + for addr := range addresses { var bal balance - var address types.Address - if err := rows.Scan(decode(&address), decode(&bal.sc), decode(&bal.immatureSC), decode(&bal.sf)); err != nil { - return err + if err := balanceRowsStmt.QueryRow(encode(addr)).Scan(decode(&bal.sc), decode(&bal.immatureSC), decode(&bal.sf)); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("updateBalances: failed to scan balance: %w", err) } - addresses[address] = bal + addresses[addr] = bal } for _, sce := range newSiacoinElements { @@ -386,17 +430,17 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco addresses[sfe.SiafundOutput.Address] = bal } - stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + stmt, err := tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES (?, ?, ?, ?) ON CONFLICT(address) - DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?, siafund_balance = ?`) + DO UPDATE set siacoin_balance = EXCLUDED.siacoin_balance, immature_siacoin_balance = EXCLUDED.immature_siacoin_balance, siafund_balance = EXCLUDED.siafund_balance`) if err != nil { return fmt.Errorf("updateBalances: failed to prepare statement: %w", err) } defer stmt.Close() for addr, bal := range addresses { - if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), encode(bal.sf), encode(bal.sc), encode(bal.immatureSC), encode(bal.sf)); err != nil { + if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), encode(bal.sf)); err != nil { return fmt.Errorf("updateBalances: failed to exec statement: %w", err) } // log.Println(addr, "=", bal.sc) @@ -405,48 +449,47 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco return nil } -func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { +func updateMaturedBalances(tx *txn, revert bool, height uint64) error { // Prevent double counting - outputs with a maturity height of 0 are // handled in updateBalances if height == 0 { return nil } - rows, err := ut.tx.Query(`SELECT address, value - FROM siacoin_elements - WHERE maturity_height = ?`, height) + rows, err := tx.Query(`SELECT address, value + FROM siacoin_elements + WHERE maturity_height = ?`, height) if err != nil { return fmt.Errorf("updateMaturedBalances: failed to query siacoin_elements: %w", err) } defer rows.Close() - var addressList []any var scos []types.SiacoinOutput + addressList := make(map[types.Address]struct{}) for rows.Next() { var sco types.SiacoinOutput if err := rows.Scan(decode(&sco.Address), decode(&sco.Value)); err != nil { return fmt.Errorf("updateMaturedBalances: failed to scan maturing outputs: %w", err) } scos = append(scos, sco) - addressList = append(addressList, encode(sco.Address)) + addressList[sco.Address] = struct{}{} } - balanceRows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance - FROM address_balance - WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) + balanceRowsStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance + FROM address_balance + WHERE address = ?`) if err != nil { - return fmt.Errorf("updateMaturedBalances: failed to query address_balance: %w", err) + return fmt.Errorf("updateMaturedBalances: failed to prepare address_balance statement: %w", err) } - defer balanceRows.Close() + defer balanceRowsStmt.Close() addresses := make(map[types.Address]balance) - for balanceRows.Next() { - var address types.Address + for addr := range addressList { var bal balance - if err := balanceRows.Scan(decode(&address), decode(&bal.sc), decode(&bal.immatureSC)); err != nil { + if err := balanceRowsStmt.QueryRow(encode(addr)).Scan(decode(&bal.sc), decode(&bal.immatureSC)); err != nil { return fmt.Errorf("updateMaturedBalances: failed to scan balance: %w", err) } - addresses[address] = bal + addresses[addr] = bal } // If the update is an apply update then we add the amounts. @@ -463,10 +506,10 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { addresses[sco.Address] = bal } - stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) - VALUES (?, ?, ?, ?) - ON CONFLICT(address) - DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?`) + stmt, err := tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + VALUES (?, ?, ?, ?) + ON CONFLICT(address) + DO UPDATE set siacoin_balance = EXCLUDED.siacoin_balance, immature_siacoin_balance = EXCLUDED.immature_siacoin_balance`) if err != nil { return fmt.Errorf("updateMaturedBalances: failed to prepare statement: %w", err) } @@ -474,7 +517,7 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { initialSF := encode(uint64(0)) for addr, bal := range addresses { - if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), initialSF, encode(bal.sc), encode(bal.immatureSC)); err != nil { + if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), initialSF); err != nil { return fmt.Errorf("updateMaturedBalances: failed to exec statement: %w", err) } } @@ -482,8 +525,8 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { return nil } -func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { - stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) +func updateStateTree(tx *txn, changes []explorer.TreeNodeUpdate) error { + stmt, err := tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -498,121 +541,153 @@ func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { return nil } -func (ut *updateTx) addSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]explorer.Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (output_id) - DO UPDATE SET spent = ?, leaf_index = ?`) - if err != nil { - return nil, fmt.Errorf("addSiacoinElements: failed to prepare siacoin_elements statement: %w", err) - } - defer stmt.Close() - +func addSiacoinElements(tx *txn, index types.ChainIndex, spentElements, newElements []explorer.SiacoinOutput) (map[types.SiacoinOutputID]int64, error) { scDBIds := make(map[types.SiacoinOutputID]int64) - for _, sce := range newElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex)) + if len(newElements) > 0 { + stmt, err := tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, source, maturity_height, address, value) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (output_id) + DO UPDATE SET leaf_index = EXCLUDED.leaf_index, spent_index = NULL + RETURNING id;`) if err != nil { - return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) + return nil, fmt.Errorf("addSiacoinElements: failed to prepare siacoin_elements statement: %w", err) } + defer stmt.Close() - dbID, err := result.LastInsertId() - if err != nil { - return nil, fmt.Errorf("addSiacoinElements: failed to get last insert ID: %w", err) - } + for _, sce := range newElements { + var dbID int64 + if err := stmt.QueryRow(encode(sce.ID), encode(index.ID), encode(sce.StateElement.LeafIndex), int(sce.Source), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) + } - scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID + scDBIds[types.SiacoinOutputID(sce.ID)] = dbID + } } - for _, sce := range spentElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex)) + if len(spentElements) > 0 { + stmt, err := tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent_index, source, maturity_height, address, value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (output_id) + DO UPDATE SET spent_index = ?, leaf_index = EXCLUDED.leaf_index + RETURNING id;`) if err != nil { - return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) + return nil, fmt.Errorf("addSiacoinElements: failed to prepare siacoin_elements statement: %w", err) } + defer stmt.Close() - dbID, err := result.LastInsertId() - if err != nil { - return nil, fmt.Errorf("addSiacoinElements: failed to get last insert ID: %w", err) - } + for _, sce := range spentElements { + var dbID int64 + if err := stmt.QueryRow(encode(sce.ID), encode(index.ID), encode(sce.StateElement.LeafIndex), encode(index), int(sce.Source), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), encode(index)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) + } - scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID + scDBIds[types.SiacoinOutputID(sce.ID)] = dbID + } } return scDBIds, nil } -func (ut *updateTx) addSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent, claim_start, address, value) - VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT - DO UPDATE SET spent = ?, leaf_index = ?`) - if err != nil { - return nil, fmt.Errorf("addSiafundElements: failed to prepare siafund_elements statement: %w", err) - } - defer stmt.Close() - +func addSiafundElements(tx *txn, index types.ChainIndex, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { sfDBIds := make(map[types.SiafundOutputID]int64) - for _, sfe := range newElements { - result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), false, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), false, encode(sfe.StateElement.LeafIndex)) + if len(newElements) > 0 { + stmt, err := tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, claim_start, address, value) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT + DO UPDATE SET leaf_index = EXCLUDED.leaf_index, spent_index = NULL + RETURNING id;`) if err != nil { - return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) + return nil, fmt.Errorf("addSiafundElements: failed to prepare siafund_elements statement: %w", err) } + defer stmt.Close() - dbID, err := result.LastInsertId() - if err != nil { - return nil, fmt.Errorf("addSiafundElements: failed to get last insert ID: %w", err) - } + for _, sfe := range newElements { + var dbID int64 + if err := stmt.QueryRow(encode(sfe.ID), encode(index.ID), encode(sfe.StateElement.LeafIndex), encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) + } - sfDBIds[types.SiafundOutputID(sfe.StateElement.ID)] = dbID + sfDBIds[types.SiafundOutputID(sfe.ID)] = dbID + } } - for _, sfe := range spentElements { - result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), true, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), true, encode(sfe.StateElement.LeafIndex)) + if len(spentElements) > 0 { + stmt, err := tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent_index, claim_start, address, value) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT + DO UPDATE SET leaf_index = EXCLUDED.leaf_index, spent_index = ? + RETURNING id;`) if err != nil { - return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) + return nil, fmt.Errorf("addSiafundElements: failed to prepare siafund_elements statement: %w", err) } + defer stmt.Close() - dbID, err := result.LastInsertId() - if err != nil { - return nil, fmt.Errorf("addSiafundElements: failed to get last insert ID: %w", err) - } + for _, sfe := range spentElements { + var dbID int64 + if err := stmt.QueryRow(encode(sfe.ID), encode(index.ID), encode(sfe.StateElement.LeafIndex), encode(index), encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), encode(index)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) + } - sfDBIds[types.SiafundOutputID(sfe.StateElement.ID)] = dbID + sfDBIds[types.SiafundOutputID(sfe.ID)] = dbID + } } - return sfDBIds, nil } -func (ut *updateTx) addEvents(events []explorer.Event) error { +func addEvents(tx *txn, bid types.BlockID, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, v2FcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId, v2TxnDBIds map[types.TransactionID]txnDBId, events []explorer.Event) error { if len(events) == 0 { return nil } - insertEventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (event_id) DO NOTHING RETURNING id`) + insertEventStmt, err := tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, block_id) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (event_id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare event statement: %w", err) } defer insertEventStmt.Close() - addrStmt, err := ut.tx.Prepare(`INSERT INTO address_balance (address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (address) DO UPDATE SET address=EXCLUDED.address RETURNING id`) + addrStmt, err := tx.Prepare(`INSERT INTO address_balance (address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (address) DO UPDATE SET address=EXCLUDED.address RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - relevantAddrStmt, err := ut.tx.Prepare(`INSERT INTO event_addresses (event_id, address_id) VALUES ($1, $2) ON CONFLICT (event_id, address_id) DO NOTHING`) + relevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id, event_maturity_height) VALUES ($1, $2, $3) ON CONFLICT (event_id, address_id) DO NOTHING`) if err != nil { return fmt.Errorf("failed to prepare relevant address statement: %w", err) } - defer addrStmt.Close() + defer relevantAddrStmt.Close() - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - for _, event := range events { - buf.Reset() - if err := enc.Encode(event.Data); err != nil { - return fmt.Errorf("failed to encode event: %w", err) - } + v1TransactionEventStmt, err := tx.Prepare(`INSERT INTO v1_transaction_events (event_id, transaction_id) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare v1 transaction event statement: %w", err) + } + defer v1TransactionEventStmt.Close() + v2TransactionEventStmt, err := tx.Prepare(`INSERT INTO v2_transaction_events (event_id, transaction_id) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare v2 transaction event statement: %w", err) + } + defer v2TransactionEventStmt.Close() + + payoutEventStmt, err := tx.Prepare(`INSERT INTO payout_events (event_id, output_id) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare minerpayout event statement: %w", err) + } + defer payoutEventStmt.Close() + + v1ContractResolutionEventStmt, err := tx.Prepare(`INSERT INTO v1_contract_resolution_events (event_id, output_id, parent_id, missed) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare v1 contract resolution event statement: %w", err) + } + defer v1ContractResolutionEventStmt.Close() + + v2ContractResolutionEventStmt, err := tx.Prepare(`INSERT INTO v2_contract_resolution_events (event_id, output_id, parent_id, missed) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare v2 contract resolution event statement: %w", err) + } + defer v2ContractResolutionEventStmt.Close() + + for _, event := range events { var eventID int64 - err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Data.EventType(), buf.String(), encode(event.Index.ID), event.Index.Height).Scan(&eventID) + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, encode(bid)).Scan(&eventID) if errors.Is(err, sql.ErrNoRows) { continue // skip if the event already exists } else if err != nil { @@ -626,152 +701,466 @@ func (ut *updateTx) addEvents(events []explorer.Event) error { } var addressID int64 - err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency), 0).Scan(&addressID) + err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency)).Scan(&addressID) if err != nil { return fmt.Errorf("failed to get address: %w", err) } - _, err = relevantAddrStmt.Exec(eventID, addressID) + _, err = relevantAddrStmt.Exec(eventID, addressID, event.MaturityHeight) if err != nil { return fmt.Errorf("failed to add relevant address: %w", err) } used[addr] = true } + + switch v := event.Data.(type) { + case explorer.EventV1Transaction: + dbID := txnDBIds[types.TransactionID(event.ID)].id + if _, err = v1TransactionEventStmt.Exec(eventID, dbID); err != nil { + return fmt.Errorf("failed to insert transaction event: %w", err) + } + case explorer.EventV2Transaction: + dbID := v2TxnDBIds[types.TransactionID(event.ID)].id + if _, err = v2TransactionEventStmt.Exec(eventID, dbID); err != nil { + return fmt.Errorf("failed to insert transaction event: %w", err) + } + case explorer.EventPayout: + _, err = payoutEventStmt.Exec(eventID, scDBIds[types.SiacoinOutputID(event.ID)]) + case explorer.EventV1ContractResolution: + _, err = v1ContractResolutionEventStmt.Exec(eventID, scDBIds[v.SiacoinElement.ID], fcDBIds[explorer.DBFileContract{ID: v.Parent.ID, RevisionNumber: v.Parent.RevisionNumber}], v.Missed) + case explorer.EventV2ContractResolution: + _, err = v2ContractResolutionEventStmt.Exec(eventID, scDBIds[v.SiacoinElement.ID], v2FcDBIds[explorer.DBFileContract{ID: v.Resolution.Parent.ID, RevisionNumber: v.Resolution.Parent.V2FileContract.RevisionNumber}], v.Missed) + default: + return fmt.Errorf("unknown event type: %T", reflect.TypeOf(event.Data)) + } + if err != nil { + return fmt.Errorf("failed to insert %v event: %w", reflect.TypeOf(event.Data), err) + } } return nil } -func (ut *updateTx) deleteBlock(bid types.BlockID) error { - _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) +func deleteBlock(tx *txn, bid types.BlockID) error { + _, err := tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) return err } -func (ut *updateTx) addFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) - VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (contract_id, revision_number) - DO UPDATE SET resolved = ?, valid = ?, leaf_index = ? - RETURNING id;`) +func updateFileContractElements(tx *txn, revert bool, index types.ChainIndex, b types.Block, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { + stmt, err := tx.Prepare(`INSERT INTO file_contract_elements(contract_id, block_id, transaction_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) + VALUES (?, ?, ?, ?, FALSE, FALSE, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (contract_id, revision_number) + DO UPDATE SET resolved = ?, valid = ?, leaf_index = EXCLUDED.leaf_index + RETURNING id;`) if err != nil { - return nil, fmt.Errorf("addFileContractElements: failed to prepare file_contract_elements statement: %w", err) + return nil, fmt.Errorf("updateFileContractElements: failed to prepare main statement: %w", err) } defer stmt.Close() - revisionStmt, err := ut.tx.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id) - VALUES (?, ?) - ON CONFLICT (contract_id) - DO UPDATE SET contract_element_id = ?`) + revisionStmt, err := tx.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id, ed25519_renter_key, ed25519_host_key, confirmation_height, confirmation_block_id, confirmation_transaction_id) + VALUES (?, ?, ?, ?, COALESCE(?, X''), COALESCE(?, X''), COALESCE(?, X'')) + ON CONFLICT (contract_id) + DO UPDATE SET contract_element_id = ?, ed25519_renter_key = COALESCE(?, ed25519_renter_key), ed25519_host_key = COALESCE(?, ed25519_host_key), confirmation_height = COALESCE(?, confirmation_height), confirmation_block_id = COALESCE(?, confirmation_block_id), confirmation_transaction_id = COALESCE(?, confirmation_transaction_id)`) if err != nil { - return nil, fmt.Errorf("addFileContractElements: failed to prepare last_contract_revision statement: %w", err) + return nil, fmt.Errorf("updateFileContractElements: failed to prepare last_contract_revision statement: %w", err) } + defer revisionStmt.Close() - var updateErr error - fcDBIds := make(map[explorer.DBFileContract]int64) - for _, update := range fces { - fce := update.FileContractElement + validOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, id, address, value) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`) + if err != nil { + return nil, fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) + } + defer validOutputsStmt.Close() + + missedOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, id, address, value) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`) + if err != nil { + return nil, fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) + } + defer missedOutputsStmt.Close() + + fcKeys := make(map[explorer.DBFileContract][2]types.PublicKey) + // populate fcKeys using revision UnlockConditions fields + for _, txn := range b.Transactions { + for _, fcr := range txn.FileContractRevisions { + fc := fcr.FileContract + uc := fcr.UnlockConditions + dbFC := explorer.DBFileContract{ID: fcr.ParentID, RevisionNumber: fc.RevisionNumber} + + // check for 2 ed25519 keys + ok := true + var result [2]types.PublicKey + for i := 0; i < 2; i++ { + // fewer than 2 keys + if i >= len(uc.PublicKeys) { + ok = false + break + } + + if uc.PublicKeys[i].Algorithm == types.SpecifierEd25519 { + result[i] = types.PublicKey(uc.PublicKeys[i].Key) + } else { + // not an ed25519 key + ok = false + } + } + if ok { + fcKeys[dbFC] = result + } + } + } + + fcTxns := make(map[explorer.DBFileContract]types.TransactionID) + for _, txn := range b.Transactions { + id := txn.ID() - fc := &fce.FileContract - if update.Revision != nil { - fc = &update.Revision.FileContract + for i, fc := range txn.FileContracts { + fcTxns[explorer.DBFileContract{ + ID: txn.FileContractID(i), + RevisionNumber: fc.RevisionNumber, + }] = id } + for _, fcr := range txn.FileContractRevisions { + fcTxns[explorer.DBFileContract{ + ID: fcr.ParentID, + RevisionNumber: fcr.FileContract.RevisionNumber, + }] = id + } + } + fcDBIds := make(map[explorer.DBFileContract]int64) + addFC := func(fcID types.FileContractID, leafIndex uint64, fc types.FileContract, confirmationTransactionID *types.TransactionID, resolved, valid, lastRevision bool) error { var dbID int64 - err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, update.Resolved, update.Valid, encode(fce.StateElement.LeafIndex)).Scan(&dbID) + dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} + err := stmt.QueryRow(encode(fcID), encode(index.ID), encode(fcTxns[dbFC]), encode(leafIndex), encode(fc.Filesize), encode(fc.FileMerkleRoot), encode(fc.WindowStart), encode(fc.WindowEnd), encode(fc.Payout), encode(fc.UnlockHash), encode(fc.RevisionNumber), resolved, valid).Scan(&dbID) if err != nil { - return nil, fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) + return fmt.Errorf("failed to execute file_contract_elements statement: %w", err) + } + + for i, sco := range fc.ValidProofOutputs { + if _, err := validOutputsStmt.Exec(dbID, i, encode(fcID.ValidOutputID(i)), encode(sco.Address), encode(sco.Value)); err != nil { + return fmt.Errorf("updateFileContractElements: failed to execute valid proof outputs statement: %w", err) + } + } + for i, sco := range fc.MissedProofOutputs { + if _, err := missedOutputsStmt.Exec(dbID, i, encode(fcID.MissedOutputID(i)), encode(sco.Address), encode(sco.Value)); err != nil { + return fmt.Errorf("updateFileContractElements: failed to execute missed proof outputs statement: %w", err) + } } - if _, err := revisionStmt.Exec(encode(fce.StateElement.ID), dbID, dbID); err != nil { - return nil, fmt.Errorf("addFileContractElements: failed to update last revision number: %w", err) + // only update if it's the most recent revision which will come from + // running ForEachFileContractElement on the update + if lastRevision { + var encodedRenterKey, encodedHostKey []byte + if keys, ok := fcKeys[dbFC]; ok { + encodedRenterKey = encode(keys[0]).([]byte) + encodedHostKey = encode(keys[1]).([]byte) + } + + var encodedHeight, encodedBlockID, encodedConfirmationTransactionID []byte + if confirmationTransactionID != nil { + encodedHeight = encode(index.Height).([]byte) + encodedBlockID = encode(index.ID).([]byte) + encodedConfirmationTransactionID = encode(*confirmationTransactionID).([]byte) + } + + if _, err := revisionStmt.Exec(encode(fcID), dbID, encodedRenterKey, encodedHostKey, encodedHeight, encodedBlockID, encodedConfirmationTransactionID, dbID, encodedRenterKey, encodedHostKey, encodedHeight, encodedBlockID, encodedConfirmationTransactionID); err != nil { + return fmt.Errorf("failed to update last revision number: %w", err) + } } - fcDBIds[explorer.DBFileContract{ID: types.FileContractID(fce.StateElement.ID), RevisionNumber: fc.RevisionNumber}] = dbID + fcDBIds[dbFC] = dbID + return nil } - return fcDBIds, updateErr + + for _, update := range fces { + var fce *types.FileContractElement + + if revert { + // Reverting + if update.Revision != nil { + // Contract revision reverted. + // We are reverting the revision, so get the contract before + // the revision. + fce = &update.FileContractElement + } else { + // Contract formation reverted. + // The contract update has no revision, therefore it refers + // to the original contract formation. + continue + } + } else { + // Applying + fce = &update.FileContractElement + if update.Revision != nil { + // Contract is revised. + // We want last_contract_revision to refer to the latest + // revision, so use the revision FCE if there is one. + fce = update.Revision + } + } + + if err := addFC( + fce.ID, + fce.StateElement.LeafIndex, + fce.FileContract, + update.ConfirmationTransactionID, + update.Resolved, + update.Valid, + true, + ); err != nil { + return nil, fmt.Errorf("updateFileContractElements: %w", err) + } + } + + if revert { + return fcDBIds, nil + } + + for _, txn := range b.Transactions { + // add in any contracts that are not the latest, i.e. contracts that + // were created and revised in the same block + for j, fc := range txn.FileContracts { + fcID := txn.FileContractID(j) + dbFC := explorer.DBFileContract{ID: txn.FileContractID(j), RevisionNumber: fc.RevisionNumber} + if _, exists := fcDBIds[dbFC]; exists { + continue + } + + if err := addFC(fcID, 0, fc, nil, false, false, false); err != nil { + return nil, fmt.Errorf("updateFileContractElements: %w", err) + } + } + // add in any revisions that are not the latest, i.e. contracts that + // were revised multiple times in one block + for _, fcr := range txn.FileContractRevisions { + fc := fcr.FileContract + dbFC := explorer.DBFileContract{ID: fcr.ParentID, RevisionNumber: fc.RevisionNumber} + if _, exists := fcDBIds[dbFC]; exists { + continue + } + + if err := addFC(fcr.ParentID, 0, fc, nil, false, false, false); err != nil { + return nil, fmt.Errorf("updateFileContractElements: %w", err) + } + } + } + + return fcDBIds, nil +} + +func updateFileContractIndices(tx *txn, revert bool, index types.ChainIndex, fces []explorer.FileContractUpdate) error { + proofIndexStmt, err := tx.Prepare(`UPDATE last_contract_revision SET proof_height = ?, proof_block_id = ?, proof_transaction_id = ? WHERE contract_id = ?`) + if err != nil { + return fmt.Errorf("updateFileContractIndices: failed to prepare proof index statement: %w", err) + } + defer proofIndexStmt.Close() + + for _, update := range fces { + // id stays the same even if revert happens so we don't need to check that here + fcID := update.FileContractElement.ID + + if revert { + if update.ProofTransactionID != nil { + if _, err := proofIndexStmt.Exec(nil, nil, nil, encode(fcID)); err != nil { + return fmt.Errorf("updateFileContractIndices: failed to update proof index: %w", err) + } + } + } else { + if update.ProofTransactionID != nil { + if _, err := proofIndexStmt.Exec(encode(index.Height), encode(index.ID), encode(update.ProofTransactionID), encode(fcID)); err != nil { + return fmt.Errorf("updateFileContractIndices: failed to update proof index: %w", err) + } + } + } + } + + return nil +} + +func addMetrics(tx *txn, s explorer.UpdateState) error { + _, err := tx.Exec(`INSERT INTO network_metrics(block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + encode(s.Metrics.Index.ID), + s.Metrics.Index.Height, + encode(s.Metrics.Difficulty), + encode(s.Metrics.SiafundTaxRevenue), + encode(s.Metrics.NumLeaves), + s.Metrics.TotalHosts, + s.Metrics.ActiveContracts, + s.Metrics.FailedContracts, + s.Metrics.SuccessfulContracts, + s.Metrics.StorageUtilization, + encode(s.Metrics.CirculatingSupply), + encode(s.Metrics.ContractRevenue), + ) + return err +} + +func (ut *updateTx) HostExists(pubkey types.PublicKey) (exists bool, err error) { + err = ut.tx.QueryRow(`SELECT EXISTS(SELECT public_key FROM host_info WHERE public_key = ?)`, encode(pubkey)).Scan(&exists) + return +} + +func (ut *updateTx) Metrics(height uint64) (explorer.Metrics, error) { + var metrics explorer.Metrics + if err := ut.tx.QueryRow("SELECT total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue from network_metrics WHERE height = ?", height).Scan(&metrics.TotalHosts, &metrics.ActiveContracts, &metrics.FailedContracts, &metrics.SuccessfulContracts, &metrics.StorageUtilization, decode(&metrics.CirculatingSupply), decode(&metrics.ContractRevenue)); err != nil && err != sql.ErrNoRows { + return explorer.Metrics{}, err + } + return metrics, nil } func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { - if err := ut.addBlock(state.Block, state.Index.Height); err != nil { + if err := addBlock(ut.tx, state.Block, state.ChainIndexElement, state.Metrics.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to add block: %w", err) - } else if err := ut.updateMaturedBalances(false, state.Index.Height); err != nil { + } else if err := updateMaturedBalances(ut.tx, false, state.Metrics.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err) } - scDBIds, err := ut.addSiacoinElements( - state.Block.ID(), - state.Sources, + txnDBIds, err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add transactions: %w", err) + } + + v2TxnDBIds, err := addV2Transactions(ut.tx, state.Block.ID(), state.Block.V2Transactions()) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add v2 transactions: %w", err) + } + + scDBIds, err := addSiacoinElements( + ut.tx, + state.Metrics.Index, append(state.SpentSiacoinElements, state.EphemeralSiacoinElements...), state.NewSiacoinElements, ) if err != nil { return fmt.Errorf("ApplyIndex: failed to add siacoin outputs: %w", err) } - sfDBIds, err := ut.addSiafundElements( - state.Block.ID(), + sfDBIds, err := addSiafundElements( + ut.tx, + state.Metrics.Index, append(state.SpentSiafundElements, state.EphemeralSiafundElements...), state.NewSiafundElements, ) if err != nil { return fmt.Errorf("ApplyIndex: failed to add siafund outputs: %w", err) } - if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { - return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) + fcDBIds, err := updateFileContractElements(ut.tx, false, state.Metrics.Index, state.Block, state.FileContractElements) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add file contracts: %w", err) } - fcDBIds, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements) + v2FcDBIds, err := updateV2FileContractElements(ut.tx, false, state.Metrics.Index, state.Block, state.V2FileContractElements) if err != nil { - return fmt.Errorf("v: failed to add file contracts: %w", err) + return fmt.Errorf("ApplyIndex: failed to add v2 file contracts: %w", err) } - if err := ut.addMinerPayouts(state.Block.ID(), state.Index.Height, state.Block.MinerPayouts, scDBIds); err != nil { + if err := addTransactionFields(ut.tx, state.Block.Transactions, scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add transaction fields: %w", err) + } else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), scDBIds, sfDBIds, v2FcDBIds, v2TxnDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add v2 transaction fields: %w", err) + } else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) + } else if err := addMinerPayouts(ut.tx, state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) - } else if err := ut.addTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { - return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) - } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { + } else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil { return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) - } else if err := ut.addEvents(state.Events); err != nil { + } else if err := addMetrics(ut.tx, state); err != nil { + return fmt.Errorf("ApplyIndex: failed to update metrics: %w", err) + } else if err := addHostAnnouncements(ut.tx, state.Block.Timestamp, state.HostAnnouncements, state.V2HostAnnouncements); err != nil { + return fmt.Errorf("ApplyIndex: failed to add host announcements: %w", err) + } else if err := updateFileContractIndices(ut.tx, false, state.Metrics.Index, state.FileContractElements); err != nil { + return fmt.Errorf("ApplyIndex: failed to update file contract element indices: %w", err) + } else if err := updateV2FileContractIndices(ut.tx, false, state.Metrics.Index, state.V2FileContractElements); err != nil { + return fmt.Errorf("ApplyIndex: failed to update v2 file contract element indices: %w", err) + } else if err := addEvents(ut.tx, state.Block.ID(), scDBIds, fcDBIds, v2FcDBIds, txnDBIds, v2TxnDBIds, state.Events); err != nil { return fmt.Errorf("ApplyIndex: failed to add events: %w", err) } return nil } -func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { - if err := ut.updateMaturedBalances(true, state.Index.Height); err != nil { - return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) - } else if _, err := ut.addSiacoinElements( - state.Block.ID(), - nil, - state.SpentSiacoinElements, - append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), - ); err != nil { - return fmt.Errorf("RevertIndex: failed to update siacoin output state: %w", err) - } else if _, err := ut.addSiafundElements( - state.Block.ID(), - state.SpentSiafundElements, - append(state.NewSiafundElements, state.EphemeralSiafundElements...), - ); err != nil { - return fmt.Errorf("RevertIndex: failed to update siafund output state: %w", err) - } else if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { - return fmt.Errorf("RevertIndex: failed to update balances: %w", err) - } else if _, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements); err != nil { - return fmt.Errorf("RevertIndex: failed to update file contract state: %w", err) - } else if err := ut.deleteBlock(state.Block.ID()); err != nil { - return fmt.Errorf("RevertIndex: failed to delete block: %w", err) - } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { - return fmt.Errorf("RevertIndex: failed to update state tree: %w", err) +func addHosts(tx *txn, hosts []explorer.Host) error { + if len(hosts) == 0 { + return nil + } + + stmt, err := tx.Prepare(`INSERT INTO host_info(public_key, v2, net_address, country_code, latitude, longitude, known_since, last_scan, last_scan_successful, last_scan_error, next_scan, failed_interactions_streak, last_announcement, total_scans, successful_interactions, failed_interactions, settings_accepting_contracts, settings_max_download_batch_size, settings_max_duration, settings_max_revise_batch_size, settings_net_address, settings_remaining_storage, settings_sector_size, settings_total_storage, settings_used_storage, settings_address, settings_window_size, settings_collateral, settings_max_collateral, settings_base_rpc_price, settings_contract_price, settings_download_bandwidth_price, settings_sector_access_price, settings_storage_price, settings_upload_bandwidth_price, settings_ephemeral_account_expiry, settings_max_ephemeral_account_balance, settings_revision_number, settings_version, settings_release, settings_sia_mux_port, price_table_uid, price_table_validity, price_table_host_block_height, price_table_update_price_table_cost, price_table_account_balance_cost, price_table_fund_account_cost, price_table_latest_revision_cost, price_table_subscription_memory_cost, price_table_subscription_notification_cost, price_table_init_base_cost, price_table_memory_time_cost, price_table_download_bandwidth_cost, price_table_upload_bandwidth_cost, price_table_drop_sectors_base_cost, price_table_drop_sectors_unit_cost, price_table_has_sector_base_cost, price_table_read_base_cost, price_table_read_length_cost, price_table_renew_contract_cost, price_table_revision_base_cost, price_table_swap_sector_base_cost, price_table_write_base_cost, price_table_write_length_cost, price_table_write_store_cost, price_table_txn_fee_min_recommended, price_table_txn_fee_max_recommended, price_table_contract_price, price_table_collateral_cost, price_table_max_collateral, price_table_max_duration, price_table_window_size, price_table_registry_entries_left, price_table_registry_entries_total, v2_settings_protocol_version, v2_settings_release, v2_settings_wallet_address, v2_settings_accepting_contracts, v2_settings_max_collateral, v2_settings_max_contract_duration, v2_settings_remaining_storage, v2_settings_total_storage, v2_settings_used_storage, v2_prices_contract_price, v2_prices_collateral_price, v2_prices_storage_price, v2_prices_ingress_price, v2_prices_egress_price, v2_prices_free_sector_price, v2_prices_tip_height, v2_prices_valid_until, v2_prices_signature) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36,$37,$38,$39,$40,$41,$42,$43,$44,$45,$46,$47,$48,$49,$50,$51,$52,$53,$54,$55,$56,$57,$58,$59,$60,$61,$62,$63,$64,$65,$66,$67,$68,$69,$70,$71,$72,$73,$74,$75,$76,$77,$78,$79,$80,$81,$82,$83,$84,$85,$86,$87,$88,$89,$90,$91,$92) ON CONFLICT (public_key) DO UPDATE SET v2 = EXCLUDED.v2, net_address = EXCLUDED.net_address, last_announcement = EXCLUDED.last_announcement, next_scan = EXCLUDED.last_announcement`) + if err != nil { + return fmt.Errorf("failed to prepare host_info stmt: %w", err) + } + defer stmt.Close() + + deleteV2AddrStmt, err := tx.Prepare(`DELETE FROM host_info_v2_netaddresses WHERE public_key = ?`) + if err != nil { + return fmt.Errorf("failed to prepare delete v2 net address stmt: %w", err) + } + defer deleteV2AddrStmt.Close() + + addV2AddrStmt, err := tx.Prepare(`INSERT INTO host_info_v2_netaddresses(public_key, netaddress_order, protocol, address) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare add v2 net address stmt: %w", err) } + defer addV2AddrStmt.Close() + + for _, host := range hosts { + s, p := host.Settings, host.PriceTable + sV2, pV2 := host.V2Settings, host.V2Settings.Prices + isV2 := len(host.V2NetAddresses) > 0 + if _, err := stmt.Exec(encode(host.PublicKey), isV2, host.NetAddress, host.Location.CountryCode, host.Location.Latitude, host.Location.Longitude, encode(host.KnownSince), encode(host.LastScan), host.LastScanSuccessful, "", encode(host.LastAnnouncement), 0, encode(host.LastAnnouncement), host.TotalScans, host.SuccessfulInteractions, host.FailedInteractions, s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV2.ProtocolVersion[:], sV2.Release, encode(sV2.WalletAddress), sV2.AcceptingContracts, encode(sV2.MaxCollateral), encode(sV2.MaxContractDuration), encode(sV2.RemainingStorage), encode(sV2.TotalStorage), encode(sV2.TotalStorage-sV2.RemainingStorage), encode(pV2.ContractPrice), encode(pV2.Collateral), encode(pV2.StoragePrice), encode(pV2.IngressPrice), encode(pV2.EgressPrice), encode(pV2.FreeSectorPrice), encode(pV2.TipHeight), encode(pV2.ValidUntil), encode(pV2.Signature)); err != nil { + return fmt.Errorf("failed to execute host_info stmt: %w", err) + } + + if isV2 { + if _, err := deleteV2AddrStmt.Exec(encode(host.PublicKey)); err != nil { + return fmt.Errorf("failed to execute delete v2 net address stmt: %w", err) + } + for i, netAddr := range host.V2NetAddresses { + if _, err := addV2AddrStmt.Exec(encode(host.PublicKey), i, netAddr.Protocol, netAddr.Address); err != nil { + return fmt.Errorf("failed to execute add v2 net address stmt: %w", err) + } + } + } + } return nil } +// AddHostScans implements explorer.Store +func (s *Store) AddHostScans(scans ...explorer.HostScan) error { + return s.transaction(func(tx *txn) error { + unsuccessfulStmt, err := tx.Prepare(`UPDATE host_info SET last_scan = ?, last_scan_successful = 0, last_scan_error = ?, next_scan = ?, total_scans = total_scans + 1, failed_interactions = failed_interactions + 1, failed_interactions_streak = failed_interactions_streak + 1 WHERE public_key = ?`) + if err != nil { + return fmt.Errorf("addHostScans: failed to prepare unsuccessful statement: %w", err) + } + defer unsuccessfulStmt.Close() + + successfulStmt, err := tx.Prepare(`UPDATE host_info SET country_code = ?, latitude = ?, longitude = ?, last_scan = ?, last_scan_successful = 1, last_scan_error = "", next_scan = ?, total_scans = total_scans + 1, successful_interactions = successful_interactions + 1, failed_interactions_streak = 0, settings_accepting_contracts = ?, settings_max_download_batch_size = ?, settings_max_duration = ?, settings_max_revise_batch_size = ?, settings_net_address = ?, settings_remaining_storage = ?, settings_sector_size = ?, settings_total_storage = ?, settings_used_storage = ?, settings_address = ?, settings_window_size = ?, settings_collateral = ?, settings_max_collateral = ?, settings_base_rpc_price = ?, settings_contract_price = ?, settings_download_bandwidth_price = ?, settings_sector_access_price = ?, settings_storage_price = ?, settings_upload_bandwidth_price = ?, settings_ephemeral_account_expiry = ?, settings_max_ephemeral_account_balance = ?, settings_revision_number = ?, settings_version = ?, settings_release = ?, settings_sia_mux_port = ?, price_table_uid = ?, price_table_validity = ?, price_table_host_block_height = ?, price_table_update_price_table_cost = ?, price_table_account_balance_cost = ?, price_table_fund_account_cost = ?, price_table_latest_revision_cost = ?, price_table_subscription_memory_cost = ?, price_table_subscription_notification_cost = ?, price_table_init_base_cost = ?, price_table_memory_time_cost = ?, price_table_download_bandwidth_cost = ?, price_table_upload_bandwidth_cost = ?, price_table_drop_sectors_base_cost = ?, price_table_drop_sectors_unit_cost = ?, price_table_has_sector_base_cost = ?, price_table_read_base_cost = ?, price_table_read_length_cost = ?, price_table_renew_contract_cost = ?, price_table_revision_base_cost = ?, price_table_swap_sector_base_cost = ?, price_table_write_base_cost = ?, price_table_write_length_cost = ?, price_table_write_store_cost = ?, price_table_txn_fee_min_recommended = ?, price_table_txn_fee_max_recommended = ?, price_table_contract_price = ?, price_table_collateral_cost = ?, price_table_max_collateral = ?, price_table_max_duration = ?, price_table_window_size = ?, price_table_registry_entries_left = ?, price_table_registry_entries_total = ?, v2_settings_protocol_version = ?, v2_settings_release = ?, v2_settings_wallet_address = ?, v2_settings_accepting_contracts = ?, v2_settings_max_collateral = ?, v2_settings_max_contract_duration = ?, v2_settings_remaining_storage = ?, v2_settings_total_storage = ?, v2_settings_used_storage = ?, v2_prices_contract_price = ?, v2_prices_collateral_price = ?, v2_prices_storage_price = ?, v2_prices_ingress_price = ?, v2_prices_egress_price = ?, v2_prices_free_sector_price = ?, v2_prices_tip_height = ?, v2_prices_valid_until = ?, v2_prices_signature = ? WHERE public_key = ?`) + if err != nil { + return fmt.Errorf("addHostScans: failed to prepare successful statement: %w", err) + } + defer successfulStmt.Close() + + for _, scan := range scans { + s, p := scan.Settings, scan.PriceTable + sV2, pV2 := scan.V2Settings, scan.V2Settings.Prices + if scan.Success { + if _, err := successfulStmt.Exec(scan.Location.CountryCode, scan.Location.Latitude, scan.Location.Longitude, encode(scan.Timestamp), encode(scan.NextScan), s.AcceptingContracts, encode(s.MaxDownloadBatchSize), encode(s.MaxDuration), encode(s.MaxReviseBatchSize), s.NetAddress, encode(s.RemainingStorage), encode(s.SectorSize), encode(s.TotalStorage), encode(s.TotalStorage-s.RemainingStorage), encode(s.Address), encode(s.WindowSize), encode(s.Collateral), encode(s.MaxCollateral), encode(s.BaseRPCPrice), encode(s.ContractPrice), encode(s.DownloadBandwidthPrice), encode(s.SectorAccessPrice), encode(s.StoragePrice), encode(s.UploadBandwidthPrice), s.EphemeralAccountExpiry, encode(s.MaxEphemeralAccountBalance), encode(s.RevisionNumber), s.Version, s.Release, s.SiaMuxPort, encode(p.UID), p.Validity, encode(p.HostBlockHeight), encode(p.UpdatePriceTableCost), encode(p.AccountBalanceCost), encode(p.FundAccountCost), encode(p.LatestRevisionCost), encode(p.SubscriptionMemoryCost), encode(p.SubscriptionNotificationCost), encode(p.InitBaseCost), encode(p.MemoryTimeCost), encode(p.DownloadBandwidthCost), encode(p.UploadBandwidthCost), encode(p.DropSectorsBaseCost), encode(p.DropSectorsUnitCost), encode(p.HasSectorBaseCost), encode(p.ReadBaseCost), encode(p.ReadLengthCost), encode(p.RenewContractCost), encode(p.RevisionBaseCost), encode(p.SwapSectorBaseCost), encode(p.WriteBaseCost), encode(p.WriteLengthCost), encode(p.WriteStoreCost), encode(p.TxnFeeMinRecommended), encode(p.TxnFeeMaxRecommended), encode(p.ContractPrice), encode(p.CollateralCost), encode(p.MaxCollateral), encode(p.MaxDuration), encode(p.WindowSize), encode(p.RegistryEntriesLeft), encode(p.RegistryEntriesTotal), sV2.ProtocolVersion[:], sV2.Release, encode(sV2.WalletAddress), sV2.AcceptingContracts, encode(sV2.MaxCollateral), encode(sV2.MaxContractDuration), encode(sV2.RemainingStorage), encode(sV2.TotalStorage), encode(sV2.TotalStorage-sV2.RemainingStorage), encode(pV2.ContractPrice), encode(pV2.Collateral), encode(pV2.StoragePrice), encode(pV2.IngressPrice), encode(pV2.EgressPrice), encode(pV2.FreeSectorPrice), encode(pV2.TipHeight), encode(pV2.ValidUntil), encode(pV2.Signature), encode(scan.PublicKey)); err != nil { + return fmt.Errorf("addHostScans: failed to execute successful statement: %w", err) + } + } else { + if _, err := unsuccessfulStmt.Exec(encode(scan.Timestamp), *scan.Error, encode(scan.NextScan), encode(scan.PublicKey)); err != nil { + return fmt.Errorf("addHostScans: failed to execute unsuccessful statement: %w", err) + } + } + } + return nil + }) +} + // UpdateChainState implements explorer.Store func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { return s.transaction(func(tx *txn) error { utx := &updateTx{ - tx: tx, - relevantAddresses: make(map[types.Address]bool), + tx: tx, } if err := explorer.UpdateChainState(utx, reverted, applied); err != nil { @@ -781,14 +1170,79 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. }) } -// Tip implements explorer.Store. -func (s *Store) Tip() (result types.ChainIndex, err error) { - const query = `SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1` - err = s.transaction(func(dbTxn *txn) error { - return dbTxn.QueryRow(query).Scan(decode(&result.ID), &result.Height) - }) - if errors.Is(err, sql.ErrNoRows) { - return types.ChainIndex{}, explorer.ErrNoTip +// ResetChainState implements explorer.Store +func (s *Store) ResetChainState() error { + if err := s.transaction(func(tx *txn) error { + if _, err := tx.Exec(`PRAGMA defer_foreign_keys=ON`); err != nil { + return fmt.Errorf("failed to defer foreign key checks: %w", err) + } + + names := []string{ + "network_metrics", + "file_contract_valid_proof_outputs", + "file_contract_missed_proof_outputs", + "miner_payouts", + "block_transactions", + "transaction_arbitrary_data", + "transaction_miner_fees", + "transaction_signatures", + "transaction_storage_proofs", + "transaction_siacoin_inputs", + "transaction_siacoin_outputs", + "transaction_siafund_inputs", + "transaction_siafund_outputs", + "transaction_file_contracts", + "transaction_file_contract_revisions", + "v2_block_transactions", + "v2_transaction_siacoin_inputs", + "v2_transaction_siacoin_outputs", + "v2_transaction_siafund_inputs", + "v2_transaction_siafund_outputs", + "v2_transaction_file_contracts", + "v2_transaction_file_contract_revisions", + "v2_transaction_file_contract_resolutions", + "v2_transaction_attestations", + "event_addresses", + "v1_transaction_events", + "v2_transaction_events", + "payout_events", + "v1_contract_resolution_events", + "v2_contract_resolution_events", + "last_contract_revision", + "v2_last_contract_revision", + "host_info_v2_netaddresses", + "file_contract_elements", + "v2_file_contract_elements", + "transactions", + "v2_transactions", + "address_balance", + "siacoin_elements", + "siafund_elements", + "events", + "blocks", + "host_info", + "state_tree", + "global_settings", + } + for _, name := range names { + if _, err := tx.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, name)); err != nil { + return fmt.Errorf("failed to drop table %s: %w", name, err) + } + } + + target := int64(len(migrations) + 1) + if err := s.initNewDatabase(tx, target); err != nil { + return fmt.Errorf("failed to reinit tables: %w", err) + } + + return nil + }); err != nil { + return fmt.Errorf("ResetChainState: failed to delete and reinit database: %w", err) } - return + + if _, err := s.db.Exec(`VACUUM`); err != nil { + return fmt.Errorf("ResetChainState: failed to vacuum database: %w", err) + } + + return nil } diff --git a/persist/sqlite/consensus_refactored_test.go b/persist/sqlite/consensus_refactored_test.go new file mode 100644 index 0000000..f5d6fe2 --- /dev/null +++ b/persist/sqlite/consensus_refactored_test.go @@ -0,0 +1,911 @@ +package sqlite_test + +import ( + "errors" + "math" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" + "go.sia.tech/explored/persist/sqlite" + "go.uber.org/zap/zaptest" +) + +type testChain struct { + db explorer.Store + store *chain.DBStore + + blocks []types.Block + states []consensus.State +} + +func newTestChain(t *testing.T, v2 bool, modifyGenesis func(*consensus.Network, types.Block)) *testChain { + log := zaptest.NewLogger(t) + dir := t.TempDir() + + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + db.Close() + }) + + var network *consensus.Network + var genesisBlock types.Block + if v2 { + network, genesisBlock = ctestutil.V2Network() + } else { + network, genesisBlock = ctestutil.Network() + } + if v2 { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + } + if modifyGenesis != nil { + modifyGenesis(network, genesisBlock) + } + + store, genesisState, err := chain.NewDBStore(chain.NewMemDB(), network, genesisBlock, nil) + if err != nil { + t.Fatal(err) + } + + bs := consensus.V1BlockSupplement{Transactions: make([]consensus.V1TransactionSupplement, len(genesisBlock.Transactions))} + _, au := consensus.ApplyBlock(network.GenesisState(), genesisBlock, bs, time.Time{}) + if err := db.UpdateChainState(nil, []chain.ApplyUpdate{{ + ApplyUpdate: au, + Block: genesisBlock, + State: genesisState, + }}); err != nil { + t.Fatal(err) + } + + return &testChain{ + db: db, + store: store, + + blocks: []types.Block{genesisBlock}, + states: []consensus.State{genesisState}, + } +} + +func (n *testChain) genesis() types.Block { + return n.blocks[0] +} + +func (n *testChain) tipState() consensus.State { + return n.states[len(n.states)-1] +} + +func (n *testChain) applyBlock(t *testing.T, b types.Block) { + cs := n.tipState() + bs := n.store.SupplementTipBlock(b) + if cs.Index.Height != math.MaxUint64 { + // don't validate genesis block + if err := consensus.ValidateBlock(cs, b, bs); err != nil { + t.Fatal(err) + } + } + + cs, au := consensus.ApplyBlock(cs, b, bs, time.Time{}) + if err := n.db.UpdateChainState(nil, []chain.ApplyUpdate{{ + ApplyUpdate: au, + Block: b, + State: cs, + }}); err != nil { + t.Fatal(err) + } + + n.states = append(n.states, cs) + n.blocks = append(n.blocks, b) +} + +func (n *testChain) revertBlock(t *testing.T) { + b := n.blocks[len(n.blocks)-1] + prevState := n.states[len(n.states)-2] + + bs := n.store.SupplementTipBlock(b) + ru := consensus.RevertBlock(prevState, b, bs) + if err := n.db.UpdateChainState([]chain.RevertUpdate{{ + RevertUpdate: ru, + Block: b, + State: prevState, + }}, nil); err != nil { + t.Fatal(err) + } + + n.states = n.states[:len(n.states)-1] + n.blocks = n.blocks[:len(n.blocks)-1] +} + +func (n *testChain) mineTransactions(t *testing.T, txns ...types.Transaction) { + b := testutil.MineBlock(n.tipState(), txns, types.VoidAddress) + n.applyBlock(t, b) +} + +func (n *testChain) mineV2Transactions(t *testing.T, txns ...types.V2Transaction) { + b := testutil.MineV2Block(n.tipState(), txns, types.VoidAddress) + n.applyBlock(t, b) +} + +func (n *testChain) assertTransactions(t *testing.T, expected ...types.Transaction) { + t.Helper() + + for _, txn := range expected { + txns, err := n.db.Transactions([]types.TransactionID{txn.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 1, len(txns)) + + testutil.CheckTransaction(t, txn, txns[0]) + } +} + +func (n *testChain) assertV2Transactions(t *testing.T, expected ...types.V2Transaction) { + t.Helper() + + for _, txn := range expected { + txns, err := n.db.V2Transactions([]types.TransactionID{txn.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 1, len(txns)) + + testutil.CheckV2Transaction(t, txn, txns[0]) + } +} + +func (n *testChain) assertChainIndices(t *testing.T, txnID types.TransactionID, expected ...types.ChainIndex) { + t.Helper() + + indices, err := n.db.TransactionChainIndices(txnID, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } else if len(indices) != len(expected) { + t.Fatalf("expected %d indices, got %d", len(expected), len(indices)) + } + + for i := range indices { + testutil.Equal(t, "index", expected[i], indices[i]) + } +} + +func (n *testChain) assertV2ChainIndices(t *testing.T, txnID types.TransactionID, expected ...types.ChainIndex) { + t.Helper() + + indices, err := n.db.V2TransactionChainIndices(txnID, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } else if len(indices) != len(expected) { + t.Fatalf("expected %d indices, got %d", len(expected), len(indices)) + } + + for i := range indices { + testutil.Equal(t, "index", expected[i], indices[i]) + } +} + +// helper to assert the Siacoin element in the db has the right source, index and output +func (n *testChain) assertSCE(t *testing.T, scID types.SiacoinOutputID, index *types.ChainIndex, sco types.SiacoinOutput) { + t.Helper() + + sces, err := n.db.SiacoinElements([]types.SiacoinOutputID{scID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sces)", 1, len(sces)) + + sce := sces[0] + testutil.Equal(t, "sce.Source", explorer.SourceTransaction, sce.Source) + testutil.Equal(t, "sce.SpentIndex", index, sce.SpentIndex) + testutil.Equal(t, "sce.SiacoinElement.SiacoinOutput", sco, sce.SiacoinOutput) +} + +// helper to assert the Siafund element in the db has the right source, index and output +func (n *testChain) assertSFE(t *testing.T, sfID types.SiafundOutputID, index *types.ChainIndex, sfo types.SiafundOutput) { + t.Helper() + + sfes, err := n.db.SiafundElements([]types.SiafundOutputID{sfID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sfes)", 1, len(sfes)) + + sfe := sfes[0] + testutil.Equal(t, "sfe.SpentIndex", index, sfe.SpentIndex) + testutil.Equal(t, "sfe.SiafundElement.SiafundOutput", sfo, sfe.SiafundOutput) +} + +func TestSiacoinOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + scID := n.genesis().Transactions[0].SiacoinOutputID(0) + + // genesis output should be unspent + // so spentIndex = nil + n.assertSCE(t, scID, nil, n.genesis().Transactions[0].SiacoinOutputs[0]) + + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: scID, + UnlockConditions: uc1, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr2, + Value: n.genesis().Transactions[0].SiacoinOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + n.mineTransactions(t, txn1) + + // genesis output should be spent + tip := n.tipState().Index + n.assertSCE(t, scID, &tip, n.genesis().Transactions[0].SiacoinOutputs[0]) + + // the output from txn1 should exist now that the block with txn1 was + // mined + n.assertSCE(t, txn1.SiacoinOutputID(0), nil, txn1.SiacoinOutputs[0]) + + n.revertBlock(t) + + // the genesis output should be unspent now because we reverted the block + // containing txn1 which spent it + n.assertSCE(t, scID, nil, n.genesis().Transactions[0].SiacoinOutputs[0]) + + // the output from txn1 should not exist after txn1 reverted + { + sces, err := n.db.SiacoinElements([]types.SiacoinOutputID{txn1.SiacoinOutputID(0)}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sces)", 0, len(sces)) + } +} + +func TestEphemeralSiacoinOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + scID := n.genesis().Transactions[0].SiacoinOutputID(0) + + // genesis output should be unspent + // so spentIndex = nil + n.assertSCE(t, scID, nil, n.genesis().Transactions[0].SiacoinOutputs[0]) + + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: scID, + UnlockConditions: uc1, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr2, + Value: n.genesis().Transactions[0].SiacoinOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: txn1.SiacoinOutputID(0), + UnlockConditions: uc2, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: types.VoidAddress, + Value: txn1.SiacoinOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk2, &txn2) + + n.mineTransactions(t, txn1, txn2) + + tip := n.tipState().Index + // genesis output should be spent + n.assertSCE(t, scID, &tip, n.genesis().Transactions[0].SiacoinOutputs[0]) + + // now that txn1 and txn2 are mined the outputs from them should exist + n.assertSCE(t, txn1.SiacoinOutputID(0), &tip, txn1.SiacoinOutputs[0]) + n.assertSCE(t, txn2.SiacoinOutputID(0), nil, txn2.SiacoinOutputs[0]) + + n.revertBlock(t) + + // genesis output should be unspent now that we reverted + n.assertSCE(t, scID, nil, n.genesis().Transactions[0].SiacoinOutputs[0]) + + // outputs from txn1 and txn2 should not exist because those transactions + // were reverted + { + sces, err := n.db.SiacoinElements([]types.SiacoinOutputID{txn1.SiacoinOutputID(0), txn2.SiacoinOutputID(0)}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sces)", 0, len(sces)) + } +} + +func TestSiafundOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + sfID := n.genesis().Transactions[0].SiafundOutputID(0) + + // genesis output should be unspent + // so spentIndex = nil + n.assertSFE(t, sfID, nil, n.genesis().Transactions[0].SiafundOutputs[0]) + + txn1 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: sfID, + UnlockConditions: uc1, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: addr2, + Value: n.genesis().Transactions[0].SiafundOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + n.mineTransactions(t, txn1) + + // genesis output should be spent + tip := n.tipState().Index + n.assertSFE(t, sfID, &tip, n.genesis().Transactions[0].SiafundOutputs[0]) + + // the output from txn1 should exist now that the block with txn1 was + // mined + n.assertSFE(t, txn1.SiafundOutputID(0), nil, txn1.SiafundOutputs[0]) + + n.revertBlock(t) + + // the genesis output should be unspent now because we reverted the block + // containing txn1 which spent it + n.assertSFE(t, sfID, nil, n.genesis().Transactions[0].SiafundOutputs[0]) + + // the output from txn1 should not exist after txn1 reverted + { + sfes, err := n.db.SiafundElements([]types.SiafundOutputID{txn1.SiafundOutputID(0)}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sfes)", 0, len(sfes)) + } +} + +func TestEphemeralSiafundOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + sfID := n.genesis().Transactions[0].SiafundOutputID(0) + + // genesis output should be unspent + // so spentIndex = nil + n.assertSFE(t, sfID, nil, n.genesis().Transactions[0].SiafundOutputs[0]) + + txn1 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: sfID, + UnlockConditions: uc1, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: addr2, + Value: n.genesis().Transactions[0].SiafundOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + txn2 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: txn1.SiafundOutputID(0), + UnlockConditions: uc2, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: types.VoidAddress, + Value: txn1.SiafundOutputs[0].Value, + }}, + } + testutil.SignTransaction(n.tipState(), pk2, &txn2) + + n.mineTransactions(t, txn1, txn2) + + tip := n.tipState().Index + // genesis output should be spent + n.assertSFE(t, sfID, &tip, n.genesis().Transactions[0].SiafundOutputs[0]) + + // now that txn1 and txn2 are mined the outputs from them should exist + n.assertSFE(t, txn1.SiafundOutputID(0), &tip, txn1.SiafundOutputs[0]) + n.assertSFE(t, txn2.SiafundOutputID(0), nil, txn2.SiafundOutputs[0]) + + n.revertBlock(t) + + // genesis output should be unspent now that we reverted + n.assertSFE(t, sfID, nil, n.genesis().Transactions[0].SiafundOutputs[0]) + + // outputs from txn1 and txn2 should not exist because those transactions + // were reverted + { + sfes, err := n.db.SiafundElements([]types.SiafundOutputID{txn1.SiafundOutputID(0), txn2.SiafundOutputID(0)}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(sfes)", 0, len(sfes)) + } +} + +func TestTransactionChainIndices(t *testing.T) { + n := newTestChain(t, false, nil) + + txn1 := types.Transaction{ + ArbitraryData: [][]byte{{0}}, + } + txn2 := types.Transaction{ + ArbitraryData: [][]byte{{0}, {1}}, + } + + // mine block with txn1 twice and txn2 + n.mineTransactions(t, txn1, txn1, txn2) + cs1 := n.tipState() + + n.assertTransactions(t, txn1, txn2) + // both transactions should only be in the first block + n.assertChainIndices(t, txn1.ID(), cs1.Index) + n.assertChainIndices(t, txn2.ID(), cs1.Index) + + // mine same block again + n.mineTransactions(t, txn1, txn1, txn2) + cs2 := n.tipState() + + // both transactions should be in the blocks + n.assertTransactions(t, txn1, txn2) + n.assertChainIndices(t, txn1.ID(), cs2.Index, cs1.Index) + n.assertChainIndices(t, txn2.ID(), cs2.Index, cs1.Index) + + n.revertBlock(t) + + // after revert both transactions should only be in the first block + n.assertTransactions(t, txn1, txn2) + n.assertChainIndices(t, txn1.ID(), cs1.Index) + n.assertChainIndices(t, txn2.ID(), cs1.Index) + + n.revertBlock(t) + + // after reverting the first block there should be no transactions + { + txns, err := n.db.Transactions([]types.TransactionID{txn1.ID(), txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 0, len(txns)) + } + n.assertChainIndices(t, txn1.ID()) + n.assertChainIndices(t, txn2.ID()) +} + +func TestV2TransactionChainIndices(t *testing.T) { + n := newTestChain(t, true, nil) + + txn1 := types.V2Transaction{ + ArbitraryData: []byte{0}, + } + txn2 := types.V2Transaction{ + ArbitraryData: []byte{0, 1}, + } + + // mine block with txn1 twice and txn2 + n.mineV2Transactions(t, txn1, txn1, txn2) + cs1 := n.tipState() + + n.assertV2Transactions(t, txn1, txn2) + // both transactions should only be in the first block + n.assertV2ChainIndices(t, txn1.ID(), cs1.Index) + n.assertV2ChainIndices(t, txn2.ID(), cs1.Index) + + // mine same block again + n.mineV2Transactions(t, txn1, txn1, txn2) + cs2 := n.tipState() + + // both transactions should be in the blocks + n.assertV2Transactions(t, txn1, txn2) + n.assertV2ChainIndices(t, txn1.ID(), cs2.Index, cs1.Index) + n.assertV2ChainIndices(t, txn2.ID(), cs2.Index, cs1.Index) + + n.revertBlock(t) + + // after revert both transactions should only be in the first block + n.assertV2Transactions(t, txn1, txn2) + n.assertV2ChainIndices(t, txn1.ID(), cs1.Index) + n.assertV2ChainIndices(t, txn2.ID(), cs1.Index) + + n.revertBlock(t) + + // after reverting the first block there should be no transactions + { + txns, err := n.db.V2Transactions([]types.TransactionID{txn1.ID(), txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 0, len(txns)) + } + n.assertV2ChainIndices(t, txn1.ID()) + n.assertV2ChainIndices(t, txn2.ID()) +} + +func TestSiacoinBalance(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + val := n.genesis().Transactions[0].SiacoinOutputs[0].Value + + checkBalance := func(addr types.Address, expectedSC, expectedImmatureSC types.Currency) { + t.Helper() + + sc, immatureSC, sf, err := n.db.Balance(addr) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "siacoins", expectedSC, sc) + testutil.Equal(t, "immature siacoins", expectedImmatureSC, immatureSC) + testutil.Equal(t, "siafunds", 0, sf) + } + + // only addr1 should have SC from genesis block + checkBalance(types.VoidAddress, types.ZeroCurrency, types.ZeroCurrency) + checkBalance(addr1, val, types.ZeroCurrency) + checkBalance(addr2, types.ZeroCurrency, types.ZeroCurrency) + + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: n.genesis().Transactions[0].SiacoinOutputID(0), + UnlockConditions: uc1, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr2, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + // send addr1 output to addr2 + b := testutil.MineBlock(n.tipState(), []types.Transaction{txn1}, types.VoidAddress) + n.applyBlock(t, b) + + // addr2 should have SC and the void address should have immature SC from + // block + checkBalance(types.VoidAddress, types.ZeroCurrency, b.MinerPayouts[0].Value) + checkBalance(addr1, types.ZeroCurrency, types.ZeroCurrency) + checkBalance(addr2, val, types.ZeroCurrency) + + n.revertBlock(t) + + // after revert, addr1 should have funds again and the void address should + // have nothing + checkBalance(types.VoidAddress, types.ZeroCurrency, types.ZeroCurrency) + checkBalance(addr1, val, types.ZeroCurrency) + checkBalance(addr2, types.ZeroCurrency, types.ZeroCurrency) +} + +func TestSiafundBalance(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + val := n.genesis().Transactions[0].SiafundOutputs[0].Value + + checkBalance := func(addr types.Address, expectedSF uint64) { + t.Helper() + + sc, immatureSC, sf, err := n.db.Balance(addr) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "siacoins", types.ZeroCurrency, sc) + if addr != types.VoidAddress { + testutil.Equal(t, "immature siacoins", types.ZeroCurrency, immatureSC) + } + testutil.Equal(t, "siafunds", expectedSF, sf) + } + + // addr1 should have SF from genesis block + checkBalance(types.VoidAddress, 0) + checkBalance(addr1, val) + checkBalance(addr2, 0) + + txn1 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: n.genesis().Transactions[0].SiafundOutputID(0), + UnlockConditions: uc1, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: addr2, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + // send addr1 SF to addr2 + n.mineTransactions(t, txn1) + + // addr2 should have SF now + checkBalance(types.VoidAddress, 0) + checkBalance(addr1, 0) + checkBalance(addr2, val) + + n.revertBlock(t) + + // after revert, addr1 should have SF again + checkBalance(types.VoidAddress, 0) + checkBalance(addr1, val) + checkBalance(addr2, 0) +} + +func TestEphemeralSiacoinBalance(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + pk3 := types.GeneratePrivateKey() + uc3 := types.StandardUnlockConditions(pk3.PublicKey()) + addr3 := uc3.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + val := n.genesis().Transactions[0].SiacoinOutputs[0].Value + + checkBalance := func(addr types.Address, expectedSC types.Currency) { + t.Helper() + + sc, immatureSC, sf, err := n.db.Balance(addr) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "siacoins", expectedSC, sc) + testutil.Equal(t, "immature siacoins", types.ZeroCurrency, immatureSC) + testutil.Equal(t, "siafunds", 0, sf) + } + + // only addr1 should have SC from genesis block + checkBalance(addr1, val) + checkBalance(addr2, types.ZeroCurrency) + checkBalance(addr3, types.ZeroCurrency) + + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: n.genesis().Transactions[0].SiacoinOutputID(0), + UnlockConditions: uc1, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr2, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: txn1.SiacoinOutputID(0), + UnlockConditions: uc2, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr3, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk2, &txn2) + + // net effect of txn1 and txn2 is to send addr1 output to addr3 + n.mineTransactions(t, txn1, txn2) + + // addr3 should have all the value now + checkBalance(addr1, types.ZeroCurrency) + checkBalance(addr2, types.ZeroCurrency) + checkBalance(addr3, val) + + n.revertBlock(t) + + // after revert, addr1 should have funds again and the others should + // have nothing + checkBalance(addr1, val) + checkBalance(addr2, types.ZeroCurrency) + checkBalance(addr3, types.ZeroCurrency) +} + +func TestEphemeralSiafundBalance(t *testing.T) { + pk1 := types.GeneratePrivateKey() + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + addr1 := uc1.UnlockHash() + + pk2 := types.GeneratePrivateKey() + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + addr2 := uc2.UnlockHash() + + pk3 := types.GeneratePrivateKey() + uc3 := types.StandardUnlockConditions(pk3.PublicKey()) + addr3 := uc3.UnlockHash() + + n := newTestChain(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + val := n.genesis().Transactions[0].SiafundOutputs[0].Value + + checkBalance := func(addr types.Address, expectedSF uint64) { + t.Helper() + + sc, immatureSC, sf, err := n.db.Balance(addr) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "siacoins", types.ZeroCurrency, sc) + testutil.Equal(t, "immature siacoins", types.ZeroCurrency, immatureSC) + testutil.Equal(t, "siafunds", expectedSF, sf) + } + + // only addr1 should have SF from genesis block + checkBalance(addr1, val) + checkBalance(addr2, 0) + checkBalance(addr3, 0) + + txn1 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: n.genesis().Transactions[0].SiafundOutputID(0), + UnlockConditions: uc1, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: addr2, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk1, &txn1) + + txn2 := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: txn1.SiafundOutputID(0), + UnlockConditions: uc2, + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: addr3, + Value: val, + }}, + } + testutil.SignTransaction(n.tipState(), pk2, &txn2) + + // net effect of txn1 and txn2 is to send addr1 output to addr3 + n.mineTransactions(t, txn1, txn2) + + // addr3 should have all the value now + checkBalance(addr1, 0) + checkBalance(addr2, 0) + checkBalance(addr3, val) + + n.revertBlock(t) + + // after revert, addr1 should have funds again and the others should + // have nothing + checkBalance(addr1, val) + checkBalance(addr2, 0) + checkBalance(addr3, 0) +} + +func TestTip(t *testing.T) { + n := newTestChain(t, false, nil) + + checkTips := func() { + t.Helper() + + tip, err := n.db.Tip() + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "tip", n.tipState().Index, tip) + + for _, state := range n.states { + best, err := n.db.BestTip(state.Index.Height) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "best tip", state.Index, best) + } + } + checkTips() + + n.mineTransactions(t) + checkTips() + + n.mineTransactions(t) + checkTips() + + n.revertBlock(t) + checkTips() + + n.revertBlock(t) + checkTips() +} + +func TestMissingTip(t *testing.T) { + n := newTestChain(t, false, nil) + + _, err := n.db.BestTip(n.tipState().Index.Height) + if err != nil { + t.Fatalf("error retrieving tip known to exist: %v", err) + } + + _, err = n.db.BestTip(n.tipState().Index.Height + 1) + if !errors.Is(err, explorer.ErrNoTip) { + t.Fatalf("should have got ErrNoTip retrieving: %v", err) + } +} + +func TestMissingBlock(t *testing.T) { + n := newTestChain(t, false, nil) + + id := n.tipState().Index.ID + _, err := n.db.Block(id) + if err != nil { + t.Fatalf("error retrieving genesis block: %v", err) + } + + id[0] ^= 255 + _, err = n.db.Block(id) + if !errors.Is(err, explorer.ErrNoBlock) { + t.Fatalf("did not get ErrNoBlock retrieving missing block: %v", err) + } +} diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index ca7e5aa..0d2edc6 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -3,119 +3,24 @@ package sqlite_test import ( "errors" "math" - "math/bits" "path/filepath" - "reflect" "testing" + "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" + ctestutil "go.sia.tech/coreutils/testutil" "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" "go.sia.tech/explored/persist/sqlite" "go.uber.org/zap/zaptest" ) -func testV1Network(giftAddr types.Address, sc types.Currency, sf uint64) (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 1000 - n.HardforkV2.RequireHeight = 1000 - genesisBlock.Transactions = []types.Transaction{{}} - if sf > 0 { - genesisBlock.Transactions[0].SiafundOutputs = []types.SiafundOutput{{ - Address: giftAddr, - Value: sf, - }} - } - if sc.Cmp(types.ZeroCurrency) == 1 { - genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{{ - Address: giftAddr, - Value: sc, - }} - } - return n, genesisBlock -} - -func testV2Network() (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 100 - n.HardforkV2.RequireHeight = 110 - return n, genesisBlock -} - -func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { - b := types.Block{ - ParentID: state.Index.ID, - Timestamp: types.CurrentTimestamp(), - Transactions: txns, - MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, - } - for b.ID().CmpWork(state.ChildTarget) < 0 { - b.Nonce += state.NonceFactor() - } - return b -} - -func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { - b := types.Block{ - ParentID: state.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, - - V2: &types.V2BlockData{ - Transactions: txns, - Height: state.Index.Height + 1, - }, - } - b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - for b.ID().CmpWork(state.ChildTarget) < 0 { - b.Nonce += state.NonceFactor() - } - return b -} - -func signTxn(cs consensus.State, pk types.PrivateKey, txn *types.Transaction) { - appendSig := func(key types.PrivateKey, pubkeyIndex uint64, parentID types.Hash256) { - sig := key.SignHash(cs.WholeSigHash(*txn, parentID, pubkeyIndex, 0, nil)) - txn.Signatures = append(txn.Signatures, types.TransactionSignature{ - ParentID: parentID, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - PublicKeyIndex: pubkeyIndex, - Signature: sig[:], - }) - } - for i := range txn.SiacoinInputs { - appendSig(pk, 0, types.Hash256(txn.SiacoinInputs[i].ParentID)) - } - for i := range txn.SiafundInputs { - appendSig(pk, 0, types.Hash256(txn.SiafundInputs[i].ParentID)) - } -} - -func check(t *testing.T, desc string, expect, got any) { - if !reflect.DeepEqual(expect, got) { - t.Fatalf("expected %v %s, got %v", expect, desc, got) - } -} +func syncDB(t *testing.T, db explorer.Store, cm *chain.Manager) { + t.Helper() -func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { index, err := db.Tip() if err != nil && !errors.Is(err, explorer.ErrNoTip) { t.Fatal(err) @@ -139,40 +44,129 @@ func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { } } -func TestBalance(t *testing.T) { +func newStore(t *testing.T, v2 bool, f func(*consensus.Network, types.Block)) (*consensus.Network, types.Block, *chain.Manager, explorer.Store) { log := zaptest.NewLogger(t) dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } - defer db.Close() bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) if err != nil { t.Fatal(err) } - defer bdb.Close() - network, genesisBlock := testV1Network(types.VoidAddress, types.ZeroCurrency, 0) + var network *consensus.Network + var genesisBlock types.Block + if v2 { + network, genesisBlock = ctestutil.V2Network() + } else { + network, genesisBlock = ctestutil.Network() + } + if f != nil { + f(network, genesisBlock) + } - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { t.Fatal(err) } - cm := chain.NewManager(store, genesisState) + syncDB(t, db, cm) - // checkBalance checks that an address has the balances we expect - checkBalance := func(addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { - sc, immatureSC, sf, err := db.Balance(addr) - if err != nil { - t.Fatal(err) + t.Cleanup(func() { + db.Close() + bdb.Close() + }) + return network, genesisBlock, cm, db +} + +// CheckMetrics checks the that the metrics from the DB match what we expect. +func CheckMetrics(t *testing.T, db explorer.Store, cm *chain.Manager, expected explorer.Metrics) { + t.Helper() + + tip, err := db.Tip() + if err != nil { + t.Fatal(err) + } + got, err := db.Metrics(tip.ID) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "index", cm.Tip(), got.Index) + testutil.Equal(t, "difficulty", cm.TipState().Difficulty, got.Difficulty) + testutil.Equal(t, "total hosts", expected.TotalHosts, got.TotalHosts) + testutil.Equal(t, "active contracts", expected.ActiveContracts, got.ActiveContracts) + testutil.Equal(t, "failed contracts", expected.FailedContracts, got.FailedContracts) + testutil.Equal(t, "successful contracts", expected.SuccessfulContracts, got.SuccessfulContracts) + testutil.Equal(t, "contract revenue", expected.ContractRevenue, got.ContractRevenue) + testutil.Equal(t, "storage utilization", expected.StorageUtilization, got.StorageUtilization) + // don't check circulating supply here because it requires a lot of accounting +} + +// CheckChainIndices checks that the chain indices that a transaction was in +// from the explorer match the expected chain indices. +func CheckChainIndices(t *testing.T, db explorer.Store, txnID types.TransactionID, expected []types.ChainIndex) { + t.Helper() + + indices, err := db.TransactionChainIndices(txnID, 0, 100) + switch { + case err != nil: + t.Fatal(err) + case len(indices) != len(expected): + t.Fatalf("expected %d indices, got %d", len(expected), len(indices)) + } + for i := range indices { + testutil.Equal(t, "index", expected[i], indices[i]) + } +} + +// CheckFCRevisions checks that the revision numbers for the file contracts match. +func CheckFCRevisions(t *testing.T, confirmationIndex types.ChainIndex, confirmationTransactionID types.TransactionID, valid, missed []types.SiacoinOutput, revisionNumbers []uint64, fcs []explorer.ExtendedFileContract) { + t.Helper() + + testutil.Equal(t, "number of revisions", len(revisionNumbers), len(fcs)) + for i := range revisionNumbers { + testutil.Equal(t, "revision number", revisionNumbers[i], fcs[i].RevisionNumber) + testutil.Equal(t, "confirmation index", confirmationIndex, fcs[i].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", confirmationTransactionID, fcs[i].ConfirmationTransactionID) + + testutil.Equal(t, "valid proof outputs", len(valid), len(fcs[i].ValidProofOutputs)) + for j := range valid { + expected := valid[j] + got := fcs[i].ValidProofOutputs[j] + + testutil.Equal(t, "id", fcs[i].ID.ValidOutputID(j), got.ID) + testutil.Equal(t, "value", expected.Value, got.Value) + testutil.Equal(t, "address", expected.Address, got.Address) + } + + testutil.Equal(t, "missed proof outputs", len(missed), len(fcs[i].MissedProofOutputs)) + for j := range missed { + expected := missed[j] + got := fcs[i].MissedProofOutputs[j] + + testutil.Equal(t, "id", fcs[i].ID.MissedOutputID(j), got.ID) + testutil.Equal(t, "value", expected.Value, got.Value) + testutil.Equal(t, "address", expected.Address, got.Address) } - check(t, "siacoins", expectSC, sc) - check(t, "immature siacoins", expectImmatureSC, immatureSC) - check(t, "siafunds", expectSF, sf) } +} + +func checkTransaction(t *testing.T, db explorer.Store, expected types.Transaction) { + txns, err := db.Transactions([]types.TransactionID{expected.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 1, len(txns)) + testutil.CheckTransaction(t, expected, txns[0]) +} + +func TestBalance(t *testing.T) { + _, _, cm, db := newStore(t, false, nil) // Generate three addresses: addr1, addr2, addr3 pk1 := types.GeneratePrivateKey() @@ -188,37 +182,49 @@ func TestBalance(t *testing.T) { maturityHeight := cm.TipState().MaturityHeight() // Mine a block sending the payout to addr1 - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr1)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, addr1)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) // Check that addr1 has the miner payout output - utxos, err := db.UnspentSiacoinOutputs(addr1, 100, 0) + utxos, err := db.UnspentSiacoinOutputs(addr1, 0, 100) if err != nil { t.Fatal(err) } - check(t, "utxos", 1, len(utxos)) - check(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxos[0].Source) + testutil.Equal(t, "utxos", 1, len(utxos)) + testutil.Equal(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxos[0].Source) + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 1, len(events)) + + ev0 := events[0].Data.(explorer.EventPayout) + testutil.Equal(t, "event 0 output ID", cm.Tip().ID.MinerOutputID(0), ev0.SiacoinElement.ID) + testutil.Equal(t, "event 0 output source", explorer.SourceMinerPayout, ev0.SiacoinElement.Source) + } // Mine until the payout matures for i := cm.Tip().Height; i < maturityHeight; i++ { - checkBalance(addr1, types.ZeroCurrency, expectedPayout, 0) - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, expectedPayout, 0) + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) } - checkBalance(addr1, expectedPayout, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr1, expectedPayout, types.ZeroCurrency, 0) // Send all of the payout except 100 SC to addr2 unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) parentTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ { - ParentID: types.SiacoinOutputID(utxos[0].ID), + ParentID: utxos[0].ID, UnlockConditions: unlockConditions, }, }, @@ -227,7 +233,7 @@ func TestBalance(t *testing.T) { {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100))}, }, } - signTxn(cm.TipState(), pk1, &parentTxn) + testutil.SignTransaction(cm.TipState(), pk1, &parentTxn) // In the same block, have addr1 send the 100 SC it still has left to // addr3 @@ -243,120 +249,18 @@ func TestBalance(t *testing.T) { {Address: addr3, Value: types.Siacoins(100)}, }, } - signTxn(cm.TipState(), pk1, &txn) - - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{parentTxn, txn}, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - checkBalance(addr2, utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100)), types.ZeroCurrency, 0) - checkBalance(addr3, types.Siacoins(100), types.ZeroCurrency, 0) -} - -func TestSiafundBalance(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // Generate three addresses: addr1, addr2, addr3 - pk1 := types.GeneratePrivateKey() - addr1 := types.StandardUnlockHash(pk1.PublicKey()) - - pk2 := types.GeneratePrivateKey() - addr2 := types.StandardUnlockHash(pk2.PublicKey()) - - pk3 := types.GeneratePrivateKey() - addr3 := types.StandardUnlockHash(pk3.PublicKey()) - - const giftSF = 10000 - network, genesisBlock := testV1Network(addr1, types.ZeroCurrency, giftSF) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - // checkBalance checks that an address has the balances we expect - checkBalance := func(addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { - sc, immatureSC, sf, err := db.Balance(addr) - if err != nil { - t.Fatal(err) - } - check(t, "siacoins", expectSC, sc) - check(t, "immature siacoins", expectImmatureSC, immatureSC) - check(t, "siafunds", expectSF, sf) - } - - // Send all of the payout except 100 SF to addr2 - unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) - parentTxn := types.Transaction{ - SiafundInputs: []types.SiafundInput{ - { - ParentID: types.SiafundOutputID(genesisBlock.Transactions[0].SiafundOutputID(0)), - UnlockConditions: unlockConditions, - }, - }, - SiafundOutputs: []types.SiafundOutput{ - {Address: addr1, Value: 100}, - {Address: addr2, Value: genesisBlock.Transactions[0].SiafundOutputs[0].Value - 100}, - }, - } - signTxn(cm.TipState(), pk1, &parentTxn) - - // In the same block, have addr1 send the 100 SF it still has left to - // addr3 - outputID := parentTxn.SiafundOutputID(0) - txn := types.Transaction{ - SiafundInputs: []types.SiafundInput{ - { - ParentID: outputID, - UnlockConditions: unlockConditions, - }, - }, - SiafundOutputs: []types.SiafundOutput{ - {Address: addr3, Value: 100}, - }, - } - signTxn(cm.TipState(), pk1, &txn) + testutil.SignTransaction(cm.TipState(), pk1, &txn) - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{parentTxn, txn}, types.VoidAddress)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{parentTxn, txn}, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) - checkBalance(addr1, types.ZeroCurrency, types.ZeroCurrency, 0) - checkBalance(addr2, types.ZeroCurrency, types.ZeroCurrency, giftSF-100) - checkBalance(addr3, types.ZeroCurrency, types.ZeroCurrency, 100) + testutil.CheckBalance(t, db, addr2, utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100)), types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, types.Siacoins(100), types.ZeroCurrency, 0) } func TestSendTransactions(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - // Generate three addresses: addr1, addr2, addr3 pk1 := types.GeneratePrivateKey() addr1 := types.StandardUnlockHash(pk1.PublicKey()) @@ -367,96 +271,42 @@ func TestSendTransactions(t *testing.T) { pk3 := types.GeneratePrivateKey() addr3 := types.StandardUnlockHash(pk3.PublicKey()) - const giftSF = 10000 - network, genesisBlock := testV1Network(addr1, types.ZeroCurrency, giftSF) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - // checkBalance checks that an address has the balances we expect - checkBalance := func(addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { - sc, immatureSC, sf, err := db.Balance(addr) - if err != nil { - t.Fatal(err) - } - check(t, "siacoins", expectSC, sc) - check(t, "immature siacoins", expectImmatureSC, immatureSC) - check(t, "siafunds", expectSF, sf) - } - - checkTransaction := func(expectTxn types.Transaction, gotTxn explorer.Transaction) { - check(t, "siacoin inputs", len(expectTxn.SiacoinInputs), len(gotTxn.SiacoinInputs)) - check(t, "siacoin outputs", len(expectTxn.SiacoinOutputs), len(gotTxn.SiacoinOutputs)) - check(t, "siafund inputs", len(expectTxn.SiafundInputs), len(gotTxn.SiafundInputs)) - check(t, "siafund outputs", len(expectTxn.SiafundOutputs), len(gotTxn.SiafundOutputs)) - - for i := range expectTxn.SiacoinInputs { - expectSci := expectTxn.SiacoinInputs[i] - gotSci := gotTxn.SiacoinInputs[i] - - check(t, "parent ID", expectSci.ParentID, gotSci.ParentID) - check(t, "unlock conditions", expectSci.UnlockConditions, gotSci.UnlockConditions) - } - for i := range expectTxn.SiacoinOutputs { - expectSco := expectTxn.SiacoinOutputs[i] - gotSco := gotTxn.SiacoinOutputs[i].SiacoinOutput - - check(t, "address", expectSco.Address, gotSco.Address) - check(t, "value", expectSco.Value, gotSco.Value) - check(t, "source", explorer.SourceTransaction, gotTxn.SiacoinOutputs[i].Source) - } - for i := range expectTxn.SiafundInputs { - expectSfi := expectTxn.SiafundInputs[i] - gotSfi := gotTxn.SiafundInputs[i] - - check(t, "parent ID", expectSfi.ParentID, gotSfi.ParentID) - check(t, "claim address", expectSfi.ClaimAddress, gotSfi.ClaimAddress) - check(t, "unlock conditions", expectSfi.UnlockConditions, gotSfi.UnlockConditions) - } - for i := range expectTxn.SiafundOutputs { - expectSfo := expectTxn.SiafundOutputs[i] - gotSfo := gotTxn.SiafundOutputs[i].SiafundOutput - - check(t, "address", expectSfo.Address, gotSfo.Address) - check(t, "value", expectSfo.Value, gotSfo.Value) - } - } + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + giftSF := genesisBlock.Transactions[0].SiafundOutputs[0].Value expectedPayout := cm.TipState().BlockReward() maturityHeight := cm.TipState().MaturityHeight() // Mine a block sending the payout to the addr1 - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr1)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, addr1)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) // Mine until the payout matures for i := cm.Tip().Height; i < maturityHeight; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) } - checkBalance(addr1, expectedPayout, types.ZeroCurrency, giftSF) - checkBalance(addr2, types.ZeroCurrency, types.ZeroCurrency, 0) - checkBalance(addr3, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr1, expectedPayout, types.ZeroCurrency, giftSF) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, types.ZeroCurrency, types.ZeroCurrency, 0) const n = 100 // Check that addr1 has the miner payout output - utxos, err := db.UnspentSiacoinOutputs(addr1, n, 0) + utxos, err := db.UnspentSiacoinOutputs(addr1, 0, n) if err != nil { t.Fatal(err) } - check(t, "utxos", 1, len(utxos)) - check(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxos[0].Source) + testutil.Equal(t, "utxos", 1, len(utxos)) + testutil.Equal(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxos[0].Source) sfOutputID := genesisBlock.Transactions[0].SiafundOutputID(0) scOutputID := utxos[0].ID @@ -469,7 +319,7 @@ func TestSendTransactions(t *testing.T) { parentTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ { - ParentID: types.SiacoinOutputID(scOutputID), + ParentID: scOutputID, UnlockConditions: unlockConditions, }, }, @@ -491,20 +341,22 @@ func TestSendTransactions(t *testing.T) { }, } - signTxn(cm.TipState(), pk1, &parentTxn) - scOutputID = types.Hash256(parentTxn.SiacoinOutputID(2)) + testutil.SignTransaction(cm.TipState(), pk1, &parentTxn) + scOutputID = parentTxn.SiacoinOutputID(2) sfOutputID = parentTxn.SiafundOutputID(2) // Mine a block with the above transaction - b := mineBlock(cm.TipState(), []types.Transaction{parentTxn}, types.VoidAddress) + b := testutil.MineBlock(cm.TipState(), []types.Transaction{parentTxn}, types.VoidAddress) if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } syncDB(t, db, cm) - checkBalance(addr1, addr1SCs, types.ZeroCurrency, addr1SFs) - checkBalance(addr2, types.Siacoins(1).Mul64(uint64(i+1)), types.ZeroCurrency, 1*uint64(i+1)) - checkBalance(addr3, types.Siacoins(2).Mul64(uint64(i+1)), types.ZeroCurrency, 2*uint64(i+1)) + CheckMetrics(t, db, cm, explorer.Metrics{}) + + testutil.CheckBalance(t, db, addr1, addr1SCs, types.ZeroCurrency, addr1SFs) + testutil.CheckBalance(t, db, addr2, types.Siacoins(1).Mul64(uint64(i+1)), types.ZeroCurrency, 1*uint64(i+1)) + testutil.CheckBalance(t, db, addr3, types.Siacoins(2).Mul64(uint64(i+1)), types.ZeroCurrency, 2*uint64(i+1)) // Ensure the block we retrieved from the database is the same as the // actual block @@ -512,28 +364,24 @@ func TestSendTransactions(t *testing.T) { if err != nil { t.Fatal(err) } - check(t, "transactions", len(b.Transactions), len(block.Transactions)) - check(t, "miner payouts", len(b.MinerPayouts), len(block.MinerPayouts)) - check(t, "nonce", b.Nonce, block.Nonce) - check(t, "timestamp", b.Timestamp, block.Timestamp) + testutil.Equal(t, "transactions", len(b.Transactions), len(block.Transactions)) + testutil.Equal(t, "miner payouts", len(b.MinerPayouts), len(block.MinerPayouts)) + testutil.Equal(t, "nonce", b.Nonce, block.Nonce) + testutil.Equal(t, "timestamp", b.Timestamp, block.Timestamp) // Ensure the miner payouts in the block match for i := range b.MinerPayouts { - check(t, "address", b.MinerPayouts[i].Address, b.MinerPayouts[i].Address) - check(t, "value", b.MinerPayouts[i].Value, b.MinerPayouts[i].Value) + testutil.Equal(t, "address", b.MinerPayouts[i].Address, b.MinerPayouts[i].Address) + testutil.Equal(t, "value", b.MinerPayouts[i].Value, b.MinerPayouts[i].Value) } // Ensure the transactions in the block and retrieved separately match // with the actual transactions for i := range b.Transactions { - checkTransaction(b.Transactions[i], block.Transactions[i]) + testutil.CheckTransaction(t, b.Transactions[i], block.Transactions[i]) + CheckChainIndices(t, db, b.Transactions[i].ID(), []types.ChainIndex{cm.Tip()}) - txns, err := db.Transactions([]types.TransactionID{b.Transactions[i].ID()}) - if err != nil { - t.Fatal(err) - } - check(t, "transactions", 1, len(txns)) - checkTransaction(b.Transactions[i], txns[0]) + checkTransaction(t, db, b.Transactions[i]) } type expectedUTXOs struct { @@ -551,26 +399,26 @@ func TestSendTransactions(t *testing.T) { {addr3, i + 1, types.Siacoins(2), i + 1, 2}, } for _, e := range expected { - sc, err := db.UnspentSiacoinOutputs(e.addr, n, 0) + sc, err := db.UnspentSiacoinOutputs(e.addr, 0, n) if err != nil { t.Fatal(err) } - sf, err := db.UnspentSiafundOutputs(e.addr, n, 0) + sf, err := db.UnspentSiafundOutputs(e.addr, 0, n) if err != nil { t.Fatal(err) } - check(t, "sc utxos", e.sc, len(sc)) - check(t, "sf utxos", e.sf, len(sf)) + testutil.Equal(t, "sc utxos", e.sc, len(sc)) + testutil.Equal(t, "sf utxos", e.sf, len(sf)) for _, sco := range sc { - check(t, "address", e.addr, sco.SiacoinOutput.Address) - check(t, "value", e.scValue, sco.SiacoinOutput.Value) - check(t, "source", explorer.SourceTransaction, sco.Source) + testutil.Equal(t, "address", e.addr, sco.SiacoinOutput.Address) + testutil.Equal(t, "value", e.scValue, sco.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceTransaction, sco.Source) } for _, sfo := range sf { - check(t, "address", e.addr, sfo.SiafundOutput.Address) - check(t, "value", e.sfValue, sfo.SiafundOutput.Value) + testutil.Equal(t, "address", e.addr, sfo.SiafundOutput.Address) + testutil.Equal(t, "value", e.sfValue, sfo.SiafundOutput.Value) } } @@ -599,190 +447,299 @@ func TestSendTransactions(t *testing.T) { } } -func TestTip(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() +func TestFileContract(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) - network, genesisBlock := testV1Network(types.VoidAddress, types.ZeroCurrency, 0) + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + scOutputID := genesisBlock.Transactions[0].SiacoinOutputID(0) + unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) + + windowStart := cm.Tip().Height + 10 + windowEnd := windowStart + 10 + fc := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), windowStart, windowEnd, addr2) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: scOutputID, + UnlockConditions: unlockConditions, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: giftSC.Sub(fc.Payout), + }}, + FileContracts: []types.FileContract{fc}, + } + fcID := txn.FileContractID(0) + testutil.SignTransaction(cm.TipState(), pk1, &txn) + + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) - cm := chain.NewManager(store, genesisState) + confirmationIndex := cm.Tip() + confirmationTransactionID := txn.ID() - const n = 100 - for i := cm.Tip().Height; i < n; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { t.Fatal(err) } - syncDB(t, db, cm) + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + testutil.Equal(t, "transaction ID", txn.ID(), dbFCs[0].TransactionID) + testutil.Equal(t, "confirmation index", cm.Tip(), dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } - tip, err := db.Tip() + { + dbFCs, err := db.ContractRevisions(fcID) if err != nil { t.Fatal(err) } - check(t, "tip", cm.Tip(), tip) + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0}, dbFCs) } - for i := 0; i < n; i++ { - best, err := db.BestTip(uint64(i)) + { + txns, err := db.Transactions([]types.TransactionID{txn.ID()}) if err != nil { t.Fatal(err) } - if cmBest, ok := cm.BestIndex(uint64(i)); !ok || cmBest != best { - t.Fatal("best tip mismatch") - } - } -} + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContracts)) + testutil.CheckFC(t, false, false, false, fc, txns[0].FileContracts[0]) -// copied from rhp/v2 to avoid import cycle -func prepareContractFormation(renterPubKey types.PublicKey, hostKey types.PublicKey, renterPayout, hostCollateral types.Currency, endHeight uint64, windowSize uint64, refundAddr types.Address) types.FileContract { - taxAdjustedPayout := func(target types.Currency) types.Currency { - guess := target.Mul64(1000).Div64(961) - mod64 := func(c types.Currency, v uint64) types.Currency { - var r uint64 - if c.Hi < v { - _, r = bits.Div64(c.Hi, c.Lo, v) - } else { - _, r = bits.Div64(0, c.Hi, v) - _, r = bits.Div64(r, c.Lo, v) - } - return types.NewCurrency64(r) - } - sfc := (consensus.State{}).SiafundCount() - tm := mod64(target, sfc) - gm := mod64(guess, sfc) - if gm.Cmp(tm) < 0 { - guess = guess.Sub(types.NewCurrency64(sfc)) - } - return guess.Add(tm).Sub(gm) + testutil.Equal(t, "transaction ID", txn.ID(), txns[0].FileContracts[0].TransactionID) + testutil.Equal(t, "confirmation index", cm.Tip(), txns[0].FileContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), txns[0].FileContracts[0].ConfirmationTransactionID) } + uc := types.UnlockConditions{ PublicKeys: []types.UnlockKey{ - renterPubKey.UnlockKey(), - hostKey.UnlockKey(), + renterPublicKey.UnlockKey(), + hostPublicKey.UnlockKey(), }, SignaturesRequired: 2, } - hostPayout := hostCollateral - payout := taxAdjustedPayout(renterPayout.Add(hostPayout)) - return types.FileContract{ - Filesize: 0, - FileMerkleRoot: types.Hash256{}, - WindowStart: endHeight, - WindowEnd: endHeight + windowSize, - Payout: payout, - UnlockHash: types.Hash256(uc.UnlockHash()), - RevisionNumber: 0, - ValidProofOutputs: []types.SiacoinOutput{ - {Value: renterPayout, Address: refundAddr}, - {Value: hostPayout, Address: types.VoidAddress}, - }, - MissedProofOutputs: []types.SiacoinOutput{ - {Value: renterPayout, Address: refundAddr}, - {Value: hostPayout, Address: types.VoidAddress}, - {Value: types.ZeroCurrency, Address: types.VoidAddress}, - }, + fc.RevisionNumber++ + reviseTxn := types.Transaction{ + FileContractRevisions: []types.FileContractRevision{{ + ParentID: fcID, + UnlockConditions: uc, + FileContract: fc, + }}, } -} - -func TestFileContract(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn) - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { + prevTip := cm.Tip() + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{reviseTxn}, types.VoidAddress)}); err != nil { t.Fatal(err) } - defer db.Close() + syncDB(t, db, cm) - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) + { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + hostContracts, err := db.ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.CheckFC(t, false, false, false, fc, renterContracts[0]) + testutil.CheckFC(t, false, false, false, fc, hostContracts[0]) + + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) + testutil.Equal(t, "confirmation index", prevTip, renterContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), renterContracts[0].ConfirmationTransactionID) + testutil.Equal(t, "confirmation index", prevTip, hostContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), hostContracts[0].ConfirmationTransactionID) } - defer bdb.Close() - pk1 := types.GeneratePrivateKey() - addr1 := types.StandardUnlockHash(pk1.PublicKey()) + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize, + }) - renterPrivateKey := types.GeneratePrivateKey() - renterPublicKey := renterPrivateKey.PublicKey() + // Explorer.Contracts should return latest revision + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + } - hostPrivateKey := types.GeneratePrivateKey() - hostPublicKey := hostPrivateKey.PublicKey() + { + dbFCs, err := db.ContractRevisions(fcID) + if err != nil { + t.Fatal(err) + } + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0, 1}, dbFCs) + } - giftSC := types.Siacoins(1000) - network, genesisBlock := testV1Network(addr1, giftSC, 0) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) + { + txns, err := db.Transactions([]types.TransactionID{reviseTxn.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContractRevisions)) + + fcr := txns[0].FileContractRevisions[0] + testutil.Equal(t, "parent id", txn.FileContractID(0), fcr.ParentID) + testutil.Equal(t, "unlock conditions", uc, fcr.UnlockConditions) + + testutil.Equal(t, "confirmation index", prevTip, fcr.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), fcr.ConfirmationTransactionID) + + testutil.CheckFC(t, false, false, false, fc, fcr.ExtendedFileContract) } - cm := chain.NewManager(store, genesisState) + for i := cm.Tip().Height; i < windowEnd; i++ { + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: 1 * testutil.ContractFilesize, + }) - scOutputID := genesisBlock.Transactions[0].SiacoinOutputID(0) - unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } - signTxn := func(txn *types.Transaction) { - appendSig := func(key types.PrivateKey, pubkeyIndex uint64, parentID types.Hash256) { - sig := key.SignHash(cm.TipState().WholeSigHash(*txn, parentID, pubkeyIndex, 0, nil)) - txn.Signatures = append(txn.Signatures, types.TransactionSignature{ - ParentID: parentID, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - PublicKeyIndex: pubkeyIndex, - Signature: sig[:], - }) + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + FailedContracts: 1, + SuccessfulContracts: 0, + StorageUtilization: 0, + }) + + { + events, err := db.AddressEvents(addr2, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) } - for i := range txn.SiacoinInputs { - appendSig(pk1, 0, types.Hash256(txn.SiacoinInputs[i].ParentID)) + testutil.Equal(t, "events", 3, len(events)) + + ev0 := events[0].Data.(explorer.EventV1ContractResolution) + testutil.Equal(t, "event 0 parent ID", fcID, ev0.Parent.ID) + testutil.Equal(t, "event 0 output ID", fcID.MissedOutputID(0), ev0.SiacoinElement.ID) + testutil.Equal(t, "event 0 output source", explorer.SourceMissedProofOutput, ev0.SiacoinElement.Source) + testutil.Equal(t, "event 0 missed", true, ev0.Missed) + + ev1 := events[1].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, reviseTxn, ev1.Transaction) + + ev2 := events[2].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, txn, ev2.Transaction) + } + + { + events, err := db.Events([]types.Hash256{types.Hash256(reviseTxn.ID()), types.Hash256(txn.ID())}) + if err != nil { + t.Fatal(err) } - for i := range txn.SiafundInputs { - appendSig(pk1, 0, types.Hash256(txn.SiafundInputs[i].ParentID)) + testutil.Equal(t, "events", 2, len(events)) + + ev0 := events[0].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, reviseTxn, ev0.Transaction) + + ev1 := events[1].Data.(explorer.EventV1Transaction) + testutil.CheckTransaction(t, txn, ev1.Transaction) + } + + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) } - for i := range txn.FileContractRevisions { - appendSig(renterPrivateKey, 0, types.Hash256(txn.FileContractRevisions[i].ParentID)) - appendSig(hostPrivateKey, 1, types.Hash256(txn.FileContractRevisions[i].ParentID)) + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, true, false, fc, dbFCs[0]) + + testutil.Equal(t, "confirmation index", prevTip, dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } + + for i := 0; i < 100; i++ { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) } + syncDB(t, db, cm) } - checkFC := func(resolved, valid bool, expected types.FileContract, got explorer.FileContract) { - check(t, "resolved state", resolved, got.Resolved) - check(t, "valid state", valid, got.Valid) - check(t, "filesize", expected.Filesize, got.Filesize) - check(t, "file merkle root", expected.FileMerkleRoot, got.FileMerkleRoot) - check(t, "window start", expected.WindowStart, got.WindowStart) - check(t, "window end", expected.WindowEnd, got.WindowEnd) - check(t, "payout", expected.Payout, got.Payout) - check(t, "unlock hash", expected.UnlockHash, got.UnlockHash) - check(t, "revision number", expected.RevisionNumber, got.RevisionNumber) - check(t, "valid proof outputs", len(expected.ValidProofOutputs), len(got.ValidProofOutputs)) - for i := range expected.ValidProofOutputs { - check(t, "valid proof output address", expected.ValidProofOutputs[i].Address, got.ValidProofOutputs[i].Address) - check(t, "valid proof output value", expected.ValidProofOutputs[i].Value, got.ValidProofOutputs[i].Value) + { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) } - check(t, "missed proof outputs", len(expected.MissedProofOutputs), len(got.MissedProofOutputs)) - for i := range expected.MissedProofOutputs { - check(t, "missed proof output address", expected.MissedProofOutputs[i].Address, got.MissedProofOutputs[i].Address) - check(t, "missed proof output value", expected.MissedProofOutputs[i].Value, got.MissedProofOutputs[i].Value) + hostContracts, err := db.ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) } - } + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.CheckFC(t, false, true, false, fc, renterContracts[0]) + testutil.CheckFC(t, false, true, false, fc, hostContracts[0]) + + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) + testutil.Equal(t, "confirmation index", prevTip, renterContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), renterContracts[0].ConfirmationTransactionID) + testutil.Equal(t, "confirmation index", prevTip, hostContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), hostContracts[0].ConfirmationTransactionID) + } + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + FailedContracts: 1, + SuccessfulContracts: 0, + StorageUtilization: 0, + }) +} + +func TestEphemeralFileContract(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + scOutputID := genesisBlock.Transactions[0].SiacoinOutputID(0) + unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) windowStart := cm.Tip().Height + 10 windowEnd := windowStart + 10 - fc := prepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), windowStart, windowEnd, types.VoidAddress) + fc := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), windowStart, windowEnd, types.VoidAddress) txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: scOutputID, @@ -795,31 +752,7 @@ func TestFileContract(t *testing.T) { FileContracts: []types.FileContract{fc}, } fcID := txn.FileContractID(0) - signTxn(&txn) - - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - { - dbFCs, err := db.Contracts([]types.FileContractID{fcID}) - if err != nil { - t.Fatal(err) - } - check(t, "fcs", 1, len(dbFCs)) - checkFC(false, true, fc, dbFCs[0]) - } - - { - txns, err := db.Transactions([]types.TransactionID{txn.ID()}) - if err != nil { - t.Fatal(err) - } - check(t, "transactions", 1, len(txns)) - check(t, "file contracts", 1, len(txns[0].FileContracts)) - checkFC(false, true, fc, txns[0].FileContracts[0]) - } + testutil.SignTransaction(cm.TipState(), pk1, &txn) uc := types.UnlockConditions{ PublicKeys: []types.UnlockKey{ @@ -828,174 +761,193 @@ func TestFileContract(t *testing.T) { }, SignaturesRequired: 2, } - fc.RevisionNumber++ - reviseTxn := types.Transaction{ + revisedFC1 := fc + revisedFC1.RevisionNumber++ + reviseTxn1 := types.Transaction{ FileContractRevisions: []types.FileContractRevision{{ ParentID: fcID, UnlockConditions: uc, - FileContract: fc, + FileContract: revisedFC1, }}, } - signTxn(&reviseTxn) + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn1) - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{reviseTxn}, types.VoidAddress)}); err != nil { + // Create a contract and revise it in the same block + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn, reviseTxn1}, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) + confirmationIndex := cm.Tip() + confirmationTransactionID := txn.ID() + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize, + }) + + { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + hostContracts, err := db.ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.CheckFC(t, true, false, false, revisedFC1, renterContracts[0]) + testutil.CheckFC(t, true, false, false, revisedFC1, hostContracts[0]) + } + // Explorer.Contracts should return latest revision { dbFCs, err := db.Contracts([]types.FileContractID{fcID}) if err != nil { t.Fatal(err) } - check(t, "fcs", 1, len(dbFCs)) - checkFC(false, true, fc, dbFCs[0]) + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, true, false, false, revisedFC1, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn1.ID(), dbFCs[0].TransactionID) } { - txns, err := db.Transactions([]types.TransactionID{reviseTxn.ID()}) + dbFCs, err := db.ContractRevisions(fcID) if err != nil { t.Fatal(err) } - check(t, "transactions", 1, len(txns)) - check(t, "file contracts", 1, len(txns[0].FileContractRevisions)) - - fcr := txns[0].FileContractRevisions[0] - check(t, "parent id", txn.FileContractID(0), fcr.ParentID) - check(t, "unlock conditions", uc, fcr.UnlockConditions) - - checkFC(false, true, fc, fcr.FileContract) + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0, 1}, dbFCs) } - for i := cm.Tip().Height; i < windowEnd+10; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + { + txns, err := db.Transactions([]types.TransactionID{txn.ID()}) + if err != nil { t.Fatal(err) } - syncDB(t, db, cm) + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContracts)) + testutil.CheckFC(t, true, false, false, fc, txns[0].FileContracts[0]) + + testutil.Equal(t, "transaction ID", txn.ID(), txns[0].FileContracts[0].TransactionID) + testutil.Equal(t, "confirmation index", cm.Tip(), txns[0].FileContracts[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), txns[0].FileContracts[0].ConfirmationTransactionID) } { - dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + txns, err := db.Transactions([]types.TransactionID{reviseTxn1.ID()}) if err != nil { t.Fatal(err) } - check(t, "fcs", 1, len(dbFCs)) - checkFC(true, false, fc, dbFCs[0]) - } -} + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContractRevisions)) -func TestRevertTip(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) + fcr := txns[0].FileContractRevisions[0] + testutil.Equal(t, "parent id", txn.FileContractID(0), fcr.ParentID) + testutil.Equal(t, "unlock conditions", uc, fcr.UnlockConditions) + + testutil.CheckFC(t, true, false, false, revisedFC1, fcr.ExtendedFileContract) } - defer db.Close() - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) + revisedFC2 := revisedFC1 + revisedFC2.RevisionNumber++ + reviseTxn2 := types.Transaction{ + FileContractRevisions: []types.FileContractRevision{{ + ParentID: fcID, + UnlockConditions: uc, + FileContract: revisedFC2, + }}, } - defer bdb.Close() + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn2) - network, genesisBlock := testV1Network(types.VoidAddress, types.ZeroCurrency, 0) + revisedFC3 := revisedFC2 + revisedFC3.RevisionNumber++ + reviseTxn3 := types.Transaction{ + FileContractRevisions: []types.FileContractRevision{{ + ParentID: fcID, + UnlockConditions: uc, + FileContract: revisedFC3, + }}, + } + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn3) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { + // Two more revisions of the same contract in the next block + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{reviseTxn2, reviseTxn3}, types.VoidAddress)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) - cm := chain.NewManager(store, genesisState) - - pk1 := types.GeneratePrivateKey() - addr1 := types.StandardUnlockHash(pk1.PublicKey()) - - pk2 := types.GeneratePrivateKey() - addr2 := types.StandardUnlockHash(pk2.PublicKey()) + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize, + }) - const n = 100 - for i := cm.Tip().Height; i < n; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr1)}); err != nil { + // Explorer.Contracts should return latest revision + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { t.Fatal(err) } - syncDB(t, db, cm) + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, true, false, false, revisedFC3, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn3.ID(), dbFCs[0].TransactionID) + } - tip, err := db.Tip() + { + dbFCs, err := db.ContractRevisions(fcID) if err != nil { t.Fatal(err) } - check(t, "tip", cm.Tip(), tip) + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0, 1, 2, 3}, dbFCs) } { - // mine to trigger a reorg - var blocks []types.Block - state := genesisState - for i := uint64(0); i < n+5; i++ { - blocks = append(blocks, mineBlock(state, nil, addr2)) - state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { t.Fatal(err) } - syncDB(t, db, cm) - - tip, err := db.Tip() + hostContracts, err := db.ContractsKey(hostPublicKey) if err != nil { t.Fatal(err) } - check(t, "tip", cm.Tip(), tip) + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.CheckFC(t, true, false, false, revisedFC3, renterContracts[0]) + testutil.CheckFC(t, true, false, false, revisedFC3, hostContracts[0]) } - for i := 0; i < n; i++ { - best, err := db.BestTip(uint64(i)) + { + txns, err := db.Transactions([]types.TransactionID{reviseTxn2.ID()}) if err != nil { t.Fatal(err) } - if cmBest, ok := cm.BestIndex(uint64(i)); !ok || cmBest != best { - t.Fatal("best tip mismatch") - } - } -} - -func TestRevertBalance(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContractRevisions)) - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress, types.ZeroCurrency, 0) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) + fcr := txns[0].FileContractRevisions[0] + testutil.Equal(t, "parent id", txn.FileContractID(0), fcr.ParentID) + testutil.Equal(t, "unlock conditions", uc, fcr.UnlockConditions) + testutil.CheckFC(t, true, false, false, revisedFC2, fcr.ExtendedFileContract) } - cm := chain.NewManager(store, genesisState) - - // checkBalance checks that an address has the balances we expect - checkBalance := func(addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { - sc, immatureSC, sf, err := db.Balance(addr) + { + txns, err := db.Transactions([]types.TransactionID{reviseTxn3.ID()}) if err != nil { t.Fatal(err) } - check(t, "siacoins", expectSC, sc) - check(t, "immature siacoins", expectImmatureSC, immatureSC) - check(t, "siafunds", expectSF, sf) + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContractRevisions)) + + fcr := txns[0].FileContractRevisions[0] + testutil.Equal(t, "parent id", txn.FileContractID(0), fcr.ParentID) + testutil.Equal(t, "unlock conditions", uc, fcr.UnlockConditions) + testutil.CheckFC(t, true, false, false, revisedFC3, fcr.ExtendedFileContract) } +} +func TestRevertBalance(t *testing.T) { // Generate three addresses: addr1, addr2, addr3 pk1 := types.GeneratePrivateKey() addr1 := types.StandardUnlockHash(pk1.PublicKey()) @@ -1006,6 +958,9 @@ func TestRevertBalance(t *testing.T) { pk3 := types.GeneratePrivateKey() addr3 := types.StandardUnlockHash(pk3.PublicKey()) + _, _, cm, db := newStore(t, false, nil) + genesisState := cm.TipState() + // t.Log("addr1:", addr1) // t.Log("addr2:", addr2) // t.Log("addr3:", addr3) @@ -1014,19 +969,19 @@ func TestRevertBalance(t *testing.T) { maturityHeight := cm.TipState().MaturityHeight() // Mine a block sending the payout to addr1 - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr1)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, addr1)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) // Check that addr1 has the miner payout output - utxos, err := db.UnspentSiacoinOutputs(addr1, 100, 0) + utxos, err := db.UnspentSiacoinOutputs(addr1, 0, 100) if err != nil { t.Fatal(err) } - check(t, "utxos", 1, len(utxos)) - check(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxos[0].Source) + testutil.Equal(t, "utxos", 1, len(utxos)) + testutil.Equal(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxos[0].Source) { // Mine to trigger a reorg @@ -1034,7 +989,7 @@ func TestRevertBalance(t *testing.T) { var blocks []types.Block state := genesisState for i := uint64(0); i < 2; i++ { - blocks = append(blocks, mineBlock(state, nil, addr2)) + blocks = append(blocks, testutil.MineBlock(state, nil, addr2)) state.Index.ID = blocks[len(blocks)-1].ID() state.Index.Height++ } @@ -1046,30 +1001,36 @@ func TestRevertBalance(t *testing.T) { // Mine until the payout matures for i := cm.Tip().Height; i < maturityHeight; i++ { - checkBalance(addr1, types.ZeroCurrency, types.ZeroCurrency, 0) - checkBalance(addr2, types.ZeroCurrency, expectedPayout.Mul64(2), 0) - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, expectedPayout.Mul64(2), 0) + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + StorageUtilization: 0, + }) } - checkBalance(addr1, types.ZeroCurrency, types.ZeroCurrency, 0) - checkBalance(addr2, expectedPayout.Mul64(1), expectedPayout.Mul64(1), 0) + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, expectedPayout.Mul64(1), expectedPayout.Mul64(1), 0) - utxos1, err := db.UnspentSiacoinOutputs(addr1, 100, 0) + utxos1, err := db.UnspentSiacoinOutputs(addr1, 0, 100) if err != nil { t.Fatal(err) } - check(t, "addr1 utxos", 0, len(utxos1)) + testutil.Equal(t, "addr1 utxos", 0, len(utxos1)) - utxos2, err := db.UnspentSiacoinOutputs(addr2, 100, 0) + utxos2, err := db.UnspentSiacoinOutputs(addr2, 0, 100) if err != nil { t.Fatal(err) } - check(t, "addr2 utxos", 2, len(utxos2)) + testutil.Equal(t, "addr2 utxos", 2, len(utxos2)) for _, utxo := range utxos2 { - check(t, "value", expectedPayout, utxo.SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxo.Source) + testutil.Equal(t, "value", expectedPayout, utxo.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxo.Source) } // Send all of the payout except 100 SC to addr3 @@ -1078,7 +1039,7 @@ func TestRevertBalance(t *testing.T) { parentTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ { - ParentID: types.SiacoinOutputID(utxos2[0].ID), + ParentID: utxos2[0].ID, UnlockConditions: unlockConditions, }, }, @@ -1087,7 +1048,7 @@ func TestRevertBalance(t *testing.T) { {Address: addr3, Value: utxos2[0].SiacoinOutput.Value.Sub(hundredSC)}, }, } - signTxn(cm.TipState(), pk2, &parentTxn) + testutil.SignTransaction(cm.TipState(), pk2, &parentTxn) // In the same block, have addr2 send the 100 SC it still has left to // addr1 @@ -1103,17 +1064,32 @@ func TestRevertBalance(t *testing.T) { {Address: addr1, Value: hundredSC}, }, } - signTxn(cm.TipState(), pk2, &txn) + testutil.SignTransaction(cm.TipState(), pk2, &txn) - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{parentTxn, txn}, types.VoidAddress)}); err != nil { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{parentTxn, txn}, types.VoidAddress)}); err != nil { t.Fatal(err) } syncDB(t, db, cm) - checkBalance(addr1, hundredSC, types.ZeroCurrency, 0) + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "spent_index", *b.Transactions[0].SiacoinOutputs[0].SpentIndex, cm.Tip()) + testutil.Equal(t, "spent_index", b.Transactions[1].SiacoinOutputs[0].SpentIndex, (*types.ChainIndex)(nil)) + } + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + StorageUtilization: 0, + }) + + testutil.CheckBalance(t, db, addr1, hundredSC, types.ZeroCurrency, 0) // second block added in reorg has now matured - checkBalance(addr2, utxos2[1].SiacoinOutput.Value, types.ZeroCurrency, 0) - checkBalance(addr3, utxos2[0].SiacoinOutput.Value.Sub(hundredSC), types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, utxos2[1].SiacoinOutput.Value, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, utxos2[0].SiacoinOutput.Value.Sub(hundredSC), types.ZeroCurrency, 0) { // Reorg everything from before @@ -1129,7 +1105,7 @@ func TestRevertBalance(t *testing.T) { } else if i == 1 { addr = addr2 } - blocks = append(blocks, mineBlock(state, nil, addr)) + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) state.Index.ID = blocks[len(blocks)-1].ID() state.Index.Height++ } @@ -1139,35 +1115,35 @@ func TestRevertBalance(t *testing.T) { syncDB(t, db, cm) } - checkBalance(addr1, expectedPayout, types.ZeroCurrency, 0) - checkBalance(addr2, expectedPayout, types.ZeroCurrency, 0) - checkBalance(addr3, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr1, expectedPayout, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, expectedPayout, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, types.ZeroCurrency, types.ZeroCurrency, 0) - utxos1, err = db.UnspentSiacoinOutputs(addr1, 100, 0) + utxos1, err = db.UnspentSiacoinOutputs(addr1, 0, 100) if err != nil { t.Fatal(err) } - check(t, "addr1 utxos", 1, len(utxos1)) + testutil.Equal(t, "addr1 utxos", 1, len(utxos1)) for _, utxo := range utxos1 { - check(t, "value", expectedPayout, utxo.SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxo.Source) + testutil.Equal(t, "value", expectedPayout, utxo.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxo.Source) } - utxos2, err = db.UnspentSiacoinOutputs(addr2, 100, 0) + utxos2, err = db.UnspentSiacoinOutputs(addr2, 0, 100) if err != nil { t.Fatal(err) } - check(t, "addr2 utxos", 1, len(utxos2)) + testutil.Equal(t, "addr2 utxos", 1, len(utxos2)) for _, utxo := range utxos2 { - check(t, "value", expectedPayout, utxo.SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxo.Source) + testutil.Equal(t, "value", expectedPayout, utxo.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxo.Source) } - utxos3, err := db.UnspentSiacoinOutputs(addr3, 100, 0) + utxos3, err := db.UnspentSiacoinOutputs(addr3, 0, 100) if err != nil { t.Fatal(err) } - check(t, "addr3 utxos", 0, len(utxos3)) + testutil.Equal(t, "addr3 utxos", 0, len(utxos3)) } func TestRevertSendTransactions(t *testing.T) { @@ -1199,70 +1175,22 @@ func TestRevertSendTransactions(t *testing.T) { // t.Log("addr2:", addr2) // t.Log("addr3:", addr3) - const giftSF = 10000 - network, genesisBlock := testV1Network(addr1, types.ZeroCurrency, giftSF) + network, genesisBlock := ctestutil.Network() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + giftSF := genesisBlock.Transactions[0].SiafundOutputs[0].Value - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { t.Fatal(err) } cm := chain.NewManager(store, genesisState) - // checkBalance checks that an address has the balances we expect - checkBalance := func(addr types.Address, expectSC, expectImmatureSC types.Currency, expectSF uint64) { - sc, immatureSC, sf, err := db.Balance(addr) - if err != nil { - t.Fatal(err) - } - check(t, "siacoins", expectSC, sc) - check(t, "immature siacoins", expectImmatureSC, immatureSC) - check(t, "siafunds", expectSF, sf) - } - - checkTransaction := func(expectTxn types.Transaction, gotTxn explorer.Transaction) { - check(t, "siacoin inputs", len(expectTxn.SiacoinInputs), len(gotTxn.SiacoinInputs)) - check(t, "siacoin outputs", len(expectTxn.SiacoinOutputs), len(gotTxn.SiacoinOutputs)) - check(t, "siafund inputs", len(expectTxn.SiafundInputs), len(gotTxn.SiafundInputs)) - check(t, "siafund outputs", len(expectTxn.SiafundOutputs), len(gotTxn.SiafundOutputs)) - - for i := range expectTxn.SiacoinInputs { - expectSci := expectTxn.SiacoinInputs[i] - gotSci := gotTxn.SiacoinInputs[i] - - check(t, "parent ID", expectSci.ParentID, gotSci.ParentID) - check(t, "unlock conditions", expectSci.UnlockConditions, gotSci.UnlockConditions) - } - for i := range expectTxn.SiacoinOutputs { - expectSco := expectTxn.SiacoinOutputs[i] - gotSco := gotTxn.SiacoinOutputs[i].SiacoinOutput - - check(t, "address", expectSco.Address, gotSco.Address) - check(t, "value", expectSco.Value, gotSco.Value) - check(t, "source", explorer.SourceTransaction, gotTxn.SiacoinOutputs[i].Source) - } - for i := range expectTxn.SiafundInputs { - expectSfi := expectTxn.SiafundInputs[i] - gotSfi := gotTxn.SiafundInputs[i] - - check(t, "parent ID", expectSfi.ParentID, gotSfi.ParentID) - check(t, "claim address", expectSfi.ClaimAddress, gotSfi.ClaimAddress) - check(t, "unlock conditions", expectSfi.UnlockConditions, gotSfi.UnlockConditions) - } - for i := range expectTxn.SiafundOutputs { - expectSfo := expectTxn.SiafundOutputs[i] - gotSfo := gotTxn.SiafundOutputs[i].SiafundOutput - - check(t, "address", expectSfo.Address, gotSfo.Address) - check(t, "value", expectSfo.Value, gotSfo.Value) - } - } - expectedPayout := cm.TipState().BlockReward() maturityHeight := cm.TipState().MaturityHeight() var blocks []types.Block - b1 := mineBlock(cm.TipState(), nil, addr1) + b1 := testutil.MineBlock(cm.TipState(), nil, addr1) // Mine a block sending the payout to the addr1 if err := cm.AddBlocks([]types.Block{b1}); err != nil { t.Fatal(err) @@ -1272,7 +1200,7 @@ func TestRevertSendTransactions(t *testing.T) { // Mine until the payout matures for i := cm.Tip().Height; i < maturityHeight; i++ { - b := mineBlock(cm.TipState(), nil, types.VoidAddress) + b := testutil.MineBlock(cm.TipState(), nil, types.VoidAddress) if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } @@ -1280,20 +1208,20 @@ func TestRevertSendTransactions(t *testing.T) { syncDB(t, db, cm) } - checkBalance(addr1, expectedPayout, types.ZeroCurrency, giftSF) - checkBalance(addr2, types.ZeroCurrency, types.ZeroCurrency, 0) - checkBalance(addr3, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr1, expectedPayout, types.ZeroCurrency, giftSF) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, types.ZeroCurrency, types.ZeroCurrency, 0) const n = 26 // Check that addr1 has the miner payout output - utxos, err := db.UnspentSiacoinOutputs(addr1, n, 0) + utxos, err := db.UnspentSiacoinOutputs(addr1, 0, n) if err != nil { t.Fatal(err) } - check(t, "utxos", 1, len(utxos)) - check(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) - check(t, "source", explorer.SourceMinerPayout, utxos[0].Source) + testutil.Equal(t, "utxos", 1, len(utxos)) + testutil.Equal(t, "value", expectedPayout, utxos[0].SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceMinerPayout, utxos[0].Source) sfOutputID := genesisBlock.Transactions[0].SiafundOutputID(0) scOutputID := utxos[0].ID @@ -1306,7 +1234,7 @@ func TestRevertSendTransactions(t *testing.T) { parentTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ { - ParentID: types.SiacoinOutputID(scOutputID), + ParentID: scOutputID, UnlockConditions: unlockConditions, }, }, @@ -1328,21 +1256,27 @@ func TestRevertSendTransactions(t *testing.T) { }, } - signTxn(cm.TipState(), pk1, &parentTxn) - scOutputID = types.Hash256(parentTxn.SiacoinOutputID(2)) + testutil.SignTransaction(cm.TipState(), pk1, &parentTxn) + scOutputID = parentTxn.SiacoinOutputID(2) sfOutputID = parentTxn.SiafundOutputID(2) // Mine a block with the above transaction - b := mineBlock(cm.TipState(), []types.Transaction{parentTxn}, types.VoidAddress) + b := testutil.MineBlock(cm.TipState(), []types.Transaction{parentTxn}, types.VoidAddress) if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } blocks = append(blocks, b) syncDB(t, db, cm) - checkBalance(addr1, addr1SCs, types.ZeroCurrency, addr1SFs) - checkBalance(addr2, types.Siacoins(1).Mul64(uint64(i+1)), types.ZeroCurrency, 1*uint64(i+1)) - checkBalance(addr3, types.Siacoins(2).Mul64(uint64(i+1)), types.ZeroCurrency, 2*uint64(i+1)) + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + StorageUtilization: 0, + }) + + testutil.CheckBalance(t, db, addr1, addr1SCs, types.ZeroCurrency, addr1SFs) + testutil.CheckBalance(t, db, addr2, types.Siacoins(1).Mul64(uint64(i+1)), types.ZeroCurrency, 1*uint64(i+1)) + testutil.CheckBalance(t, db, addr3, types.Siacoins(2).Mul64(uint64(i+1)), types.ZeroCurrency, 2*uint64(i+1)) // Ensure the block we retrieved from the database is the same as the // actual block @@ -1350,28 +1284,24 @@ func TestRevertSendTransactions(t *testing.T) { if err != nil { t.Fatal(err) } - check(t, "transactions", len(b.Transactions), len(block.Transactions)) - check(t, "miner payouts", len(b.MinerPayouts), len(block.MinerPayouts)) - check(t, "nonce", b.Nonce, block.Nonce) - check(t, "timestamp", b.Timestamp, block.Timestamp) + testutil.Equal(t, "transactions", len(b.Transactions), len(block.Transactions)) + testutil.Equal(t, "miner payouts", len(b.MinerPayouts), len(block.MinerPayouts)) + testutil.Equal(t, "nonce", b.Nonce, block.Nonce) + testutil.Equal(t, "timestamp", b.Timestamp, block.Timestamp) // Ensure the miner payouts in the block match for i := range b.MinerPayouts { - check(t, "address", b.MinerPayouts[i].Address, b.MinerPayouts[i].Address) - check(t, "value", b.MinerPayouts[i].Value, b.MinerPayouts[i].Value) + testutil.Equal(t, "address", b.MinerPayouts[i].Address, b.MinerPayouts[i].Address) + testutil.Equal(t, "value", b.MinerPayouts[i].Value, b.MinerPayouts[i].Value) } // Ensure the transactions in the block and retrieved separately match // with the actual transactions for i := range b.Transactions { - checkTransaction(b.Transactions[i], block.Transactions[i]) + testutil.CheckTransaction(t, b.Transactions[i], block.Transactions[i]) + CheckChainIndices(t, db, b.Transactions[i].ID(), []types.ChainIndex{cm.Tip()}) - txns, err := db.Transactions([]types.TransactionID{b.Transactions[i].ID()}) - if err != nil { - t.Fatal(err) - } - check(t, "transactions", 1, len(txns)) - checkTransaction(b.Transactions[i], txns[0]) + checkTransaction(t, db, b.Transactions[i]) } type expectedUTXOs struct { @@ -1389,26 +1319,26 @@ func TestRevertSendTransactions(t *testing.T) { {addr3, i + 1, types.Siacoins(2), i + 1, 2}, } for _, e := range expected { - sc, err := db.UnspentSiacoinOutputs(e.addr, n, 0) + sc, err := db.UnspentSiacoinOutputs(e.addr, 0, n) if err != nil { t.Fatal(err) } - sf, err := db.UnspentSiafundOutputs(e.addr, n, 0) + sf, err := db.UnspentSiafundOutputs(e.addr, 0, n) if err != nil { t.Fatal(err) } - check(t, "sc utxos", e.sc, len(sc)) - check(t, "sf utxos", e.sf, len(sf)) + testutil.Equal(t, "sc utxos", e.sc, len(sc)) + testutil.Equal(t, "sf utxos", e.sf, len(sf)) for _, sco := range sc { - check(t, "address", e.addr, sco.SiacoinOutput.Address) - check(t, "value", e.scValue, sco.SiacoinOutput.Value) - check(t, "source", explorer.SourceTransaction, sco.Source) + testutil.Equal(t, "address", e.addr, sco.SiacoinOutput.Address) + testutil.Equal(t, "value", e.scValue, sco.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceTransaction, sco.Source) } for _, sfo := range sf { - check(t, "address", e.addr, sfo.SiafundOutput.Address) - check(t, "value", e.sfValue, sfo.SiafundOutput.Value) + testutil.Equal(t, "address", e.addr, sfo.SiafundOutput.Address) + testutil.Equal(t, "value", e.sfValue, sfo.SiafundOutput.Value) } } } @@ -1423,7 +1353,7 @@ func TestRevertSendTransactions(t *testing.T) { t.Fatal("no such block") } for i := 0; i < 3+1; i++ { - newBlocks = append(newBlocks, mineBlock(state, nil, types.VoidAddress)) + newBlocks = append(newBlocks, testutil.MineBlock(state, nil, types.VoidAddress)) state.Index.ID = newBlocks[len(newBlocks)-1].ID() state.Index.Height++ } @@ -1436,71 +1366,990 @@ func TestRevertSendTransactions(t *testing.T) { addr1SCs := expectedPayout.Sub(types.Siacoins(1 + 2).Mul64(uint64(n - 3))) addr1SFs := giftSF - (1+2)*uint64(n-3) - checkBalance(addr1, addr1SCs, types.ZeroCurrency, addr1SFs) - checkBalance(addr2, types.Siacoins(1).Mul64(uint64(n-3)), types.ZeroCurrency, 1*uint64(n-3)) - checkBalance(addr3, types.Siacoins(2).Mul64(uint64(n-3)), types.ZeroCurrency, 2*uint64(n-3)) + testutil.CheckBalance(t, db, addr1, addr1SCs, types.ZeroCurrency, addr1SFs) + testutil.CheckBalance(t, db, addr2, types.Siacoins(1).Mul64(uint64(n-3)), types.ZeroCurrency, 1*uint64(n-3)) + testutil.CheckBalance(t, db, addr3, types.Siacoins(2).Mul64(uint64(n-3)), types.ZeroCurrency, 2*uint64(n-3)) - scUtxos1, err := db.UnspentSiacoinOutputs(addr1, n, 0) + scUtxos1, err := db.UnspentSiacoinOutputs(addr1, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr1 sc utxos", 1, len(scUtxos1)) + testutil.Equal(t, "addr1 sc utxos", 1, len(scUtxos1)) for _, sce := range scUtxos1 { - check(t, "address", addr1, sce.SiacoinOutput.Address) - check(t, "value", addr1SCs, sce.SiacoinOutput.Value) - check(t, "source", explorer.SourceTransaction, sce.Source) + testutil.Equal(t, "address", addr1, sce.SiacoinOutput.Address) + testutil.Equal(t, "value", addr1SCs, sce.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceTransaction, sce.Source) } - scUtxos2, err := db.UnspentSiacoinOutputs(addr2, n, 0) + scUtxos2, err := db.UnspentSiacoinOutputs(addr2, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr2 sc utxos", n-3, len(scUtxos2)) + testutil.Equal(t, "addr2 sc utxos", n-3, len(scUtxos2)) for _, sce := range scUtxos2 { - check(t, "address", addr2, sce.SiacoinOutput.Address) - check(t, "value", types.Siacoins(1), sce.SiacoinOutput.Value) - check(t, "source", explorer.SourceTransaction, sce.Source) + testutil.Equal(t, "address", addr2, sce.SiacoinOutput.Address) + testutil.Equal(t, "value", types.Siacoins(1), sce.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceTransaction, sce.Source) } - scUtxos3, err := db.UnspentSiacoinOutputs(addr3, n, 0) + scUtxos3, err := db.UnspentSiacoinOutputs(addr3, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr3 sc utxos", n-3, len(scUtxos3)) + testutil.Equal(t, "addr3 sc utxos", n-3, len(scUtxos3)) for _, sce := range scUtxos3 { - check(t, "address", addr3, sce.SiacoinOutput.Address) - check(t, "value", types.Siacoins(2), sce.SiacoinOutput.Value) - check(t, "source", explorer.SourceTransaction, sce.Source) + testutil.Equal(t, "address", addr3, sce.SiacoinOutput.Address) + testutil.Equal(t, "value", types.Siacoins(2), sce.SiacoinOutput.Value) + testutil.Equal(t, "source", explorer.SourceTransaction, sce.Source) } - sfUtxos1, err := db.UnspentSiafundOutputs(addr1, n, 0) + sfUtxos1, err := db.UnspentSiafundOutputs(addr1, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr1 sf utxos", 1, len(sfUtxos1)) + testutil.Equal(t, "addr1 sf utxos", 1, len(sfUtxos1)) for _, sfe := range sfUtxos1 { - check(t, "address", addr1, sfe.SiafundOutput.Address) - check(t, "value", addr1SFs, sfe.SiafundOutput.Value) + testutil.Equal(t, "address", addr1, sfe.SiafundOutput.Address) + testutil.Equal(t, "value", addr1SFs, sfe.SiafundOutput.Value) } - sfUtxos2, err := db.UnspentSiafundOutputs(addr2, n, 0) + sfUtxos2, err := db.UnspentSiafundOutputs(addr2, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr2 sf utxos", n-3, len(sfUtxos2)) + testutil.Equal(t, "addr2 sf utxos", n-3, len(sfUtxos2)) for _, sfe := range sfUtxos2 { - check(t, "address", addr2, sfe.SiafundOutput.Address) - check(t, "value", uint64(1), sfe.SiafundOutput.Value) + testutil.Equal(t, "address", addr2, sfe.SiafundOutput.Address) + testutil.Equal(t, "value", uint64(1), sfe.SiafundOutput.Value) } - sfUtxos3, err := db.UnspentSiafundOutputs(addr3, n, 0) + sfUtxos3, err := db.UnspentSiafundOutputs(addr3, 0, n) if err != nil { t.Fatal(err) } - check(t, "addr3 sf utxos", n-3, len(sfUtxos3)) + testutil.Equal(t, "addr3 sf utxos", n-3, len(sfUtxos3)) for _, sfe := range sfUtxos3 { - check(t, "address", addr3, sfe.SiafundOutput.Address) - check(t, "value", uint64(2), sfe.SiafundOutput.Value) + testutil.Equal(t, "address", addr3, sfe.SiafundOutput.Address) + testutil.Equal(t, "value", uint64(2), sfe.SiafundOutput.Value) } } + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + StorageUtilization: 0, + }) +} + +func TestHostAnnouncement(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + + pk2 := types.GeneratePrivateKey() + pk3 := types.GeneratePrivateKey() + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + + hostPubkeys := func(pks []types.PublicKey) ([]explorer.Host, error) { + return db.QueryHosts(explorer.HostQuery{PublicKeys: pks}, explorer.HostSortPublicKey, explorer.HostSortAsc, 0, math.MaxInt64) + } + + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: genesisBlock.Transactions[0].SiacoinOutputID(0), + UnlockConditions: uc1, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: genesisBlock.Transactions[0].SiacoinOutputs[0].Value, + }}, + ArbitraryData: [][]byte{ + testutil.CreateAnnouncement(pk1, "127.0.0.1:1234"), + }, + } + testutil.SignTransaction(cm.TipState(), pk1, &txn1) + + // Mine a block containing host announcement + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 1, + ActiveContracts: 0, + StorageUtilization: 0, + }) + + txn2 := types.Transaction{ + ArbitraryData: [][]byte{ + testutil.CreateAnnouncement(pk1, "127.0.0.1:5678"), + }, + } + txn3 := types.Transaction{ + ArbitraryData: [][]byte{ + testutil.CreateAnnouncement(pk2, "127.0.0.1:9999"), + }, + } + txn4 := types.Transaction{ + ArbitraryData: [][]byte{ + testutil.CreateAnnouncement(pk3, "127.0.0.1:9999"), + }, + } + + // Mine a block containing host announcement + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn2, txn3, txn4}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 3, + ActiveContracts: 0, + StorageUtilization: 0, + }) + + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 3, len(b.Transactions)) + testutil.Equal(t, "txns[0].ID", txn2.ID(), b.Transactions[0].ID) + testutil.Equal(t, "txns[1].ID", txn3.ID(), b.Transactions[1].ID) + testutil.Equal(t, "txns[2].ID", txn4.ID(), b.Transactions[2].ID) + } + + checkTransaction(t, db, txn1) + checkTransaction(t, db, txn2) + checkTransaction(t, db, txn3) + checkTransaction(t, db, txn4) + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 2, len(events)) + testutil.CheckTransaction(t, txn1, events[0].Data.(explorer.EventV1Transaction).Transaction) + testutil.CheckTransaction(t, genesisBlock.Transactions[0], events[1].Data.(explorer.EventV1Transaction).Transaction) + } + + checkTransaction(t, db, txn1) + checkTransaction(t, db, txn2) + checkTransaction(t, db, txn3) + + hosts, err := db.HostsForScanning(time.Unix(0, 0), 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(hosts)", 3, len(hosts)) + + { + scans, err := hostPubkeys([]types.PublicKey{hosts[0].PublicKey}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scans)", 1, len(scans)) + } + + scan1 := explorer.HostScan{ + PublicKey: hosts[0].PublicKey, + Success: true, + Timestamp: time.Now(), + } + scan2 := explorer.HostScan{ + PublicKey: hosts[0].PublicKey, + Success: false, + Timestamp: time.Now(), + Error: func() *string { + x := "error" + return &x + }(), + } + + { + if err := db.AddHostScans([]explorer.HostScan{scan1}...); err != nil { + t.Fatal(err) + } + + scans, err := hostPubkeys([]types.PublicKey{hosts[0].PublicKey}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scans)", 1, len(scans)) + + scan := scans[0] + testutil.Equal(t, "last scan", scan1.Timestamp.Unix(), scan.LastScan.Unix()) + testutil.Equal(t, "last scan successful", scan1.Success, scan.LastScanSuccessful) + testutil.Equal(t, "total scans", 1, scan.TotalScans) + testutil.Equal(t, "successful interactions", 1, scan.SuccessfulInteractions) + testutil.Equal(t, "failed interactions", 0, scan.FailedInteractions) + } + + { + if err := db.AddHostScans([]explorer.HostScan{scan2}...); err != nil { + t.Fatal(err) + } + + scans, err := hostPubkeys([]types.PublicKey{hosts[0].PublicKey}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(scans)", 1, len(scans)) + + scan := scans[0] + testutil.Equal(t, "last scan", scan2.Timestamp.Unix(), scan.LastScan.Unix()) + testutil.Equal(t, "last scan successful", scan2.Success, scan.LastScanSuccessful) + testutil.Equal(t, "total scans", 2, scan.TotalScans) + testutil.Equal(t, "successful interactions", 1, scan.SuccessfulInteractions) + testutil.Equal(t, "failed interactions", 1, scan.FailedInteractions) + } +} + +func TestMultipleReorg(t *testing.T) { + // Generate three addresses: addr1, addr2, addr3 + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + pk3 := types.GeneratePrivateKey() + addr3 := types.StandardUnlockHash(pk3.PublicKey()) + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + giftSF := genesisBlock.Transactions[0].SiafundOutputs[0].Value + + uc1 := types.StandardUnlockConditions(pk1.PublicKey()) + // transfer gift from addr1 to addr2 + // element gets added at height 1 + txn1 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: genesisBlock.Transactions[0].SiacoinOutputID(0), + UnlockConditions: uc1, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: giftSC}, + }, + SiafundInputs: []types.SiafundInput{ + { + ParentID: genesisBlock.Transactions[0].SiafundOutputID(0), + UnlockConditions: uc1, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr2, Value: giftSF}, + }, + } + testutil.SignTransaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 0, + StorageUtilization: 0, + }) + + { + // addr2 should have all the SC + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, giftSC, types.ZeroCurrency, giftSF) + testutil.CheckBalance(t, db, addr3, types.ZeroCurrency, types.ZeroCurrency, 0) + + scUtxos1, err := db.UnspentSiacoinOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sc utxos", 0, len(scUtxos1)) + + scUtxos2, err := db.UnspentSiacoinOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sc utxos", 1, len(scUtxos2)) + + scUtxos3, err := db.UnspentSiacoinOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sc utxos", 0, len(scUtxos3)) + + sfUtxos1, err := db.UnspentSiafundOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sf utxos", 0, len(sfUtxos1)) + + sfUtxos2, err := db.UnspentSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sf utxos", 1, len(sfUtxos2)) + + sfUtxos3, err := db.UnspentSiafundOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sf utxos", 0, len(sfUtxos3)) + } + + for i := 0; i < 10; i++ { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + uc2 := types.StandardUnlockConditions(pk2.PublicKey()) + // element gets spent at height 12 + // transfer gift from addr2 to addr3 + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: txn1.SiacoinOutputID(0), + UnlockConditions: uc2, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr3, Value: giftSC}, + }, + SiafundInputs: []types.SiafundInput{ + { + ParentID: txn1.SiafundOutputID(0), + UnlockConditions: uc2, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr3, Value: giftSF}, + }, + } + testutil.SignTransaction(cm.TipState(), pk2, &txn2) + + prevState1 := cm.TipState() + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + prevState2 := cm.TipState() + + { + // addr3 should have all the SC + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, giftSC, types.ZeroCurrency, giftSF) + + scUtxos1, err := db.UnspentSiacoinOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sc utxos", 0, len(scUtxos1)) + + scUtxos2, err := db.UnspentSiacoinOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sc utxos", 0, len(scUtxos2)) + + scUtxos3, err := db.UnspentSiacoinOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sc utxos", 1, len(scUtxos3)) + + sfUtxos1, err := db.UnspentSiafundOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sf utxos", 0, len(sfUtxos1)) + + sfUtxos2, err := db.UnspentSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sf utxos", 0, len(sfUtxos2)) + + sfUtxos3, err := db.UnspentSiafundOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sf utxos", 1, len(sfUtxos3)) + } + + // revert block 12 with increasingly large reorgs and sanity check results + for reorg := 0; reorg < 2; reorg++ { + // revert block 12 (the addr2 -> addr3 transfer), unspending the + // element + { + var blocks []types.Block + state := prevState1 + for i := 0; i < reorg+2; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // we should be back in state before block 12 (addr2 has all the SC + // instead of addr3) + { + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, giftSC, types.ZeroCurrency, giftSF) + testutil.CheckBalance(t, db, addr3, types.ZeroCurrency, types.ZeroCurrency, 0) + + scUtxos1, err := db.UnspentSiacoinOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sc utxos", 0, len(scUtxos1)) + + scUtxos2, err := db.UnspentSiacoinOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sc utxos", 1, len(scUtxos2)) + + scUtxos3, err := db.UnspentSiacoinOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sc utxos", 0, len(scUtxos3)) + + sfUtxos1, err := db.UnspentSiafundOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sf utxos", 0, len(sfUtxos1)) + + sfUtxos2, err := db.UnspentSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sf utxos", 1, len(sfUtxos2)) + + sfUtxos3, err := db.UnspentSiafundOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sf utxos", 0, len(sfUtxos3)) + } + } + + // now make the original chain where addr3 got the coins the longest + // and make sure addr3 ends up with the coins + extra := cm.Tip().Height - prevState2.Index.Height + 1 + for reorg := uint64(0); reorg < 2; reorg++ { + { + var blocks []types.Block + state := prevState2 + for i := uint64(0); i < reorg+extra; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // we should be back in state before the reverts (addr3 has all the SC + // instead of addr2) + { + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr3, giftSC, types.ZeroCurrency, giftSF) + + scUtxos1, err := db.UnspentSiacoinOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sc utxos", 0, len(scUtxos1)) + + scUtxos2, err := db.UnspentSiacoinOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sc utxos", 0, len(scUtxos2)) + + scUtxos3, err := db.UnspentSiacoinOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sc utxos", 1, len(scUtxos3)) + + sfUtxos1, err := db.UnspentSiafundOutputs(addr1, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr1 sf utxos", 0, len(sfUtxos1)) + + sfUtxos2, err := db.UnspentSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr2 sf utxos", 0, len(sfUtxos2)) + + sfUtxos3, err := db.UnspentSiafundOutputs(addr3, 0, 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "addr3 sf utxos", 1, len(sfUtxos3)) + } + } +} + +func TestMultipleReorgFileContract(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + genesisState := cm.TipState() + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + scOutputID := genesisBlock.Transactions[0].SiacoinOutputID(0) + unlockConditions := types.StandardUnlockConditions(pk1.PublicKey()) + + windowStart := cm.Tip().Height + 10 + windowEnd := windowStart + 10 + fc := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), windowStart, windowEnd, types.VoidAddress) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: scOutputID, + UnlockConditions: unlockConditions, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: giftSC.Sub(fc.Payout), + }}, + FileContracts: []types.FileContract{fc}, + } + fcID := txn.FileContractID(0) + testutil.SignTransaction(cm.TipState(), pk1, &txn) + + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + confirmationIndex := cm.Tip() + confirmationTransactionID := txn.ID() + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize, + }) + + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + + testutil.Equal(t, "confirmation index", cm.Tip(), dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } + + { + dbFCs, err := db.ContractRevisions(fcID) + if err != nil { + t.Fatal(err) + } + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0}, dbFCs) + } + + { + txns, err := db.Transactions([]types.TransactionID{txn.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContracts)) + testutil.CheckFC(t, false, false, false, fc, txns[0].FileContracts[0]) + } + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 2, len(events)) + testutil.CheckTransaction(t, txn, events[0].Data.(explorer.EventV1Transaction).Transaction) + testutil.CheckTransaction(t, genesisBlock.Transactions[0], events[1].Data.(explorer.EventV1Transaction).Transaction) + } + + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{ + renterPublicKey.UnlockKey(), + hostPublicKey.UnlockKey(), + }, + SignaturesRequired: 2, + } + revFC := fc + // add 10 bytes to filesize and increment revision number + revFC.Filesize += 10 + revFC.RevisionNumber++ + reviseTxn := types.Transaction{ + FileContractRevisions: []types.FileContractRevision{{ + ParentID: fcID, + UnlockConditions: uc, + FileContract: revFC, + }}, + } + testutil.SignTransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &reviseTxn) + + // state before revision + prevState1 := cm.TipState() + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{reviseTxn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + prevState2 := cm.TipState() + + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize + 10, + }) + + // Explorer.Contracts should return latest revision + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, revFC, dbFCs[0]) + + testutil.Equal(t, "transaction ID", reviseTxn.ID(), dbFCs[0].TransactionID) + testutil.Equal(t, "confirmation index", prevState1.Index, dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } + + { + txns, err := db.Transactions([]types.TransactionID{reviseTxn.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "transactions", 1, len(txns)) + testutil.Equal(t, "file contracts", 1, len(txns[0].FileContractRevisions)) + + fcr := txns[0].FileContractRevisions[0] + testutil.Equal(t, "parent id", txn.FileContractID(0), fcr.ParentID) + testutil.Equal(t, "unlock conditions", uc, fcr.UnlockConditions) + + testutil.CheckFC(t, false, false, false, revFC, fcr.ExtendedFileContract) + } + + { + dbFCs, err := db.ContractRevisions(fcID) + if err != nil { + t.Fatal(err) + } + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0, 1}, dbFCs) + } + + { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + hostContracts, err := db.ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) + testutil.CheckFC(t, false, false, false, revFC, renterContracts[0]) + testutil.CheckFC(t, false, false, false, revFC, hostContracts[0]) + } + + extra := cm.Tip().Height - prevState1.Index.Height + 1 + for reorg := uint64(0); reorg < 2; reorg++ { + // revert the revision + { + var blocks []types.Block + state := prevState1 + for i := uint64(0); i < reorg+extra; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // we should be back in state before the revision + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + + testutil.Equal(t, "transaction ID", txn.ID(), dbFCs[0].TransactionID) + testutil.Equal(t, "confirmation index", prevState1.Index, dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } + + { + dbFCs, err := db.ContractRevisions(fcID) + if err != nil { + t.Fatal(err) + } + CheckFCRevisions(t, confirmationIndex, confirmationTransactionID, fc.ValidProofOutputs, fc.MissedProofOutputs, []uint64{0}, dbFCs) + } + + // storage utilization should be back to testutil.ContractFilesize instead of + // testutil.ContractFilesize + 10 + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize, + }) + } + + extra = cm.Tip().Height - prevState2.Index.Height + 1 + for reorg := uint64(0); reorg < 2; reorg++ { + // bring the revision back + { + var blocks []types.Block + state := prevState2 + for i := uint64(0); i < reorg+extra; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // revision should be applied + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 1, len(dbFCs)) + testutil.CheckFC(t, false, false, false, revFC, dbFCs[0]) + + testutil.Equal(t, "transaction ID", reviseTxn.ID(), dbFCs[0].TransactionID) + testutil.Equal(t, "confirmation index", prevState1.Index, dbFCs[0].ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn.ID(), dbFCs[0].ConfirmationTransactionID) + } + + // should have revision filesize + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + ActiveContracts: 1, + StorageUtilization: testutil.ContractFilesize + 10, + }) + } + + extra = cm.Tip().Height - genesisState.Index.Height + 1 + for reorg := uint64(0); reorg < 2; reorg++ { + { + var blocks []types.Block + state := genesisState + for i := uint64(0); i < reorg+extra; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // contract should no longer exist + { + dbFCs, err := db.Contracts([]types.FileContractID{fcID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "fcs", 0, len(dbFCs)) + } + + { + renterContracts, err := db.ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + hostContracts, err := db.ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) + testutil.Equal(t, "len(contracts)", 0, len(renterContracts)) + } + + { + _, err := db.ContractRevisions(fcID) + if err != explorer.ErrContractNotFound { + t.Fatal(err) + } + } + + // no more contracts or storage utilization + CheckMetrics(t, db, cm, explorer.Metrics{ + TotalHosts: 0, + }) + } + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 1, len(events)) + testutil.CheckTransaction(t, genesisBlock.Transactions[0], events[0].Data.(explorer.EventV1Transaction).Transaction) + } +} + +func TestMetricCirculatingSupply(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + + _, genesisBlock, cm, db := newStore(t, false, func(network *consensus.Network, genesisBlock types.Block) { + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + genesisState := cm.TipState() + + var circulatingSupply types.Currency + if foundationSubsidy, ok := genesisState.FoundationSubsidy(); ok { + circulatingSupply = circulatingSupply.Add(foundationSubsidy.Value) + } + for _, txn := range genesisBlock.Transactions { + for _, sco := range txn.SiacoinOutputs { + circulatingSupply = circulatingSupply.Add(sco.Value) + } + } + + var rewards []types.Currency + prev := cm.TipState() + for i := 0; i < 10; i++ { + state := cm.TipState() + rewards = append(rewards, state.BlockReward()) + circulatingSupply = circulatingSupply.Add(state.BlockReward()) + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(state, nil, addr1)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + metrics, err := db.Metrics(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "circulating supply", circulatingSupply, metrics.CirculatingSupply) + } + } + + { + var blocks []types.Block + state := prev + + // remove reverted rewards + for _, reward := range rewards { + circulatingSupply = circulatingSupply.Sub(reward) + } + rewards = rewards[:0] + + for i := uint64(0); i < 15; i++ { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + blocks = append(blocks, testutil.MineBlock(state, nil, addr)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + + rewards = append(rewards, state.BlockReward()) + circulatingSupply = circulatingSupply.Add(state.BlockReward()) + } + + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + { + metrics, err := db.Metrics(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + + testutil.Equal(t, "circulating supply", circulatingSupply, metrics.CirculatingSupply) + } +} + +func TestBlockSameTransaction(t *testing.T) { + _, _, cm, db := newStore(t, false, nil) + + txn1 := types.Transaction{ + ArbitraryData: [][]byte{{0}}, + } + txn2 := types.Transaction{ + ArbitraryData: [][]byte{{0}, {1}}, + } + + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), []types.Transaction{txn1, txn1, txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkTransaction(t, db, txn1) + checkTransaction(t, db, txn2) } diff --git a/persist/sqlite/contracts.go b/persist/sqlite/contracts.go index b17b3ea..cb3241c 100644 --- a/persist/sqlite/contracts.go +++ b/persist/sqlite/contracts.go @@ -7,20 +7,43 @@ import ( "go.sia.tech/explored/explorer" ) -// Contracts implements explorer.Store. -func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileContract, err error) { - encodedIDs := func(ids []types.FileContractID) []any { - result := make([]any, len(ids)) - for i, id := range ids { - result[i] = encode(id) - } - return result +func encodedIDs(ids []types.FileContractID) []any { + result := make([]any, len(ids)) + for i, id := range ids { + result[i] = encode(id) + } + return result +} + +func scanFileContract(tx *txn, s scanner) (contractID int64, fc explorer.ExtendedFileContract, err error) { + var proofIndex types.ChainIndex + var proofTransactionID types.TransactionID + err = s.Scan(&contractID, decode(&fc.ID), &fc.Resolved, &fc.Valid, decode(&fc.TransactionID), decode(&fc.ConfirmationIndex.Height), decode(&fc.ConfirmationIndex.ID), decode(&fc.ConfirmationTransactionID), decodeNull(&proofIndex.Height), decodeNull(&proofIndex.ID), decodeNull(&proofTransactionID), decode(&fc.Filesize), decode(&fc.FileMerkleRoot), decode(&fc.WindowStart), decode(&fc.WindowEnd), decode(&fc.Payout), decode(&fc.UnlockHash), decode(&fc.RevisionNumber)) + if err != nil { + return } + fc.ValidProofOutputs, fc.MissedProofOutputs, err = fileContractOutputs(tx, contractID) + if err != nil { + return + } + + if proofIndex != (types.ChainIndex{}) { + fc.ProofIndex = &proofIndex + } + if proofTransactionID != (types.TransactionID{}) { + fc.ProofTransactionID = &proofTransactionID + } + + return +} + +// Contracts implements explorer.Store. +func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.ExtendedFileContract, err error) { err = s.transaction(func(tx *txn) error { - query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number + query := `SELECT fc1.id, fc1.contract_id, fc1.resolved, fc1.valid, fc1.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.proof_height, rev.proof_block_id, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number FROM file_contract_elements fc1 - INNER JOIN last_contract_revision rev ON (rev.contract_element_id = fc1.id) + INNER JOIN last_contract_revision rev ON rev.contract_element_id = fc1.id WHERE rev.contract_id IN (` + queryPlaceHolders(len(ids)) + `)` rows, err := tx.Query(query, encodedIDs(ids)...) if err != nil { @@ -28,31 +51,72 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon } defer rows.Close() - var contractIDs []int64 - idContract := make(map[int64]explorer.FileContract) for rows.Next() { - var contractID int64 - var fc explorer.FileContract - if err := rows.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { - return fmt.Errorf("failed to scan transaction: %w", err) + _, fc, err := scanFileContract(tx, rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + result = append(result, fc) + } + + return nil + }) + + return +} + +// ContractRevisions implements explorer.Store. +func (s *Store) ContractRevisions(id types.FileContractID) (revisions []explorer.ExtendedFileContract, err error) { + err = s.transaction(func(tx *txn) error { + query := `SELECT fc.id, fc.contract_id, fc.resolved, fc.valid, fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.proof_height, rev.proof_block_id, rev.proof_transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + FROM file_contract_elements fc + JOIN last_contract_revision rev ON rev.contract_id = fc.contract_id + WHERE fc.contract_id = ? + ORDER BY fc.revision_number ASC` + rows, err := tx.Query(query, encode(id)) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + _, fc, err := scanFileContract(tx, rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) } + revisions = append(revisions, fc) + } - idContract[contractID] = fc - contractIDs = append(contractIDs, contractID) + if len(revisions) == 0 { + return explorer.ErrContractNotFound } + return nil + }) + return +} - proofOutputs, err := fileContractOutputs(tx, contractIDs) +// ContractsKey implements explorer.Store. +func (s *Store) ContractsKey(key types.PublicKey) (result []explorer.ExtendedFileContract, err error) { + err = s.transaction(func(tx *txn) error { + query := `SELECT fc1.id, fc1.contract_id, fc1.resolved, fc1.valid, fc1.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.proof_height, rev.proof_block_id, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number + FROM file_contract_elements fc1 + INNER JOIN last_contract_revision rev ON rev.contract_element_id = fc1.id + WHERE rev.ed25519_renter_key = ? OR rev.ed25519_host_key = ?` + rows, err := tx.Query(query, encode(key), encode(key)) if err != nil { - return fmt.Errorf("failed to get file contract outputs: %w", err) + return err } - for contractID, output := range proofOutputs { - fc := idContract[contractID] - fc.ValidProofOutputs = output.valid - fc.MissedProofOutputs = output.missed + defer rows.Close() + + for rows.Next() { + _, fc, err := scanFileContract(tx, rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } result = append(result, fc) } - return nil + return rows.Err() }) return diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 966c6b0..0952c38 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -8,7 +8,9 @@ import ( "fmt" "time" + "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" ) func encode(obj any) any { @@ -25,9 +27,23 @@ func encode(obj any) any { obj.EncodeTo(e) e.Flush() return buf.Bytes() + case rhp.SettingsID: + return obj[:] + case []types.Hash256: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + types.EncodeSlice(e, obj) + e.Flush() + return buf.Bytes() + case []chain.NetAddress: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + types.EncodeSlice(e, obj) + e.Flush() + return buf.Bytes() case uint64: b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, obj) + binary.BigEndian.PutUint64(b, obj) return b case time.Time: return obj.Unix() @@ -55,12 +71,22 @@ func (d *decodable) Scan(src any) error { } v.Hi = binary.BigEndian.Uint64(src) v.Lo = binary.BigEndian.Uint64(src[8:]) + case *rhp.SettingsID: + *v = rhp.SettingsID(src) case types.DecoderFrom: dec := types.NewBufDecoder(src) v.DecodeFrom(dec) return dec.Err() + case *[]types.Hash256: + dec := types.NewBufDecoder(src) + types.DecodeSlice(dec, v) + return dec.Err() + case *[]chain.NetAddress: + dec := types.NewBufDecoder(src) + types.DecodeSlice(dec, v) + return dec.Err() case *uint64: - *v = binary.LittleEndian.Uint64(src) + *v = binary.BigEndian.Uint64(src) default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } @@ -71,6 +97,8 @@ func (d *decodable) Scan(src any) error { *v = uint64(src) case *time.Time: *v = time.Unix(src, 0).UTC() + case *time.Duration: + *v = time.Duration(src) default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } @@ -84,45 +112,20 @@ func decode(obj any) sql.Scanner { return &decodable{obj} } -type decodableSlice[T any] struct { - v *[]T -} - -func (d *decodableSlice[T]) Scan(src any) error { - switch src := src.(type) { - case []byte: - dec := types.NewBufDecoder(src) - s := make([]T, dec.ReadPrefix()) - for i := range s { - dv, ok := any(&s[i]).(types.DecoderFrom) - if !ok { - panic(fmt.Errorf("cannot decode %T", s[i])) - } - dv.DecodeFrom(dec) - } - if err := dec.Err(); err != nil { - return err - } - *d.v = s - return nil - default: - return fmt.Errorf("cannot scan %T to []byte", src) - } +type nullDecodable struct { + v any } -func decodeSlice[T any](v *[]T) sql.Scanner { - return &decodableSlice[T]{v: v} +func decodeNull(obj any) sql.Scanner { + return &nullDecodable{obj} } -func encodeSlice[T types.EncoderTo](v []T) []byte { - var buf bytes.Buffer - enc := types.NewEncoder(&buf) - enc.WritePrefix(len(v)) - for _, e := range v { - e.EncodeTo(enc) - } - if err := enc.Flush(); err != nil { - panic(err) +// Scan implements the sql.Scanner interface. +func (d *nullDecodable) Scan(src any) error { + if src == nil { + return nil } - return buf.Bytes() + + dd := decode(d.v) + return dd.Scan(src) } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go new file mode 100644 index 0000000..637b440 --- /dev/null +++ b/persist/sqlite/events.go @@ -0,0 +1,282 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/wallet" + "go.sia.tech/explored/explorer" +) + +// Events returns the events with the given event IDs. If an event is not found, +// it is skipped. +func (s *Store) Events(eventIDs []types.Hash256) (events []explorer.Event, err error) { + err = s.transaction(func(tx *txn) error { + // sqlite doesn't have easy support for IN clauses, use a statement since + // the number of event IDs is likely to be small instead of dynamically + // building the query + const query = ` +WITH last_chain_index (height) AS ( + SELECT MAX(height) FROM blocks +) +SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + b.height, + b.id, + CASE + WHEN last_chain_index.height < b.height THEN 0 + ELSE last_chain_index.height - b.height + END AS confirmations, + ev.event_type +FROM events ev +INNER JOIN blocks b ON (ev.block_id = b.id) +CROSS JOIN last_chain_index +WHERE ev.event_id = $1` + + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]explorer.Event, 0, len(eventIDs)) + for _, id := range eventIDs { + event, _, err := scanEvent(tx, stmt.QueryRow(encode(id))) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return fmt.Errorf("failed to query transaction %q: %w", id, err) + } + events = append(events, event) + } + return nil + }) + return +} + +func scanEvent(tx *txn, s scanner) (ev explorer.Event, eventID int64, err error) { + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Confirmations, &ev.Type) + if err != nil { + return + } + + switch ev.Type { + case wallet.EventTypeV1Transaction: + var txnID int64 + err = tx.QueryRow(`SELECT transaction_id FROM v1_transaction_events WHERE event_id = ?`, eventID).Scan(&txnID) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to fetch v1 transaction ID: %w", err) + } + txns, err := getTransactions(tx, []types.TransactionID{types.TransactionID(ev.ID)}) + if err != nil || len(txns) == 0 { + return explorer.Event{}, 0, fmt.Errorf("failed to fetch v1 transaction: %w", err) + } + ev.Data = explorer.EventV1Transaction{ + Transaction: txns[0], + } + case wallet.EventTypeV2Transaction: + txns, err := getV2Transactions(tx, []types.TransactionID{types.TransactionID(ev.ID)}) + if err != nil || len(txns) == 0 { + return explorer.Event{}, 0, fmt.Errorf("failed to fetch v2 transaction: %w", err) + } + ev.Data = explorer.EventV2Transaction(txns[0]) + case wallet.EventTypeV1ContractResolution: + var resolution explorer.EventV1ContractResolution + var spentIndex types.ChainIndex + fce, sce := &resolution.Parent, &resolution.SiacoinElement + err := tx.QueryRow(`SELECT sce.output_id, sce.leaf_index, sce.source, sce.spent_index, sce.maturity_height, sce.address, sce.value, fce.contract_id, fce.filesize, fce.file_merkle_root, fce.window_start, fce.window_end, fce.payout, fce.unlock_hash, fce.revision_number, ev.missed + FROM v1_contract_resolution_events ev + JOIN siacoin_elements sce ON ev.output_id = sce.id + JOIN file_contract_elements fce ON ev.parent_id = fce.id + WHERE ev.event_id = ?`, eventID).Scan(decode(&sce.ID), decode(&sce.StateElement.LeafIndex), &sce.Source, decodeNull(&spentIndex), &sce.MaturityHeight, decode(&sce.SiacoinOutput.Address), decode(&sce.SiacoinOutput.Value), decode(&fce.ID), decode(&fce.Filesize), decode(&fce.FileMerkleRoot), decode(&fce.WindowStart), decode(&fce.WindowEnd), decode(&fce.Payout), decode(&fce.UnlockHash), decode(&fce.RevisionNumber), &resolution.Missed) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to retrieve v1 resolution event: %w", err) + } + if spentIndex != (types.ChainIndex{}) { + sce.SpentIndex = &spentIndex + } + ev.Data = resolution + case wallet.EventTypeV2ContractResolution: + var resolution explorer.EventV2ContractResolution + var parentContractID types.FileContractID + var resolutionTransactionID types.TransactionID + var spentIndex types.ChainIndex + sce := &resolution.SiacoinElement + err := tx.QueryRow(`SELECT sce.output_id, sce.leaf_index, sce.source, sce.spent_index, sce.maturity_height, sce.address, sce.value, rev.contract_id, rev.resolution_transaction_id, ev.missed + FROM v2_contract_resolution_events ev + JOIN siacoin_elements sce ON ev.output_id = sce.id + JOIN v2_file_contract_elements fce ON ev.parent_id = fce.id + JOIN v2_last_contract_revision rev ON fce.contract_id = rev.contract_id + WHERE ev.event_id = ?`, eventID).Scan(decode(&sce.ID), decode(&sce.StateElement.LeafIndex), &sce.Source, decodeNull(&spentIndex), &sce.MaturityHeight, decode(&sce.SiacoinOutput.Address), decode(&sce.SiacoinOutput.Value), decode(&parentContractID), decode(&resolutionTransactionID), &resolution.Missed) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to retrieve v2 resolution event: %w", err) + } + if spentIndex != (types.ChainIndex{}) { + sce.SpentIndex = &spentIndex + } + + resolutionTxns, err := getV2Transactions(tx, []types.TransactionID{resolutionTransactionID}) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to get transaction with v2 resolution: %w", err) + } else if len(resolutionTxns) == 0 { + return explorer.Event{}, 0, fmt.Errorf("v2 resolution transaction not found") + } + txn := resolutionTxns[0] + + found := false + for _, fcr := range txn.FileContractResolutions { + if fcr.Parent.ID == parentContractID { + found = true + resolution.Resolution = fcr + break + } + } + if !found { + return explorer.Event{}, 0, fmt.Errorf("failed to find resolution in v2 resolution transaction") + } + + ev.Data = resolution + case wallet.EventTypeSiafundClaim, wallet.EventTypeMinerPayout, wallet.EventTypeFoundationSubsidy: + var payout explorer.EventPayout + payout.SiacoinElement, err = scanSiacoinOutput(tx.QueryRow(`SELECT sce.output_id, sce.leaf_index, sce.source, sce.spent_index, sce.maturity_height, sce.address, sce.value + FROM payout_events ev + JOIN siacoin_elements sce ON ev.output_id = sce.id + WHERE ev.event_id = ?`, eventID)) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to retrieve payout event: %w", err) + } + ev.Data = payout + default: + return explorer.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) + } + + return +} + +// UnconfirmedEvents annotates a list of unconfirmed transactions. +func (s *Store) UnconfirmedEvents(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (events []explorer.Event, err error) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v explorer.EventData, relevant []types.Address) { + // dedup relevant addresses + seen := make(map[types.Address]bool) + unique := relevant[:0] + for _, addr := range relevant { + if !seen[addr] { + unique = append(unique, addr) + seen[addr] = true + } + } + + events = append(events, explorer.Event{ + ID: id, + Timestamp: timestamp, + Index: index, + MaturityHeight: maturityHeight, + Relevant: unique, + Type: eventType, + Data: v, + }) + } + + var scIDs []types.SiacoinOutputID + for _, txn := range v1 { + for _, sci := range txn.SiacoinInputs { + scIDs = append(scIDs, sci.ParentID) + } + } + sces, err := s.SiacoinElements(scIDs) + if err != nil { + return nil, fmt.Errorf("failed to retrieve sces: %w", err) + } + sceCache := make(map[types.SiacoinOutputID]explorer.SiacoinOutput) + for _, sce := range sces { + sceCache[sce.ID] = sce + } + + var sfIDs []types.SiafundOutputID + for _, txn := range v1 { + for _, sfi := range txn.SiafundInputs { + sfIDs = append(sfIDs, sfi.ParentID) + } + } + sfes, err := s.SiafundElements(sfIDs) + if err != nil { + return nil, fmt.Errorf("failed to retrieve sfes: %w", err) + } + + sfeCache := make(map[types.SiafundOutputID]explorer.SiafundOutput) + for _, sfe := range sfes { + sfeCache[sfe.ID] = sfe + } + + for _, txn := range v1 { + id := txn.ID() + evTxn := explorer.CoreToExplorerV1Transaction(txn) + for i := range evTxn.SiacoinInputs { + sci := &evTxn.SiacoinInputs[i] + sce, ok := sceCache[sci.ParentID] + if !ok { + // We could have an ephemeral output, which SiacoinElements + // won't return because it hasn't been in a block yet. In + // which case this is not erroneous, and we should just leave + // these details unfilled. + continue + } + sci.Address = sce.SiacoinElement.SiacoinOutput.Address + sci.Value = sce.SiacoinElement.SiacoinOutput.Value + } + for i := range evTxn.SiafundInputs { + sfi := &evTxn.SiafundInputs[i] + sfe, ok := sfeCache[sfi.ParentID] + if !ok { + // We could have an ephemeral output, which SiacoinElements + // won't return because it hasn't been in a block yet. In + // which case this is not erroneous, and we should just leave + // these details unfilled. + continue + } + sfi.Address = sfe.SiafundElement.SiafundOutput.Address + sfi.Value = sfe.SiafundElement.SiafundOutput.Value + } + for i := range evTxn.FileContracts { + fc := &evTxn.FileContracts[i] + fc.ConfirmationIndex = index + fc.ConfirmationTransactionID = id + } + for i := range evTxn.FileContractRevisions { + fcr := &evTxn.FileContractRevisions[i] + fcr.ExtendedFileContract.ConfirmationIndex = index + fcr.ExtendedFileContract.ConfirmationTransactionID = id + } + relevant := explorer.RelevantAddressesV1(txn) + ev := explorer.EventV1Transaction{Transaction: evTxn} + addEvent(types.Hash256(txn.ID()), index.Height, wallet.EventTypeV1Transaction, ev, relevant) // transaction maturity height is the current block height + } + + // handle v2 transactions + for _, txn := range v2 { + id := txn.ID() + evTxn := explorer.CoreToExplorerV2Transaction(txn) + for i := range evTxn.FileContracts { + fc := &evTxn.FileContracts[i] + fc.ConfirmationIndex = index + fc.ConfirmationTransactionID = id + } + for i := range evTxn.FileContractRevisions { + fcr := &evTxn.FileContractRevisions[i] + fcr.Revision.ConfirmationIndex = index + fcr.Revision.ConfirmationTransactionID = id + } + + relevant := explorer.RelevantAddressesV2(txn) + ev := explorer.EventV2Transaction(evTxn) + addEvent(types.Hash256(txn.ID()), index.Height, wallet.EventTypeV2Transaction, ev, relevant) // transaction maturity height is the current block height + } + + return +} diff --git a/persist/sqlite/hosts.go b/persist/sqlite/hosts.go new file mode 100644 index 0000000..76aedc3 --- /dev/null +++ b/persist/sqlite/hosts.go @@ -0,0 +1,266 @@ +package sqlite + +import ( + "fmt" + "strings" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/explorer" +) + +// HostsForScanning returns hosts ordered by their time to next scan. Hosts +// which are repeatedly offline will face an exponentially growing next scan +// time to avoid wasting resources. +// Note that only the PublicKey, V2, NetAddress, V2NetAddresses, +// FailedInteractionsStreak fields are populated. +func (s *Store) HostsForScanning(minLastAnnouncement time.Time, limit uint64) (result []explorer.UnscannedHost, err error) { + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT public_key, v2, net_address, failed_interactions_streak FROM host_info WHERE next_scan <= ? AND last_announcement >= ? ORDER BY next_scan ASC LIMIT ?`, encode(types.CurrentTimestamp()), encode(minLastAnnouncement), limit) + if err != nil { + return err + } + defer rows.Close() + + v2AddrStmt, err := tx.Prepare(`SELECT protocol,address FROM host_info_v2_netaddresses WHERE public_key = ? ORDER BY netaddress_order`) + if err != nil { + return err + } + defer v2AddrStmt.Close() + + for rows.Next() { + var host explorer.UnscannedHost + if err := rows.Scan(decode(&host.PublicKey), &host.V2, &host.NetAddress, &host.FailedInteractionsStreak); err != nil { + return err + } + + if host.V2 { + err := func() error { + v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey)) + if err != nil { + return err + } + defer v2AddrRows.Close() + for v2AddrRows.Next() { + var netAddr chain.NetAddress + if err := v2AddrRows.Scan(&netAddr.Protocol, &netAddr.Address); err != nil { + return err + } + host.V2NetAddresses = append(host.V2NetAddresses, netAddr) + } + return nil + }() + if err != nil { + return err + } + } + result = append(result, host) + } + return nil + }) + return +} + +// QueryHosts returns the hosts matching the query parameters in the order +// specified by dir. +func (st *Store) QueryHosts(params explorer.HostQuery, sortBy explorer.HostSortColumn, dir explorer.HostSortDir, offset, limit uint64) (result []explorer.Host, err error) { + err = st.transaction(func(tx *txn) error { + var args []any + var filters []string + + if params.V2 != nil { + if *params.V2 { + filters = append(filters, `v2 = 1`) + } else { + filters = append(filters, `v2 = 0`) + } + } + + if len(params.PublicKeys) > 0 { + filter := `public_key IN (` + queryPlaceHolders(len(params.PublicKeys)) + `)` + for _, pk := range params.PublicKeys { + args = append(args, encode(pk)) + } + filters = append(filters, filter) + } + + if len(params.NetAddresses) > 0 { + var addrFilters []string + if params.V2 == nil || !*params.V2 { + for _, netAddress := range params.NetAddresses { + args = append(args, any(netAddress)) + } + addrFilters = append(addrFilters, `net_address IN (`+queryPlaceHolders(len(params.NetAddresses))+`)`) + } + if params.V2 == nil || *params.V2 { + netAddresses := make([]any, 0, len(params.NetAddresses)) + for _, netAddress := range params.NetAddresses { + netAddresses = append(netAddresses, any(netAddress)) + } + rows, err := tx.Query(`SELECT public_key FROM host_info_v2_netaddresses WHERE address IN (`+queryPlaceHolders(len(params.NetAddresses))+`)`, netAddresses...) + if err != nil { + return fmt.Errorf("failed to get query public keys for given net addresses: %w", err) + } + defer rows.Close() + + var pks []any + for rows.Next() { + var pk types.PublicKey + if err := rows.Scan(decode(&pk)); err != nil { + return fmt.Errorf("failed to scan public key: %w", err) + } + pks = append(pks, encode(pk)) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("error retrieving public keys for given net addresses: %w", err) + } + + args = append(args, pks...) + addrFilters = append(addrFilters, `public_key IN (`+queryPlaceHolders(len(pks))+`)`) + } + filters = append(filters, `(`+strings.Join(addrFilters, ` OR `)+`)`) + } + + const uptimeValue = `(successful_interactions * 1.0 / MAX(1, total_scans))` + if params.MinUptime != nil { + filters = append(filters, uptimeValue+` >= ?`) + args = append(args, *params.MinUptime/100.0) + } + if params.MinDuration != nil { + filters = append(filters, `CASE WHEN v2=1 THEN v2_settings_max_contract_duration >= ? ELSE settings_max_duration >= ? END`) + args = append(args, encode(*params.MinDuration), encode(*params.MinDuration)) + } + if params.MaxStoragePrice != nil { + filters = append(filters, `CASE WHEN v2=1 THEN v2_prices_storage_price <= ? ELSE settings_storage_price <= ? END`) + args = append(args, encode(*params.MaxStoragePrice), encode(*params.MaxStoragePrice)) + } + if params.MaxContractPrice != nil { + filters = append(filters, `CASE WHEN v2=1 THEN v2_prices_contract_price <= ? ELSE settings_contract_price <= ? END`) + args = append(args, encode(*params.MaxContractPrice), encode(*params.MaxContractPrice)) + } + if params.MaxUploadPrice != nil { + filters = append(filters, `CASE WHEN v2=1 THEN v2_prices_ingress_price <= ? ELSE settings_upload_bandwidth_price <= ? END`) + args = append(args, encode(*params.MaxUploadPrice), encode(*params.MaxUploadPrice)) + } + if params.MaxDownloadPrice != nil { + filters = append(filters, `CASE WHEN v2=1 THEN v2_prices_egress_price <= ? ELSE settings_download_bandwidth_price <= ? END`) + args = append(args, encode(*params.MaxDownloadPrice), encode(*params.MaxDownloadPrice)) + } + if params.MaxBaseRPCPrice != nil { + filters = append(filters, `settings_base_rpc_price <= ?`) + args = append(args, encode(*params.MaxBaseRPCPrice)) + } + if params.MaxSectorAccessPrice != nil { + filters = append(filters, `settings_sector_access_price <= ?`) + args = append(args, encode(*params.MaxSectorAccessPrice)) + } + if params.AcceptContracts != nil { + v := 0 + if *params.AcceptContracts { + v = 1 + } + filters = append(filters, fmt.Sprintf(`CASE WHEN v2=1 THEN v2_settings_accepting_contracts = %d ELSE settings_accepting_contracts = %d END`, v, v)) + } + if params.Online != nil { + v := 0 + if *params.Online { + v = 1 + } + filters = append(filters, fmt.Sprintf(`last_scan_successful = %d`, v)) + } + args = append(args, limit, offset) + + var sortColumn string + switch sortBy { + case explorer.HostSortDateCreated: + sortColumn = `known_since` + case explorer.HostSortNetAddress: + sortColumn = `net_address` + case explorer.HostSortPublicKey: + sortColumn = `public_key` + case explorer.HostSortUptime: + sortColumn = uptimeValue + case explorer.HostSortAcceptingContracts: + sortColumn = `CASE WHEN v2=1 THEN v2_settings_accepting_contracts ELSE settings_accepting_contracts END` + case explorer.HostSortStoragePrice: + sortColumn = `CASE WHEN v2=1 THEN v2_prices_storage_price ELSE settings_storage_price END` + case explorer.HostSortContractPrice: + sortColumn = `CASE WHEN v2=1 THEN v2_prices_contract_price ELSE settings_contract_price END` + case explorer.HostSortDownloadPrice: + sortColumn = `CASE WHEN v2=1 THEN v2_prices_egress_price ELSE settings_download_bandwidth_price END` + case explorer.HostSortUploadPrice: + sortColumn = `CASE WHEN v2=1 THEN v2_prices_ingress_price ELSE settings_upload_bandwidth_price END` + case explorer.HostSortUsedStorage: + sortColumn = `CASE WHEN v2=1 THEN v2_settings_used_storage ELSE settings_used_storage END` + case explorer.HostSortTotalStorage: + sortColumn = `CASE WHEN v2=1 THEN v2_settings_total_storage ELSE settings_total_storage END` + default: + return fmt.Errorf("%w: %s", explorer.ErrNoSortColumn, sortBy) + } + + var whereClause string + if len(filters) > 0 { + whereClause = "WHERE " + strings.Join(filters, " AND ") + } + query := fmt.Sprintf(` + SELECT public_key,v2,net_address,country_code,latitude,longitude,known_since,last_scan,last_scan_successful,last_scan_error,last_announcement,next_scan,total_scans,successful_interactions,failed_interactions_streak,settings_accepting_contracts,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_net_address,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_address,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,settings_version,settings_release,settings_sia_mux_port,price_table_uid,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,v2_settings_protocol_version,v2_settings_release,v2_settings_wallet_address,v2_settings_accepting_contracts,v2_settings_max_collateral,v2_settings_max_contract_duration,v2_settings_remaining_storage,v2_settings_total_storage,v2_prices_contract_price,v2_prices_collateral_price,v2_prices_storage_price,v2_prices_ingress_price,v2_prices_egress_price,v2_prices_free_sector_price,v2_prices_tip_height,v2_prices_valid_until,v2_prices_signature FROM host_info + %s + ORDER BY (%s) %s + LIMIT ? OFFSET ?`, + whereClause, sortColumn, dir, + ) + + v2AddrStmt, err := tx.Prepare(`SELECT protocol,address FROM host_info_v2_netaddresses WHERE public_key = ? ORDER BY netaddress_order`) + if err != nil { + return err + } + defer v2AddrStmt.Close() + + rows, err := tx.Query(query, args...) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + if err := func() error { + var host explorer.Host + var protocolVersion []uint8 + var lastScanError string + + s, p := &host.Settings, &host.PriceTable + sV2, pV2 := &host.V2Settings, &host.V2Settings.Prices + if err := rows.Scan(decode(&host.PublicKey), &host.V2, &host.NetAddress, &host.Location.CountryCode, &host.Location.Latitude, &host.Location.Longitude, decode(&host.KnownSince), decode(&host.LastScan), &host.LastScanSuccessful, &lastScanError, decode(&host.LastAnnouncement), decode(&host.NextScan), &host.TotalScans, &host.SuccessfulInteractions, &host.FailedInteractions, &s.AcceptingContracts, decode(&s.MaxDownloadBatchSize), decode(&s.MaxDuration), decode(&s.MaxReviseBatchSize), &s.NetAddress, decode(&s.RemainingStorage), decode(&s.SectorSize), decode(&s.TotalStorage), decode(&s.Address), decode(&s.WindowSize), decode(&s.Collateral), decode(&s.MaxCollateral), decode(&s.BaseRPCPrice), decode(&s.ContractPrice), decode(&s.DownloadBandwidthPrice), decode(&s.SectorAccessPrice), decode(&s.StoragePrice), decode(&s.UploadBandwidthPrice), &s.EphemeralAccountExpiry, decode(&s.MaxEphemeralAccountBalance), decode(&s.RevisionNumber), &s.Version, &s.Release, &s.SiaMuxPort, decode(&p.UID), &p.Validity, decode(&p.HostBlockHeight), decode(&p.UpdatePriceTableCost), decode(&p.AccountBalanceCost), decode(&p.FundAccountCost), decode(&p.LatestRevisionCost), decode(&p.SubscriptionMemoryCost), decode(&p.SubscriptionNotificationCost), decode(&p.InitBaseCost), decode(&p.MemoryTimeCost), decode(&p.DownloadBandwidthCost), decode(&p.UploadBandwidthCost), decode(&p.DropSectorsBaseCost), decode(&p.DropSectorsUnitCost), decode(&p.HasSectorBaseCost), decode(&p.ReadBaseCost), decode(&p.ReadLengthCost), decode(&p.RenewContractCost), decode(&p.RevisionBaseCost), decode(&p.SwapSectorBaseCost), decode(&p.WriteBaseCost), decode(&p.WriteLengthCost), decode(&p.WriteStoreCost), decode(&p.TxnFeeMinRecommended), decode(&p.TxnFeeMaxRecommended), decode(&p.ContractPrice), decode(&p.CollateralCost), decode(&p.MaxCollateral), decode(&p.MaxDuration), decode(&p.WindowSize), decode(&p.RegistryEntriesLeft), decode(&p.RegistryEntriesTotal), &protocolVersion, &sV2.Release, decode(&sV2.WalletAddress), &sV2.AcceptingContracts, decode(&sV2.MaxCollateral), decode(&sV2.MaxContractDuration), decode(&sV2.RemainingStorage), decode(&sV2.TotalStorage), decode(&pV2.ContractPrice), decode(&pV2.Collateral), decode(&pV2.StoragePrice), decode(&pV2.IngressPrice), decode(&pV2.EgressPrice), decode(&pV2.FreeSectorPrice), decode(&pV2.TipHeight), decode(&pV2.ValidUntil), decode(&pV2.Signature)); err != nil { + return err + } + sV2.ProtocolVersion = [3]uint8(protocolVersion) + if lastScanError != "" { + host.LastScanError = &lastScanError + } + + if host.V2 { + v2AddrRows, err := v2AddrStmt.Query(encode(host.PublicKey)) + if err != nil { + return err + } + defer v2AddrRows.Close() + for v2AddrRows.Next() { + var netAddr chain.NetAddress + if err := v2AddrRows.Scan(&netAddr.Protocol, &netAddr.Address); err != nil { + return err + } + host.V2NetAddresses = append(host.V2NetAddresses, netAddr) + } + } + + result = append(result, host) + return nil + }(); err != nil { + return err + } + } + return rows.Err() + }) + return +} diff --git a/persist/sqlite/hosts_test.go b/persist/sqlite/hosts_test.go new file mode 100644 index 0000000..ac901f7 --- /dev/null +++ b/persist/sqlite/hosts_test.go @@ -0,0 +1,630 @@ +package sqlite + +import ( + "math" + "path/filepath" + "testing" + "time" + + "go.sia.tech/coreutils/rhp/v4/siamux" + + rhpv2 "go.sia.tech/core/rhp/v2" + rhpv4 "go.sia.tech/core/rhp/v4" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/geoip" + "go.uber.org/zap/zaptest" +) + +func TestQueryHosts(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + + db, err := OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + const ( + netAddr1 = "host1.com:9982" + netAddr2 = "host2.com:9982" + netAddr3 = "host3.com:9982" + netAddr4 = "host4.com:9982" + ) + + pk1 := types.GeneratePrivateKey().PublicKey() + pk2 := types.GeneratePrivateKey().PublicKey() + pk3 := types.GeneratePrivateKey().PublicKey() + pk4 := types.GeneratePrivateKey().PublicKey() + + tm := time.Now() + hosts := []explorer.Host{ + { + PublicKey: pk1, + V2: false, + NetAddress: netAddr1, + Location: geoip.Location{ + CountryCode: "US", + Latitude: 0.01, + Longitude: -0.02, + }, + KnownSince: tm.Add(-4 * time.Hour), + LastScan: tm, + LastScanSuccessful: true, + SuccessfulInteractions: 75, + TotalScans: 100, + Settings: rhpv2.HostSettings{ + AcceptingContracts: true, + MaxDuration: 1000, + StoragePrice: types.Siacoins(1), + ContractPrice: types.Siacoins(2), + DownloadBandwidthPrice: types.Siacoins(3), + UploadBandwidthPrice: types.Siacoins(4), + BaseRPCPrice: types.Siacoins(5), + SectorAccessPrice: types.Siacoins(6), + TotalStorage: 2000, + RemainingStorage: 1000, + }, + }, + { + PublicKey: pk2, + V2: false, + NetAddress: netAddr2, + Location: geoip.Location{ + CountryCode: "US", + Latitude: 0.01, + Longitude: -0.02, + }, + KnownSince: tm.Add(-3 * time.Hour), + LastScan: tm, + LastScanSuccessful: true, + SuccessfulInteractions: 90, + TotalScans: 100, + Settings: rhpv2.HostSettings{ + AcceptingContracts: false, + MaxDuration: 10000, + StoragePrice: types.Siacoins(60), + ContractPrice: types.Siacoins(50), + DownloadBandwidthPrice: types.Siacoins(40), + UploadBandwidthPrice: types.Siacoins(30), + BaseRPCPrice: types.Siacoins(20), + SectorAccessPrice: types.Siacoins(10), + TotalStorage: 1000, + RemainingStorage: 500, + }, + }, + { + PublicKey: pk3, + V2: true, + V2NetAddresses: []chain.NetAddress{{Protocol: siamux.Protocol, Address: netAddr3}}, + Location: geoip.Location{ + CountryCode: "DE", + Latitude: 0.05, + Longitude: -0.10, + }, + KnownSince: tm.Add(-2 * time.Hour), + LastScan: tm, + LastScanSuccessful: false, + SuccessfulInteractions: 95, + TotalScans: 100, + V2Settings: rhpv4.HostSettings{ + AcceptingContracts: true, + MaxContractDuration: 1000, + TotalStorage: 2000, + RemainingStorage: 1000, + Prices: rhpv4.HostPrices{ + StoragePrice: types.Siacoins(10), + ContractPrice: types.Siacoins(20), + EgressPrice: types.Siacoins(30), + IngressPrice: types.Siacoins(40), + }, + }, + }, + { + PublicKey: pk4, + V2: true, + V2NetAddresses: []chain.NetAddress{{Protocol: siamux.Protocol, Address: netAddr4}}, + Location: geoip.Location{ + CountryCode: "DE", + Latitude: 0.05, + Longitude: -0.10, + }, + KnownSince: tm.Add(-1 * time.Hour), + LastScan: tm, + LastScanSuccessful: false, + SuccessfulInteractions: 75, + TotalScans: 100, + V2Settings: rhpv4.HostSettings{ + AcceptingContracts: false, + MaxContractDuration: 10000, + TotalStorage: 1000, + RemainingStorage: 500, + Prices: rhpv4.HostPrices{ + StoragePrice: types.Siacoins(1), + ContractPrice: types.Siacoins(2), + EgressPrice: types.Siacoins(3), + IngressPrice: types.Siacoins(4), + }, + }, + }, + } + + // Add hosts to database + if err := db.transaction(func(tx *txn) error { + return addHosts(tx, hosts) + }); err != nil { + t.Fatal(err) + } + + uint64Ptr := func(x uint64) *uint64 { + return &x + } + trueBool, falseBool := true, false + tests := []struct { + name string + query explorer.HostQuery + sortBy explorer.HostSortColumn + dir explorer.HostSortDir + offset uint64 + want []types.PublicKey // Expected host public keys in order + }{ + { + name: "all hosts", + query: explorer.HostQuery{}, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2, pk3, pk4}, + }, + { + name: "all hosts pubkey", + query: explorer.HostQuery{PublicKeys: []types.PublicKey{pk1, pk2, pk3, pk4}}, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2, pk3, pk4}, + }, + { + name: "all hosts accepting contracts", + query: explorer.HostQuery{AcceptContracts: &trueBool}, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk3}, + }, + { + name: "all hosts pubkey accepting contracts", + query: explorer.HostQuery{AcceptContracts: &trueBool, PublicKeys: []types.PublicKey{pk1, pk2, pk3, pk4}}, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk3}, + }, + + { + name: "v1 asc AcceptingContracts", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk2, pk1}, + }, + { + name: "v2 asc AcceptingContracts", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + { + name: "v1 desc AcceptingContracts", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v1 desc AcceptingContracts offset", + query: explorer.HostQuery{ + V2: &falseBool, + }, + offset: 1, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk2}, + }, + { + name: "v2 desc AcceptingContracts", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk3, pk4}, + }, + { + name: "v2 desc AcceptingContracts offset", + query: explorer.HostQuery{ + V2: &trueBool, + }, + offset: 1, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk4}, + }, + + { + name: "v1 asc DateCreated", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc DateCreated", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk3, pk4}, + }, + + // host1.com:9982 < host2.com:9982 + { + name: "v1 asc NetAddress", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortNetAddress, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + + { + name: "v1 asc Uptime", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortUptime, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc Uptime", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortUptime, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + + { + name: "v1 asc StoragePrice", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortStoragePrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc StoragePrice", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortStoragePrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + { + name: "v1 asc ContractPrice", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortContractPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc ContractPrice", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortContractPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + { + name: "v1 asc DownloadPrice", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortDownloadPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc DownloadPrice", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortDownloadPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + { + name: "v1 asc UploadPrice", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortUploadPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v2 asc UploadPrice", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortUploadPrice, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + + { + name: "v1 asc TotalStorage", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortTotalStorage, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk2, pk1}, + }, + { + name: "v2 asc TotalStorage", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortTotalStorage, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + + { + name: "v1 asc UsedStorage", + query: explorer.HostQuery{ + V2: &falseBool, + }, + sortBy: explorer.HostSortUsedStorage, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk2, pk1}, + }, + { + name: "v2 asc UsedStorage", + query: explorer.HostQuery{ + V2: &trueBool, + }, + sortBy: explorer.HostSortUsedStorage, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk4, pk3}, + }, + { + name: "v2 asc UsedStorage offset 1", + query: explorer.HostQuery{ + V2: &trueBool, + }, + offset: 1, + sortBy: explorer.HostSortUsedStorage, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk3}, + }, + { + name: "v2 desc UsedStorage offset 1", + query: explorer.HostQuery{ + V2: &trueBool, + }, + offset: 1, + sortBy: explorer.HostSortUsedStorage, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk4}, + }, + { + name: "v2 desc UsedStorage offset 2", + query: explorer.HostQuery{ + V2: &trueBool, + }, + offset: 2, + sortBy: explorer.HostSortUsedStorage, + dir: explorer.HostSortDesc, + want: []types.PublicKey{}, + }, + + { + name: "v1 min duration 1000", + query: explorer.HostQuery{ + V2: &falseBool, + MinDuration: uint64Ptr(1000), + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk1, pk2}, + }, + { + name: "v1 min duration 5000", + query: explorer.HostQuery{ + V2: &falseBool, + MinDuration: uint64Ptr(5000), + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk2}, + }, + { + name: "v2 min duration 1000", + query: explorer.HostQuery{ + V2: &trueBool, + MinDuration: uint64Ptr(1000), + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk3, pk4}, + }, + { + name: "v2 min duration 5000", + query: explorer.HostQuery{ + V2: &trueBool, + MinDuration: uint64Ptr(5000), + }, + sortBy: explorer.HostSortAcceptingContracts, + dir: explorer.HostSortDesc, + want: []types.PublicKey{pk4}, + }, + + { + name: "net address 1 2 3 4", + query: explorer.HostQuery{ + NetAddresses: []string{netAddr1, netAddr2, netAddr3, netAddr4}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2, pk3, pk4}, + }, + { + name: "net address 1 2 3", + query: explorer.HostQuery{ + NetAddresses: []string{netAddr1, netAddr2, netAddr3}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk2, pk3}, + }, + { + name: "net address pubkey 1 2 3", + query: explorer.HostQuery{ + PublicKeys: []types.PublicKey{pk1, pk2, pk3}, + NetAddresses: []string{netAddr1, netAddr3}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1, pk3}, + }, + { + name: "net address v2 1 2 3 4", + query: explorer.HostQuery{ + V2: &trueBool, + NetAddresses: []string{netAddr1, netAddr2, netAddr3, netAddr4}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk3, pk4}, + }, + { + name: "net address v2 3", + query: explorer.HostQuery{ + V2: &trueBool, + NetAddresses: []string{netAddr3}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk3}, + }, + { + name: "net address v1 1", + query: explorer.HostQuery{ + V2: &falseBool, + NetAddresses: []string{netAddr1}, + }, + sortBy: explorer.HostSortDateCreated, + dir: explorer.HostSortAsc, + want: []types.PublicKey{pk1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := db.QueryHosts(tt.query, tt.sortBy, tt.dir, tt.offset, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + if len(got) != len(tt.want) { + t.Errorf("got %d results, want %d", len(got), len(tt.want)) + return + } + + for i, want := range tt.want { + if got[i].PublicKey != want { + t.Errorf("%d got %v, want %v", i, got[i].PublicKey, want) + } + } + + if tt.sortBy == explorer.HostSortStoragePrice || + tt.sortBy == explorer.HostSortContractPrice || + tt.sortBy == explorer.HostSortDownloadPrice || + tt.sortBy == explorer.HostSortUploadPrice { + verifyCurrencySort(t, got, tt.sortBy, tt.dir) + } + }) + } +} + +// verifyCurrencySort ensures that currency values are properly sorted numerically +func verifyCurrencySort(t *testing.T, hosts []explorer.Host, sortBy explorer.HostSortColumn, dir explorer.HostSortDir) { + if len(hosts) < 2 { + return + } + + for i := 1; i < len(hosts); i++ { + var prev, curr types.Currency + switch sortBy { + case explorer.HostSortStoragePrice: + if hosts[i].V2 { + prev = hosts[i-1].V2Settings.Prices.StoragePrice + curr = hosts[i].V2Settings.Prices.StoragePrice + } else { + prev = hosts[i-1].Settings.StoragePrice + curr = hosts[i].Settings.StoragePrice + } + case explorer.HostSortContractPrice: + if hosts[i].V2 { + prev = hosts[i-1].V2Settings.Prices.ContractPrice + curr = hosts[i].V2Settings.Prices.ContractPrice + } else { + prev = hosts[i-1].Settings.ContractPrice + curr = hosts[i].Settings.ContractPrice + } + case explorer.HostSortDownloadPrice: + if hosts[i].V2 { + prev = hosts[i-1].V2Settings.Prices.EgressPrice + curr = hosts[i].V2Settings.Prices.EgressPrice + } else { + prev = hosts[i-1].Settings.DownloadBandwidthPrice + curr = hosts[i].Settings.DownloadBandwidthPrice + } + case explorer.HostSortUploadPrice: + if hosts[i].V2 { + prev = hosts[i-1].V2Settings.Prices.IngressPrice + curr = hosts[i].V2Settings.Prices.IngressPrice + } else { + prev = hosts[i-1].Settings.UploadBandwidthPrice + curr = hosts[i].Settings.UploadBandwidthPrice + } + } + + if dir == explorer.HostSortAsc { + if prev.Cmp(curr) > 0 { + t.Errorf("Ascending sort failed: %v > %v", prev, curr) + } + } else { + if prev.Cmp(curr) < 0 { + t.Errorf("Descending sort failed: %v < %v", prev, curr) + } + } + } +} diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index 4d02da9..59df5a5 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -16,15 +16,13 @@ import ( //go:embed init.sql var initDatabase string -func (s *Store) initNewDatabase(target int64) error { - return s.transaction(func(tx *txn) error { - if _, err := tx.Exec(initDatabase); err != nil { - return fmt.Errorf("failed to initialize database: %w", err) - } else if err := setDBVersion(tx, target); err != nil { - return fmt.Errorf("failed to set initial database version: %w", err) - } - return nil - }) +func (s *Store) initNewDatabase(tx *txn, target int64) error { + if _, err := tx.Exec(initDatabase); err != nil { + return fmt.Errorf("failed to initialize database: %w", err) + } else if err := setDBVersion(tx, target); err != nil { + return fmt.Errorf("failed to set initial database version: %w", err) + } + return nil } func (s *Store) upgradeDatabase(current, target int64) error { @@ -68,7 +66,9 @@ func (s *Store) init() error { version := getDBVersion(s.db) switch { case version == 0: - return s.initNewDatabase(target) + return s.transaction(func(tx *txn) error { + return s.initNewDatabase(tx, target) + }) case version < target: return s.upgradeDatabase(version, target) case version > target: diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 0454713..215c72c 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -1,132 +1,168 @@ CREATE TABLE global_settings ( - id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row - db_version INTEGER NOT NULL -- used for migrations + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL -- used for migrations ); CREATE TABLE blocks ( - id BLOB NOT NULL PRIMARY KEY, - height INTEGER NOT NULL, - parent_id BLOB NOT NULL, - nonce BLOB NOT NULL, - timestamp INTEGER NOT NULL + id BLOB NOT NULL PRIMARY KEY, + height INTEGER NOT NULL, + parent_id BLOB NOT NULL, + nonce BLOB NOT NULL, + timestamp INTEGER NOT NULL, + leaf_index BLOB NOT NULL, + + v2_height INTEGER, + v2_commitment BLOB ); - CREATE INDEX blocks_height_index ON blocks(height); +CREATE TABLE network_metrics ( + block_id BLOB PRIMARY KEY REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + + height INTEGER NOT NULL, + difficulty BLOB NOT NULL, + siafund_tax_revenue BLOB NOT NULL, + num_leaves BLOB NOT NULL, + total_hosts INTEGER NOT NULL, + active_contracts INTEGER NOT NULL, + failed_contracts INTEGER NOT NULL, + successful_contracts INTEGER NOT NULL, + storage_utilization INTEGER NOT NULL, + circulating_supply BLOB NOT NULL, + contract_revenue BLOB NOT NULL +); + +CREATE INDEX network_metrics_height_index ON network_metrics(height); + CREATE TABLE address_balance ( - id INTEGER PRIMARY KEY, - address BLOB UNIQUE NOT NULL, - siacoin_balance BLOB NOT NULL, - immature_siacoin_balance BLOB NOT NULL, - siafund_balance BLOB NOT NULL + id INTEGER PRIMARY KEY, + address BLOB UNIQUE NOT NULL, + siacoin_balance BLOB NOT NULL, + immature_siacoin_balance BLOB NOT NULL, + siafund_balance BLOB NOT NULL ); --- CREATE INDEX address_balance_address_index ON address_balance(address); +CREATE INDEX address_balance_address_index ON address_balance(address); CREATE TABLE siacoin_elements ( - id INTEGER PRIMARY KEY, - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + id INTEGER PRIMARY KEY, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, - output_id BLOB UNIQUE NOT NULL, - leaf_index BLOB NOT NULL, + output_id BLOB UNIQUE NOT NULL, + leaf_index BLOB NOT NULL, - spent INTEGER NOT NULL, - source INTEGER NOT NULL, - maturity_height INTEGER NOT NULL, - address BLOB NOT NULL, - value BLOB NOT NULL + spent_index BLOB, + source INTEGER NOT NULL, + maturity_height INTEGER NOT NULL, + address BLOB NOT NULL, + value BLOB NOT NULL ); +CREATE INDEX siacoin_elements_maturity_height_index ON siacoin_elements(maturity_height); CREATE INDEX siacoin_elements_output_id_index ON siacoin_elements(output_id); -CREATE INDEX siacoin_elements_address_spent_index ON siacoin_elements(address, spent); +CREATE INDEX siacoin_elements_address_spent_index ON siacoin_elements(address, spent_index); CREATE TABLE siafund_elements ( - id INTEGER PRIMARY KEY, - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + id INTEGER PRIMARY KEY, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, - output_id BLOB UNIQUE NOT NULL, - leaf_index BLOB NOT NULL, + output_id BLOB UNIQUE NOT NULL, + leaf_index BLOB NOT NULL, - spent INTEGER NOT NULL, - claim_start BLOB NOT NULL, - address BLOB NOT NULL, - value BLOB NOT NULL + spent_index BLOB, + claim_start BLOB NOT NULL, + address BLOB NOT NULL, + value BLOB NOT NULL ); CREATE INDEX siafund_elements_output_id_index ON siafund_elements(output_id); -CREATE INDEX siafund_elements_address_spent_index ON siafund_elements(address, spent); +CREATE INDEX siafund_elements_address_spent_index ON siafund_elements(address, spent_index); CREATE TABLE file_contract_elements ( - id INTEGER PRIMARY KEY, - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + id INTEGER PRIMARY KEY, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + transaction_id BLOB REFERENCES transactions(transaction_id) ON DELETE CASCADE NOT NULL, + + contract_id BLOB NOT NULL, + leaf_index BLOB NOT NULL, + + resolved INTEGER NOT NULL, + valid INTEGER NOT NULL, + + filesize BLOB NOT NULL, + file_merkle_root BLOB NOT NULL, + window_start BLOB NOT NULL, + window_end BLOB NOT NULL, + payout BLOB NOT NULL, + unlock_hash BLOB NOT NULL, + revision_number BLOB NOT NULL, + UNIQUE(contract_id, revision_number) +); +CREATE INDEX file_contract_elements_contract_id_revision_number_index ON file_contract_elements(contract_id, revision_number); - contract_id BLOB NOT NULL, - leaf_index BLOB NOT NULL, +CREATE TABLE last_contract_revision ( + contract_id BLOB PRIMARY KEY NOT NULL, - resolved INTEGER NOT NULL, - valid INTEGER NOT NULL, + ed25519_renter_key BLOB, + ed25519_host_key BLOB, - filesize INTEGER NOT NULL, - file_merkle_root BLOB NOT NULL, - window_start INTEGER NOT NULL, - window_end INTEGER NOT NULL, - payout BLOB NOT NULL, - unlock_hash BLOB NOT NULL, - revision_number INTEGER NOT NULL, - UNIQUE(contract_id, revision_number) -); + confirmation_height BLOB NOT NULL, + confirmation_block_id BLOB NOT NULL REFERENCES blocks(id) ON DELETE CASCADE, + confirmation_transaction_id BLOB NOT NULL REFERENCES transactions(transaction_id), -CREATE INDEX file_contract_elements_contract_id_index ON file_contract_elements(contract_id); + proof_height BLOB, + proof_block_id BLOB, + proof_transaction_id BLOB REFERENCES transactions(transaction_id), -CREATE TABLE last_contract_revision ( - contract_id BLOB PRIMARY KEY NOT NULL, - contract_element_id INTEGER UNIQUE REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL + contract_element_id INTEGER UNIQUE REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL ); CREATE TABLE file_contract_valid_proof_outputs ( - contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, - contract_order INTEGER NOT NULL, - address BLOB NOT NULL, - value BLOB NOT NULL, - UNIQUE(contract_id, contract_order) + contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, + contract_order INTEGER NOT NULL, + id BLOB NOT NULL, + address BLOB NOT NULL, + value BLOB NOT NULL, + UNIQUE(contract_id, contract_order) ); CREATE INDEX file_contract_valid_proof_outputs_contract_id_index ON file_contract_valid_proof_outputs(contract_id); CREATE TABLE file_contract_missed_proof_outputs ( - contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, - contract_order INTEGER NOT NULL, - address BLOB NOT NULL, - value BLOB NOT NULL, - UNIQUE(contract_id, contract_order) + contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, + contract_order INTEGER NOT NULL, + id BLOB NOT NULL, + address BLOB NOT NULL, + value BLOB NOT NULL, + UNIQUE(contract_id, contract_order) ); CREATE INDEX file_contract_missed_proof_outputs_contract_id_index ON file_contract_missed_proof_outputs(contract_id); CREATE TABLE miner_payouts ( - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, - block_order INTEGER NOT NULL, - output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, - UNIQUE(block_id, block_order) + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + block_order INTEGER NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + UNIQUE(block_id, block_order) ); CREATE INDEX miner_payouts_block_id_index ON miner_payouts(block_id); CREATE TABLE transactions ( - id INTEGER PRIMARY KEY, - transaction_id BLOB UNIQUE NOT NULL + id INTEGER PRIMARY KEY, + transaction_id BLOB UNIQUE NOT NULL ); - CREATE INDEX transactions_transaction_id_index ON transactions(transaction_id); CREATE TABLE block_transactions ( - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - block_order INTEGER NOT NULL, - UNIQUE(block_id, block_order) + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + block_order INTEGER NOT NULL, + UNIQUE(block_id, block_order) ); - CREATE INDEX block_transactions_block_id_index ON block_transactions(block_id); +CREATE INDEX block_transactions_transaction_id_index ON block_transactions(transaction_id); +CREATE INDEX block_transactions_transaction_id_block_id ON block_transactions(transaction_id, block_id); CREATE TABLE transaction_addresses ( transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, @@ -137,80 +173,431 @@ CREATE TABLE transaction_addresses ( CREATE INDEX transaction_addresses_transaction_id_index ON transaction_addresses(transaction_id); CREATE TABLE transaction_arbitrary_data ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - data BLOB NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + data BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) ); CREATE INDEX transaction_arbitrary_data_transaction_id_index ON transaction_arbitrary_data(transaction_id); -CREATE TABLE transaction_siacoin_inputs ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - parent_id BLOB NOT NULL, - unlock_conditions BLOB NOT NULL, - UNIQUE(transaction_id, transaction_order) +CREATE TABLE transaction_miner_fees ( + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + fee BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); + +CREATE INDEX transaction_miner_fees_transaction_id_index ON transaction_miner_fees(transaction_id); + +CREATE TABLE transaction_signatures ( + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id BLOB NOT NULL, + public_key_index INTEGER NOT NULL, + timelock INTEGER NOT NULL, + covered_fields BLOB NOT NULL, + signature BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) ); +CREATE INDEX transaction_signatures_transaction_id_index ON transaction_signatures(transaction_id); + +CREATE TABLE transaction_storage_proofs ( + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id BLOB REFERENCES last_contract_revision(contract_id) ON DELETE CASCADE NOT NULL, + leaf BLOB NOT NULL, + proof BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX transaction_storage_proofs_transaction_id_index ON transaction_storage_proofs(transaction_id); +CREATE INDEX transaction_storage_proofs_parent_id_index ON transaction_storage_proofs(parent_id); + +CREATE TABLE transaction_siacoin_inputs ( + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + unlock_conditions BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); CREATE INDEX transaction_siacoin_inputs_transaction_id_index ON transaction_siacoin_inputs(transaction_id); CREATE TABLE transaction_siacoin_outputs ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + UNIQUE(transaction_id, transaction_order) ); - CREATE INDEX transaction_siacoin_outputs_transaction_id_index ON transaction_siacoin_outputs(transaction_id); CREATE TABLE transaction_siafund_inputs ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - parent_id BLOB NOT NULL, - unlock_conditions BLOB NOT NULL, - claim_address BLOB NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id INTEGER REFERENCES siafund_elements(id) ON DELETE CASCADE NOT NULL, + unlock_conditions BLOB NOT NULL, + claim_address BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) ); - CREATE INDEX transaction_siafund_inputs_transaction_id_index ON transaction_siafund_inputs(transaction_id); CREATE TABLE transaction_siafund_outputs ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - output_id INTEGER REFERENCES siafund_elements(id) ON DELETE CASCADE NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + output_id INTEGER REFERENCES siafund_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + UNIQUE(transaction_id, transaction_order) ); - CREATE INDEX transaction_siafund_outputs_transaction_id_index ON transaction_siafund_outputs(transaction_id); CREATE TABLE transaction_file_contracts ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + UNIQUE(transaction_id, transaction_order) ); - CREATE INDEX transaction_file_contracts_transaction_id_index ON transaction_file_contracts(transaction_id); CREATE TABLE transaction_file_contract_revisions ( - transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, - transaction_order INTEGER NOT NULL, - contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, - parent_id BLOB UNIQUE NOT NULL, - unlock_conditions BLOB NOT NULL, - UNIQUE(transaction_id, transaction_order) + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + parent_id BLOB NOT NULL, + unlock_conditions BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) ); - CREATE INDEX transaction_file_contract_revisions_transaction_id_index ON transaction_file_contract_revisions(transaction_id); +CREATE TABLE v2_transactions ( + id INTEGER PRIMARY KEY, + transaction_id BLOB UNIQUE NOT NULL, + + new_foundation_address BLOB, + miner_fee BLOB NOT NULL, + arbitrary_data BLOB +); +CREATE INDEX v2_transactions_transaction_id_index ON v2_transactions(transaction_id); + +CREATE TABLE v2_block_transactions ( + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + block_order INTEGER NOT NULL, + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + UNIQUE(block_id, block_order) +); +CREATE INDEX v2_block_transactions_block_id_index ON v2_block_transactions(block_id); +CREATE INDEX v2_block_transactions_transaction_id_block_id ON v2_block_transactions(transaction_id, block_id); + +CREATE TABLE v2_transaction_siacoin_inputs ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + satisfied_policy BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_siacoin_inputs_transaction_id_index ON v2_transaction_siacoin_inputs(transaction_id); + +CREATE TABLE v2_transaction_siacoin_outputs ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_siacoin_outputs_transaction_id_index ON v2_transaction_siacoin_outputs(transaction_id); + +CREATE TABLE v2_transaction_siafund_inputs ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_id INTEGER REFERENCES siafund_elements(id) ON DELETE CASCADE NOT NULL, + claim_address BLOB NOT NULL, + satisfied_policy BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_siafund_inputs_transaction_id_index ON v2_transaction_siafund_inputs(transaction_id); + +CREATE TABLE v2_transaction_siafund_outputs ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + output_id INTEGER REFERENCES siafund_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_siafund_outputs_transaction_id_index ON v2_transaction_siafund_outputs(transaction_id); + +CREATE TABLE v2_transaction_file_contracts ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + contract_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_file_contracts_transaction_id_index ON v2_transaction_file_contracts(transaction_id); + +CREATE TABLE v2_transaction_file_contract_revisions ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_contract_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + revision_contract_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_file_contract_revisions_transaction_id_index ON v2_transaction_file_contract_revisions(transaction_id); + +CREATE TABLE v2_transaction_file_contract_resolutions ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + parent_contract_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL, -- add an index to all foreign keys + + -- See explorer.V2Resolution for enum values. + resolution_type INTEGER NOT NULL, + + -- V2FileContractRenewal + renewal_new_contract_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE, + renewal_final_renter_output_address BLOB, + renewal_final_renter_output_value BLOB, + renewal_final_host_output_address BLOB, + renewal_final_host_output_value BLOB, + renewal_renter_rollover BLOB, + renewal_host_rollover BLOB, + renewal_renter_signature BLOB, + renewal_host_signature BLOB, + + -- V2StorageProof + storage_proof_proof_index BLOB, + storage_proof_leaf BLOB, + storage_proof_proof BLOB, + + -- V2FileContractExpiration + -- no fields + + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_file_contract_resolutions_transaction_id_index ON v2_transaction_file_contract_resolutions(transaction_id); + +CREATE TABLE v2_transaction_attestations ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + public_key BLOB NOT NULL, + key TEXT NOT NULL, + value BLOB NOT NULL, + signature BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_transaction_attestations_transaction_id_index ON v2_transaction_attestations(transaction_id); + CREATE TABLE state_tree ( - row INTEGER NOT NULL, - column INTEGER NOT NULL, - value BLOB NOT NULL, - PRIMARY KEY(row, column) + row INTEGER NOT NULL, + column INTEGER NOT NULL, + value BLOB NOT NULL, + PRIMARY KEY(row, column) +); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + block_id BLOB NOT NULL REFERENCES blocks(id) ON DELETE CASCADE, + event_id BLOB UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, + date_created INTEGER NOT NULL, + event_type TEXT NOT NULL +); +CREATE INDEX events_block_id_idx ON events (block_id); +CREATE INDEX events_maturity_height_id_idx ON events (maturity_height DESC, id DESC); + +CREATE TABLE event_addresses ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES address_balance (id), + event_maturity_height INTEGER NOT NULL, -- flattened from events to improve query performance + PRIMARY KEY (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_event_maturity_height_event_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); + +CREATE TABLE v1_transaction_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE CASCADE NOT NULL ); +CREATE TABLE v2_transaction_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL +); + +CREATE TABLE payout_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL +); + +CREATE TABLE v1_contract_resolution_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + parent_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + missed INTEGER NOT NULL +); + +CREATE TABLE v2_contract_resolution_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + parent_id INTEGER REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL, + output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL, + missed INTEGER NOT NULL +); + +CREATE TABLE v2_file_contract_elements ( + id INTEGER PRIMARY KEY, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + transaction_id BLOB REFERENCES v2_transactions(transaction_id) ON DELETE CASCADE NOT NULL, + + contract_id BLOB NOT NULL, + leaf_index BLOB NOT NULL, + + capacity BLOB NOT NULL, + filesize BLOB NOT NULL, + file_merkle_root BLOB NOT NULL, + proof_height BLOB NOT NULL, + expiration_height BLOB NOT NULL, + renter_output_address BLOB NOT NULL, + renter_output_value BLOB NOT NULL, + host_output_address BLOB NOT NULL, + host_output_value BLOB NOT NULL, + missed_host_value BLOB NOT NULL, + total_collateral BLOB NOT NULL, + renter_public_key BLOB NOT NULL, + host_public_key BLOB NOT NULL, + revision_number BLOB NOT NULL, + + renter_signature BLOB NOT NULL, + host_signature BLOB NOT NULL, + + UNIQUE(contract_id, revision_number) +); +CREATE INDEX v2_file_contract_elements_contract_id_revision_number_index ON v2_file_contract_elements(contract_id, revision_number); + +CREATE TABLE v2_last_contract_revision ( + contract_id BLOB PRIMARY KEY NOT NULL, + + confirmation_height BLOB NOT NULL, + confirmation_block_id BLOB NOT NULL REFERENCES blocks(id) ON DELETE CASCADE, + confirmation_transaction_id BLOB NOT NULL REFERENCES v2_transactions(transaction_id), + + -- See explorer.V2Resolution for enum values. + resolution_type INTEGER, + resolution_height BLOB, + resolution_block_id BLOB, + resolution_transaction_id BLOB REFERENCES v2_transactions(transaction_id), + renewed_from BLOB, + renewed_to BLOB, + + contract_element_id INTEGER UNIQUE REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL +); + +CREATE TABLE host_info ( + public_key BLOB PRIMARY KEY NOT NULL, + v2 INTEGER NOT NULL, + net_address TEXT NOT NULL, + country_code TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + known_since INTEGER NOT NULL, + last_scan INTEGER NOT NULL, + last_scan_successful INTEGER NOT NULL, + last_scan_error TEXT NOT NULL, + next_scan INTEGER NOT NULL, + last_announcement INTEGER NOT NULL, + total_scans INTEGER NOT NULL, + successful_interactions INTEGER NOT NULL, + failed_interactions INTEGER NOT NULL, + -- number of failed interactions since the last successful interaction + failed_interactions_streak INTEGER NOT NULL, + -- settings + settings_accepting_contracts INTEGER NOT NULL, + settings_max_download_batch_size BLOB NOT NULL, + settings_max_duration BLOB NOT NULL, + settings_max_revise_batch_size BLOB NOT NULL, + settings_net_address TEXT NOT NULL, + settings_remaining_storage BLOB NOT NULL, + settings_sector_size BLOB NOT NULL, + settings_total_storage BLOB NOT NULL, + settings_used_storage BLOB NOT NULL, -- needed so we can sort by this because there's no clean way of subtracting binary encoded uint64s (total and remaining storage) in sqlite + settings_address BLOB NOT NULL, + settings_window_size BLOB NOT NULL, + settings_collateral BLOB NOT NULL, + settings_max_collateral BLOB NOT NULL, + settings_base_rpc_price BLOB NOT NULL, + settings_contract_price BLOB NOT NULL, + settings_download_bandwidth_price BLOB NOT NULL, + settings_sector_access_price BLOB NOT NULL, + settings_storage_price BLOB NOT NULL, + settings_upload_bandwidth_price BLOB NOT NULL, + settings_ephemeral_account_expiry INTEGER NOT NULL, + settings_max_ephemeral_account_balance BLOB NOT NULL, + settings_revision_number BLOB NOT NULL, + settings_version TEXT NOT NULL, + settings_release TEXT NOT NULL, + settings_sia_mux_port TEXT NOT NULL, + -- price table + price_table_uid BLOB NOT NULL, + price_table_validity INTEGER NOT NULL, + price_table_host_block_height BLOB NOT NULL, + price_table_update_price_table_cost BLOB NOT NULL, + price_table_account_balance_cost BLOB NOT NULL, + price_table_fund_account_cost BLOB NOT NULL, + price_table_latest_revision_cost BLOB NOT NULL, + price_table_subscription_memory_cost BLOB NOT NULL, + price_table_subscription_notification_cost BLOB NOT NULL, + price_table_init_base_cost BLOB NOT NULL, + price_table_memory_time_cost BLOB NOT NULL, + price_table_download_bandwidth_cost BLOB NOT NULL, + price_table_upload_bandwidth_cost BLOB NOT NULL, + price_table_drop_sectors_base_cost BLOB NOT NULL, + price_table_drop_sectors_unit_cost BLOB NOT NULL, + price_table_has_sector_base_cost BLOB NOT NULL, + price_table_read_base_cost BLOB NOT NULL, + price_table_read_length_cost BLOB NOT NULL, + price_table_renew_contract_cost BLOB NOT NULL, + price_table_revision_base_cost BLOB NOT NULL, + price_table_swap_sector_base_cost BLOB NOT NULL, + price_table_write_base_cost BLOB NOT NULL, + price_table_write_length_cost BLOB NOT NULL, + price_table_write_store_cost BLOB NOT NULL, + price_table_txn_fee_min_recommended BLOB NOT NULL, + price_table_txn_fee_max_recommended BLOB NOT NULL, + price_table_contract_price BLOB NOT NULL, + price_table_collateral_cost BLOB NOT NULL, + price_table_max_collateral BLOB NOT NULL, + price_table_max_duration BLOB NOT NULL, + price_table_window_size BLOB NOT NULL, + price_table_registry_entries_left BLOB NOT NULL, + price_table_registry_entries_total BLOB NOT NULL, + -- rhp4 settings + v2_settings_protocol_version BLOB NOT NULL, + v2_settings_release TEXT NOT NULL, + v2_settings_wallet_address BLOB NOT NULL, + v2_settings_accepting_contracts INTEGER NOT NULL, + v2_settings_max_collateral BLOB NOT NULL, + v2_settings_max_contract_duration BLOB NOT NULL, + v2_settings_remaining_storage BLOB NOT NULL, + v2_settings_total_storage BLOB NOT NULL, + v2_settings_used_storage BLOB NOT NULL, -- needed so we can sort by this because there's no clean way of subtracting binary encoded uint64s (total and remaining storage) in sqlite + -- rhp4 prices + v2_prices_contract_price BLOB NOT NULL, + v2_prices_collateral_price BLOB NOT NULL, + v2_prices_storage_price BLOB NOT NULL, + v2_prices_ingress_price BLOB NOT NULL, + v2_prices_egress_price BLOB NOT NULL, + v2_prices_free_sector_price BLOB NOT NULL, + v2_prices_tip_height BLOB NOT NULL, + v2_prices_valid_until BLOB NOT NULL, + v2_prices_signature BLOB NOT NULL +); +CREATE INDEX host_info_net_address ON host_info(net_address); + +CREATE TABLE host_info_v2_netaddresses( + public_key BLOB REFERENCES host_info(public_key) ON DELETE CASCADE NOT NULL, + netaddress_order INTEGER NOT NULL, + protocol TEXT NOT NULL, + address TEXT NOT NULL, + + PRIMARY KEY(public_key, netaddress_order) +); + +CREATE INDEX host_info_v2_netaddresses_public_key ON host_info_v2_netaddresses(public_key); +CREATE INDEX host_info_v2_netaddresses_address ON host_info_v2_netaddresses(address); + CREATE TABLE events ( id INTEGER PRIMARY KEY, event_id BLOB UNIQUE NOT NULL, @@ -232,4 +619,4 @@ CREATE INDEX event_addresses_event_id_index ON event_addresses(event_id); CREATE INDEX event_addresses_address_id_index ON event_addresses(address_id); -- initialize the global settings table -INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed +INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed \ No newline at end of file diff --git a/persist/sqlite/merkle.go b/persist/sqlite/merkle.go index 4e8f19e..bcbd47a 100644 --- a/persist/sqlite/merkle.go +++ b/persist/sqlite/merkle.go @@ -10,12 +10,12 @@ import ( func (s *Store) MerkleProof(leafIndex uint64) (proof []types.Hash256, err error) { err = s.transaction(func(tx *txn) error { var numLeaves uint64 - if err := tx.QueryRow("SELECT COUNT(*) FROM state_tree WHERE i = 0").Scan(&numLeaves); err != nil { + if err := tx.QueryRow("SELECT num_leaves FROM network_metrics ORDER BY height DESC LIMIT 1").Scan(decode(&numLeaves)); err != nil { return err } pos := leafIndex - stmt, err := tx.Prepare("SELECT hash FROM state_tree WHERE row = ? AND column = ?") + stmt, err := tx.Prepare("SELECT value FROM state_tree WHERE row = ? AND column = ?") if err != nil { return err } diff --git a/persist/sqlite/metrics.go b/persist/sqlite/metrics.go new file mode 100644 index 0000000..92c6b91 --- /dev/null +++ b/persist/sqlite/metrics.go @@ -0,0 +1,223 @@ +package sqlite + +import ( + "fmt" + "slices" + "time" + + proto4 "go.sia.tech/core/rhp/v4" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +// Metrics implements explorer.Store +func (s *Store) Metrics(id types.BlockID) (result explorer.Metrics, err error) { + err = s.transaction(func(tx *txn) error { + err = tx.QueryRow(`SELECT block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue FROM network_metrics WHERE block_id = ?`, encode(id)).Scan(decode(&result.Index.ID), &result.Index.Height, decode(&result.Difficulty), decode(&result.SiafundTaxRevenue), decode(&result.NumLeaves), &result.TotalHosts, &result.ActiveContracts, &result.FailedContracts, &result.SuccessfulContracts, &result.StorageUtilization, decode(&result.CirculatingSupply), decode(&result.ContractRevenue)) + if err != nil { + return fmt.Errorf("failed to get metrics: %w", err) + } + return nil + }) + return +} + +// HostMetrics implements explorer.Store +func (s *Store) HostMetrics() (result explorer.HostMetrics, err error) { + medianUint64 := func(x []uint64) uint64 { + if len(x) == 0 { + return 0 + } + + slices.Sort(x) + if len(x)%2 == 1 { + return x[len(x)/2] + } + return (x[(len(x)/2)-1] + x[(len(x)/2)]) / 2 + } + + medianCurrency := func(x []types.Currency) types.Currency { + if len(x) == 0 { + return types.ZeroCurrency + } + + slices.SortFunc(x, func(a, b types.Currency) int { + return a.Cmp(b) + }) + if len(x)%2 == 1 { + return x[len(x)/2] + } + return (x[(len(x)/2)-1].Add(x[(len(x) / 2)])).Div64(2) + } + + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT v2,settings_max_download_batch_size,settings_max_duration,settings_max_revise_batch_size,settings_remaining_storage,settings_sector_size,settings_total_storage,settings_window_size,settings_collateral,settings_max_collateral,settings_base_rpc_price,settings_contract_price,settings_download_bandwidth_price,settings_sector_access_price,settings_storage_price,settings_upload_bandwidth_price,settings_ephemeral_account_expiry,settings_max_ephemeral_account_balance,settings_revision_number,price_table_validity,price_table_host_block_height,price_table_update_price_table_cost,price_table_account_balance_cost,price_table_fund_account_cost,price_table_latest_revision_cost,price_table_subscription_memory_cost,price_table_subscription_notification_cost,price_table_init_base_cost,price_table_memory_time_cost,price_table_download_bandwidth_cost,price_table_upload_bandwidth_cost,price_table_drop_sectors_base_cost,price_table_drop_sectors_unit_cost,price_table_has_sector_base_cost,price_table_read_base_cost,price_table_read_length_cost,price_table_renew_contract_cost,price_table_revision_base_cost,price_table_swap_sector_base_cost,price_table_write_base_cost,price_table_write_length_cost,price_table_write_store_cost,price_table_txn_fee_min_recommended,price_table_txn_fee_max_recommended,price_table_contract_price,price_table_collateral_cost,price_table_max_collateral,price_table_max_duration,price_table_window_size,price_table_registry_entries_left,price_table_registry_entries_total,v2_settings_max_collateral,v2_settings_max_contract_duration,v2_settings_remaining_storage,v2_settings_total_storage,v2_prices_contract_price,v2_prices_collateral_price,v2_prices_storage_price,v2_prices_ingress_price,v2_prices_egress_price,v2_prices_free_sector_price,v2_prices_tip_height,v2_prices_valid_until FROM host_info WHERE last_scan_successful = 1`) + if err != nil { + return fmt.Errorf("failed to get hosts: %w", err) + } + defer rows.Close() + + var count uint64 + var settingsMaxDownloadBatchSize, settingsMaxDuration, settingsMaxReviseBatchSize, settingsRemainingStorage, settingsSectorSize, settingsTotalStorage, settingsWindowSize, settingsRevisionNumber, priceTableHostBlockHeight, priceTableMaxDuration, priceTableWindowSize, priceTableRegistryEntriesLeft, priceTableRegistryEntriesTotal, settingsEphemeralAccountExpiry, priceTableValidity, v2MaxContractDuration, v2RemainingStorage, v2TotalStorage, v2PricesTipHeight, v2PricesValidUntil []uint64 + var settingsCollateral, settingsMaxCollateral, settingsBaseRPCPrice, settingsContractPrice, settingsDownloadBandwidthPrice, settingsSectorAccessPrice, settingsStoragePrice, settingsUploadBandwidthPrice, settingsMaxEphemeralAccountBalance, priceTableUpdatePriceTableCost, priceTableAccountBalanceCost, priceTableFundAccountCost, priceTableLatestRevisionCost, priceTableSubscriptionMemoryCost, priceTableSubscriptionNotificationCost, priceTableInitBaseCost, priceTableMemoryTimeCost, priceTableDownloadBandwidthCost, priceTableUploadBandwidthCost, priceTableDropSectorsBaseCost, priceTableDropSectorsUnitCost, priceTableHasSectorBaseCost, priceTableReadBaseCost, priceTableReadLengthCost, priceTableRenewContractCost, priceTableRevisionBaseCost, priceTableSwapSectorBaseCost, priceTableWriteBaseCost, priceTableWriteLengthCost, priceTableWriteStoreCost, priceTableTxnFeeMinRecommended, priceTableTxnFeeMaxRecommended, priceTableContractPrice, priceTableCollateralCost, priceTableMaxCollateral, v2MaxCollateral, v2PricesContractPrice, v2PricesCollateral, v2PricesStoragePrice, v2PricesIngressPrice, v2PricesEgressPrice, v2PricesFreeSectorPrice []types.Currency + for rows.Next() { + var host explorer.Host + if err := rows.Scan(&host.V2, decode(&host.Settings.MaxDownloadBatchSize), decode(&host.Settings.MaxDuration), decode(&host.Settings.MaxReviseBatchSize), decode(&host.Settings.RemainingStorage), decode(&host.Settings.SectorSize), decode(&host.Settings.TotalStorage), decode(&host.Settings.WindowSize), decode(&host.Settings.Collateral), decode(&host.Settings.MaxCollateral), decode(&host.Settings.BaseRPCPrice), decode(&host.Settings.ContractPrice), decode(&host.Settings.DownloadBandwidthPrice), decode(&host.Settings.SectorAccessPrice), decode(&host.Settings.StoragePrice), decode(&host.Settings.UploadBandwidthPrice), decode(&host.Settings.EphemeralAccountExpiry), decode(&host.Settings.MaxEphemeralAccountBalance), decode(&host.Settings.RevisionNumber), decode(&host.PriceTable.Validity), decode(&host.PriceTable.HostBlockHeight), decode(&host.PriceTable.UpdatePriceTableCost), decode(&host.PriceTable.AccountBalanceCost), decode(&host.PriceTable.FundAccountCost), decode(&host.PriceTable.LatestRevisionCost), decode(&host.PriceTable.SubscriptionMemoryCost), decode(&host.PriceTable.SubscriptionNotificationCost), decode(&host.PriceTable.InitBaseCost), decode(&host.PriceTable.MemoryTimeCost), decode(&host.PriceTable.DownloadBandwidthCost), decode(&host.PriceTable.UploadBandwidthCost), decode(&host.PriceTable.DropSectorsBaseCost), decode(&host.PriceTable.DropSectorsUnitCost), decode(&host.PriceTable.HasSectorBaseCost), decode(&host.PriceTable.ReadBaseCost), decode(&host.PriceTable.ReadLengthCost), decode(&host.PriceTable.RenewContractCost), decode(&host.PriceTable.RevisionBaseCost), decode(&host.PriceTable.SwapSectorBaseCost), decode(&host.PriceTable.WriteBaseCost), decode(&host.PriceTable.WriteLengthCost), decode(&host.PriceTable.WriteStoreCost), decode(&host.PriceTable.TxnFeeMinRecommended), decode(&host.PriceTable.TxnFeeMaxRecommended), decode(&host.PriceTable.ContractPrice), decode(&host.PriceTable.CollateralCost), decode(&host.PriceTable.MaxCollateral), decode(&host.PriceTable.MaxDuration), decode(&host.PriceTable.WindowSize), decode(&host.PriceTable.RegistryEntriesLeft), decode(&host.PriceTable.RegistryEntriesTotal), decode(&host.V2Settings.MaxCollateral), decode(&host.V2Settings.MaxContractDuration), decode(&host.V2Settings.RemainingStorage), decode(&host.V2Settings.TotalStorage), decode(&host.V2Settings.Prices.ContractPrice), decode(&host.V2Settings.Prices.Collateral), decode(&host.V2Settings.Prices.StoragePrice), decode(&host.V2Settings.Prices.IngressPrice), decode(&host.V2Settings.Prices.EgressPrice), decode(&host.V2Settings.Prices.FreeSectorPrice), decode(&host.V2Settings.Prices.TipHeight), decode(&host.V2Settings.Prices.ValidUntil)); err != nil { + return fmt.Errorf("failed to scan host: %w", err) + } + + if host.V2 { + result.TotalStorage += proto4.SectorSize * host.V2Settings.TotalStorage + result.RemainingStorage += proto4.SectorSize * host.V2Settings.RemainingStorage + + v2MaxCollateral = append(v2MaxCollateral, host.V2Settings.MaxCollateral) + v2MaxContractDuration = append(v2MaxContractDuration, host.V2Settings.MaxContractDuration) + v2RemainingStorage = append(v2RemainingStorage, host.V2Settings.RemainingStorage) + v2TotalStorage = append(v2TotalStorage, host.V2Settings.TotalStorage) + + v2PricesContractPrice = append(v2PricesContractPrice, host.V2Settings.Prices.ContractPrice) + v2PricesCollateral = append(v2PricesCollateral, host.V2Settings.Prices.Collateral) + v2PricesStoragePrice = append(v2PricesStoragePrice, host.V2Settings.Prices.StoragePrice) + v2PricesIngressPrice = append(v2PricesIngressPrice, host.V2Settings.Prices.IngressPrice) + v2PricesEgressPrice = append(v2PricesEgressPrice, host.V2Settings.Prices.EgressPrice) + v2PricesFreeSectorPrice = append(v2PricesFreeSectorPrice, host.V2Settings.Prices.FreeSectorPrice) + v2PricesTipHeight = append(v2PricesTipHeight, host.V2Settings.Prices.TipHeight) + v2PricesValidUntil = append(v2PricesValidUntil, uint64(host.V2Settings.Prices.ValidUntil.Unix())) + } else { + result.TotalStorage += host.Settings.TotalStorage + result.RemainingStorage += host.Settings.RemainingStorage + + settingsMaxDownloadBatchSize = append(settingsMaxDownloadBatchSize, host.Settings.MaxDownloadBatchSize) + settingsMaxDuration = append(settingsMaxDuration, host.Settings.MaxDuration) + settingsMaxReviseBatchSize = append(settingsMaxReviseBatchSize, host.Settings.MaxReviseBatchSize) + settingsRemainingStorage = append(settingsRemainingStorage, host.Settings.RemainingStorage) + settingsSectorSize = append(settingsSectorSize, host.Settings.SectorSize) + settingsTotalStorage = append(settingsTotalStorage, host.Settings.TotalStorage) + settingsWindowSize = append(settingsWindowSize, host.Settings.WindowSize) + settingsCollateral = append(settingsCollateral, host.Settings.Collateral) + settingsMaxCollateral = append(settingsMaxCollateral, host.Settings.MaxCollateral) + settingsBaseRPCPrice = append(settingsBaseRPCPrice, host.Settings.BaseRPCPrice) + settingsContractPrice = append(settingsContractPrice, host.Settings.ContractPrice) + settingsDownloadBandwidthPrice = append(settingsDownloadBandwidthPrice, host.Settings.DownloadBandwidthPrice) + settingsSectorAccessPrice = append(settingsSectorAccessPrice, host.Settings.SectorAccessPrice) + settingsStoragePrice = append(settingsStoragePrice, host.Settings.StoragePrice) + settingsUploadBandwidthPrice = append(settingsUploadBandwidthPrice, host.Settings.UploadBandwidthPrice) + settingsEphemeralAccountExpiry = append(settingsEphemeralAccountExpiry, uint64(host.Settings.EphemeralAccountExpiry)) + settingsMaxEphemeralAccountBalance = append(settingsMaxEphemeralAccountBalance, host.Settings.MaxEphemeralAccountBalance) + settingsRevisionNumber = append(settingsRevisionNumber, host.Settings.RevisionNumber) + + priceTableValidity = append(priceTableValidity, uint64(host.PriceTable.Validity)) + priceTableHostBlockHeight = append(priceTableHostBlockHeight, host.PriceTable.HostBlockHeight) + priceTableUpdatePriceTableCost = append(priceTableUpdatePriceTableCost, host.PriceTable.UpdatePriceTableCost) + priceTableAccountBalanceCost = append(priceTableAccountBalanceCost, host.PriceTable.AccountBalanceCost) + priceTableFundAccountCost = append(priceTableFundAccountCost, host.PriceTable.FundAccountCost) + priceTableLatestRevisionCost = append(priceTableLatestRevisionCost, host.PriceTable.LatestRevisionCost) + priceTableSubscriptionMemoryCost = append(priceTableSubscriptionMemoryCost, host.PriceTable.SubscriptionMemoryCost) + priceTableSubscriptionNotificationCost = append(priceTableSubscriptionNotificationCost, host.PriceTable.SubscriptionNotificationCost) + priceTableInitBaseCost = append(priceTableInitBaseCost, host.PriceTable.InitBaseCost) + priceTableMemoryTimeCost = append(priceTableMemoryTimeCost, host.PriceTable.MemoryTimeCost) + priceTableDownloadBandwidthCost = append(priceTableDownloadBandwidthCost, host.PriceTable.DownloadBandwidthCost) + priceTableUploadBandwidthCost = append(priceTableUploadBandwidthCost, host.PriceTable.UploadBandwidthCost) + priceTableDropSectorsBaseCost = append(priceTableDropSectorsBaseCost, host.PriceTable.DropSectorsBaseCost) + priceTableDropSectorsUnitCost = append(priceTableDropSectorsUnitCost, host.PriceTable.DropSectorsUnitCost) + priceTableHasSectorBaseCost = append(priceTableHasSectorBaseCost, host.PriceTable.HasSectorBaseCost) + priceTableReadBaseCost = append(priceTableReadBaseCost, host.PriceTable.ReadBaseCost) + priceTableReadLengthCost = append(priceTableReadLengthCost, host.PriceTable.ReadLengthCost) + priceTableRenewContractCost = append(priceTableRenewContractCost, host.PriceTable.RenewContractCost) + priceTableRevisionBaseCost = append(priceTableRevisionBaseCost, host.PriceTable.RevisionBaseCost) + priceTableSwapSectorBaseCost = append(priceTableSwapSectorBaseCost, host.PriceTable.SwapSectorBaseCost) + priceTableWriteBaseCost = append(priceTableWriteBaseCost, host.PriceTable.WriteBaseCost) + priceTableWriteLengthCost = append(priceTableWriteLengthCost, host.PriceTable.WriteLengthCost) + priceTableWriteStoreCost = append(priceTableWriteStoreCost, host.PriceTable.WriteStoreCost) + priceTableTxnFeeMinRecommended = append(priceTableTxnFeeMinRecommended, host.PriceTable.TxnFeeMinRecommended) + priceTableTxnFeeMaxRecommended = append(priceTableTxnFeeMaxRecommended, host.PriceTable.TxnFeeMaxRecommended) + priceTableContractPrice = append(priceTableContractPrice, host.PriceTable.ContractPrice) + priceTableCollateralCost = append(priceTableCollateralCost, host.PriceTable.CollateralCost) + priceTableMaxCollateral = append(priceTableMaxCollateral, host.PriceTable.MaxCollateral) + priceTableMaxDuration = append(priceTableMaxDuration, host.PriceTable.MaxDuration) + priceTableWindowSize = append(priceTableWindowSize, host.PriceTable.WindowSize) + priceTableRegistryEntriesLeft = append(priceTableRegistryEntriesLeft, host.PriceTable.RegistryEntriesLeft) + priceTableRegistryEntriesTotal = append(priceTableRegistryEntriesTotal, host.PriceTable.RegistryEntriesTotal) + } + + count++ + } + if err := rows.Err(); err != nil { + return err + } + + if count > 0 { + result.ActiveHosts = count + + result.Settings.MaxDownloadBatchSize = medianUint64(settingsMaxDownloadBatchSize) + result.Settings.MaxDuration = medianUint64(settingsMaxDuration) + result.Settings.MaxReviseBatchSize = medianUint64(settingsMaxReviseBatchSize) + result.Settings.RemainingStorage = medianUint64(settingsRemainingStorage) + result.Settings.SectorSize = medianUint64(settingsSectorSize) + result.Settings.TotalStorage = medianUint64(settingsTotalStorage) + result.Settings.WindowSize = medianUint64(settingsWindowSize) + result.Settings.Collateral = medianCurrency(settingsCollateral) + result.Settings.MaxCollateral = medianCurrency(settingsMaxCollateral) + result.Settings.BaseRPCPrice = medianCurrency(settingsBaseRPCPrice) + result.Settings.ContractPrice = medianCurrency(settingsContractPrice) + result.Settings.DownloadBandwidthPrice = medianCurrency(settingsDownloadBandwidthPrice) + result.Settings.SectorAccessPrice = medianCurrency(settingsSectorAccessPrice) + result.Settings.StoragePrice = medianCurrency(settingsStoragePrice) + result.Settings.UploadBandwidthPrice = medianCurrency(settingsUploadBandwidthPrice) + result.Settings.EphemeralAccountExpiry = time.Duration(medianUint64(settingsEphemeralAccountExpiry)) + result.Settings.MaxEphemeralAccountBalance = medianCurrency(settingsMaxEphemeralAccountBalance) + result.Settings.RevisionNumber = medianUint64(settingsRevisionNumber) + + result.PriceTable.Validity = time.Duration(medianUint64(priceTableValidity)) + result.PriceTable.HostBlockHeight = medianUint64(priceTableHostBlockHeight) + result.PriceTable.UpdatePriceTableCost = medianCurrency(priceTableUpdatePriceTableCost) + result.PriceTable.AccountBalanceCost = medianCurrency(priceTableAccountBalanceCost) + result.PriceTable.FundAccountCost = medianCurrency(priceTableFundAccountCost) + result.PriceTable.LatestRevisionCost = medianCurrency(priceTableLatestRevisionCost) + result.PriceTable.SubscriptionMemoryCost = medianCurrency(priceTableSubscriptionMemoryCost) + result.PriceTable.SubscriptionNotificationCost = medianCurrency(priceTableSubscriptionNotificationCost) + result.PriceTable.InitBaseCost = medianCurrency(priceTableInitBaseCost) + result.PriceTable.MemoryTimeCost = medianCurrency(priceTableMemoryTimeCost) + result.PriceTable.DownloadBandwidthCost = medianCurrency(priceTableDownloadBandwidthCost) + result.PriceTable.UploadBandwidthCost = medianCurrency(priceTableUploadBandwidthCost) + result.PriceTable.DropSectorsBaseCost = medianCurrency(priceTableDropSectorsBaseCost) + result.PriceTable.DropSectorsUnitCost = medianCurrency(priceTableDropSectorsUnitCost) + result.PriceTable.HasSectorBaseCost = medianCurrency(priceTableHasSectorBaseCost) + result.PriceTable.ReadBaseCost = medianCurrency(priceTableReadBaseCost) + result.PriceTable.ReadLengthCost = medianCurrency(priceTableReadLengthCost) + result.PriceTable.RenewContractCost = medianCurrency(priceTableRenewContractCost) + result.PriceTable.RevisionBaseCost = medianCurrency(priceTableRevisionBaseCost) + result.PriceTable.SwapSectorBaseCost = medianCurrency(priceTableSwapSectorBaseCost) + result.PriceTable.WriteBaseCost = medianCurrency(priceTableWriteBaseCost) + result.PriceTable.WriteLengthCost = medianCurrency(priceTableWriteLengthCost) + result.PriceTable.WriteStoreCost = medianCurrency(priceTableWriteStoreCost) + result.PriceTable.TxnFeeMinRecommended = medianCurrency(priceTableTxnFeeMinRecommended) + result.PriceTable.TxnFeeMaxRecommended = medianCurrency(priceTableTxnFeeMaxRecommended) + result.PriceTable.ContractPrice = medianCurrency(priceTableContractPrice) + result.PriceTable.CollateralCost = medianCurrency(priceTableCollateralCost) + result.PriceTable.MaxCollateral = medianCurrency(priceTableMaxCollateral) + result.PriceTable.MaxDuration = medianUint64(priceTableMaxDuration) + result.PriceTable.WindowSize = medianUint64(priceTableWindowSize) + result.PriceTable.RegistryEntriesLeft = medianUint64(priceTableRegistryEntriesLeft) + result.PriceTable.RegistryEntriesTotal = medianUint64(priceTableRegistryEntriesTotal) + + result.V2Settings.MaxCollateral = medianCurrency(v2MaxCollateral) + result.V2Settings.MaxContractDuration = medianUint64(v2MaxContractDuration) + result.V2Settings.RemainingStorage = medianUint64(v2RemainingStorage) + result.V2Settings.TotalStorage = medianUint64(v2TotalStorage) + + result.V2Settings.Prices.ContractPrice = medianCurrency(v2PricesContractPrice) + result.V2Settings.Prices.Collateral = medianCurrency(v2PricesCollateral) + result.V2Settings.Prices.StoragePrice = medianCurrency(v2PricesStoragePrice) + result.V2Settings.Prices.IngressPrice = medianCurrency(v2PricesIngressPrice) + result.V2Settings.Prices.EgressPrice = medianCurrency(v2PricesEgressPrice) + result.V2Settings.Prices.FreeSectorPrice = medianCurrency(v2PricesFreeSectorPrice) + result.V2Settings.Prices.TipHeight = medianUint64(v2PricesTipHeight) + result.V2Settings.Prices.ValidUntil = time.Unix(int64(medianUint64(v2PricesValidUntil)), 0) + } + + return nil + }) + return +} diff --git a/persist/sqlite/revert.go b/persist/sqlite/revert.go new file mode 100644 index 0000000..237bbf7 --- /dev/null +++ b/persist/sqlite/revert.go @@ -0,0 +1,94 @@ +package sqlite + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +// deleteV1Transactions deletes the transactions from the database if they are +// not referenced in any blocks. +func deleteV1Transactions(tx *txn, txns []types.Transaction) error { + stmt, err := tx.Prepare(`DELETE FROM transactions AS t +WHERE t.transaction_id = ? + AND NOT EXISTS ( + SELECT 1 + FROM block_transactions bt + WHERE bt.transaction_id = t.id +);`) + if err != nil { + return fmt.Errorf("deleteV1Transactions: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, txn := range txns { + if _, err := stmt.Exec(encode(txn.ID())); err != nil { + return fmt.Errorf("deleteV1Transactions: failed to execute: %w", err) + } + } + return nil +} + +// deleteV2Transactions deletes the transactions from the database if they are +// not referenced in any blocks. +func deleteV2Transactions(tx *txn, txns []types.V2Transaction) error { + stmt, err := tx.Prepare(`DELETE FROM v2_transactions AS t +WHERE t.transaction_id = ? + AND NOT EXISTS ( + SELECT 1 + FROM v2_block_transactions bt + WHERE bt.transaction_id = t.id +);`) + if err != nil { + return fmt.Errorf("deleteV2Transactions: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, txn := range txns { + if _, err := stmt.Exec(encode(txn.ID())); err != nil { + return fmt.Errorf("deleteV2Transactions: failed to execute: %w", err) + } + } + return nil +} + +func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { + if err := updateMaturedBalances(ut.tx, true, state.Metrics.Index.Height); err != nil { + return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) + } else if _, err := addSiacoinElements( + ut.tx, + state.Metrics.Index, + state.SpentSiacoinElements, + append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), + ); err != nil { + return fmt.Errorf("RevertIndex: failed to update siacoin output state: %w", err) + } else if _, err := addSiafundElements( + ut.tx, + state.Metrics.Index, + state.SpentSiafundElements, + append(state.NewSiafundElements, state.EphemeralSiafundElements...), + ); err != nil { + return fmt.Errorf("RevertIndex: failed to update siafund output state: %w", err) + } else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update balances: %w", err) + } else if _, err := updateFileContractElements(ut.tx, true, state.Metrics.Index, state.Block, state.FileContractElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update file contract state: %w", err) + } else if _, err := updateV2FileContractElements(ut.tx, true, state.Metrics.Index, state.Block, state.V2FileContractElements); err != nil { + return fmt.Errorf("RevertIndex: failed to add v2 file contracts: %w", err) + } else if err := deleteBlock(ut.tx, state.Block.ID()); err != nil { + return fmt.Errorf("RevertIndex: failed to delete block: %w", err) + } else if err := updateFileContractIndices(ut.tx, true, state.Metrics.Index, state.FileContractElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update file contract element indices: %w", err) + } else if err := updateV2FileContractIndices(ut.tx, true, state.Metrics.Index, state.V2FileContractElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update v2 file contract element indices: %w", err) + } else if err := deleteV1Transactions(ut.tx, state.Block.Transactions); err != nil { + return fmt.Errorf("RevertIndex: failed to delete v1 transactions: %w", err) + } else if err := deleteV2Transactions(ut.tx, state.Block.V2Transactions()); err != nil { + return fmt.Errorf("RevertIndex: failed to delete v2 transactions: %w", err) + } else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil { + return fmt.Errorf("RevertIndex: failed to update state tree: %w", err) + } + + return nil +} diff --git a/persist/sqlite/scan_test.go b/persist/sqlite/scan_test.go new file mode 100644 index 0000000..c1e754d --- /dev/null +++ b/persist/sqlite/scan_test.go @@ -0,0 +1,646 @@ +package sqlite_test + +import ( + "context" + "encoding/json" + "errors" + "net" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + proto2 "go.sia.tech/core/rhp/v2" + proto3 "go.sia.tech/core/rhp/v3" + proto4 "go.sia.tech/core/rhp/v4" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + crhpv4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/coreutils/rhp/v4/siamux" + "go.sia.tech/coreutils/syncer" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/coreutils/wallet" + "go.sia.tech/explored/config" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" + "go.sia.tech/explored/persist/sqlite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +func startTestNode(tb testing.TB, n *consensus.Network, genesis types.Block) (*chain.Manager, *syncer.Syncer, *wallet.SingleAddressWallet) { + db, tipstate, err := chain.NewDBStore(chain.NewMemDB(), n, genesis, chain.NewZapMigrationLogger(zap.NewNop())) + if err != nil { + tb.Fatal(err) + } + cm := chain.NewManager(db, tipstate) + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { syncerListener.Close() }) + + s := syncer.New(syncerListener, cm, ctestutil.NewEphemeralPeerStore(), gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: "localhost:1234", + }) + go s.Run() + tb.Cleanup(func() { s.Close() }) + + ws := ctestutil.NewEphemeralWalletStore() + w, err := wallet.NewSingleAddressWallet(types.GeneratePrivateKey(), cm, ws) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { w.Close() }) + + reorgCh := make(chan struct{}, 1) + tb.Cleanup(func() { close(reorgCh) }) + + go func() { + for range reorgCh { + reverted, applied, err := cm.UpdatesSince(w.Tip(), 1000) + if err != nil { + tb.Error(err) + } + + err = ws.UpdateChainState(func(tx wallet.UpdateTx) error { + return w.UpdateChainState(tx, reverted, applied) + }) + if err != nil { + tb.Error(err) + } + } + }() + + stop := cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgCh <- struct{}{}: + default: + } + }) + tb.Cleanup(stop) + + return cm, s, w +} + +func testV2Host(tb testing.TB, hostKey types.PrivateKey, cm crhpv4.ChainManager, s crhpv4.Syncer, w crhpv4.Wallet, c crhpv4.Contractor, sr crhpv4.Settings, ss crhpv4.Sectors, log *zap.Logger) (string, crhpv4.TransportClient) { + rs := crhpv4.NewServer(hostKey, cm, s, c, w, sr, ss, crhpv4.WithPriceTableValidity(2*time.Minute)) + hostAddr := ctestutil.ServeSiaMux(tb, rs, log.Named("siamux")) + + transport, err := siamux.Dial(context.Background(), hostAddr, hostKey.PublicKey()) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { transport.Close() }) + + return hostAddr, transport +} + +func testV1Host(tb testing.TB, hostKey types.PrivateKey, hostSettings *proto2.HostSettings, priceTable *proto3.HostPriceTable) (rhp2Addr string, rhp3Addr string) { + rhp2Listener, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + rhp3Listener, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + rhp2Addr, rhp3Addr = rhp2Listener.Addr().String(), rhp3Listener.Addr().String() + + hostSettings.NetAddress = rhp2Listener.Addr().String() + _, hostSettings.SiaMuxPort, err = net.SplitHostPort(rhp3Listener.Addr().String()) + if err != nil { + tb.Fatal(err) + } + + tb.Cleanup(func() { + rhp2Listener.Close() + rhp3Listener.Close() + }) + + go func() { + for { + func() { + conn, err := rhp2Listener.Accept() + if errors.Is(err, net.ErrClosed) { + return + } else if err != nil { + tb.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + + transport, err := proto2.NewHostTransport(conn, hostKey) + if err != nil { + tb.Fatal(err) + } + defer transport.Close() + + id, err := transport.ReadID() + if err != nil { + tb.Fatal(err) + } else if id != proto2.RPCSettingsID { + tb.Fatal("received non settings RPC") + } + + encoded, err := json.Marshal(hostSettings) + if err != nil { + tb.Fatal(err) + } + if err := transport.WriteResponse(&proto2.RPCSettingsResponse{ + Settings: encoded, + }); err != nil { + tb.Fatal(err) + } + }() + } + }() + + go func() { + for { + func() { + conn, err := rhp3Listener.Accept() + if errors.Is(err, net.ErrClosed) { + return + } else if err != nil { + tb.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + + transport, err := proto3.NewHostTransport(conn, hostKey) + if err != nil { + tb.Fatal(err) + } + defer transport.Close() + + stream, err := transport.AcceptStream() + if err != nil { + tb.Fatal(err) + } + defer stream.Close() + + id, err := stream.ReadID() + if err != nil { + tb.Fatal(err) + } else if id != proto3.RPCUpdatePriceTableID { + tb.Fatal("received non price table RPC") + } + + encoded, err := json.Marshal(priceTable) + if err != nil { + tb.Fatal(err) + } + if err := stream.WriteResponse(&proto3.RPCUpdatePriceTableResponse{ + PriceTableJSON: encoded, + }); err != nil { + tb.Fatal(err) + } + }() + } + }() + + return +} + +func TestScan(t *testing.T) { + if testing.Short() { + t.Skip() + } + randSC := func() types.Currency { + return types.NewCurrency64(frand.Uint64n(10000)) + } + + log := zaptest.NewLogger(t) + dir := t.TempDir() + + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + network, genesisBlock := ctestutil.V2Network() + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 5 + + cm, s, w := startTestNode(t, network, genesisBlock) + + settings := proto2.HostSettings{ + AcceptingContracts: true, + MaxDownloadBatchSize: 10, + MaxDuration: 20, + MaxReviseBatchSize: 30, + RemainingStorage: 50, + SectorSize: 60, + TotalStorage: 70, + + Collateral: randSC(), + MaxCollateral: randSC(), + BaseRPCPrice: randSC(), + ContractPrice: randSC(), + DownloadBandwidthPrice: randSC(), + SectorAccessPrice: randSC(), + StoragePrice: randSC(), + UploadBandwidthPrice: randSC(), + + EphemeralAccountExpiry: time.Duration(80), + RevisionNumber: 90, + Version: "version", + Release: "release", + } + table := proto3.HostPriceTable{ + Validity: time.Duration(100), + HostBlockHeight: cm.Tip().Height, + + UpdatePriceTableCost: randSC(), + AccountBalanceCost: randSC(), + FundAccountCost: randSC(), + LatestRevisionCost: randSC(), + SubscriptionMemoryCost: randSC(), + SubscriptionNotificationCost: randSC(), + InitBaseCost: randSC(), + MemoryTimeCost: randSC(), + DownloadBandwidthCost: randSC(), + UploadBandwidthCost: randSC(), + DropSectorsBaseCost: randSC(), + DropSectorsUnitCost: randSC(), + HasSectorBaseCost: randSC(), + ReadBaseCost: randSC(), + ReadLengthCost: randSC(), + RenewContractCost: randSC(), + RevisionBaseCost: randSC(), + SwapSectorBaseCost: randSC(), + WriteBaseCost: randSC(), + WriteLengthCost: randSC(), + WriteStoreCost: randSC(), + TxnFeeMinRecommended: randSC(), + TxnFeeMaxRecommended: randSC(), + ContractPrice: randSC(), + CollateralCost: randSC(), + MaxCollateral: randSC(), + + MaxDuration: 20, + WindowSize: 30, + RegistryEntriesLeft: 40, + RegistryEntriesTotal: 50, + } + v2Settings := proto4.HostSettings{ + ProtocolVersion: [3]uint8{4, 0, 0}, + Release: "test", + AcceptingContracts: true, + WalletAddress: w.Address(), + MaxCollateral: randSC(), + MaxContractDuration: 1000, + RemainingStorage: 100, + TotalStorage: 100, + Prices: proto4.HostPrices{ + ContractPrice: randSC(), + StoragePrice: randSC(), + IngressPrice: randSC(), + EgressPrice: randSC(), + Collateral: randSC(), + }, + } + + sr := ctestutil.NewEphemeralSettingsReporter() + sr.Update(v2Settings) + ss := ctestutil.NewEphemeralSectorStore() + c := ctestutil.NewEphemeralContractor(cm) + + var pks [4]types.PrivateKey + var pubkeys [4]types.PublicKey + for i := range pks { + pks[i] = types.GeneratePrivateKey() + pubkeys[i] = pks[i].PublicKey() + } + + rhp2Addr, _ := testV1Host(t, pks[0], &settings, &table) + v4Addr, _ := testV2Host(t, pks[2], cm, s, w, c, sr, ss, zap.NewNop()) + + cfg := config.Scanner{ + NumThreads: 100, + ScanTimeout: 100 * time.Millisecond, + ScanFrequency: 100 * time.Millisecond, + ScanInterval: 3 * time.Hour, + MinLastAnnouncement: 90 * 24 * time.Hour, + } + + e, err := explorer.NewExplorer(cm, db, config.Index{BatchSize: 1000}, cfg, log) + if err != nil { + t.Fatal(err) + } + timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 60*time.Second) + defer timeoutCancel() + defer e.Shutdown(timeoutCtx) + + ha1 := chain.HostAnnouncement{PublicKey: pubkeys[0], NetAddress: rhp2Addr} + ha2 := chain.HostAnnouncement{PublicKey: pubkeys[1], NetAddress: "127.0.0.1:9999"} + txn1 := types.Transaction{ + ArbitraryData: [][]byte{ + ha1.ToArbitraryData(pks[0]), + ha2.ToArbitraryData(pks[1]), + }, + } + + b1 := testutil.MineBlock(cm.TipState(), []types.Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + + ha3 := chain.V2HostAnnouncement{{Protocol: siamux.Protocol, Address: v4Addr}} + txn2 := types.V2Transaction{ + Attestations: []types.Attestation{ + ha3.ToAttestation(cm.TipState(), pks[2]), + }, + } + testutil.SignV2Transaction(cm.TipState(), pks[2], &txn2) + + ha4 := chain.V2HostAnnouncement{{Protocol: siamux.Protocol, Address: "127.0.0.1:9999"}} + txn3 := types.V2Transaction{ + Attestations: []types.Attestation{ + ha4.ToAttestation(cm.TipState(), pks[3]), + }, + } + testutil.SignV2Transaction(cm.TipState(), pks[3], &txn3) + + b2 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2, txn3}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + + waitForTip := func() { + t.Helper() + + for { + if tip, err := e.Tip(); err != nil { + t.Fatal(err) + } else if tip == cm.Tip() { + break + } + time.Sleep(time.Second) + } + } + waitForTip() + time.Sleep(2 * cfg.ScanTimeout) + + type hostTest struct { + name string + pubkey types.PublicKey + totalScans uint64 + lastScanSuccessful bool + knownSince time.Time + lastAnnounce time.Time + nextScanFactor int + + expectedV2NetAddresses []chain.NetAddress + expectedNetAddress *string + expectedV2Settings *proto4.HostSettings + expectedSettings *proto2.HostSettings + expectedPriceTable *proto3.HostPriceTable + } + + runTests := func(tests []hostTest) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hosts, err := e.Hosts([]types.PublicKey{tt.pubkey}) + if err != nil { + t.Fatal(err) + } else if len(hosts) != 1 { + t.Fatalf("can't find host %s (%v) in DB", tt.name, tt.pubkey) + } + + h := hosts[0] + testutil.Equal(t, "PublicKey", tt.pubkey, h.PublicKey) + testutil.Equal(t, "TotalScans", tt.totalScans, h.TotalScans) + if tt.lastScanSuccessful { + testutil.Equal(t, "SuccessfulInteractions", tt.totalScans, h.SuccessfulInteractions) + testutil.Equal(t, "FailedInteractions", 0, h.FailedInteractions) + } else { + testutil.Equal(t, "FailedInteractions", tt.totalScans, h.FailedInteractions) + testutil.Equal(t, "SuccessfulInteractions", 0, h.SuccessfulInteractions) + if h.LastScanError == nil || *h.LastScanError == "" { + t.Fatal("empty last scan error when last scan was unsuccessful") + } + } + testutil.Equal(t, "LastScanSuccessful", tt.lastScanSuccessful, h.LastScanSuccessful) + testutil.Equal(t, "KnownSince", tt.knownSince, h.KnownSince) + testutil.Equal(t, "LastAnnouncement", tt.lastAnnounce, h.LastAnnouncement) + testutil.Equal(t, "NextScan", h.LastScan.Add(time.Duration(tt.nextScanFactor)*cfg.ScanInterval), h.NextScan) + + if tt.expectedV2NetAddresses != nil { + testutil.Equal(t, "V2NetAddresses", tt.expectedV2NetAddresses, h.V2NetAddresses) + } + if tt.expectedNetAddress != nil { + testutil.Equal(t, "NetAddress", *tt.expectedNetAddress, h.NetAddress) + } + if tt.expectedV2Settings != nil { + h.V2Settings.Prices.ValidUntil = time.Time{} + h.V2Settings.Prices.TipHeight = 0 + h.V2Settings.Prices.Signature = types.Signature{} + testutil.Equal(t, "V2Settings", *tt.expectedV2Settings, h.V2Settings) + } + if tt.expectedSettings != nil { + testutil.Equal(t, "Settings", *tt.expectedSettings, h.Settings) + } + if tt.expectedPriceTable != nil { + testutil.Equal(t, "PriceTable", *tt.expectedPriceTable, h.PriceTable) + } + }) + } + } + + runTests([]hostTest{ + { + name: "offline v2 host", + pubkey: pubkeys[3], + totalScans: 1, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b1.Timestamp, + nextScanFactor: 2, + expectedV2NetAddresses: ha4, + }, + { + name: "online v2 host", + pubkey: pubkeys[2], + totalScans: 1, + lastScanSuccessful: true, + knownSince: b2.Timestamp, + lastAnnounce: b2.Timestamp, + nextScanFactor: 1, + expectedV2NetAddresses: ha3, + expectedV2Settings: &v2Settings, + }, + { + name: "offline v1 host", + pubkey: pubkeys[1], + totalScans: 1, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b1.Timestamp, + nextScanFactor: 2, + expectedNetAddress: &ha2.NetAddress, + }, + { + name: "online v1 host", + pubkey: pubkeys[0], + totalScans: 1, + lastScanSuccessful: true, + knownSince: b1.Timestamp, + lastAnnounce: b1.Timestamp, + nextScanFactor: 1, + expectedNetAddress: &ha1.NetAddress, + expectedSettings: &settings, + expectedPriceTable: &table, + }, + }) + // Test host scanning after reannouncement + b3 := testutil.MineBlock(cm.TipState(), []types.Transaction{txn1}, types.VoidAddress) + cm.AddBlocks([]types.Block{b3}) + b4 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2, txn3}, types.VoidAddress) + cm.AddBlocks([]types.Block{b4}) + + waitForTip() + time.Sleep(2 * cfg.ScanTimeout) + + runTests([]hostTest{ + { + name: "offline v2 host", + pubkey: pubkeys[3], + totalScans: 2, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b4.Timestamp, + nextScanFactor: 4, + expectedV2NetAddresses: ha4, + }, + { + name: "online v2 host", + pubkey: pubkeys[2], + totalScans: 2, + lastScanSuccessful: true, + knownSince: b2.Timestamp, + lastAnnounce: b4.Timestamp, + nextScanFactor: 1, + expectedV2NetAddresses: ha3, + expectedV2Settings: &v2Settings, + }, + { + name: "offline v1 host", + pubkey: pubkeys[1], + totalScans: 2, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b3.Timestamp, + nextScanFactor: 4, + expectedNetAddress: &ha2.NetAddress, + }, + { + name: "online v1 host", + pubkey: pubkeys[0], + totalScans: 2, + lastScanSuccessful: true, + knownSince: b1.Timestamp, + lastAnnounce: b3.Timestamp, + nextScanFactor: 1, + expectedNetAddress: &ha1.NetAddress, + expectedSettings: &settings, + expectedPriceTable: &table, + }, + }) + + // Check that we have no more hosts to scan + now := types.CurrentTimestamp() + dbHosts, err := db.HostsForScanning(now.Add(-cfg.MinLastAnnouncement), 100) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "hostsForScanning", 0, len(dbHosts)) + + { + hosts, err := e.Hosts([]types.PublicKey{pubkeys[0], pubkeys[2]}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(hosts)", 2, len(hosts)) + + v1Host, v2Host := hosts[0], hosts[1] + if v1Host.V2 { + v1Host, v2Host = v2Host, v1Host + } + + // we only have one v1 host and one v2 host, so the median will just be + // whatever the values for that host are + metrics, err := e.HostMetrics() + if err != nil { + t.Fatal(err) + } + + // zero out fields we can't take median of + v2Host.V2Settings.ProtocolVersion, v2Host.V2Settings.AcceptingContracts, v2Host.V2Settings.Release, v2Host.V2Settings.WalletAddress, v2Host.V2Settings.Prices.Signature = [3]uint8{}, false, "", types.VoidAddress, types.Signature{} + v1Host.Settings.AcceptingContracts, v1Host.Settings.NetAddress, v1Host.Settings.Address, v1Host.Settings.Version, v1Host.Settings.Release, v1Host.Settings.SiaMuxPort, v1Host.PriceTable.UID = false, "", types.VoidAddress, "", "", "", proto3.SettingsID{} + + testutil.Equal(t, "metrics.TotalStorage", proto4.SectorSize*v2Host.V2Settings.TotalStorage+v1Host.Settings.TotalStorage, metrics.TotalStorage) + testutil.Equal(t, "metrics.RemainingStorage", proto4.SectorSize*v2Host.V2Settings.RemainingStorage+v1Host.Settings.RemainingStorage, metrics.RemainingStorage) + testutil.Equal(t, "metrics.V2Settings", v2Host.V2Settings, metrics.V2Settings) + testutil.Equal(t, "metrics.Settings", v1Host.Settings, metrics.Settings) + testutil.Equal(t, "metrics.PriceTable", v1Host.PriceTable, metrics.PriceTable) + } + + // Manually scan all the hosts + if _, err := e.ScanHosts(pubkeys[:]...); err != nil { + t.Fatal(err) + } + + runTests([]hostTest{ + { + name: "offline v2 host", + pubkey: pubkeys[3], + totalScans: 3, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b4.Timestamp, + nextScanFactor: 1, + expectedV2NetAddresses: ha4, + }, + { + name: "online v2 host", + pubkey: pubkeys[2], + totalScans: 3, + lastScanSuccessful: true, + knownSince: b2.Timestamp, + lastAnnounce: b4.Timestamp, + nextScanFactor: 1, + expectedV2NetAddresses: ha3, + expectedV2Settings: &v2Settings, + }, + { + name: "offline v1 host", + pubkey: pubkeys[1], + totalScans: 3, + lastScanSuccessful: false, + knownSince: b1.Timestamp, + lastAnnounce: b3.Timestamp, + nextScanFactor: 1, + expectedNetAddress: &ha2.NetAddress, + }, + { + name: "online v1 host", + pubkey: pubkeys[0], + totalScans: 3, + lastScanSuccessful: true, + knownSince: b1.Timestamp, + lastAnnounce: b3.Timestamp, + nextScanFactor: 1, + expectedNetAddress: &ha1.NetAddress, + expectedSettings: &settings, + expectedPriceTable: &table, + }, + }) +} diff --git a/persist/sqlite/search.go b/persist/sqlite/search.go new file mode 100644 index 0000000..71dc63e --- /dev/null +++ b/persist/sqlite/search.go @@ -0,0 +1,71 @@ +package sqlite + +import ( + "encoding/hex" + "errors" + "fmt" + "strings" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +// Search implements explorer.Store. +func (s *Store) Search(input string) (explorer.SearchType, error) { + decodeHex := func(input string) ([]byte, error) { + // Strip prefix (i.e., "txid:") if present + if idx := strings.Index(input, ":"); idx != -1 && len(input) >= idx { + input = input[idx+1:] + } + decoded, err := hex.DecodeString(input) + if err != nil { + return nil, err + } + + const idLen = len(types.Hash256{}) + if len(decoded) < len(types.Hash256{}) { + return nil, errors.New("should have hex encoded 32 byte input") + } + return decoded[:idLen], nil + } + + id, err := decodeHex(input) + if err != nil { + return explorer.SearchTypeInvalid, fmt.Errorf("%w: %w", explorer.ErrSearchParse, err) + } + + var result explorer.SearchType + err = s.transaction(func(tx *txn) error { + var exists bool + queries := []struct { + query string + typ explorer.SearchType + }{ + {`SELECT EXISTS(SELECT 1 FROM address_balance WHERE address=?)`, explorer.SearchTypeAddress}, + {`SELECT EXISTS(SELECT 1 FROM blocks WHERE id=?)`, explorer.SearchTypeBlock}, + {`SELECT EXISTS(SELECT 1 FROM transactions WHERE transaction_id=?)`, explorer.SearchTypeTransaction}, + {`SELECT EXISTS(SELECT 1 FROM v2_transactions WHERE transaction_id=?)`, explorer.SearchTypeV2Transaction}, + {`SELECT EXISTS(SELECT 1 FROM siacoin_elements WHERE output_id=?)`, explorer.SearchTypeSiacoinElement}, + {`SELECT EXISTS(SELECT 1 FROM siafund_elements WHERE output_id=?)`, explorer.SearchTypeSiafundElement}, + {`SELECT EXISTS(SELECT 1 FROM last_contract_revision WHERE contract_id=?)`, explorer.SearchTypeContract}, + {`SELECT EXISTS(SELECT 1 FROM v2_last_contract_revision WHERE contract_id=?)`, explorer.SearchTypeV2Contract}, + {`SELECT EXISTS(SELECT 1 FROM host_info WHERE public_key=?)`, explorer.SearchTypeHost}, + } + + for _, q := range queries { + err := tx.QueryRow(q.query, id).Scan(&exists) + if err != nil { + return err + } + if exists { + result = q.typ + return nil + } + } + return explorer.ErrNoSearchResults + }) + if err != nil { + return explorer.SearchTypeInvalid, err + } + return result, nil +} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 7c03a7c..55a2d8d 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -88,9 +88,9 @@ func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { Tx: tx, log: log, } - if err = fn(ltx); err != nil { + if err := fn(ltx); err != nil { return err - } else if err = tx.Commit(); err != nil { + } else if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } return nil @@ -103,6 +103,7 @@ func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { if err != nil { return nil, err } + store := &Store{ db: db, log: log, diff --git a/persist/sqlite/transactions.go b/persist/sqlite/transactions.go index 17f1338..0fa27b5 100644 --- a/persist/sqlite/transactions.go +++ b/persist/sqlite/transactions.go @@ -1,306 +1,500 @@ package sqlite import ( + "database/sql" + "errors" "fmt" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/explored/explorer" ) +// TransactionChainIndices returns the chain indices of the blocks the transaction +// was included in. If the transaction has not been included in any blocks, the +// result will be nil,nil. +func (s *Store) TransactionChainIndices(txnID types.TransactionID, offset, limit uint64) (indices []types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT DISTINCT b.id, b.height FROM blocks b +INNER JOIN block_transactions bt ON bt.block_id = b.id +INNER JOIN transactions t ON t.id = bt.transaction_id +WHERE t.transaction_id = ? +ORDER BY b.height DESC +LIMIT ? OFFSET ?`, encode(txnID), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var index types.ChainIndex + if err := rows.Scan(decode(&index.ID), decode(&index.Height)); err != nil { + return fmt.Errorf("failed to scan chain index: %w", err) + } + indices = append(indices, index) + } + return rows.Err() + }) + return +} + +// transactionMinerFee returns the miner fees for each transaction. +func transactionMinerFee(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT fee +FROM transaction_miner_fees +WHERE transaction_id = ? +ORDER BY transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var fee types.Currency + if err := rows.Scan(decode(&fee)); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + txns[i].MinerFees = append(txns[i].MinerFees, fee) + } + return rows.Err() + }() + if err != nil { + return err + } + } + return nil +} + // transactionArbitraryData returns the arbitrary data for each transaction. -func transactionArbitraryData(tx *txn, txnIDs []int64) (map[int64][][]byte, error) { - query := `SELECT transaction_id, data +func transactionArbitraryData(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT data FROM transaction_arbitrary_data -WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +WHERE transaction_id = ? +ORDER BY transaction_order ASC`) if err != nil { - return nil, err + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - result := make(map[int64][][]byte) - for rows.Next() { - var txnID int64 - var data []byte - if err := rows.Scan(&txnID, &data); err != nil { - return nil, fmt.Errorf("failed to scan arbitrary data: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var data []byte + if err := rows.Scan(&data); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + txns[i].ArbitraryData = append(txns[i].ArbitraryData, data) + } + return rows.Err() + }() + if err != nil { + return err } - result[txnID] = append(result[txnID], data) } - return result, nil + return nil +} + +// transactionSignatures returns the signatures for each transaction. +func transactionSignatures(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT parent_id, public_key_index, timelock, covered_fields, signature +FROM transaction_signatures +WHERE transaction_id = ? +ORDER BY transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var sig types.TransactionSignature + if err := rows.Scan(decode(&sig.ParentID), &sig.PublicKeyIndex, decode(&sig.Timelock), decode(&sig.CoveredFields), &sig.Signature); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + txns[i].Signatures = append(txns[i].Signatures, sig) + } + return rows.Err() + }() + if err != nil { + return err + } + } + return nil } // transactionSiacoinOutputs returns the siacoin outputs for each transaction. -func transactionSiacoinOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiacoinOutput, error) { - query := `SELECT ts.transaction_id, sc.output_id, sc.leaf_index, sc.source, sc.maturity_height, sc.address, sc.value +func transactionSiacoinOutputs(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT sc.output_id, sc.leaf_index, sc.spent_index, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc -INNER JOIN transaction_siacoin_outputs ts ON (ts.output_id = sc.id) -WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY ts.transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +INNER JOIN transaction_siacoin_outputs ts ON ts.output_id = sc.id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, fmt.Errorf("failed to query siacoin output ids: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - // map transaction ID to output list - result := make(map[int64][]explorer.SiacoinOutput) - for rows.Next() { - var txnID int64 - var sco explorer.SiacoinOutput - if err := rows.Scan(&txnID, decode(&sco.StateElement.ID), decode(&sco.LeafIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { - return nil, fmt.Errorf("failed to scan siacoin output: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var spentIndex types.ChainIndex + var sco explorer.SiacoinOutput + if err := rows.Scan(decode(&sco.ID), decode(&sco.StateElement.LeafIndex), decodeNull(&spentIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { + return fmt.Errorf("failed to scan siacoin output: %w", err) + } + if spentIndex != (types.ChainIndex{}) { + sco.SpentIndex = &spentIndex + } + txns[i].SiacoinOutputs = append(txns[i].SiacoinOutputs, sco) + } + return rows.Err() + }() + if err != nil { + return err } - result[txnID] = append(result[txnID], sco) } - return result, nil + return nil } // transactionSiacoinInputs returns the siacoin inputs for each transaction. -func transactionSiacoinInputs(tx *txn, txnIDs []int64) (map[int64][]types.SiacoinInput, error) { - query := `SELECT transaction_id, parent_id, unlock_conditions -FROM transaction_siacoin_inputs -WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +func transactionSiacoinInputs(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT sc.output_id, ts.unlock_conditions, sc.value +FROM siacoin_elements sc +INNER JOIN transaction_siacoin_inputs ts ON ts.parent_id = sc.id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, err + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - result := make(map[int64][]types.SiacoinInput) - for rows.Next() { - var txnID int64 - var sci types.SiacoinInput - if err := rows.Scan(&txnID, decode(&sci.ParentID), decode(&sci.UnlockConditions)); err != nil { - return nil, fmt.Errorf("failed to scan siacoin input: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var sci explorer.SiacoinInput + if err := rows.Scan(decode(&sci.ParentID), decode(&sci.UnlockConditions), decode(&sci.Value)); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + sci.Address = sci.UnlockConditions.UnlockHash() + txns[i].SiacoinInputs = append(txns[i].SiacoinInputs, sci) + } + return rows.Err() + }() + if err != nil { + return err } - result[txnID] = append(result[txnID], sci) } - return result, nil + return nil } // transactionSiafundInputs returns the siafund inputs for each transaction. -func transactionSiafundInputs(tx *txn, txnIDs []int64) (map[int64][]types.SiafundInput, error) { - query := `SELECT transaction_id, parent_id, unlock_conditions, claim_address -FROM transaction_siafund_inputs -WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +func transactionSiafundInputs(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT sf.output_id, ts.unlock_conditions, ts.claim_address, sf.value +FROM siafund_elements sf +INNER JOIN transaction_siafund_inputs ts ON ts.parent_id = sf.id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, err + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - result := make(map[int64][]types.SiafundInput) - for rows.Next() { - var txnID int64 - var sfi types.SiafundInput - if err := rows.Scan(&txnID, decode(&sfi.ParentID), decode(&sfi.UnlockConditions), decode(&sfi.ClaimAddress)); err != nil { - return nil, fmt.Errorf("failed to scan siafund input: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var sfi explorer.SiafundInput + if err := rows.Scan(decode(&sfi.ParentID), decode(&sfi.UnlockConditions), decode(&sfi.ClaimAddress), decode(&sfi.Value)); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + sfi.Address = sfi.UnlockConditions.UnlockHash() + txns[i].SiafundInputs = append(txns[i].SiafundInputs, sfi) + } + return rows.Err() + }() + if err != nil { + return err } - result[txnID] = append(result[txnID], sfi) } - return result, nil + return nil } // transactionSiafundOutputs returns the siafund outputs for each transaction. -func transactionSiafundOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiafundOutput, error) { - query := `SELECT ts.transaction_id, sf.output_id, sf.leaf_index, sf.claim_start, sf.address, sf.value +func transactionSiafundOutputs(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT sf.output_id, sf.leaf_index, sf.spent_index, sf.claim_start, sf.address, sf.value FROM siafund_elements sf -INNER JOIN transaction_siafund_outputs ts ON (ts.output_id = sf.id) -WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY ts.transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +INNER JOIN transaction_siafund_outputs ts ON ts.output_id = sf.id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, fmt.Errorf("failed to query siafund output ids: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - // map transaction ID to output list - result := make(map[int64][]explorer.SiafundOutput) - for rows.Next() { - var txnID int64 - var sfo explorer.SiafundOutput - if err := rows.Scan(&txnID, decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { - return nil, fmt.Errorf("failed to scan siafund output: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var spentIndex types.ChainIndex + var sfo explorer.SiafundOutput + if err := rows.Scan(decode(&sfo.ID), decode(&sfo.StateElement.LeafIndex), decodeNull(&spentIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + + if spentIndex != (types.ChainIndex{}) { + sfo.SpentIndex = &spentIndex + } + txns[i].SiafundOutputs = append(txns[i].SiafundOutputs, sfo) + } + return rows.Err() + }() + if err != nil { + return err } - result[txnID] = append(result[txnID], sfo) } - return result, nil -} - -type fileContractProofOutputs struct { - valid []types.SiacoinOutput - missed []types.SiacoinOutput + return nil } -func fileContractOutputs(tx *txn, contractIDs []int64) (map[int64]fileContractProofOutputs, error) { - result := make(map[int64]fileContractProofOutputs) - - validQuery := `SELECT contract_id, address, value -FROM file_contract_valid_proof_outputs -WHERE contract_id IN (` + queryPlaceHolders(len(contractIDs)) + `) -ORDER BY contract_order` - validRows, err := tx.Query(validQuery, queryArgs(contractIDs)...) +func fileContractOutputs(tx *txn, contractID int64) (valid []explorer.ContractSiacoinOutput, missed []explorer.ContractSiacoinOutput, err error) { + validRows, err := tx.Query(`SELECT id, address, value + FROM file_contract_valid_proof_outputs + WHERE contract_id = ? + ORDER BY contract_order`, contractID) if err != nil { - return nil, err + return nil, nil, err } defer validRows.Close() for validRows.Next() { - var contractID int64 - var sco types.SiacoinOutput - if err := validRows.Scan(&contractID, decode(&sco.Address), decode(&sco.Value)); err != nil { - return nil, fmt.Errorf("failed to scan valid proof output: %w", err) + var sco explorer.ContractSiacoinOutput + if err := validRows.Scan(decode(&sco.ID), decode(&sco.Address), decode(&sco.Value)); err != nil { + return nil, nil, fmt.Errorf("failed to scan valid proof output: %w", err) } - - r := result[contractID] - r.valid = append(r.valid, sco) - result[contractID] = r + valid = append(valid, sco) } - missedQuery := `SELECT contract_id, address, value + missedRows, err := tx.Query(`SELECT id, address, value FROM file_contract_missed_proof_outputs -WHERE contract_id IN (` + queryPlaceHolders(len(contractIDs)) + `) -ORDER BY contract_order` - missedRows, err := tx.Query(missedQuery, queryArgs(contractIDs)...) +WHERE contract_id = ? +ORDER BY contract_order`, contractID) if err != nil { - return nil, err + return nil, nil, err } defer missedRows.Close() for missedRows.Next() { - var contractID int64 - var sco types.SiacoinOutput - if err := missedRows.Scan(&contractID, decode(&sco.Address), decode(&sco.Value)); err != nil { - return nil, fmt.Errorf("failed to scan missed proof output: %w", err) + var sco explorer.ContractSiacoinOutput + if err := missedRows.Scan(decode(&sco.ID), decode(&sco.Address), decode(&sco.Value)); err != nil { + return nil, nil, fmt.Errorf("failed to scan valid proof output: %w", err) } - - r := result[contractID] - r.missed = append(r.missed, sco) - result[contractID] = r + missed = append(missed, sco) } - - return result, nil -} - -type contractOrder struct { - txnID int64 - transactionOrder int64 + return valid, missed, nil } // transactionFileContracts returns the file contracts for each transaction. -func transactionFileContracts(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContract, error) { - query := `SELECT ts.transaction_id, fc.id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number +func transactionFileContracts(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT fc.id, fc.contract_id, fc.resolved, fc.valid, fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.proof_height, rev.proof_block_id, rev.proof_transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc -INNER JOIN transaction_file_contracts ts ON (ts.contract_id = fc.id) -WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY ts.transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +INNER JOIN transaction_file_contracts ts ON ts.contract_id = fc.id +INNER JOIN last_contract_revision rev ON rev.contract_id = fc.contract_id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, fmt.Errorf("failed to query contract output ids: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - var contractIDs []int64 - // map transaction ID to contract list - result := make(map[int64][]explorer.FileContract) - // map contract ID to transaction ID - contractTransaction := make(map[int64]contractOrder) - for rows.Next() { - var txnID, contractID int64 - var fc explorer.FileContract - if err := rows.Scan(&txnID, &contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { - return nil, fmt.Errorf("failed to scan file contract: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + _, fc, err := scanFileContract(tx, rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + txns[i].FileContracts = append(txns[i].FileContracts, fc) + } + return rows.Err() + }() + if err != nil { + return err } - - result[txnID] = append(result[txnID], fc) - contractIDs = append(contractIDs, contractID) - contractTransaction[contractID] = contractOrder{txnID, int64(len(result[txnID])) - 1} - } - - proofOutputs, err := fileContractOutputs(tx, contractIDs) - if err != nil { - return nil, fmt.Errorf("failed to get file contract outputs: %w", err) } - for contractID, output := range proofOutputs { - index := contractTransaction[contractID] - result[index.txnID][index.transactionOrder].ValidProofOutputs = output.valid - result[index.txnID][index.transactionOrder].MissedProofOutputs = output.missed - } - - return result, nil + return nil } // transactionFileContracts returns the file contract revisions for each transaction. -func transactionFileContractRevisions(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContractRevision, error) { - query := `SELECT ts.transaction_id, fc.id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number +func transactionFileContractRevisions(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT fc.id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.proof_height, rev.proof_block_id, rev.proof_transaction_id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.resolved, fc.valid, fc.transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc -INNER JOIN transaction_file_contract_revisions ts ON (ts.contract_id = fc.id) -WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) -ORDER BY ts.transaction_order ASC` - rows, err := tx.Query(query, queryArgs(txnIDs)...) +INNER JOIN transaction_file_contract_revisions ts ON ts.contract_id = fc.id +INNER JOIN last_contract_revision rev ON rev.contract_id = fc.contract_id +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) if err != nil { - return nil, fmt.Errorf("failed to query contract output ids: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - var contractIDs []int64 - // map transaction ID to contract list - result := make(map[int64][]explorer.FileContractRevision) - // map contract ID to transaction ID - contractTransaction := make(map[int64]contractOrder) - for rows.Next() { - var txnID, contractID int64 - var fc explorer.FileContractRevision - if err := rows.Scan(&txnID, &contractID, decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { - return nil, fmt.Errorf("failed to scan file contract: %w", err) + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var contractID int64 + var fc explorer.FileContractRevision + + var proofIndex types.ChainIndex + var proofTransactionID types.TransactionID + if err := rows.Scan(&contractID, decode(&fc.ConfirmationIndex.Height), decode(&fc.ConfirmationIndex.ID), decode(&fc.ConfirmationTransactionID), decodeNull(&proofIndex.Height), decodeNull(&proofIndex.ID), decodeNull(&proofTransactionID), decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.ID), &fc.Resolved, &fc.Valid, decode(&fc.TransactionID), decode(&fc.ExtendedFileContract.Filesize), decode(&fc.ExtendedFileContract.FileMerkleRoot), decode(&fc.ExtendedFileContract.WindowStart), decode(&fc.ExtendedFileContract.WindowEnd), decode(&fc.ExtendedFileContract.Payout), decode(&fc.ExtendedFileContract.UnlockHash), decode(&fc.ExtendedFileContract.RevisionNumber)); err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + fc.ValidProofOutputs, fc.MissedProofOutputs, err = fileContractOutputs(tx, contractID) + if err != nil { + return fmt.Errorf("failed to get contract proof outputs: %w", err) + } + + if proofIndex != (types.ChainIndex{}) { + fc.ProofIndex = &proofIndex + } + if proofTransactionID != (types.TransactionID{}) { + fc.ProofTransactionID = &proofTransactionID + } + + txns[i].FileContractRevisions = append(txns[i].FileContractRevisions, fc) + } + return rows.Err() + }() + if err != nil { + return err } - - result[txnID] = append(result[txnID], fc) - contractIDs = append(contractIDs, contractID) - contractTransaction[contractID] = contractOrder{txnID, int64(len(result[txnID])) - 1} } + return nil +} - proofOutputs, err := fileContractOutputs(tx, contractIDs) +// transactionStorageProofs returns the storage proofs for each transaction. +func transactionStorageProofs(tx *txn, dbIDs []int64, txns []explorer.Transaction) error { + stmt, err := tx.Prepare(`SELECT transaction_id, parent_id, leaf, proof +FROM transaction_storage_proofs +WHERE transaction_id = ? +ORDER BY transaction_order ASC`) if err != nil { - return nil, fmt.Errorf("failed to get file contract outputs: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - for contractID, output := range proofOutputs { - index := contractTransaction[contractID] - result[index.txnID][index.transactionOrder].ValidProofOutputs = output.valid - result[index.txnID][index.transactionOrder].MissedProofOutputs = output.missed + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var proof types.StorageProof + leaf := make([]byte, 64) + if err := rows.Scan(decode(&proof.ParentID), &leaf, decode(&proof.Proof)); err != nil { + return fmt.Errorf("failed to scan: %w", err) + } + proof.Leaf = [64]byte(leaf) + txns[i].StorageProofs = append(txns[i].StorageProofs, proof) + } + return rows.Err() + }() + if err != nil { + return err + } } + return nil +} - return result, nil +type transactionID struct { + id types.TransactionID + dbID int64 } // blockTransactionIDs returns the database ID for each transaction in the // block. -func blockTransactionIDs(tx *txn, blockID types.BlockID) (dbIDs []int64, err error) { - rows, err := tx.Query(`SELECT transaction_id FROM block_transactions WHERE block_id = ? ORDER BY block_order`, encode(blockID)) +func blockTransactionIDs(tx *txn, blockID types.BlockID) (txnIDs []types.TransactionID, err error) { + rows, err := tx.Query(`SELECT t.transaction_id +FROM block_transactions bt +INNER JOIN transactions t ON t.id = bt.transaction_id +WHERE block_id = ? ORDER BY block_order ASC`, encode(blockID)) if err != nil { return nil, err } defer rows.Close() for rows.Next() { - var dbID int64 - if err := rows.Scan(&dbID); err != nil { + var txnID types.TransactionID + if err := rows.Scan(decode(&txnID)); err != nil { return nil, fmt.Errorf("failed to scan block transaction: %w", err) } - dbIDs = append(dbIDs, dbID) + txnIDs = append(txnIDs, txnID) } + return } // blockMinerPayouts returns the miner payouts for the block. func blockMinerPayouts(tx *txn, blockID types.BlockID) ([]explorer.SiacoinOutput, error) { - query := `SELECT sc.output_id, sc.leaf_index, sc.source, sc.maturity_height, sc.address, sc.value + query := `SELECT sc.output_id, sc.leaf_index, sc.spent_index, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc -INNER JOIN miner_payouts mp ON (mp.output_id = sc.id) +INNER JOIN miner_payouts mp ON mp.output_id = sc.id WHERE mp.block_id = ? ORDER BY mp.block_order ASC` rows, err := tx.Query(query, encode(blockID)) @@ -311,105 +505,85 @@ ORDER BY mp.block_order ASC` var result []explorer.SiacoinOutput for rows.Next() { + var spentIndex types.ChainIndex var output explorer.SiacoinOutput - if err := rows.Scan(decode(&output.StateElement.ID), decode(&output.StateElement.LeafIndex), &output.Source, &output.MaturityHeight, decode(&output.SiacoinOutput.Address), decode(&output.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(decode(&output.ID), decode(&output.StateElement.LeafIndex), decodeNull(&spentIndex), &output.Source, &output.MaturityHeight, decode(&output.SiacoinOutput.Address), decode(&output.SiacoinOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan miner payout: %w", err) } + if spentIndex != (types.ChainIndex{}) { + output.SpentIndex = &spentIndex + } result = append(result, output) } return result, nil } // transactionDatabaseIDs returns the database ID for each transaction. -func transactionDatabaseIDs(tx *txn, txnIDs []types.TransactionID) (dbIDs []int64, err error) { - encodedIDs := func(ids []types.TransactionID) []any { - result := make([]any, len(ids)) - for i, id := range ids { - result[i] = encode(id) - } - return result - } - - query := `SELECT id FROM transactions WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `)` - rows, err := tx.Query(query, encodedIDs(txnIDs)...) +func transactionDatabaseIDs(tx *txn, txnIDs []types.TransactionID) (dbIDs []int64, txns []explorer.Transaction, err error) { + stmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) if err != nil { - return nil, err + return nil, nil, fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - for rows.Next() { + for _, txnID := range txnIDs { var dbID int64 - if err := rows.Scan(&dbID); err != nil { - return nil, fmt.Errorf("failed to scan transaction: %w", err) + if err := stmt.QueryRow(encode(txnID)).Scan(&dbID); errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return nil, nil, fmt.Errorf("failed to get transaction database ID: %w", err) } + dbIDs = append(dbIDs, dbID) + txns = append(txns, explorer.Transaction{ + ID: txnID, + }) } return } -func (s *Store) getTransactions(tx *txn, dbIDs []int64) ([]explorer.Transaction, error) { - txnArbitraryData, err := transactionArbitraryData(tx, dbIDs) +func getTransactions(tx *txn, ids []types.TransactionID) ([]explorer.Transaction, error) { + dbIDs, txns, err := transactionDatabaseIDs(tx, ids) if err != nil { + return nil, fmt.Errorf("getTransactions: failed to get base transactions: %w", err) + } else if err := transactionArbitraryData(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get arbitrary data: %w", err) - } - - txnSiacoinInputs, err := transactionSiacoinInputs(tx, dbIDs) - if err != nil { + } else if err := transactionMinerFee(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getTransactions: failed to get miner fees: %w", err) + } else if err := transactionSignatures(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getTransactions: failed to get signatures: %w", err) + } else if err := transactionSiacoinInputs(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get siacoin inputs: %w", err) - } - - txnSiacoinOutputs, err := transactionSiacoinOutputs(tx, dbIDs) - if err != nil { + } else if err := transactionSiacoinOutputs(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get siacoin outputs: %w", err) - } - - txnSiafundInputs, err := transactionSiafundInputs(tx, dbIDs) - if err != nil { + } else if err := transactionSiafundInputs(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get siafund inputs: %w", err) - } - - txnSiafundOutputs, err := transactionSiafundOutputs(tx, dbIDs) - if err != nil { + } else if err := transactionSiafundOutputs(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get siafund outputs: %w", err) - } - - txnFileContracts, err := transactionFileContracts(tx, dbIDs) - if err != nil { + } else if err := transactionFileContracts(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get file contracts: %w", err) - } - - txnFileContractRevisions, err := transactionFileContractRevisions(tx, dbIDs) - if err != nil { + } else if err := transactionFileContractRevisions(tx, dbIDs, txns); err != nil { return nil, fmt.Errorf("getTransactions: failed to get file contract revisions: %w", err) + } else if err := transactionStorageProofs(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getTransactions: failed to get storage proofs: %w", err) } - // TODO: storage proofs - // TODO: signatures - - var results []explorer.Transaction - for _, dbID := range dbIDs { - txn := explorer.Transaction{ - ArbitraryData: txnArbitraryData[dbID], - SiacoinInputs: txnSiacoinInputs[dbID], - SiacoinOutputs: txnSiacoinOutputs[dbID], - SiafundInputs: txnSiafundInputs[dbID], - SiafundOutputs: txnSiafundOutputs[dbID], - FileContracts: txnFileContracts[dbID], - FileContractRevisions: txnFileContractRevisions[dbID], + for i := range txns { + for _, arb := range txns[i].ArbitraryData { + var ha chain.HostAnnouncement + if ha.FromArbitraryData(arb) { + txns[i].HostAnnouncements = append(txns[i].HostAnnouncements, ha) + } } - results = append(results, txn) } - return results, nil + + return txns, nil } // Transactions implements explorer.Store. func (s *Store) Transactions(ids []types.TransactionID) (results []explorer.Transaction, err error) { err = s.transaction(func(tx *txn) error { - dbIDs, err := transactionDatabaseIDs(tx, ids) - if err != nil { - return fmt.Errorf("failed to get transaction IDs: %w", err) - } - results, err = s.getTransactions(tx, dbIDs) + results, err = getTransactions(tx, ids) if err != nil { return fmt.Errorf("failed to get transactions: %w", err) } diff --git a/persist/sqlite/v2consensus.go b/persist/sqlite/v2consensus.go new file mode 100644 index 0000000..0d0100d --- /dev/null +++ b/persist/sqlite/v2consensus.go @@ -0,0 +1,562 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +func addV2Transactions(tx *txn, bid types.BlockID, txns []types.V2Transaction) (map[types.TransactionID]txnDBId, error) { + checkTransactionStmt, err := tx.Prepare(`SELECT id FROM v2_transactions WHERE transaction_id = ?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare check v2_transaction statement: %v", err) + } + defer checkTransactionStmt.Close() + + insertTransactionStmt, err := tx.Prepare(`INSERT INTO v2_transactions (transaction_id, new_foundation_address, miner_fee, arbitrary_data) VALUES (?, ?, ?, ?)`) + if err != nil { + return nil, fmt.Errorf("failed to prepare insert v2_transaction statement: %v", err) + } + defer insertTransactionStmt.Close() + + blockTransactionsStmt, err := tx.Prepare(`INSERT INTO v2_block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) + if err != nil { + return nil, fmt.Errorf("failed to prepare v2_block_transactions statement: %w", err) + } + defer blockTransactionsStmt.Close() + + txnDBIds := make(map[types.TransactionID]txnDBId) + for i, txn := range txns { + var exist bool + var dbID int64 + txnID := txn.ID() + if err := checkTransactionStmt.QueryRow(encode(txnID)).Scan(&dbID); err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to insert v2 transaction ID: %w", err) + } else if err == nil { + exist = true + } + + if !exist { + var newFoundationAddress any + if txn.NewFoundationAddress != nil { + newFoundationAddress = encode(txn.NewFoundationAddress) + } + + result, err := insertTransactionStmt.Exec(encode(txnID), newFoundationAddress, encode(txn.MinerFee), txn.ArbitraryData) + if err != nil { + return nil, fmt.Errorf("failed to insert into v2_transactions: %w", err) + } + dbID, err = result.LastInsertId() + if err != nil { + return nil, fmt.Errorf("failed to get v2 transaction ID: %w", err) + } + } + + // If we have the same transaction multiple times in one block, exist + // will be true after the above query after the first time the + // transaction is encountered by this loop. So we only set the value in + // the map for each transaction once. + if _, ok := txnDBIds[txnID]; !ok { + txnDBIds[txnID] = txnDBId{id: dbID, exist: exist} + } + + if _, err := blockTransactionsStmt.Exec(encode(bid), dbID, i); err != nil { + return nil, fmt.Errorf("failed to insert into v2_block_transactions: %w", err) + } + } + return txnDBIds, nil +} + +func updateV2FileContractElements(tx *txn, revert bool, index types.ChainIndex, b types.Block, fces []explorer.V2FileContractUpdate) (map[explorer.DBFileContract]int64, error) { + stmt, err := tx.Prepare(`INSERT INTO v2_file_contract_elements(contract_id, block_id, transaction_id, leaf_index, capacity, filesize, file_merkle_root, proof_height, expiration_height, renter_output_address, renter_output_value, host_output_address, host_output_value, missed_host_value, total_collateral, renter_public_key, host_public_key, revision_number, renter_signature, host_signature) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (contract_id, revision_number) + DO UPDATE SET leaf_index = EXCLUDED.leaf_index + RETURNING id;`) + if err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to prepare main statement: %w", err) + } + defer stmt.Close() + + revisionStmt, err := tx.Prepare(`INSERT INTO v2_last_contract_revision(contract_id, contract_element_id, confirmation_height, confirmation_block_id, confirmation_transaction_id) + VALUES (?, ?, COALESCE(?, X''), COALESCE(?, X''), COALESCE(?, X'')) + ON CONFLICT (contract_id) + DO UPDATE SET contract_element_id = ?, confirmation_height = COALESCE(?, confirmation_height), confirmation_block_id = COALESCE(?, confirmation_block_id), confirmation_transaction_id = COALESCE(?, confirmation_transaction_id)`) + if err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to prepare last_contract_revision statement: %w", err) + } + defer revisionStmt.Close() + + // so we can get the ids of revision parents to add to the DB + parentStmt, err := tx.Prepare(`SELECT id FROM v2_file_contract_elements WHERE contract_id = ? AND revision_number = ?`) + if err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to prepare parent statement: %w", err) + } + defer parentStmt.Close() + + fcTxns := make(map[explorer.DBFileContract]types.TransactionID) + for _, txn := range b.V2Transactions() { + id := txn.ID() + + for i, fc := range txn.FileContracts { + fcTxns[explorer.DBFileContract{ + ID: txn.V2FileContractID(id, i), + RevisionNumber: fc.RevisionNumber, + }] = id + } + for _, fcr := range txn.FileContractRevisions { + fcTxns[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID), + RevisionNumber: fcr.Revision.RevisionNumber, + }] = id + } + for _, fcr := range txn.FileContractResolutions { + if v, ok := fcr.Resolution.(*types.V2FileContractRenewal); ok { + fcTxns[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID).V2RenewalID(), + RevisionNumber: v.NewContract.RevisionNumber, + }] = id + } + } + } + + fcDBIds := make(map[explorer.DBFileContract]int64) + addFC := func(fcID types.FileContractID, leafIndex uint64, fc types.V2FileContract, confirmationTransactionID *types.TransactionID, lastRevision bool) error { + var dbID int64 + dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} + err := stmt.QueryRow(encode(fcID), encode(b.ID()), encode(fcTxns[dbFC]), encode(leafIndex), encode(fc.Capacity), encode(fc.Filesize), encode(fc.FileMerkleRoot), encode(fc.ProofHeight), encode(fc.ExpirationHeight), encode(fc.RenterOutput.Address), encode(fc.RenterOutput.Value), encode(fc.HostOutput.Address), encode(fc.HostOutput.Value), encode(fc.MissedHostValue), encode(fc.TotalCollateral), encode(fc.RenterPublicKey), encode(fc.HostPublicKey), encode(fc.RevisionNumber), encode(fc.RenterSignature), encode(fc.HostSignature)).Scan(&dbID) + if err != nil { + return fmt.Errorf("failed to execute v2_file_contract_elements statement: %w", err) + } + + // only update if it's the most recent revision which will come from + // running ForEachFileContractElement on the update + if lastRevision { + var encodedHeight, encodedBlockID, encodedConfirmationTransactionID []byte + if confirmationTransactionID != nil { + encodedHeight = encode(index.Height).([]byte) + encodedBlockID = encode(index.ID).([]byte) + encodedConfirmationTransactionID = encode(*confirmationTransactionID).([]byte) + } + + if _, err := revisionStmt.Exec(encode(fcID), dbID, encodedHeight, encodedBlockID, encodedConfirmationTransactionID, dbID, encodedHeight, encodedBlockID, encodedConfirmationTransactionID); err != nil { + return fmt.Errorf("failed to update last revision number: %w", err) + } + } + + fcDBIds[dbFC] = dbID + return nil + } + + for _, update := range fces { + var fce *types.V2FileContractElement + + if revert { + // Reverting + if update.Resolution != nil { + fce = &update.FileContractElement + } else if update.Revision != nil { + // Contract revision reverted. + // We are reverting the revision, so get the contract before + // the revision. + fce = &update.FileContractElement + } else { + // Contract formation reverted. + // The contract update has no revision, therefore it refers + // to the original contract formation. + continue + } + } else { + // Applying + fce = &update.FileContractElement + if update.Revision != nil { + // Contract is revised. + // We want last_contract_revision to refer to the latest + // revision, so use the revision FCE if there is one. + fce = update.Revision + } + } + + if err := addFC( + types.FileContractID(fce.ID), + fce.StateElement.LeafIndex, + fce.V2FileContract, + update.ConfirmationTransactionID, + true, + ); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: %w", err) + } + } + + if revert { + return fcDBIds, nil + } + + for _, txn := range b.V2Transactions() { + // add in any contracts that are not the latest, i.e. contracts that + // were created and revised in the same block + for j, fc := range txn.FileContracts { + fcID := txn.V2FileContractID(txn.ID(), j) + dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} + if _, exists := fcDBIds[dbFC]; exists { + continue + } + + if err := addFC(fcID, 0, fc, nil, false); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: %w", err) + } + } + // add in any revisions that are not the latest, i.e. contracts that + // were revised multiple times in one block + for _, fcr := range txn.FileContractRevisions { + fc := fcr.Revision + fcID := types.FileContractID(fcr.Parent.ID) + dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} + if _, exists := fcDBIds[dbFC]; exists { + continue + } + + if err := addFC(fcID, 0, fc, nil, false); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: %w", err) + } + } + // Add the new renewal contracts + for _, fcr := range txn.FileContractResolutions { + if v, ok := fcr.Resolution.(*types.V2FileContractRenewal); ok { + { + // Add NewContract if we have not seen it already. + // Only way this could happen is if the renewal is revised + // in the same block so that the initial renewal is not the + // "latest" revision of it. + fc := v.NewContract + fcID := types.FileContractID(fcr.Parent.ID).V2RenewalID() + dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} + if _, exists := fcDBIds[dbFC]; exists { + continue + } + + if err := addFC(fcID, 0, fc, nil, false); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to add new contract: %w", err) + } + } + } + } + // don't add anything, just set parent db IDs in fcDBIds map + for _, fcr := range txn.FileContractRevisions { + fcID := types.FileContractID(fcr.Parent.ID) + parentDBFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fcr.Parent.V2FileContract.RevisionNumber} + + var dbID int64 + if err := parentStmt.QueryRow(encode(fcID), encode(parentDBFC.RevisionNumber)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to get parent contract ID: %w", err) + } + fcDBIds[parentDBFC] = dbID + } + // don't add anything, just set parent db IDs in fcDBIds map + for _, fcr := range txn.FileContractResolutions { + fcID := types.FileContractID(fcr.Parent.ID) + parentDBFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fcr.Parent.V2FileContract.RevisionNumber} + + var dbID int64 + if err := parentStmt.QueryRow(encode(fcID), encode(parentDBFC.RevisionNumber)).Scan(&dbID); err != nil { + return nil, fmt.Errorf("updateV2FileContractElements: failed to get parent contract ID: %w", err) + } + fcDBIds[parentDBFC] = dbID + } + } + + return fcDBIds, nil +} + +func updateV2FileContractIndices(tx *txn, revert bool, index types.ChainIndex, fces []explorer.V2FileContractUpdate) error { + resolutionIndexStmt, err := tx.Prepare(`UPDATE v2_last_contract_revision SET resolution_type = ?, resolution_height = ?, resolution_block_id = ?, resolution_transaction_id = ?, renewed_to = ? WHERE contract_id = ?`) + if err != nil { + return fmt.Errorf("updateV2FileContractIndices: failed to prepare resolution index statement: %w", err) + } + defer resolutionIndexStmt.Close() + + renewedFromStmt, err := tx.Prepare(`UPDATE v2_last_contract_revision SET renewed_from = ? WHERE contract_id = ?`) + if err != nil { + return fmt.Errorf("updateV2FileContractIndices: failed to prepare renewed from statement: %w", err) + } + defer renewedFromStmt.Close() + + for _, update := range fces { + // id stays the same even if revert happens so we don't need to check that here + fcID := update.FileContractElement.ID + + if revert { + if update.ResolutionTransactionID != nil { + if _, err := resolutionIndexStmt.Exec(explorer.V2ResolutionInvalid, nil, nil, nil, nil, encode(fcID)); err != nil { + return fmt.Errorf("updateV2FileContractIndices: failed to update resolution index: %w", err) + } + } + } else { + if update.ResolutionTransactionID != nil { + var renewalToID any + if _, ok := update.Resolution.(*types.V2FileContractRenewal); ok { + renewalToID = encode(fcID.V2RenewalID()) + } + + resolutionType := explorer.V2ResolutionType(update.Resolution) + if _, err := resolutionIndexStmt.Exec(resolutionType, encode(index.Height), encode(index.ID), encode(update.ResolutionTransactionID), renewalToID, encode(fcID)); err != nil { + return fmt.Errorf("updateV2FileContractIndices: failed to update resolution index: %w", err) + } + if renewalToID != nil { + if _, err := renewedFromStmt.Exec(encode(fcID), renewalToID); err != nil { + return fmt.Errorf("updateV2FileContractIndices: failed to update renewed from ID: %w", err) + } + } + } + } + } + + return nil +} + +func addV2SiacoinInputs(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, satisfied_policy) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2SiacoinInputs: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, sci := range txn.SiacoinInputs { + dbID, ok := dbIDs[types.SiacoinOutputID(sci.Parent.ID)] + if !ok { + return errors.New("addV2SiacoinInputs: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, dbID, encode(sci.SatisfiedPolicy)); err != nil { + return fmt.Errorf("addV2SiacoinInputs: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2SiacoinOutputs(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2SiacoinOutputs: failed to prepare statement: %w", err) + } + defer stmt.Close() + + id := txn.ID() + for i := range txn.SiacoinOutputs { + dbID, ok := dbIDs[txn.SiacoinOutputID(id, i)] + if !ok { + return errors.New("addV2SiacoinOutputs: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, dbID); err != nil { + return fmt.Errorf("addV2SiacoinOutputs: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2SiafundInputs(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_siafund_inputs(transaction_id, transaction_order, parent_id, claim_address, satisfied_policy) VALUES (?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2SiafundInputs: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, sfi := range txn.SiafundInputs { + dbID, ok := dbIDs[types.SiafundOutputID(sfi.Parent.ID)] + if !ok { + return errors.New("addV2SiafundInputs: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, dbID, encode(sfi.ClaimAddress), encode(sfi.SatisfiedPolicy)); err != nil { + return fmt.Errorf("addV2SiafundInputs: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2SiafundOutputs(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2SiafundOutputs: failed to prepare statement: %w", err) + } + defer stmt.Close() + + id := txn.ID() + for i := range txn.SiafundOutputs { + dbID, ok := dbIDs[txn.SiafundOutputID(id, i)] + if !ok { + return errors.New("addV2SiafundOutputs: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, dbID); err != nil { + return fmt.Errorf("addV2SiafundOutputs: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2FileContracts(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2FileContracts: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, fc := range txn.FileContracts { + dbID, ok := dbIDs[explorer.DBFileContract{ + ID: txn.V2FileContractID(txn.ID(), i), + RevisionNumber: fc.RevisionNumber, + }] + if !ok { + return errors.New("addV2FileContracts: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, dbID); err != nil { + return fmt.Errorf("addV2FileContracts: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2FileContractRevisions(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_file_contract_revisions(transaction_id, transaction_order, parent_contract_id, revision_contract_id) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2FileContractRevisions: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, fcr := range txn.FileContractRevisions { + parentDBID, ok := dbIDs[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID), + RevisionNumber: fcr.Parent.V2FileContract.RevisionNumber, + }] + if !ok { + return errors.New("addV2FileContractRevisions: parent dbID not in map") + } + + dbID, ok := dbIDs[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID), + RevisionNumber: fcr.Revision.RevisionNumber, + }] + if !ok { + return errors.New("addV2FileContractRevisions: dbID not in map") + } + + if _, err := stmt.Exec(txnID, i, parentDBID, dbID); err != nil { + return fmt.Errorf("addV2FileContractRevisions: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2FileContractResolutions(tx *txn, txnID int64, txn types.V2Transaction, dbIDs map[explorer.DBFileContract]int64) error { + renewalStmt, err := tx.Prepare(`INSERT INTO v2_transaction_file_contract_resolutions(transaction_id, transaction_order, parent_contract_id, resolution_type, renewal_new_contract_id, renewal_final_renter_output_address, renewal_final_renter_output_value, renewal_final_host_output_address, renewal_final_host_output_value, renewal_renter_rollover, renewal_host_rollover, renewal_renter_signature, renewal_host_signature) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to prepare renewal statement: %w", err) + } + defer renewalStmt.Close() + + storageProofStmt, err := tx.Prepare(`INSERT INTO v2_transaction_file_contract_resolutions(transaction_id, transaction_order, parent_contract_id, resolution_type, storage_proof_proof_index, storage_proof_leaf, storage_proof_proof) VALUES (?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to prepare storage proof statement: %w", err) + } + defer storageProofStmt.Close() + + expirationStmt, err := tx.Prepare(`INSERT INTO v2_transaction_file_contract_resolutions(transaction_id, transaction_order, parent_contract_id, resolution_type) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to prepare expiration statement: %w", err) + } + defer expirationStmt.Close() + + for i, fcr := range txn.FileContractResolutions { + parentDBID, ok := dbIDs[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID), + RevisionNumber: fcr.Parent.V2FileContract.RevisionNumber, + }] + if !ok { + return errors.New("addV2FileContractResolutions: parent dbID not in map") + } + + resolutionType := explorer.V2ResolutionType(fcr.Resolution) + switch v := fcr.Resolution.(type) { + case *types.V2FileContractRenewal: + newDBID, ok := dbIDs[explorer.DBFileContract{ + ID: types.FileContractID(fcr.Parent.ID).V2RenewalID(), + RevisionNumber: v.NewContract.RevisionNumber, + }] + if !ok { + return errors.New("addV2FileContractResolutions: renewal dbID not in map") + } + + if _, err := renewalStmt.Exec(txnID, i, parentDBID, resolutionType, newDBID, encode(v.FinalRenterOutput.Address), encode(v.FinalRenterOutput.Value), encode(v.FinalHostOutput.Address), encode(v.FinalHostOutput.Value), encode(v.RenterRollover), encode(v.HostRollover), encode(v.RenterSignature), encode(v.HostSignature)); err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to execute renewal statement: %w", err) + } + case *types.V2StorageProof: + if _, err := storageProofStmt.Exec(txnID, i, parentDBID, resolutionType, encode(v.ProofIndex), v.Leaf[:], encode(v.Proof)); err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to execute storage proof statement: %w", err) + } + case *types.V2FileContractExpiration: + if _, err := expirationStmt.Exec(txnID, i, parentDBID, resolutionType); err != nil { + return fmt.Errorf("addV2FileContractResolutions: failed to execute expiration statement: %w", err) + } + } + } + return nil +} + +func addV2Attestations(tx *txn, txnID int64, txn types.V2Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_attestations(transaction_id, transaction_order, public_key, key, value, signature) VALUES (?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("addV2Attestations: failed to prepare statement: %w", err) + } + defer stmt.Close() + + for i, attestation := range txn.Attestations { + if _, err := stmt.Exec(txnID, i, encode(attestation.PublicKey), attestation.Key, attestation.Value, encode(attestation.Signature)); err != nil { + return fmt.Errorf("addV2Attestations: failed to execute statement: %w", err) + } + } + return nil +} + +func addV2TransactionFields(tx *txn, txns []types.V2Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, v2FcDBIds map[explorer.DBFileContract]int64, v2TxnDBIds map[types.TransactionID]txnDBId) error { + for _, txn := range txns { + txnID := txn.ID() + dbID, ok := v2TxnDBIds[txnID] + if !ok { + panic(fmt.Errorf("txn %v should be in txnDBIds", txn.ID())) + } + + // transaction already exists, don't reinsert its fields + if dbID.exist { + continue + } + // set exist = true so we don't re-insert fields in case we have + // multiple of the same transaction in a block + v2TxnDBIds[txnID] = txnDBId{id: dbID.id, exist: true} + + if err := addV2Attestations(tx, dbID.id, txn); err != nil { + return fmt.Errorf("addV2TransactionFields: failed to add attestations: %w", err) + } else if err := addV2SiacoinInputs(tx, dbID.id, txn, scDBIds); err != nil { + return fmt.Errorf("failed to add siacoin inputs: %w", err) + } else if err := addV2SiacoinOutputs(tx, dbID.id, txn, scDBIds); err != nil { + return fmt.Errorf("failed to add siacoin outputs: %w", err) + } else if err := addV2SiafundInputs(tx, dbID.id, txn, sfDBIds); err != nil { + return fmt.Errorf("failed to add siafund inputs: %w", err) + } else if err := addV2SiafundOutputs(tx, dbID.id, txn, sfDBIds); err != nil { + return fmt.Errorf("failed to add siafund outputs: %w", err) + } else if err := addV2FileContracts(tx, dbID.id, txn, v2FcDBIds); err != nil { + return fmt.Errorf("failed to add file contracts: %w", err) + } else if err := addV2FileContractRevisions(tx, dbID.id, txn, v2FcDBIds); err != nil { + return fmt.Errorf("failed to add file contract revisions: %w", err) + } else if err := addV2FileContractResolutions(tx, dbID.id, txn, v2FcDBIds); err != nil { + return fmt.Errorf("failed to add file contract resolutions: %w", err) + } + } + + return nil +} diff --git a/persist/sqlite/v2consensus_test.go b/persist/sqlite/v2consensus_test.go new file mode 100644 index 0000000..36816a5 --- /dev/null +++ b/persist/sqlite/v2consensus_test.go @@ -0,0 +1,1610 @@ +package sqlite_test + +import ( + "bytes" + "math" + "testing" + "time" + + "go.sia.tech/core/consensus" + rhp2 "go.sia.tech/core/rhp/v2" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/explorer" + "go.sia.tech/explored/internal/testutil" +) + +func getSCE(t *testing.T, db explorer.Store, scid types.SiacoinOutputID) types.SiacoinElement { + t.Helper() + + sces, err := db.SiacoinElements([]types.SiacoinOutputID{scid}) + if err != nil { + t.Fatal(err) + } else if len(sces) == 0 { + t.Fatal("can't find sce") + } + return sces[0].SiacoinElement +} + +func getSFE(t *testing.T, db explorer.Store, sfid types.SiafundOutputID) types.SiafundElement { + t.Helper() + + sfes, err := db.SiafundElements([]types.SiafundOutputID{sfid}) + if err != nil { + t.Fatal(err) + } else if len(sfes) == 0 { + t.Fatal("can't find sfe") + } + return sfes[0].SiafundElement +} + +func getFCE(t *testing.T, db explorer.Store, fcid types.FileContractID) types.V2FileContractElement { + t.Helper() + + fces, err := db.V2Contracts([]types.FileContractID{fcid}) + if err != nil { + t.Fatal(err) + } else if len(fces) == 0 { + t.Fatal("can't find fces") + } + return fces[0].V2FileContractElement +} + +func getCIE(t *testing.T, db explorer.Store, bid types.BlockID) types.ChainIndexElement { + t.Helper() + + b, err := db.Block(bid) + if err != nil { + t.Fatal(err) + } + + merkleProof, err := db.MerkleProof(b.LeafIndex) + if err != nil { + t.Fatal(err) + } + return types.ChainIndexElement{ + ID: bid, + StateElement: types.StateElement{ + LeafIndex: b.LeafIndex, + MerkleProof: merkleProof, + }, + ChainIndex: types.ChainIndex{ID: bid, Height: b.Height}, + } +} + +func checkV2Transaction(t *testing.T, db explorer.Store, expected types.V2Transaction) { + txns, err := db.V2Transactions([]types.TransactionID{expected.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(txns)", 1, len(txns)) + testutil.CheckV2Transaction(t, expected, txns[0]) +} + +func TestV2ArbitraryData(t *testing.T) { + _, _, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + }) + + txn1 := types.V2Transaction{ + ArbitraryData: []byte("hello"), + } + + txn2 := types.V2Transaction{ + ArbitraryData: []byte("world"), + } + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1, txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + prev := cm.Tip() + + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "v2 height", b.V2.Height, 1) + testutil.CheckV2Transaction(t, txn1, b.V2.Transactions[0]) + testutil.CheckV2Transaction(t, txn2, b.V2.Transactions[1]) + } + + checkV2Transaction(t, db, txn1) + checkV2Transaction(t, db, txn2) + + txn3 := types.V2Transaction{ + ArbitraryData: []byte("12345"), + } + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1, txn2, txn3}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "v2 height", b.V2.Height, 2) + testutil.CheckV2Transaction(t, txn1, b.V2.Transactions[0]) + testutil.CheckV2Transaction(t, txn2, b.V2.Transactions[1]) + testutil.CheckV2Transaction(t, txn3, b.V2.Transactions[2]) + } + + checkV2Transaction(t, db, txn1) + checkV2Transaction(t, db, txn2) + checkV2Transaction(t, db, txn3) + + testutil.CheckV2ChainIndices(t, db, txn1.ID(), []types.ChainIndex{cm.Tip(), prev}) + testutil.CheckV2ChainIndices(t, db, txn2.ID(), []types.ChainIndex{cm.Tip(), prev}) + testutil.CheckV2ChainIndices(t, db, txn3.ID(), []types.ChainIndex{cm.Tip()}) +} + +func TestV2MinerFee(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + txn1 := types.V2Transaction{ + ArbitraryData: []byte("hello"), + MinerFee: giftSC, + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + } + testutil.SignV2Transaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) +} + +func TestV2FoundationAddress(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + network.HardforkFoundation.FailsafeAddress = addr1 + network.HardforkFoundation.PrimaryAddress = addr1 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + MinerFee: giftSC, + NewFoundationAddress: &addr2, + } + testutil.SignV2Transaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 3, len(events)) + + testutil.Equal(t, "event 0 type", "foundation", events[0].Type) + testutil.Equal(t, "event 1 type", "v2Transaction", events[1].Type) + testutil.Equal(t, "event 2 type", "v1Transaction", events[2].Type) + } +} + +func TestV2Attestations(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + pk2 := types.GeneratePrivateKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + cs := cm.TipState() + + ha1 := chain.V2HostAnnouncement{{ + Protocol: "http", + Address: "127.0.0.1:4444", + }} + ha2 := chain.V2HostAnnouncement{{ + Protocol: "http", + Address: "127.0.0.1:8888", + }} + + otherAttestation := types.Attestation{ + PublicKey: pk1.PublicKey(), + Key: "hello", + Value: []byte("world"), + } + otherAttestation.Signature = pk1.SignHash(cs.AttestationSigHash(otherAttestation)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + MinerFee: giftSC, + Attestations: []types.Attestation{ha1.ToAttestation(cs, pk1), otherAttestation, ha2.ToAttestation(cs, pk2)}, + } + testutil.SignV2Transaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cs, []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + events, err := db.AddressEvents(addr1, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 2, len(events)) + + testutil.CheckV2Transaction(t, txn1, explorer.V2Transaction(events[0].Data.(explorer.EventV2Transaction))) + testutil.CheckTransaction(t, genesisBlock.Transactions[0], events[1].Data.(explorer.EventV1Transaction).Transaction) + } + + checkV2Transaction(t, db, txn1) +} + +func TestV2SiacoinOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + txn1 := types.V2Transaction{ + ArbitraryData: []byte("hello"), + SiacoinOutputs: []types.SiacoinOutput{ + { + Value: giftSC.Div64(2), + Address: addr1, + }, + { + Value: giftSC.Div64(2), + Address: addr2, + }, + }, + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + } + testutil.SignV2Transaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + testutil.CheckBalance(t, db, addr1, giftSC.Div64(2), types.ZeroCurrency, 0) + testutil.CheckBalance(t, db, addr2, giftSC.Div64(2), types.ZeroCurrency, 0) +} + +func TestV2SiafundOutput(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + }) + giftSF := genesisBlock.Transactions[0].SiafundOutputs[0].Value + + txn1 := types.V2Transaction{ + ArbitraryData: []byte("hello"), + SiafundOutputs: []types.SiafundOutput{ + { + Value: giftSF / 2, + Address: addr1, + }, + { + Value: giftSF / 2, + Address: addr2, + }, + }, + SiafundInputs: []types.V2SiafundInput{{ + Parent: getSFE(t, db, genesisBlock.Transactions[0].SiafundOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + } + testutil.SignV2Transaction(cm.TipState(), pk1, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, giftSF/2) + testutil.CheckBalance(t, db, addr2, types.ZeroCurrency, types.ZeroCurrency, giftSF/2) +} + +func TestV2FileContract(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + v1FC := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, types.VoidAddress) + v1FC.Filesize = 65 + v2FC := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: 20, + ExpirationHeight: 30, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC.RenterOutput.Value.Add(v2FC.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut), + Address: addr1, + }}, + + FileContracts: []types.V2FileContract{v2FC}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + txn2 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, txn1.SiacoinOutputID(txn1.ID(), 0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + // 1 for txn1, 2 for this transaction + Value: giftSC.Sub(fcOut.Mul64(3)), + Address: addr1, + }}, + + FileContracts: []types.V2FileContract{v2FC, v2FC}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn2) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn2) + + for i := cm.Tip().Height; i < v2FC.ExpirationHeight; i++ { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + + checkV2Transaction(t, db, txn1) + checkV2Transaction(t, db, txn2) +} + +func TestV2FileContractRevert(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + prevState := cm.TipState() + + v1FC := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, types.VoidAddress) + v1FC.Filesize = 65 + v2FC := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: 20, + ExpirationHeight: 30, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC.RenterOutput.Value.Add(v2FC.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut).Sub(fcOut), + Address: addr1, + }}, + + FileContracts: []types.V2FileContract{v2FC, v2FC}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + b1 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn1.V2FileContractID(txn1.ID(), 0), txn1.V2FileContractID(txn1.ID(), 1)}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn1.FileContracts[1], fcs[1]) + } + + { + fcs, err := db.V2ContractRevisions(txn1.V2FileContractID(txn1.ID(), 0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + } + + // revert the block + { + state := prevState + extra := cm.Tip().Height - state.Index.Height + 1 + + var blocks []types.Block + for i := uint64(0); i < extra; i++ { + var bs consensus.V1BlockSupplement + block := testutil.MineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + + if err := consensus.ValidateBlock(state, block, bs); err != nil { + t.Fatal(err) + } + state, _ = consensus.ApplyBlock(state, block, bs, time.Time{}) + } + + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + { + _, err := db.Block(b1.ID()) + if err == nil { + t.Fatal("block should not exist") + } + } + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn1.V2FileContractID(txn1.ID(), 0)}) + if err != nil { + t.Fatal(err) + } + if len(fcs) > 0 { + t.Fatal("contract should not exist") + } + } + + // See if we can spend the genesis input that was spent in reverted block + // We should be able to + txn2 := txn1 + txn2.FileContracts = txn2.FileContracts[:1] + txn2.SiacoinOutputs[0].Value = txn2.SiacoinOutputs[0].Value.Add(fcOut) + txn2.SiacoinInputs[0].Parent = getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)) + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn2) + b2 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + _, err := db.Block(b1.ID()) + if err == nil { + t.Fatal("block should not exist") + } + } + + { + b, err := db.Block(b2.ID()) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn2, b.V2.Transactions[0]) + } + + checkV2Transaction(t, db, txn2) + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn2.V2FileContractID(txn2.ID(), 0)}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractRevisions(txn2.V2FileContractID(txn2.ID(), 0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[0]) + } +} + +func TestV2FileContractKey(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + renterPrivateKey1 := types.GeneratePrivateKey() + renterPublicKey1 := renterPrivateKey1.PublicKey() + + renterPrivateKey2 := types.GeneratePrivateKey() + renterPublicKey2 := renterPrivateKey2.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + v1FC := testutil.PrepareContractFormation(renterPublicKey1, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, types.VoidAddress) + v1FC.Filesize = 65 + v2FC1 := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: 20, + ExpirationHeight: 30, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey1, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC1.RenterOutput.Value.Add(v2FC1.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC1)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut), + Address: addr1, + }}, + FileContracts: []types.V2FileContract{v2FC1}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey1, hostPrivateKey, &txn1) + + b1 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn1.V2FileContractID(txn1.ID(), 0)}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractRevisions(txn1.V2FileContractID(txn1.ID(), 0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + } + + v2FC2 := v2FC1 + v2FC2.RenterPublicKey = renterPublicKey2 + + txn2 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, txn1.SiacoinOutputID(txn1.ID(), 0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Value: txn1.SiacoinOutputs[0].Value.Sub(fcOut), + Address: addr1, + }}, + FileContracts: []types.V2FileContract{v2FC2}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey2, hostPrivateKey, &txn2) + + b2 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn2) + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn2.V2FileContractID(txn2.ID(), 0)}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractRevisions(txn2.V2FileContractID(txn2.ID(), 0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractsKey(renterPublicKey1) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractsKey(renterPublicKey2) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[0]) + } + + { + fcs, err := db.V2ContractsKey(hostPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn2.FileContracts[0], fcs[1]) + } +} + +func TestV2FileContractRevision(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + v1FC := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, types.VoidAddress) + v1FC.Filesize = 65 + v2FC := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: 20, + ExpirationHeight: 30, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC.RenterOutput.Value.Add(v2FC.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut), + Address: addr1, + }}, + FileContracts: []types.V2FileContract{v2FC}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + v2FCRevision := v2FC + v2FCRevision.RevisionNumber++ + txn2 := types.V2Transaction{ + FileContractRevisions: []types.V2FileContractRevision{{ + Parent: getFCE(t, db, txn1.V2FileContractID(txn1.ID(), 0)), + Revision: v2FCRevision, + }}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn2) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn2) + + { + fcs, err := db.V2Contracts([]types.FileContractID{txn1.V2FileContractID(txn1.ID(), 0)}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContractRevisions[0].Revision, fcs[0]) + } + + { + fcs, err := db.V2ContractRevisions(txn1.V2FileContractID(txn1.ID(), 0)) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn2.FileContractRevisions[0].Revision, fcs[1]) + } + + { + fcs, err := db.V2ContractsKey(renterPublicKey) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn2.FileContractRevisions[0].Revision, fcs[0]) + } +} + +func TestV2FileContractResolution(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + v1FC := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, addr2) + v1FC.Filesize = 65 + + data := make([]byte, 2*rhp2.LeafSize) + data[0], data[rhp2.LeafSize] = 1, 1 + v1FC.FileMerkleRoot, _ = rhp2.ReaderRoot(bytes.NewReader(data)) + + v2FC := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: cm.Tip().Height + 3, + ExpirationHeight: cm.Tip().Height + 4, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC.RenterOutput.Value.Add(v2FC.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC)) + + // use identical contracts except for revision number so it is apparent if + // wrong data is retrieved + v2FC0 := v2FC + v2FC0.RevisionNumber = 0 + + v2FC1 := v2FC + v2FC1.RevisionNumber = 1 + + v2FC2 := v2FC + v2FC2.RevisionNumber = 2 + + v2FC3 := v2FC + v2FC3.RevisionNumber = 3 + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut.Mul64(4)), + Address: addr1, + }}, + FileContracts: []types.V2FileContract{v2FC0, v2FC1, v2FC2, v2FC3}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + b1 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + tip1 := cm.Tip() + + v2FC0ID := txn1.V2FileContractID(txn1.ID(), 0) + v2FC1ID := txn1.V2FileContractID(txn1.ID(), 1) + v2FC2ID := txn1.V2FileContractID(txn1.ID(), 2) + v2FC3ID := txn1.V2FileContractID(txn1.ID(), 3) + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC1ID, v2FC2ID, v2FC3ID}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn1.FileContracts[1], fcs[1]) + testutil.CheckV2FC(t, txn1.FileContracts[2], fcs[2]) + testutil.CheckV2FC(t, txn1.FileContracts[3], fcs[3]) + } + + // check that they are all not resolved before resolving + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC1ID, v2FC2ID, v2FC3ID}) + if err != nil { + t.Fatal(err) + } + for _, fc := range fcs { + testutil.Equal(t, "confirmation index", tip1, fc.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fc.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fc.ResolutionType) + testutil.Equal(t, "resolution index", nil, fc.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fc.ResolutionTransactionID) + testutil.Equal(t, "renewed to", nil, fc.RenewedTo) + } + } + + // we will revert back here when we undo the resolutions + prevState := cm.TipState() + + checkV2Transaction(t, db, txn1) + + v2FCFinalRevision := v2FC + v2FCFinalRevision.Filesize-- + v2FCFinalRevision.RevisionNumber = types.MaxRevisionNumber + v2FCNewContract := v2FC + v2FCNewContract.RevisionNumber = 10 + renewal := &types.V2FileContractRenewal{ + NewContract: v2FCNewContract, + FinalRenterOutput: v2FCFinalRevision.RenterOutput, + FinalHostOutput: v2FCFinalRevision.HostOutput, + RenterRollover: types.ZeroCurrency, + HostRollover: types.ZeroCurrency, + } + sce1 := getSCE(t, db, txn1.SiacoinOutputID(txn1.ID(), 0)) + txn2 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce1, + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: sce1.SiacoinOutput.Value.Sub(fcOut), + }}, + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: getFCE(t, db, v2FC1ID), + Resolution: renewal, + }, + }, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn2) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn2, dbTxns[0]) + + fcr := dbTxns[0].FileContractResolutions[0] + testutil.Equal(t, "confirmation index", tip1, fcr.Parent.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fcr.Parent.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionRenewal, *fcr.Parent.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.Parent.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn2.ID(), *fcr.Parent.ResolutionTransactionID) + testutil.Equal(t, "renewed to", v2FC1ID.V2RenewalID(), *fcr.Parent.RenewedTo) + testutil.Equal(t, "renewed from", v2FC1ID, *fcr.Resolution.(*explorer.V2FileContractRenewal).NewContract.RenewedFrom) + } + + b2 := testutil.MineV2Block(cm.TipState(), nil, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + storageProof := &types.V2StorageProof{ + ProofIndex: getCIE(t, db, b2.ID()), + Leaf: [64]byte{1}, + Proof: []types.Hash256{cm.TipState().StorageProofLeafHash([]byte{1})}, + } + + txn3 := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: getFCE(t, db, v2FC2ID), + Resolution: storageProof, + }, + }, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn3) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn3}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn3.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn3, dbTxns[0]) + + fcr := dbTxns[0].FileContractResolutions[0] + testutil.Equal(t, "confirmation index", tip1, fcr.Parent.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fcr.Parent.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionStorageProof, *fcr.Parent.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.Parent.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn3.ID(), *fcr.Parent.ResolutionTransactionID) + } + + txn4 := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: getFCE(t, db, v2FC3ID), + Resolution: new(types.V2FileContractExpiration), + }, + }, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn4) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn4}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn4.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn4, dbTxns[0]) + + fcr := dbTxns[0].FileContractResolutions[0] + testutil.Equal(t, "confirmation index", tip1, fcr.Parent.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fcr.Parent.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionExpiration, *fcr.Parent.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.Parent.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn4.ID(), *fcr.Parent.ResolutionTransactionID) + } + + { + events, err := db.AddressEvents(addr2, 0, math.MaxInt64) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 7, len(events)) + + ev0 := events[0].Data.(explorer.EventV2ContractResolution) + testutil.Equal(t, "event 0 parent ID", v2FC3ID, ev0.Resolution.Parent.ID) + testutil.Equal(t, "event 0 output ID", v2FC3ID.V2RenterOutputID(), ev0.SiacoinElement.ID) + testutil.Equal(t, "event 0 output source", explorer.SourceMissedProofOutput, ev0.SiacoinElement.Source) + testutil.Equal(t, "event 0 missed", true, ev0.Missed) + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn4.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "event 0 resolution", dbTxns[0].FileContractResolutions[0], ev0.Resolution) + } + + ev1 := events[1].Data.(explorer.EventV2ContractResolution) + testutil.Equal(t, "event 1 parent ID", v2FC2ID, ev1.Resolution.Parent.ID) + testutil.Equal(t, "event 1 output ID", v2FC2ID.V2RenterOutputID(), ev1.SiacoinElement.ID) + testutil.Equal(t, "event 1 output source", explorer.SourceValidProofOutput, ev1.SiacoinElement.Source) + testutil.Equal(t, "event 1 missed", false, ev1.Missed) + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn3.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "event 1 resolution", dbTxns[0].FileContractResolutions[0], ev1.Resolution) + } + + ev2 := events[2].Data.(explorer.EventV2ContractResolution) + testutil.Equal(t, "event 2 parent ID", v2FC1ID, ev2.Resolution.Parent.ID) + testutil.Equal(t, "event 2 output ID", v2FC1ID.V2RenterOutputID(), ev2.SiacoinElement.ID) + testutil.Equal(t, "event 2 output source", explorer.SourceValidProofOutput, ev2.SiacoinElement.Source) + testutil.Equal(t, "event 2 missed", false, ev2.Missed) + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "event 2 resolution", dbTxns[0].FileContractResolutions[0], ev2.Resolution) + } + + ev3 := events[3].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn4, explorer.V2Transaction(ev3)) + + ev4 := events[4].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn3, explorer.V2Transaction(ev4)) + + ev5 := events[5].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn2, explorer.V2Transaction(ev5)) + + ev6 := events[6].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn1, explorer.V2Transaction(ev6)) + } + + { + events, err := db.Events([]types.Hash256{types.Hash256(txn4.ID()), types.Hash256(txn3.ID()), types.Hash256(txn2.ID()), types.Hash256(txn1.ID())}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "events", 4, len(events)) + + ev0 := events[0].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn4, explorer.V2Transaction(ev0)) + + ev1 := events[1].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn3, explorer.V2Transaction(ev1)) + + ev2 := events[2].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn2, explorer.V2Transaction(ev2)) + + ev3 := events[3].Data.(explorer.EventV2Transaction) + testutil.CheckV2Transaction(t, txn1, explorer.V2Transaction(ev3)) + } + + // revert the block + { + state := prevState + extra := cm.Tip().Height - state.Index.Height + 1 + + var blocks []types.Block + for i := uint64(0); i < extra; i++ { + var bs consensus.V1BlockSupplement + block := testutil.MineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + + if err := consensus.ValidateBlock(state, block, bs); err != nil { + t.Fatal(err) + } + state, _ = consensus.ApplyBlock(state, block, bs, time.Time{}) + } + + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + // check that they are all not resolved after reverting resolution + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC1ID, v2FC2ID, v2FC3ID}) + if err != nil { + t.Fatal(err) + } + for _, fc := range fcs { + testutil.Equal(t, "confirmation index", tip1, fc.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fc.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fc.ResolutionType) + testutil.Equal(t, "resolution index", nil, fc.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fc.ResolutionTransactionID) + testutil.Equal(t, "renewed to", nil, fc.RenewedTo) + } + } + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC1ID, v2FC2ID, v2FC3ID}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn1.FileContracts[1], fcs[1]) + testutil.CheckV2FC(t, txn1.FileContracts[2], fcs[2]) + testutil.CheckV2FC(t, txn1.FileContracts[3], fcs[3]) + } + + tip, err := db.BestTip(v2FC3.ProofHeight) + if err != nil { + t.Fatal(err) + } + storageProof.ProofIndex = getCIE(t, db, tip.ID) + txn3.FileContractResolutions[0].Parent = getFCE(t, db, v2FC2ID) + txn3.FileContractResolutions[0].Resolution = storageProof + + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn3) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn3}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn3.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn3, dbTxns[0]) + + fcr := dbTxns[0].FileContractResolutions[0] + testutil.Equal(t, "confirmation index", tip1, fcr.Parent.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fcr.Parent.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionStorageProof, *fcr.Parent.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.Parent.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn3.ID(), *fcr.Parent.ResolutionTransactionID) + } + + txn4.FileContractResolutions[0].Parent = getFCE(t, db, v2FC3ID) + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn4) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn4}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn4.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn4, dbTxns[0]) + + fcr := dbTxns[0].FileContractResolutions[0] + testutil.Equal(t, "confirmation index", tip1, fcr.Parent.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fcr.Parent.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionExpiration, *fcr.Parent.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.Parent.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn4.ID(), *fcr.Parent.ResolutionTransactionID) + } + + { + // If we re-added the renewal after the revert this would fail because + // the revision number for v2FC0 would be types.MaxRevisionNumber. + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC1ID, v2FC2ID, v2FC3ID}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + testutil.CheckV2FC(t, txn1.FileContracts[1], fcs[1]) + + testutil.CheckV2FC(t, txn1.FileContracts[2], fcs[2]) + testutil.Equal(t, "renewed to", nil, fcs[2].RenewedTo) + + testutil.CheckV2FC(t, txn1.FileContracts[3], fcs[3]) + } + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC2ID.V2RenewalID()}) + if err != nil { + t.Fatal(err) + } else if len(fcs) != 0 { + t.Fatal("renewed contract should not exist after revert") + } + } +} + +func TestV2FileContractRenewedToFrom(t *testing.T) { + pk1 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr1Policy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk1.PublicKey()))} + + renterPrivateKey := types.GeneratePrivateKey() + renterPublicKey := renterPrivateKey.PublicKey() + + hostPrivateKey := types.GeneratePrivateKey() + hostPublicKey := hostPrivateKey.PublicKey() + + _, genesisBlock, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 + }) + giftSC := genesisBlock.Transactions[0].SiacoinOutputs[0].Value + + // Add a file contract that we will renew + v1FC := testutil.PrepareContractFormation(renterPublicKey, hostPublicKey, types.Siacoins(1), types.Siacoins(1), 100, 105, types.VoidAddress) + v2FC := types.V2FileContract{ + Capacity: v1FC.Filesize, + Filesize: v1FC.Filesize, + FileMerkleRoot: v1FC.FileMerkleRoot, + ProofHeight: cm.Tip().Height + 3, + ExpirationHeight: cm.Tip().Height + 4, + RenterOutput: v1FC.ValidProofOutputs[0], + HostOutput: v1FC.ValidProofOutputs[1], + MissedHostValue: v1FC.MissedProofOutputs[1].Value, + TotalCollateral: v1FC.ValidProofOutputs[0].Value, + RenterPublicKey: renterPublicKey, + HostPublicKey: hostPublicKey, + } + fcOut := v2FC.RenterOutput.Value.Add(v2FC.HostOutput.Value).Add(cm.TipState().V2FileContractTax(v2FC)) + + txn1 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: getSCE(t, db, genesisBlock.Transactions[0].SiacoinOutputID(0)), + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Value: giftSC.Sub(fcOut), + Address: addr1, + }}, + FileContracts: []types.V2FileContract{v2FC}, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn1) + + b1 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + tip1 := cm.Tip() + + v2FC0ID := txn1.V2FileContractID(txn1.ID(), 0) + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2FC(t, txn1.FileContracts[0], fcs[0]) + } + + // Check that it is not resolved and does not have RenewedTo set before + // renewing + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(fcs)", 1, len(fcs)) + + fc := fcs[0] + testutil.Equal(t, "confirmation index", tip1, fc.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fc.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fc.ResolutionType) + testutil.Equal(t, "resolution index", nil, fc.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fc.ResolutionTransactionID) + testutil.Equal(t, "renewed from", nil, fc.RenewedFrom) + testutil.Equal(t, "renewed to", nil, fc.RenewedTo) + } + + // Renew contract + v2FCFinalRevision := v2FC + v2FCFinalRevision.Filesize-- + v2FCFinalRevision.RevisionNumber = types.MaxRevisionNumber + v2FCNewContract := v2FC + v2FCNewContract.RevisionNumber = 10 + renewal := &types.V2FileContractRenewal{ + NewContract: v2FCNewContract, + FinalRenterOutput: v2FCFinalRevision.RenterOutput, + FinalHostOutput: v2FCFinalRevision.HostOutput, + RenterRollover: types.ZeroCurrency, + HostRollover: types.ZeroCurrency, + } + sce1 := getSCE(t, db, txn1.SiacoinOutputID(txn1.ID(), 0)) + txn2 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce1, + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: sce1.SiacoinOutput.Value.Sub(fcOut), + }}, + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: getFCE(t, db, v2FC0ID), + Resolution: renewal, + }, + }, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn2) + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + tip2 := cm.Tip() + // We will revert back here when we undo the resolutions + // This is the point after the first renewal, but before the second renewal + prevState := cm.TipState() + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC0ID.V2RenewalID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(fcs)", 2, len(fcs)) + + fc := fcs[0] + testutil.Equal(t, "confirmation index", tip1, fc.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fc.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionRenewal, *fc.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fc.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn2.ID(), *fc.ResolutionTransactionID) + testutil.Equal(t, "renewed from", nil, fc.RenewedFrom) + testutil.Equal(t, "renewed to", v2FC0ID.V2RenewalID(), *fc.RenewedTo) + + fcr := fcs[1] + testutil.Equal(t, "confirmation index", cm.Tip(), fcr.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn2.ID(), fcr.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fcr.ResolutionType) + testutil.Equal(t, "resolution index", nil, fcr.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fcr.ResolutionTransactionID) + testutil.Equal(t, "renewed from", v2FC0ID, *fcr.RenewedFrom) + testutil.Equal(t, "renewed to", nil, fcr.RenewedTo) + } + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID.V2RenewalID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(fcs)", 1, len(fcs)) + } + + // Renew again + sce2 := getSCE(t, db, txn2.SiacoinOutputID(txn2.ID(), 0)) + txn3 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce2, + SatisfiedPolicy: types.SatisfiedPolicy{Policy: addr1Policy}, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: addr1, + Value: sce2.SiacoinOutput.Value.Sub(fcOut), + }}, + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: getFCE(t, db, v2FC0ID.V2RenewalID()), + Resolution: renewal, + }, + }, + } + testutil.SignV2TransactionWithContracts(cm.TipState(), pk1, renterPrivateKey, hostPrivateKey, &txn3) + + b3 := testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn3}, types.VoidAddress) + if err := cm.AddBlocks([]types.Block{b3}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID.V2RenewalID(), v2FC0ID.V2RenewalID().V2RenewalID()}) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "len(fcs)", 2, len(fcs)) + + fcr := fcs[0] + testutil.Equal(t, "confirmation index", tip2, fcr.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn2.ID(), fcr.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionRenewal, *fcr.ResolutionType) + testutil.Equal(t, "resolution index", cm.Tip(), *fcr.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn3.ID(), *fcr.ResolutionTransactionID) + testutil.Equal(t, "renewed from", v2FC0ID, *fcr.RenewedFrom) + testutil.Equal(t, "renewed to", v2FC0ID.V2RenewalID().V2RenewalID(), *fcr.RenewedTo) + + fcrr := fcs[1] + testutil.Equal(t, "confirmation index", cm.Tip(), fcrr.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn3.ID(), fcrr.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fcrr.ResolutionType) + testutil.Equal(t, "resolution index", nil, fcrr.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fcrr.ResolutionTransactionID) + testutil.Equal(t, "renewed from", v2FC0ID.V2RenewalID(), *fcrr.RenewedFrom) + testutil.Equal(t, "renewed to", nil, fcrr.RenewedTo) + } + + // Revert the second renewal + { + state := prevState + extra := cm.Tip().Height - state.Index.Height + 1 + + var blocks []types.Block + for i := uint64(0); i < extra; i++ { + var bs consensus.V1BlockSupplement + block := testutil.MineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + + if err := consensus.ValidateBlock(state, block, bs); err != nil { + t.Fatal(err) + } + state, _ = consensus.ApplyBlock(state, block, bs, time.Time{}) + } + + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + } + + { + fcs, err := db.V2Contracts([]types.FileContractID{v2FC0ID, v2FC0ID.V2RenewalID(), v2FC0ID.V2RenewalID().V2RenewalID()}) + if err != nil { + t.Fatal(err) + } + // Second renewal should not exist so we should only expect 2 results + testutil.Equal(t, "len(fcs)", 2, len(fcs)) + + fc := fcs[0] + testutil.Equal(t, "confirmation index", tip1, fc.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn1.ID(), fc.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", explorer.V2ResolutionRenewal, *fc.ResolutionType) + testutil.Equal(t, "resolution index", tip2, *fc.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", txn2.ID(), *fc.ResolutionTransactionID) + testutil.Equal(t, "renewed from", nil, fc.RenewedFrom) + testutil.Equal(t, "renewed to", v2FC0ID.V2RenewalID(), *fc.RenewedTo) + + fcr := fcs[1] + testutil.Equal(t, "confirmation index", tip2, fcr.ConfirmationIndex) + testutil.Equal(t, "confirmation transaction ID", txn2.ID(), fcr.ConfirmationTransactionID) + testutil.Equal(t, "resolution type", nil, fcr.ResolutionType) + testutil.Equal(t, "resolution index", nil, fcr.ResolutionIndex) + testutil.Equal(t, "resolution transaction ID", nil, fcr.ResolutionTransactionID) + testutil.Equal(t, "renewed from", v2FC0ID, *fcr.RenewedFrom) + testutil.Equal(t, "renewed to", nil, fcr.RenewedTo) + } +} + +func TestBlockSameV2Transaction(t *testing.T) { + pk1 := types.GeneratePrivateKey() + + _, _, cm, db := newStore(t, true, func(network *consensus.Network, genesisBlock types.Block) { + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + }) + cs := cm.TipState() + + att1 := types.Attestation{ + PublicKey: pk1.PublicKey(), + Key: "hello", + Value: []byte("world"), + } + att1.Signature = pk1.SignHash(cs.AttestationSigHash(att1)) + + att2 := types.Attestation{ + PublicKey: pk1.PublicKey(), + Key: "123", + Value: []byte("456"), + } + att2.Signature = pk1.SignHash(cs.AttestationSigHash(att2)) + + txn1 := types.V2Transaction{ + Attestations: []types.Attestation{att1}, + } + txn2 := types.V2Transaction{ + Attestations: []types.Attestation{att1, att2}, + } + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cs, []types.V2Transaction{txn1, txn1, txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + checkV2Transaction(t, db, txn1) + checkV2Transaction(t, db, txn2) +} diff --git a/persist/sqlite/v2contracts.go b/persist/sqlite/v2contracts.go new file mode 100644 index 0000000..e54389f --- /dev/null +++ b/persist/sqlite/v2contracts.go @@ -0,0 +1,133 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +func scanV2FileContract(s scanner) (fce explorer.V2FileContract, err error) { + var resolutionType sql.Null[explorer.V2Resolution] + var resolutionIndex types.ChainIndex + var resolutionTransactionID types.TransactionID + var renewedFrom, renewedTo types.FileContractID + + fc := &fce.V2FileContractElement.V2FileContract + if err = s.Scan(decode(&fce.TransactionID), decode(&fce.ConfirmationIndex.Height), decode(&fce.ConfirmationIndex.ID), decode(&fce.ConfirmationTransactionID), &resolutionType, decodeNull(&resolutionIndex.Height), decodeNull(&resolutionIndex.ID), decodeNull(&resolutionTransactionID), decodeNull(&renewedFrom), decodeNull(&renewedTo), decode(&fce.V2FileContractElement.ID), decode(&fce.V2FileContractElement.StateElement.LeafIndex), decode(&fc.Capacity), decode(&fc.Filesize), decode(&fc.FileMerkleRoot), decode(&fc.ProofHeight), decode(&fc.ExpirationHeight), decode(&fc.RenterOutput.Address), decode(&fc.RenterOutput.Value), decode(&fc.HostOutput.Address), decode(&fc.HostOutput.Value), decode(&fc.MissedHostValue), decode(&fc.TotalCollateral), decode(&fc.RenterPublicKey), decode(&fc.HostPublicKey), decode(&fc.RevisionNumber), decode(&fc.RenterSignature), decode(&fc.HostSignature)); err != nil { + return + } + + if resolutionType.Valid && resolutionType.V != explorer.V2ResolutionInvalid { + fce.ResolutionType = &resolutionType.V + } + if resolutionIndex != (types.ChainIndex{}) { + fce.ResolutionIndex = &resolutionIndex + } + if resolutionTransactionID != (types.TransactionID{}) { + fce.ResolutionTransactionID = &resolutionTransactionID + } + if renewedFrom != (types.FileContractID{}) { + fce.RenewedFrom = &renewedFrom + } + if renewedTo != (types.FileContractID{}) { + fce.RenewedTo = &renewedTo + } + + return +} + +// V2Contracts implements explorer.Store. +func (s *Store) V2Contracts(ids []types.FileContractID) (result []explorer.V2FileContract, err error) { + err = s.transaction(func(tx *txn) error { + stmt, err := tx.Prepare(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_last_contract_revision rev +INNER JOIN v2_file_contract_elements fc ON rev.contract_element_id = fc.id +WHERE rev.contract_id = ? +`) + if err != nil { + return err + } + defer stmt.Close() + + for _, id := range ids { + fc, err := scanV2FileContract(stmt.QueryRow(encode(id))) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to scan file contract: %w", err) + } else if err == nil { + fc.V2FileContractElement.StateElement.MerkleProof, err = s.MerkleProof(fc.V2FileContractElement.StateElement.LeafIndex) + if err != nil { + return fmt.Errorf("failed to get contract merkle proof: %w", err) + } + result = append(result, fc) + } + } + + return nil + }) + + return +} + +// V2ContractRevisions implements explorer.Store. +func (s *Store) V2ContractRevisions(id types.FileContractID) (revisions []explorer.V2FileContract, err error) { + err = s.transaction(func(tx *txn) error { + query := `SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_file_contract_elements fc +INNER JOIN v2_last_contract_revision rev ON rev.contract_id = fc.contract_id +WHERE fc.contract_id = ? +ORDER BY fc.revision_number ASC +` + rows, err := tx.Query(query, encode(id)) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + fc, err := scanV2FileContract(rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + + revisions = append(revisions, fc) + } + + if len(revisions) == 0 { + return explorer.ErrContractNotFound + } + return nil + }) + return +} + +// V2ContractsKey implements explorer.Store. +func (s *Store) V2ContractsKey(key types.PublicKey) (result []explorer.V2FileContract, err error) { + err = s.transaction(func(tx *txn) error { + encoded := encode(key) + rows, err := tx.Query(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_last_contract_revision rev +INNER JOIN v2_file_contract_elements fc ON rev.contract_element_id = fc.id +WHERE fc.renter_public_key = ? OR fc.host_public_key = ? +ORDER BY rev.confirmation_height ASC +`, encoded, encoded) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + fc, err := scanV2FileContract(rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + result = append(result, fc) + } + + return nil + }) + + return +} diff --git a/persist/sqlite/v2transactions.go b/persist/sqlite/v2transactions.go new file mode 100644 index 0000000..f7136a4 --- /dev/null +++ b/persist/sqlite/v2transactions.go @@ -0,0 +1,557 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/explored/explorer" +) + +// V2TransactionChainIndices returns the chain indices of the blocks the v2 +// transaction was included in. If the transaction has not been included in +// any blocks, the result will be nil,nil. +func (s *Store) V2TransactionChainIndices(txnID types.TransactionID, offset, limit uint64) (indices []types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT DISTINCT b.id, b.height FROM blocks b +INNER JOIN v2_block_transactions bt ON (bt.block_id = b.id) +INNER JOIN v2_transactions t ON (t.id = bt.transaction_id) +WHERE t.transaction_id = ? +ORDER BY b.height DESC +LIMIT ? OFFSET ?`, encode(txnID), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var index types.ChainIndex + if err := rows.Scan(decode(&index.ID), decode(&index.Height)); err != nil { + return fmt.Errorf("failed to scan chain index: %w", err) + } + indices = append(indices, index) + } + return rows.Err() + }) + return +} + +// blockV2TransactionIDs returns the transaction id as a types.TransactionID +// for each v2 transaction in the block. +func blockV2TransactionIDs(tx *txn, blockID types.BlockID) (ids []types.TransactionID, err error) { + rows, err := tx.Query(`SELECT t.transaction_id +FROM v2_block_transactions bt +INNER JOIN v2_transactions t ON (t.id = bt.transaction_id) +WHERE block_id = ? ORDER BY block_order ASC`, encode(blockID)) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var id types.TransactionID + if err := rows.Scan(decode(&id)); err != nil { + return nil, fmt.Errorf("failed to scan block transaction: %w", err) + } + ids = append(ids, id) + } + return +} + +// getV2Transactions fetches v2 transactions in the correct order using +// prepared statements. +func getV2Transactions(tx *txn, ids []types.TransactionID) ([]explorer.V2Transaction, error) { + dbIDs, txns, err := getV2TransactionBase(tx, ids) + if err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get base transactions: %w", err) + } else if err := fillV2TransactionAttestations(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get attestations: %w", err) + } else if err := fillV2TransactionSiacoinInputs(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get siacoin inputs: %w", err) + } else if err := fillV2TransactionSiacoinOutputs(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get siacoin outputs: %w", err) + } else if err := fillV2TransactionSiafundInputs(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get siafund inputs: %w", err) + } else if err := fillV2TransactionSiafundOutputs(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get siafund outputs: %w", err) + } else if err := fillV2TransactionFileContracts(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get file contracts: %w", err) + } else if err := fillV2TransactionFileContractRevisions(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get file contract revisions: %w", err) + } else if err := fillV2TransactionFileContractResolutions(tx, dbIDs, txns); err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get file contract resolutions: %w", err) + } + + // add host announcements if we have any + for i := range txns { + for _, attestation := range txns[i].Attestations { + var ha chain.V2HostAnnouncement + if ha.FromAttestation(attestation) == nil { + txns[i].HostAnnouncements = append(txns[i].HostAnnouncements, explorer.V2HostAnnouncement{ + V2HostAnnouncement: ha, + PublicKey: attestation.PublicKey, + }) + } + } + } + return txns, nil +} + +// getV2TransactionBase fetches the base transaction data for a given list of +// transaction IDs. +func getV2TransactionBase(tx *txn, txnIDs []types.TransactionID) ([]int64, []explorer.V2Transaction, error) { + stmt, err := tx.Prepare(`SELECT id, transaction_id, new_foundation_address, miner_fee, arbitrary_data FROM v2_transactions WHERE transaction_id = ?`) + if err != nil { + return nil, nil, fmt.Errorf("getV2TransactionBase: failed to prepare statement: %w", err) + } + defer stmt.Close() + + var dbID int64 + dbIDs := make([]int64, 0, len(txnIDs)) + txns := make([]explorer.V2Transaction, 0, len(txnIDs)) + for _, id := range txnIDs { + var txn explorer.V2Transaction + var newFoundationAddress types.Address + if err := stmt.QueryRow(encode(id)).Scan(&dbID, decode(&txn.ID), decodeNull(&newFoundationAddress), decode(&txn.MinerFee), &txn.ArbitraryData); errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return nil, nil, fmt.Errorf("failed to scan base transaction: %w", err) + } + if (newFoundationAddress != types.Address{}) { + txn.NewFoundationAddress = &newFoundationAddress + } + + dbIDs = append(dbIDs, dbID) + txns = append(txns, txn) + } + return dbIDs, txns, nil +} + +// fillV2TransactionAttestations fills in the attestations for each +// transaction. +func fillV2TransactionAttestations(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT public_key, key, value, signature FROM v2_transaction_attestations WHERE transaction_id = ? ORDER BY transaction_order`) + if err != nil { + return fmt.Errorf("failed to prepare attestations statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query attestations: %w", err) + } + defer rows.Close() + + for rows.Next() { + var attestation types.Attestation + if err := rows.Scan(decode(&attestation.PublicKey), &attestation.Key, &attestation.Value, decode(&attestation.Signature)); err != nil { + return fmt.Errorf("failed to scan attestation: %w", err) + } + txns[i].Attestations = append(txns[i].Attestations, attestation) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionSiacoinInputs fills in the siacoin inputs for each +// transaction. +func fillV2TransactionSiacoinInputs(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT ts.satisfied_policy, sc.output_id, sc.leaf_index, sc.maturity_height, sc.address, sc.value +FROM siacoin_elements sc +INNER JOIN v2_transaction_siacoin_inputs ts ON (ts.parent_id = sc.id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin inputs statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query siacoin inputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + var sci types.V2SiacoinInput + if err := rows.Scan(decode(&sci.SatisfiedPolicy), decode(&sci.Parent.ID), decode(&sci.Parent.StateElement.LeafIndex), &sci.Parent.MaturityHeight, decode(&sci.Parent.SiacoinOutput.Address), decode(&sci.Parent.SiacoinOutput.Value)); err != nil { + return fmt.Errorf("failed to scan siacoin inputs: %w", err) + } + + txns[i].SiacoinInputs = append(txns[i].SiacoinInputs, sci) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionSiacoinOutputs fills in the siacoin outputs for each +// transaction. +func fillV2TransactionSiacoinOutputs(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT sc.output_id, sc.leaf_index, sc.spent_index, sc.source, sc.maturity_height, sc.address, sc.value +FROM siacoin_elements sc +INNER JOIN v2_transaction_siacoin_outputs ts ON (ts.output_id = sc.id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin outputs statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query siacoin outputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + var spentIndex types.ChainIndex + var sco explorer.SiacoinOutput + if err := rows.Scan(decode(&sco.ID), decode(&sco.StateElement.LeafIndex), decodeNull(&spentIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { + return fmt.Errorf("failed to scan siacoin output: %w", err) + } + + if spentIndex != (types.ChainIndex{}) { + sco.SpentIndex = &spentIndex + } + txns[i].SiacoinOutputs = append(txns[i].SiacoinOutputs, sco) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionSiafundInputs fills in the siacoin inputs for each +// transaction. +func fillV2TransactionSiafundInputs(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT ts.satisfied_policy, ts.claim_address, sf.output_id, sf.leaf_index, sf.address, sf.value +FROM siafund_elements sf +INNER JOIN v2_transaction_siafund_inputs ts ON (ts.parent_id = sf.id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin inputs statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query siacoin inputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + var sfi types.V2SiafundInput + if err := rows.Scan(decode(&sfi.SatisfiedPolicy), decode(&sfi.ClaimAddress), decode(&sfi.Parent.ID), decode(&sfi.Parent.StateElement.LeafIndex), decode(&sfi.Parent.SiafundOutput.Address), decode(&sfi.Parent.SiafundOutput.Value)); err != nil { + return fmt.Errorf("failed to scan siacoin inputs: %w", err) + } + + txns[i].SiafundInputs = append(txns[i].SiafundInputs, sfi) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionSiafundOutputs fills in the siafund outputs for each +// transaction. +func fillV2TransactionSiafundOutputs(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT sf.output_id, sf.leaf_index, sf.spent_index, sf.claim_start, sf.address, sf.value +FROM siafund_elements sf +INNER JOIN v2_transaction_siafund_outputs ts ON (ts.output_id = sf.id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare siafund outputs statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query siafund outputs: %w", err) + } + defer rows.Close() + + for rows.Next() { + var spentIndex types.ChainIndex + var sfo explorer.SiafundOutput + if err := rows.Scan(decode(&sfo.ID), decode(&sfo.StateElement.LeafIndex), decodeNull(&spentIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { + return fmt.Errorf("failed to scan siafund output: %w", err) + } + if spentIndex != (types.ChainIndex{}) { + sfo.SpentIndex = &spentIndex + } + + txns[i].SiafundOutputs = append(txns[i].SiafundOutputs, sfo) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionFileContracts fills in the file contracts for each +// transaction. +func fillV2TransactionFileContracts(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + stmt, err := tx.Prepare(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_file_contract_elements fc +INNER JOIN v2_transaction_file_contracts ts ON (ts.contract_id = fc.id) +INNER JOIN v2_last_contract_revision rev ON (rev.contract_id = fc.contract_id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare file contracts statement: %w", err) + } + defer stmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := stmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query file contracts: %w", err) + } + defer rows.Close() + + for rows.Next() { + fce, err := scanV2FileContract(rows) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + + txns[i].FileContracts = append(txns[i].FileContracts, fce) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// fillV2TransactionFileContractRevisions fills in the file contract revisions +// for each transaction. +func fillV2TransactionFileContractRevisions(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + parentStmt, err := tx.Prepare(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_file_contract_elements fc +INNER JOIN v2_transaction_file_contract_revisions ts ON (ts.parent_contract_id = fc.id) +INNER JOIN v2_last_contract_revision rev ON (rev.contract_id = fc.contract_id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare file contracts parent statement: %w", err) + } + defer parentStmt.Close() + + revisionStmt, err := tx.Prepare(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_file_contract_elements fc +INNER JOIN v2_transaction_file_contract_revisions ts ON (ts.revision_contract_id = fc.id) +INNER JOIN v2_last_contract_revision rev ON (rev.contract_id = fc.contract_id) +WHERE ts.transaction_id = ? +ORDER BY ts.transaction_order ASC`) + if err != nil { + return fmt.Errorf("failed to prepare file contracts revision statement: %w", err) + } + defer revisionStmt.Close() + + collectFileContracts := func(stmt *stmt, dbID int64) ([]explorer.V2FileContract, error) { + rows, err := stmt.Query(dbID) + if err != nil { + return nil, fmt.Errorf("failed to query file contracts: %w", err) + } + defer rows.Close() + + var contracts []explorer.V2FileContract + for rows.Next() { + fce, err := scanV2FileContract(rows) + if err != nil { + return nil, fmt.Errorf("failed to scan file contract: %w", err) + } + contracts = append(contracts, fce) + } + return contracts, nil + } + + for i, dbID := range dbIDs { + parents, err := collectFileContracts(parentStmt, dbID) + if err != nil { + return err + } + + revisions, err := collectFileContracts(revisionStmt, dbID) + if err != nil { + return err + } + + for j := range parents { + fcr := explorer.V2FileContractRevision{ + Parent: parents[j], + } + if j < len(revisions) { + fcr.Revision = revisions[j] + } + + txns[i].FileContractRevisions = append(txns[i].FileContractRevisions, fcr) + } + } + + return nil +} + +// fillV2TransactionFileContractResolutions fills in the file contract +// resolutions for each transaction. +func fillV2TransactionFileContractResolutions(tx *txn, dbIDs []int64, txns []explorer.V2Transaction) error { + consolidatedStmt, err := tx.Prepare(` + SELECT + parent_contract_id, resolution_type, + renewal_new_contract_id, + renewal_final_renter_output_address, renewal_final_renter_output_value, + renewal_final_host_output_address, renewal_final_host_output_value, + renewal_renter_rollover, renewal_host_rollover, + renewal_renter_signature, renewal_host_signature, + storage_proof_proof_index, storage_proof_leaf, storage_proof_proof + FROM v2_transaction_file_contract_resolutions + WHERE transaction_id = ? + ORDER BY transaction_order + `) + if err != nil { + return fmt.Errorf("failed to prepare consolidated statement: %w", err) + } + defer consolidatedStmt.Close() + + // get a v2 FC by id + fcStmt, err := tx.Prepare(`SELECT fc.transaction_id, rev.confirmation_height, rev.confirmation_block_id, rev.confirmation_transaction_id, rev.resolution_type, rev.resolution_height, rev.resolution_block_id, rev.resolution_transaction_id, rev.renewed_from, rev.renewed_to, fc.contract_id, fc.leaf_index, fc.capacity, fc.filesize, fc.file_merkle_root, fc.proof_height, fc.expiration_height, fc.renter_output_address, fc.renter_output_value, fc.host_output_address, fc.host_output_value, fc.missed_host_value, fc.total_collateral, fc.renter_public_key, fc.host_public_key, fc.revision_number, fc.renter_signature, fc.host_signature +FROM v2_file_contract_elements fc +INNER JOIN v2_last_contract_revision rev ON (rev.contract_id = fc.contract_id) +WHERE fc.id = ?`) + if err != nil { + return fmt.Errorf("failed to prepare file contracts statement: %w", err) + } + defer fcStmt.Close() + + for i, dbID := range dbIDs { + err := func() error { + rows, err := consolidatedStmt.Query(dbID) + if err != nil { + return fmt.Errorf("failed to query file contract resolutions: %w", err) + } + defer rows.Close() + + for rows.Next() { + // all + var parentContractID, resolutionType int64 + // renewal + var renewalNewContractID sql.NullInt64 + var finalRenterOutput, finalHostOutput types.SiacoinOutput + var renewalRenterRollover, renewalHostRollover types.Currency + var renewalRenterSignature, renewalHostSignature types.Signature + // storage proof + var storageProofProofIndex types.ChainIndexElement + var storageProofProof []types.Hash256 + var storageProofLeaf []byte + + // Scan all fields, some of which may be NULL + if err := rows.Scan( + &parentContractID, &resolutionType, + &renewalNewContractID, + decodeNull(&finalRenterOutput.Address), decodeNull(&finalRenterOutput.Value), + decodeNull(&finalHostOutput.Address), decodeNull(&finalHostOutput.Value), + decodeNull(&renewalRenterRollover), decodeNull(&renewalHostRollover), + decodeNull(&renewalRenterSignature), decodeNull(&renewalHostSignature), + decodeNull(&storageProofProofIndex), &storageProofLeaf, decodeNull(&storageProofProof)); err != nil { + return fmt.Errorf("failed to scan resolution metadata: %w", err) + } + + // Retrieve parent contract element + parent, err := scanV2FileContract(fcStmt.QueryRow(parentContractID)) + if err != nil { + return fmt.Errorf("failed to scan file contract: %w", err) + } + + fcr := explorer.V2FileContractResolution{ + Parent: parent, + Type: explorer.V2Resolution(resolutionType), + } + switch fcr.Type { + case explorer.V2ResolutionRenewal: + renewal := &explorer.V2FileContractRenewal{ + FinalRenterOutput: finalRenterOutput, + FinalHostOutput: finalHostOutput, + RenterRollover: renewalRenterRollover, + HostRollover: renewalHostRollover, + RenterSignature: renewalRenterSignature, + HostSignature: renewalHostSignature, + } + if renewalNewContractID.Valid { + renewal.NewContract, err = scanV2FileContract(fcStmt.QueryRow(renewalNewContractID.Int64)) + if err != nil { + return fmt.Errorf("failed to scan new contract: %w", err) + } + } + fcr.Resolution = renewal + case explorer.V2ResolutionStorageProof: + proof := &types.V2StorageProof{ + ProofIndex: storageProofProofIndex, + Proof: storageProofProof, + Leaf: [64]byte(storageProofLeaf), + } + fcr.Resolution = proof + case explorer.V2ResolutionExpiration: + fcr.Resolution = new(types.V2FileContractExpiration) + } + + // Append the resolution to the transaction. + txns[i].FileContractResolutions = append(txns[i].FileContractResolutions, fcr) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} + +// V2Transactions implements explorer.Store. +func (s *Store) V2Transactions(ids []types.TransactionID) (results []explorer.V2Transaction, err error) { + err = s.transaction(func(tx *txn) error { + results, err = getV2Transactions(tx, ids) + if err != nil { + return fmt.Errorf("failed to get transactions: %w", err) + } + return err + }) + return +}